1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  22 /*        All Rights Reserved   */
  23 
  24 /*
  25  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
  26  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  27  */
  28 
  29 #include <sys/param.h>
  30 #include <sys/t_lock.h>
  31 #include <sys/types.h>
  32 #include <sys/tuneable.h>
  33 #include <sys/sysmacros.h>
  34 #include <sys/systm.h>
  35 #include <sys/cpuvar.h>
  36 #include <sys/lgrp.h>
  37 #include <sys/user.h>
  38 #include <sys/proc.h>
  39 #include <sys/callo.h>
  40 #include <sys/kmem.h>
  41 #include <sys/var.h>
  42 #include <sys/cmn_err.h>
  43 #include <sys/swap.h>
  44 #include <sys/vmsystm.h>
  45 #include <sys/class.h>
  46 #include <sys/time.h>
  47 #include <sys/debug.h>
  48 #include <sys/vtrace.h>
  49 #include <sys/spl.h>
  50 #include <sys/atomic.h>
  51 #include <sys/dumphdr.h>
  52 #include <sys/archsystm.h>
  53 #include <sys/fs/swapnode.h>
  54 #include <sys/panic.h>
  55 #include <sys/disp.h>
  56 #include <sys/msacct.h>
  57 #include <sys/mem_cage.h>
  58 
  59 #include <vm/page.h>
  60 #include <vm/anon.h>
  61 #include <vm/rm.h>
  62 #include <sys/cyclic.h>
  63 #include <sys/cpupart.h>
  64 #include <sys/rctl.h>
  65 #include <sys/task.h>
  66 #include <sys/sdt.h>
  67 #include <sys/ddi_timer.h>
  68 #include <sys/random.h>
  69 #include <sys/modctl.h>
  70 #include <sys/zone.h>
  71 
  72 /*
  73  * for NTP support
  74  */
  75 #include <sys/timex.h>
  76 #include <sys/inttypes.h>
  77 
  78 #include <sys/sunddi.h>
  79 #include <sys/clock_impl.h>
  80 
  81 /*
  82  * clock() is called straight from the clock cyclic; see clock_init().
  83  *
  84  * Functions:
  85  *      reprime clock
  86  *      maintain date
  87  *      jab the scheduler
  88  */
  89 
  90 extern kcondvar_t       fsflush_cv;
  91 extern sysinfo_t        sysinfo;
  92 extern vminfo_t vminfo;
  93 extern int      idleswtch;      /* flag set while idle in pswtch() */
  94 extern hrtime_t volatile devinfo_freeze;
  95 
  96 /*
  97  * high-precision avenrun values.  These are needed to make the
  98  * regular avenrun values accurate.
  99  */
 100 static uint64_t hp_avenrun[3];
 101 int     avenrun[3];             /* FSCALED average run queue lengths */
 102 time_t  time;   /* time in seconds since 1970 - for compatibility only */
 103 
 104 static struct loadavg_s loadavg;
 105 /*
 106  * Phase/frequency-lock loop (PLL/FLL) definitions
 107  *
 108  * The following variables are read and set by the ntp_adjtime() system
 109  * call.
 110  *
 111  * time_state shows the state of the system clock, with values defined
 112  * in the timex.h header file.
 113  *
 114  * time_status shows the status of the system clock, with bits defined
 115  * in the timex.h header file.
 116  *
 117  * time_offset is used by the PLL/FLL to adjust the system time in small
 118  * increments.
 119  *
 120  * time_constant determines the bandwidth or "stiffness" of the PLL.
 121  *
 122  * time_tolerance determines maximum frequency error or tolerance of the
 123  * CPU clock oscillator and is a property of the architecture; however,
 124  * in principle it could change as result of the presence of external
 125  * discipline signals, for instance.
 126  *
 127  * time_precision is usually equal to the kernel tick variable; however,
 128  * in cases where a precision clock counter or external clock is
 129  * available, the resolution can be much less than this and depend on
 130  * whether the external clock is working or not.
 131  *
 132  * time_maxerror is initialized by a ntp_adjtime() call and increased by
 133  * the kernel once each second to reflect the maximum error bound
 134  * growth.
 135  *
 136  * time_esterror is set and read by the ntp_adjtime() call, but
 137  * otherwise not used by the kernel.
 138  */
 139 int32_t time_state = TIME_OK;   /* clock state */
 140 int32_t time_status = STA_UNSYNC;       /* clock status bits */
 141 int32_t time_offset = 0;                /* time offset (us) */
 142 int32_t time_constant = 0;              /* pll time constant */
 143 int32_t time_tolerance = MAXFREQ;       /* frequency tolerance (scaled ppm) */
 144 int32_t time_precision = 1;     /* clock precision (us) */
 145 int32_t time_maxerror = MAXPHASE;       /* maximum error (us) */
 146 int32_t time_esterror = MAXPHASE;       /* estimated error (us) */
 147 
 148 /*
 149  * The following variables establish the state of the PLL/FLL and the
 150  * residual time and frequency offset of the local clock. The scale
 151  * factors are defined in the timex.h header file.
 152  *
 153  * time_phase and time_freq are the phase increment and the frequency
 154  * increment, respectively, of the kernel time variable.
 155  *
 156  * time_freq is set via ntp_adjtime() from a value stored in a file when
 157  * the synchronization daemon is first started. Its value is retrieved
 158  * via ntp_adjtime() and written to the file about once per hour by the
 159  * daemon.
 160  *
 161  * time_adj is the adjustment added to the value of tick at each timer
 162  * interrupt and is recomputed from time_phase and time_freq at each
 163  * seconds rollover.
 164  *
 165  * time_reftime is the second's portion of the system time at the last
 166  * call to ntp_adjtime(). It is used to adjust the time_freq variable
 167  * and to increase the time_maxerror as the time since last update
 168  * increases.
 169  */
 170 int32_t time_phase = 0;         /* phase offset (scaled us) */
 171 int32_t time_freq = 0;          /* frequency offset (scaled ppm) */
 172 int32_t time_adj = 0;           /* tick adjust (scaled 1 / hz) */
 173 int32_t time_reftime = 0;               /* time at last adjustment (s) */
 174 
 175 /*
 176  * The scale factors of the following variables are defined in the
 177  * timex.h header file.
 178  *
 179  * pps_time contains the time at each calibration interval, as read by
 180  * microtime(). pps_count counts the seconds of the calibration
 181  * interval, the duration of which is nominally pps_shift in powers of
 182  * two.
 183  *
 184  * pps_offset is the time offset produced by the time median filter
 185  * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
 186  * this filter.
 187  *
 188  * pps_freq is the frequency offset produced by the frequency median
 189  * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
 190  * by this filter.
 191  *
 192  * pps_usec is latched from a high resolution counter or external clock
 193  * at pps_time. Here we want the hardware counter contents only, not the
 194  * contents plus the time_tv.usec as usual.
 195  *
 196  * pps_valid counts the number of seconds since the last PPS update. It
 197  * is used as a watchdog timer to disable the PPS discipline should the
 198  * PPS signal be lost.
 199  *
 200  * pps_glitch counts the number of seconds since the beginning of an
 201  * offset burst more than tick/2 from current nominal offset. It is used
 202  * mainly to suppress error bursts due to priority conflicts between the
 203  * PPS interrupt and timer interrupt.
 204  *
 205  * pps_intcnt counts the calibration intervals for use in the interval-
 206  * adaptation algorithm. It's just too complicated for words.
 207  */
 208 struct timeval pps_time;        /* kernel time at last interval */
 209 int32_t pps_tf[] = {0, 0, 0};   /* pps time offset median filter (us) */
 210 int32_t pps_offset = 0;         /* pps time offset (us) */
 211 int32_t pps_jitter = MAXTIME;   /* time dispersion (jitter) (us) */
 212 int32_t pps_ff[] = {0, 0, 0};   /* pps frequency offset median filter */
 213 int32_t pps_freq = 0;           /* frequency offset (scaled ppm) */
 214 int32_t pps_stabil = MAXFREQ;   /* frequency dispersion (scaled ppm) */
 215 int32_t pps_usec = 0;           /* microsec counter at last interval */
 216 int32_t pps_valid = PPS_VALID;  /* pps signal watchdog counter */
 217 int32_t pps_glitch = 0;         /* pps signal glitch counter */
 218 int32_t pps_count = 0;          /* calibration interval counter (s) */
 219 int32_t pps_shift = PPS_SHIFT;  /* interval duration (s) (shift) */
 220 int32_t pps_intcnt = 0;         /* intervals at current duration */
 221 
 222 /*
 223  * PPS signal quality monitors
 224  *
 225  * pps_jitcnt counts the seconds that have been discarded because the
 226  * jitter measured by the time median filter exceeds the limit MAXTIME
 227  * (100 us).
 228  *
 229  * pps_calcnt counts the frequency calibration intervals, which are
 230  * variable from 4 s to 256 s.
 231  *
 232  * pps_errcnt counts the calibration intervals which have been discarded
 233  * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
 234  * calibration interval jitter exceeds two ticks.
 235  *
 236  * pps_stbcnt counts the calibration intervals that have been discarded
 237  * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
 238  */
 239 int32_t pps_jitcnt = 0;         /* jitter limit exceeded */
 240 int32_t pps_calcnt = 0;         /* calibration intervals */
 241 int32_t pps_errcnt = 0;         /* calibration errors */
 242 int32_t pps_stbcnt = 0;         /* stability limit exceeded */
 243 
 244 kcondvar_t lbolt_cv;
 245 
 246 /*
 247  * Hybrid lbolt implementation:
 248  *
 249  * The service historically provided by the lbolt and lbolt64 variables has
 250  * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
 251  * original symbols removed from the system. The once clock driven variables are
 252  * now implemented in an event driven fashion, backed by gethrtime() coarsed to
 253  * the appropriate clock resolution. The default event driven implementation is
 254  * complemented by a cyclic driven one, active only during periods of intense
 255  * activity around the DDI lbolt routines, when a lbolt specific cyclic is
 256  * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
 257  * rely on the original low cost of consulting a memory position.
 258  *
 259  * The implementation uses the number of calls to these routines and the
 260  * frequency of these to determine when to transition from event to cyclic
 261  * driven and vice-versa. These values are kept on a per CPU basis for
 262  * scalability reasons and to prevent CPUs from constantly invalidating a single
 263  * cache line when modifying a global variable. The transition from event to
 264  * cyclic mode happens once the thresholds are crossed, and activity on any CPU
 265  * can cause such transition.
 266  *
 267  * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
 268  * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
 269  * lbolt_cyclic_driven() according to the current mode. When the thresholds
 270  * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
 271  * fire at a nsec_per_tick interval and increment an internal variable at
 272  * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
 273  * will simply return the value of such variable. lbolt_cyclic() will attempt
 274  * to shut itself off at each threshold interval (sampling period for calls
 275  * to the DDI lbolt routines), and return to the event driven mode, but will
 276  * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
 277  *
 278  * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
 279  * for the cyclic subsystem to be intialized.
 280  *
 281  */
 282 int64_t lbolt_bootstrap(void);
 283 int64_t lbolt_event_driven(void);
 284 int64_t lbolt_cyclic_driven(void);
 285 int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
 286 uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
 287 
 288 /*
 289  * lbolt's cyclic, installed by clock_init().
 290  */
 291 static void lbolt_cyclic(void);
 292 
 293 /*
 294  * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
 295  * from switching back to event driven, once it reaches cyclic mode.
 296  */
 297 static boolean_t lbolt_cyc_only = B_FALSE;
 298 
 299 /*
 300  * Cache aligned, per CPU structure with lbolt usage statistics.
 301  */
 302 static lbolt_cpu_t *lb_cpu;
 303 
 304 /*
 305  * Single, cache aligned, structure with all the information required by
 306  * the lbolt implementation.
 307  */
 308 lbolt_info_t *lb_info;
 309 
 310 
 311 int one_sec = 1; /* turned on once every second */
 312 static int fsflushcnt;  /* counter for t_fsflushr */
 313 int     dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
 314 int     tod_needsync = 0;       /* need to sync tod chip with software time */
 315 static int tod_broken = 0;      /* clock chip doesn't work */
 316 time_t  boot_time = 0;          /* Boot time in seconds since 1970 */
 317 cyclic_id_t clock_cyclic;       /* clock()'s cyclic_id */
 318 cyclic_id_t deadman_cyclic;     /* deadman()'s cyclic_id */
 319 cyclic_id_t ddi_timer_cyclic;   /* cyclic_timer()'s cyclic_id */
 320 
 321 extern void     clock_tick_schedule(int);
 322 
 323 static int lgrp_ticks;          /* counter to schedule lgrp load calcs */
 324 
 325 /*
 326  * for tod fault detection
 327  */
 328 #define TOD_REF_FREQ            ((longlong_t)(NANOSEC))
 329 #define TOD_STALL_THRESHOLD     (TOD_REF_FREQ * 3 / 2)
 330 #define TOD_JUMP_THRESHOLD      (TOD_REF_FREQ / 2)
 331 #define TOD_FILTER_N            4
 332 #define TOD_FILTER_SETTLE       (4 * TOD_FILTER_N)
 333 static int tod_faulted = TOD_NOFAULT;
 334 
 335 static int tod_status_flag = 0;         /* used by tod_validate() */
 336 
 337 static hrtime_t prev_set_tick = 0;      /* gethrtime() prior to tod_set() */
 338 static time_t prev_set_tod = 0;         /* tv_sec value passed to tod_set() */
 339 
 340 /* patchable via /etc/system */
 341 int tod_validate_enable = 1;
 342 
 343 /* Diagnose/Limit messages about delay(9F) called from interrupt context */
 344 int                     delay_from_interrupt_diagnose = 0;
 345 volatile uint32_t       delay_from_interrupt_msg = 20;
 346 
 347 /*
 348  * On non-SPARC systems, TOD validation must be deferred until gethrtime
 349  * returns non-zero values (after mach_clkinit's execution).
 350  * On SPARC systems, it must be deferred until after hrtime_base
 351  * and hres_last_tick are set (in the first invocation of hres_tick).
 352  * Since in both cases the prerequisites occur before the invocation of
 353  * tod_get() in clock(), the deferment is lifted there.
 354  */
 355 static boolean_t tod_validate_deferred = B_TRUE;
 356 
 357 /*
 358  * tod_fault_table[] must be aligned with
 359  * enum tod_fault_type in systm.h
 360  */
 361 static char *tod_fault_table[] = {
 362         "Reversed",                     /* TOD_REVERSED */
 363         "Stalled",                      /* TOD_STALLED */
 364         "Jumped",                       /* TOD_JUMPED */
 365         "Changed in Clock Rate",        /* TOD_RATECHANGED */
 366         "Is Read-Only"                  /* TOD_RDONLY */
 367         /*
 368          * no strings needed for TOD_NOFAULT
 369          */
 370 };
 371 
 372 /*
 373  * test hook for tod broken detection in tod_validate
 374  */
 375 int tod_unit_test = 0;
 376 time_t tod_test_injector;
 377 
 378 #define CLOCK_ADJ_HIST_SIZE     4
 379 
 380 static int      adj_hist_entry;
 381 
 382 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
 383 
 384 static void calcloadavg(int, uint64_t *);
 385 static int genloadavg(struct loadavg_s *);
 386 static void loadavg_update();
 387 
 388 void (*cmm_clock_callout)() = NULL;
 389 void (*cpucaps_clock_callout)() = NULL;
 390 
 391 extern clock_t clock_tick_proc_max;
 392 
 393 static int64_t deadman_counter = 0;
 394 
 395 static void
 396 clock(void)
 397 {
 398         kthread_t       *t;
 399         uint_t  nrunnable;
 400         uint_t  w_io;
 401         cpu_t   *cp;
 402         cpupart_t *cpupart;
 403         extern  void    set_freemem();
 404         void    (*funcp)();
 405         int32_t ltemp;
 406         int64_t lltemp;
 407         int s;
 408         int do_lgrp_load;
 409         int i;
 410         clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
 411 
 412         if (panicstr)
 413                 return;
 414 
 415         /*
 416          * Make sure that 'freemem' do not drift too far from the truth
 417          */
 418         set_freemem();
 419 
 420 
 421         /*
 422          * Before the section which is repeated is executed, we do
 423          * the time delta processing which occurs every clock tick
 424          *
 425          * There is additional processing which happens every time
 426          * the nanosecond counter rolls over which is described
 427          * below - see the section which begins with : if (one_sec)
 428          *
 429          * This section marks the beginning of the precision-kernel
 430          * code fragment.
 431          *
 432          * First, compute the phase adjustment. If the low-order bits
 433          * (time_phase) of the update overflow, bump the higher order
 434          * bits (time_update).
 435          */
 436         time_phase += time_adj;
 437         if (time_phase <= -FINEUSEC) {
 438                 ltemp = -time_phase / SCALE_PHASE;
 439                 time_phase += ltemp * SCALE_PHASE;
 440                 s = hr_clock_lock();
 441                 timedelta -= ltemp * (NANOSEC/MICROSEC);
 442                 hr_clock_unlock(s);
 443         } else if (time_phase >= FINEUSEC) {
 444                 ltemp = time_phase / SCALE_PHASE;
 445                 time_phase -= ltemp * SCALE_PHASE;
 446                 s = hr_clock_lock();
 447                 timedelta += ltemp * (NANOSEC/MICROSEC);
 448                 hr_clock_unlock(s);
 449         }
 450 
 451         /*
 452          * End of precision-kernel code fragment which is processed
 453          * every timer interrupt.
 454          *
 455          * Continue with the interrupt processing as scheduled.
 456          */
 457         /*
 458          * Count the number of runnable threads and the number waiting
 459          * for some form of I/O to complete -- gets added to
 460          * sysinfo.waiting.  To know the state of the system, must add
 461          * wait counts from all CPUs.  Also add up the per-partition
 462          * statistics.
 463          */
 464         w_io = 0;
 465         nrunnable = 0;
 466 
 467         /*
 468          * keep track of when to update lgrp/part loads
 469          */
 470 
 471         do_lgrp_load = 0;
 472         if (lgrp_ticks++ >= hz / 10) {
 473                 lgrp_ticks = 0;
 474                 do_lgrp_load = 1;
 475         }
 476 
 477         if (one_sec) {
 478                 loadavg_update();
 479                 deadman_counter++;
 480         }
 481 
 482         /*
 483          * First count the threads waiting on kpreempt queues in each
 484          * CPU partition.
 485          */
 486 
 487         cpupart = cp_list_head;
 488         do {
 489                 uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
 490 
 491                 cpupart->cp_updates++;
 492                 nrunnable += cpupart_nrunnable;
 493                 cpupart->cp_nrunnable_cum += cpupart_nrunnable;
 494                 if (one_sec) {
 495                         cpupart->cp_nrunning = 0;
 496                         cpupart->cp_nrunnable = cpupart_nrunnable;
 497                 }
 498         } while ((cpupart = cpupart->cp_next) != cp_list_head);
 499 
 500 
 501         /* Now count the per-CPU statistics. */
 502         cp = cpu_list;
 503         do {
 504                 uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
 505 
 506                 nrunnable += cpu_nrunnable;
 507                 cpupart = cp->cpu_part;
 508                 cpupart->cp_nrunnable_cum += cpu_nrunnable;
 509                 if (one_sec) {
 510                         cpupart->cp_nrunnable += cpu_nrunnable;
 511                         /*
 512                          * Update user, system, and idle cpu times.
 513                          */
 514                         cpupart->cp_nrunning++;
 515                         /*
 516                          * w_io is used to update sysinfo.waiting during
 517                          * one_second processing below.  Only gather w_io
 518                          * information when we walk the list of cpus if we're
 519                          * going to perform one_second processing.
 520                          */
 521                         w_io += CPU_STATS(cp, sys.iowait);
 522                 }
 523 
 524                 if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
 525                         int i, load, change;
 526                         hrtime_t intracct, intrused;
 527                         const hrtime_t maxnsec = 1000000000;
 528                         const int precision = 100;
 529 
 530                         /*
 531                          * Estimate interrupt load on this cpu each second.
 532                          * Computes cpu_intrload as %utilization (0-99).
 533                          */
 534 
 535                         /* add up interrupt time from all micro states */
 536                         for (intracct = 0, i = 0; i < NCMSTATES; i++)
 537                                 intracct += cp->cpu_intracct[i];
 538                         scalehrtime(&intracct);
 539 
 540                         /* compute nsec used in the past second */
 541                         intrused = intracct - cp->cpu_intrlast;
 542                         cp->cpu_intrlast = intracct;
 543 
 544                         /* limit the value for safety (and the first pass) */
 545                         if (intrused >= maxnsec)
 546                                 intrused = maxnsec - 1;
 547 
 548                         /* calculate %time in interrupt */
 549                         load = (precision * intrused) / maxnsec;
 550                         ASSERT(load >= 0 && load < precision);
 551                         change = cp->cpu_intrload - load;
 552 
 553                         /* jump to new max, or decay the old max */
 554                         if (change < 0)
 555                                 cp->cpu_intrload = load;
 556                         else if (change > 0)
 557                                 cp->cpu_intrload -= (change + 3) / 4;
 558 
 559                         DTRACE_PROBE3(cpu_intrload,
 560                             cpu_t *, cp,
 561                             hrtime_t, intracct,
 562                             hrtime_t, intrused);
 563                 }
 564 
 565                 if (do_lgrp_load &&
 566                     (cp->cpu_flags & CPU_EXISTS)) {
 567                         /*
 568                          * When updating the lgroup's load average,
 569                          * account for the thread running on the CPU.
 570                          * If the CPU is the current one, then we need
 571                          * to account for the underlying thread which
 572                          * got the clock interrupt not the thread that is
 573                          * handling the interrupt and caculating the load
 574                          * average
 575                          */
 576                         t = cp->cpu_thread;
 577                         if (CPU == cp)
 578                                 t = t->t_intr;
 579 
 580                         /*
 581                          * Account for the load average for this thread if
 582                          * it isn't the idle thread or it is on the interrupt
 583                          * stack and not the current CPU handling the clock
 584                          * interrupt
 585                          */
 586                         if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
 587                             CPU_ON_INTR(cp))) {
 588                                 if (t->t_lpl == cp->cpu_lpl) {
 589                                         /* local thread */
 590                                         cpu_nrunnable++;
 591                                 } else {
 592                                         /*
 593                                          * This is a remote thread, charge it
 594                                          * against its home lgroup.  Note that
 595                                          * we notice that a thread is remote
 596                                          * only if it's currently executing.
 597                                          * This is a reasonable approximation,
 598                                          * since queued remote threads are rare.
 599                                          * Note also that if we didn't charge
 600                                          * it to its home lgroup, remote
 601                                          * execution would often make a system
 602                                          * appear balanced even though it was
 603                                          * not, and thread placement/migration
 604                                          * would often not be done correctly.
 605                                          */
 606                                         lgrp_loadavg(t->t_lpl,
 607                                             LGRP_LOADAVG_IN_THREAD_MAX, 0);
 608                                 }
 609                         }
 610                         lgrp_loadavg(cp->cpu_lpl,
 611                             cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
 612                 }
 613         } while ((cp = cp->cpu_next) != cpu_list);
 614 
 615         clock_tick_schedule(one_sec);
 616 
 617         /*
 618          * Check for a callout that needs be called from the clock
 619          * thread to support the membership protocol in a clustered
 620          * system.  Copy the function pointer so that we can reset
 621          * this to NULL if needed.
 622          */
 623         if ((funcp = cmm_clock_callout) != NULL)
 624                 (*funcp)();
 625 
 626         if ((funcp = cpucaps_clock_callout) != NULL)
 627                 (*funcp)();
 628 
 629         /*
 630          * Wakeup the cageout thread waiters once per second.
 631          */
 632         if (one_sec)
 633                 kcage_tick();
 634 
 635         if (one_sec) {
 636 
 637                 int drift, absdrift;
 638                 timestruc_t tod;
 639                 int s;
 640 
 641                 /*
 642                  * Beginning of precision-kernel code fragment executed
 643                  * every second.
 644                  *
 645                  * On rollover of the second the phase adjustment to be
 646                  * used for the next second is calculated.  Also, the
 647                  * maximum error is increased by the tolerance.  If the
 648                  * PPS frequency discipline code is present, the phase is
 649                  * increased to compensate for the CPU clock oscillator
 650                  * frequency error.
 651                  *
 652                  * On a 32-bit machine and given parameters in the timex.h
 653                  * header file, the maximum phase adjustment is +-512 ms
 654                  * and maximum frequency offset is (a tad less than)
 655                  * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
 656                  */
 657                 time_maxerror += time_tolerance / SCALE_USEC;
 658 
 659                 /*
 660                  * Leap second processing. If in leap-insert state at
 661                  * the end of the day, the system clock is set back one
 662                  * second; if in leap-delete state, the system clock is
 663                  * set ahead one second. The microtime() routine or
 664                  * external clock driver will insure that reported time
 665                  * is always monotonic. The ugly divides should be
 666                  * replaced.
 667                  */
 668                 switch (time_state) {
 669 
 670                 case TIME_OK:
 671                         if (time_status & STA_INS)
 672                                 time_state = TIME_INS;
 673                         else if (time_status & STA_DEL)
 674                                 time_state = TIME_DEL;
 675                         break;
 676 
 677                 case TIME_INS:
 678                         if (hrestime.tv_sec % 86400 == 0) {
 679                                 s = hr_clock_lock();
 680                                 hrestime.tv_sec--;
 681                                 hr_clock_unlock(s);
 682                                 time_state = TIME_OOP;
 683                         }
 684                         break;
 685 
 686                 case TIME_DEL:
 687                         if ((hrestime.tv_sec + 1) % 86400 == 0) {
 688                                 s = hr_clock_lock();
 689                                 hrestime.tv_sec++;
 690                                 hr_clock_unlock(s);
 691                                 time_state = TIME_WAIT;
 692                         }
 693                         break;
 694 
 695                 case TIME_OOP:
 696                         time_state = TIME_WAIT;
 697                         break;
 698 
 699                 case TIME_WAIT:
 700                         if (!(time_status & (STA_INS | STA_DEL)))
 701                                 time_state = TIME_OK;
 702                 default:
 703                         break;
 704                 }
 705 
 706                 /*
 707                  * Compute the phase adjustment for the next second. In
 708                  * PLL mode, the offset is reduced by a fixed factor
 709                  * times the time constant. In FLL mode the offset is
 710                  * used directly. In either mode, the maximum phase
 711                  * adjustment for each second is clamped so as to spread
 712                  * the adjustment over not more than the number of
 713                  * seconds between updates.
 714                  */
 715                 if (time_offset == 0)
 716                         time_adj = 0;
 717                 else if (time_offset < 0) {
 718                         lltemp = -time_offset;
 719                         if (!(time_status & STA_FLL)) {
 720                                 if ((1 << time_constant) >= SCALE_KG)
 721                                         lltemp *= (1 << time_constant) /
 722                                             SCALE_KG;
 723                                 else
 724                                         lltemp = (lltemp / SCALE_KG) >>
 725                                             time_constant;
 726                         }
 727                         if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
 728                                 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
 729                         time_offset += lltemp;
 730                         time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
 731                 } else {
 732                         lltemp = time_offset;
 733                         if (!(time_status & STA_FLL)) {
 734                                 if ((1 << time_constant) >= SCALE_KG)
 735                                         lltemp *= (1 << time_constant) /
 736                                             SCALE_KG;
 737                                 else
 738                                         lltemp = (lltemp / SCALE_KG) >>
 739                                             time_constant;
 740                         }
 741                         if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
 742                                 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
 743                         time_offset -= lltemp;
 744                         time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
 745                 }
 746 
 747                 /*
 748                  * Compute the frequency estimate and additional phase
 749                  * adjustment due to frequency error for the next
 750                  * second. When the PPS signal is engaged, gnaw on the
 751                  * watchdog counter and update the frequency computed by
 752                  * the pll and the PPS signal.
 753                  */
 754                 pps_valid++;
 755                 if (pps_valid == PPS_VALID) {
 756                         pps_jitter = MAXTIME;
 757                         pps_stabil = MAXFREQ;
 758                         time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
 759                             STA_PPSWANDER | STA_PPSERROR);
 760                 }
 761                 lltemp = time_freq + pps_freq;
 762 
 763                 if (lltemp)
 764                         time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
 765 
 766                 /*
 767                  * End of precision kernel-code fragment
 768                  *
 769                  * The section below should be modified if we are planning
 770                  * to use NTP for synchronization.
 771                  *
 772                  * Note: the clock synchronization code now assumes
 773                  * the following:
 774                  *   - if dosynctodr is 1, then compute the drift between
 775                  *      the tod chip and software time and adjust one or
 776                  *      the other depending on the circumstances
 777                  *
 778                  *   - if dosynctodr is 0, then the tod chip is independent
 779                  *      of the software clock and should not be adjusted,
 780                  *      but allowed to free run.  this allows NTP to sync.
 781                  *      hrestime without any interference from the tod chip.
 782                  */
 783 
 784                 tod_validate_deferred = B_FALSE;
 785                 mutex_enter(&tod_lock);
 786                 tod = tod_get();
 787                 drift = tod.tv_sec - hrestime.tv_sec;
 788                 absdrift = (drift >= 0) ? drift : -drift;
 789                 if (tod_needsync || absdrift > 1) {
 790                         int s;
 791                         if (absdrift > 2) {
 792                                 if (!tod_broken && tod_faulted == TOD_NOFAULT) {
 793                                         s = hr_clock_lock();
 794                                         hrestime = tod;
 795                                         membar_enter(); /* hrestime visible */
 796                                         timedelta = 0;
 797                                         timechanged++;
 798                                         tod_needsync = 0;
 799                                         hr_clock_unlock(s);
 800                                         callout_hrestime();
 801 
 802                                 }
 803                         } else {
 804                                 if (tod_needsync || !dosynctodr) {
 805                                         gethrestime(&tod);
 806                                         tod_set(tod);
 807                                         s = hr_clock_lock();
 808                                         if (timedelta == 0)
 809                                                 tod_needsync = 0;
 810                                         hr_clock_unlock(s);
 811                                 } else {
 812                                         /*
 813                                          * If the drift is 2 seconds on the
 814                                          * money, then the TOD is adjusting
 815                                          * the clock;  record that.
 816                                          */
 817                                         clock_adj_hist[adj_hist_entry++ %
 818                                             CLOCK_ADJ_HIST_SIZE] = now;
 819                                         s = hr_clock_lock();
 820                                         timedelta = (int64_t)drift*NANOSEC;
 821                                         hr_clock_unlock(s);
 822                                 }
 823                         }
 824                 }
 825                 one_sec = 0;
 826                 time = gethrestime_sec();  /* for crusty old kmem readers */
 827                 mutex_exit(&tod_lock);
 828 
 829                 /*
 830                  * Some drivers still depend on this... XXX
 831                  */
 832                 cv_broadcast(&lbolt_cv);
 833 
 834                 vminfo.freemem += freemem;
 835                 {
 836                         pgcnt_t maxswap, resv, free;
 837                         pgcnt_t avail =
 838                             MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
 839 
 840                         maxswap = k_anoninfo.ani_mem_resv +
 841                             k_anoninfo.ani_max +avail;
 842                         /* Update ani_free */
 843                         set_anoninfo();
 844                         free = k_anoninfo.ani_free + avail;
 845                         resv = k_anoninfo.ani_phys_resv +
 846                             k_anoninfo.ani_mem_resv;
 847 
 848                         vminfo.swap_resv += resv;
 849                         /* number of reserved and allocated pages */
 850 #ifdef  DEBUG
 851                         if (maxswap < free)
 852                                 cmn_err(CE_WARN, "clock: maxswap < free");
 853                         if (maxswap < resv)
 854                                 cmn_err(CE_WARN, "clock: maxswap < resv");
 855 #endif
 856                         vminfo.swap_alloc += maxswap - free;
 857                         vminfo.swap_avail += maxswap - resv;
 858                         vminfo.swap_free += free;
 859                 }
 860                 vminfo.updates++;
 861                 if (nrunnable) {
 862                         sysinfo.runque += nrunnable;
 863                         sysinfo.runocc++;
 864                 }
 865                 if (nswapped) {
 866                         sysinfo.swpque += nswapped;
 867                         sysinfo.swpocc++;
 868                 }
 869                 sysinfo.waiting += w_io;
 870                 sysinfo.updates++;
 871 
 872                 /*
 873                  * Wake up fsflush to write out DELWRI
 874                  * buffers, dirty pages and other cached
 875                  * administrative data, e.g. inodes.
 876                  */
 877                 if (--fsflushcnt <= 0) {
 878                         fsflushcnt = tune.t_fsflushr;
 879                         cv_signal(&fsflush_cv);
 880                 }
 881 
 882                 vmmeter();
 883                 calcloadavg(genloadavg(&loadavg), hp_avenrun);
 884                 for (i = 0; i < 3; i++)
 885                         /*
 886                          * At the moment avenrun[] can only hold 31
 887                          * bits of load average as it is a signed
 888                          * int in the API. We need to ensure that
 889                          * hp_avenrun[i] >> (16 - FSHIFT) will not be
 890                          * too large. If it is, we put the largest value
 891                          * that we can use into avenrun[i]. This is
 892                          * kludgey, but about all we can do until we
 893                          * avenrun[] is declared as an array of uint64[]
 894                          */
 895                         if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
 896                                 avenrun[i] = (int32_t)(hp_avenrun[i] >>
 897                                     (16 - FSHIFT));
 898                         else
 899                                 avenrun[i] = 0x7fffffff;
 900 
 901                 cpupart = cp_list_head;
 902                 do {
 903                         calcloadavg(genloadavg(&cpupart->cp_loadavg),
 904                             cpupart->cp_hp_avenrun);
 905                 } while ((cpupart = cpupart->cp_next) != cp_list_head);
 906 
 907                 /*
 908                  * Wake up the swapper thread if necessary.
 909                  */
 910                 if (runin ||
 911                     (runout && (avefree < desfree || wake_sched_sec))) {
 912                         t = &t0;
 913                         thread_lock(t);
 914                         if (t->t_state == TS_STOPPED) {
 915                                 runin = runout = 0;
 916                                 wake_sched_sec = 0;
 917                                 t->t_whystop = 0;
 918                                 t->t_whatstop = 0;
 919                                 t->t_schedflag &= ~TS_ALLSTART;
 920                                 THREAD_TRANSITION(t);
 921                                 setfrontdq(t);
 922                         }
 923                         thread_unlock(t);
 924                 }
 925         }
 926 
 927         /*
 928          * Wake up the swapper if any high priority swapped-out threads
 929          * became runable during the last tick.
 930          */
 931         if (wake_sched) {
 932                 t = &t0;
 933                 thread_lock(t);
 934                 if (t->t_state == TS_STOPPED) {
 935                         runin = runout = 0;
 936                         wake_sched = 0;
 937                         t->t_whystop = 0;
 938                         t->t_whatstop = 0;
 939                         t->t_schedflag &= ~TS_ALLSTART;
 940                         THREAD_TRANSITION(t);
 941                         setfrontdq(t);
 942                 }
 943                 thread_unlock(t);
 944         }
 945 }
 946 
 947 void
 948 clock_init(void)
 949 {
 950         cyc_handler_t clk_hdlr, lbolt_hdlr;
 951         cyc_time_t clk_when, lbolt_when;
 952         int i, sz;
 953         intptr_t buf;
 954 
 955         /*
 956          * Setup handler and timer for the clock cyclic.
 957          */
 958         clk_hdlr.cyh_func = (cyc_func_t)clock;
 959         clk_hdlr.cyh_level = CY_LOCK_LEVEL;
 960         clk_hdlr.cyh_arg = NULL;
 961 
 962         clk_when.cyt_when = 0;
 963         clk_when.cyt_interval = nsec_per_tick;
 964 
 965         /*
 966          * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
 967          * interval to satisfy performance needs of the DDI lbolt consumers.
 968          * It is off by default.
 969          */
 970         lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
 971         lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
 972         lbolt_hdlr.cyh_arg = NULL;
 973 
 974         lbolt_when.cyt_interval = nsec_per_tick;
 975 
 976         /*
 977          * Allocate cache line aligned space for the per CPU lbolt data and
 978          * lbolt info structures, and initialize them with their default
 979          * values. Note that these structures are also cache line sized.
 980          */
 981         sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
 982         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 983         lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
 984 
 985         if (hz != HZ_DEFAULT)
 986                 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
 987                     hz/HZ_DEFAULT;
 988         else
 989                 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
 990 
 991         lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
 992 
 993         sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
 994         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 995         lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
 996 
 997         for (i = 0; i < max_ncpus; i++)
 998                 lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
 999 
1000         /*
1001          * Install the softint used to switch between event and cyclic driven
1002          * lbolt. We use a soft interrupt to make sure the context of the
1003          * cyclic reprogram call is safe.
1004          */
1005         lbolt_softint_add();
1006 
1007         /*
1008          * Since the hybrid lbolt implementation is based on a hardware counter
1009          * that is reset at every hardware reboot and that we'd like to have
1010          * the lbolt value starting at zero after both a hardware and a fast
1011          * reboot, we calculate the number of clock ticks the system's been up
1012          * and store it in the lbi_debug_time field of the lbolt info structure.
1013          * The value of this field will be subtracted from lbolt before
1014          * returning it.
1015          */
1016         lb_info->lbi_internal = lb_info->lbi_debug_time =
1017             (gethrtime()/nsec_per_tick);
1018 
1019         /*
1020          * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
1021          * and lbolt_debug_{enter,return} use this value as an indication that
1022          * the initializaion above hasn't been completed. Setting lbolt_hybrid
1023          * to either lbolt_{cyclic,event}_driven here signals those code paths
1024          * that the lbolt related structures can be used.
1025          */
1026         if (lbolt_cyc_only) {
1027                 lbolt_when.cyt_when = 0;
1028                 lbolt_hybrid = lbolt_cyclic_driven;
1029         } else {
1030                 lbolt_when.cyt_when = CY_INFINITY;
1031                 lbolt_hybrid = lbolt_event_driven;
1032         }
1033 
1034         /*
1035          * Grab cpu_lock and install all three cyclics.
1036          */
1037         mutex_enter(&cpu_lock);
1038 
1039         clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
1040         lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
1041 
1042         mutex_exit(&cpu_lock);
1043 }
1044 
1045 /*
1046  * Called before calcloadavg to get 10-sec moving loadavg together
1047  */
1048 
1049 static int
1050 genloadavg(struct loadavg_s *avgs)
1051 {
1052         int avg;
1053         int spos; /* starting position */
1054         int cpos; /* moving current position */
1055         int i;
1056         int slen;
1057         hrtime_t hr_avg;
1058 
1059         /* 10-second snapshot, calculate first positon */
1060         if (avgs->lg_len == 0) {
1061                 return (0);
1062         }
1063         slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1064 
1065         spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1066             S_LOADAVG_SZ + (avgs->lg_cur - 1);
1067         for (i = hr_avg = 0; i < slen; i++) {
1068                 cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1069                 hr_avg += avgs->lg_loads[cpos];
1070         }
1071 
1072         hr_avg = hr_avg / slen;
1073         avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1074 
1075         return (avg);
1076 }
1077 
1078 /*
1079  * Run every second from clock () to update the loadavg count available to the
1080  * system and cpu-partitions.
1081  *
1082  * This works by sampling the previous usr, sys, wait time elapsed,
1083  * computing a delta, and adding that delta to the elapsed usr, sys,
1084  * wait increase.
1085  */
1086 
1087 static void
1088 loadavg_update()
1089 {
1090         cpu_t *cp;
1091         cpupart_t *cpupart;
1092         hrtime_t cpu_total;
1093         int prev;
1094 
1095         cp = cpu_list;
1096         loadavg.lg_total = 0;
1097 
1098         /*
1099          * first pass totals up per-cpu statistics for system and cpu
1100          * partitions
1101          */
1102 
1103         do {
1104                 struct loadavg_s *lavg;
1105 
1106                 lavg = &cp->cpu_loadavg;
1107 
1108                 cpu_total = cp->cpu_acct[CMS_USER] +
1109                     cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1110                 /* compute delta against last total */
1111                 scalehrtime(&cpu_total);
1112                 prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1113                     S_LOADAVG_SZ + (lavg->lg_cur - 1);
1114                 if (lavg->lg_loads[prev] <= 0) {
1115                         lavg->lg_loads[lavg->lg_cur] = cpu_total;
1116                         cpu_total = 0;
1117                 } else {
1118                         lavg->lg_loads[lavg->lg_cur] = cpu_total;
1119                         cpu_total = cpu_total - lavg->lg_loads[prev];
1120                         if (cpu_total < 0)
1121                                 cpu_total = 0;
1122                 }
1123 
1124                 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1125                 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1126                     lavg->lg_len + 1 : S_LOADAVG_SZ;
1127 
1128                 loadavg.lg_total += cpu_total;
1129                 cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1130 
1131         } while ((cp = cp->cpu_next) != cpu_list);
1132 
1133         loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1134         loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1135         loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1136             loadavg.lg_len + 1 : S_LOADAVG_SZ;
1137         /*
1138          * Second pass updates counts
1139          */
1140         cpupart = cp_list_head;
1141 
1142         do {
1143                 struct loadavg_s *lavg;
1144 
1145                 lavg = &cpupart->cp_loadavg;
1146                 lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1147                 lavg->lg_total = 0;
1148                 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1149                 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1150                     lavg->lg_len + 1 : S_LOADAVG_SZ;
1151 
1152         } while ((cpupart = cpupart->cp_next) != cp_list_head);
1153 
1154         /*
1155          * Third pass totals up per-zone statistics.
1156          */
1157         zone_loadavg_update();
1158 }
1159 
1160 /*
1161  * clock_update() - local clock update
1162  *
1163  * This routine is called by ntp_adjtime() to update the local clock
1164  * phase and frequency. The implementation is of an
1165  * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1166  * routine computes new time and frequency offset estimates for each
1167  * call.  The PPS signal itself determines the new time offset,
1168  * instead of the calling argument.  Presumably, calls to
1169  * ntp_adjtime() occur only when the caller believes the local clock
1170  * is valid within some bound (+-128 ms with NTP). If the caller's
1171  * time is far different than the PPS time, an argument will ensue,
1172  * and it's not clear who will lose.
1173  *
1174  * For uncompensated quartz crystal oscillatores and nominal update
1175  * intervals less than 1024 s, operation should be in phase-lock mode
1176  * (STA_FLL = 0), where the loop is disciplined to phase. For update
1177  * intervals greater than this, operation should be in frequency-lock
1178  * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1179  *
1180  * Note: mutex(&tod_lock) is in effect.
1181  */
1182 void
1183 clock_update(int offset)
1184 {
1185         int ltemp, mtemp, s;
1186 
1187         ASSERT(MUTEX_HELD(&tod_lock));
1188 
1189         if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1190                 return;
1191         ltemp = offset;
1192         if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1193                 ltemp = pps_offset;
1194 
1195         /*
1196          * Scale the phase adjustment and clamp to the operating range.
1197          */
1198         if (ltemp > MAXPHASE)
1199                 time_offset = MAXPHASE * SCALE_UPDATE;
1200         else if (ltemp < -MAXPHASE)
1201                 time_offset = -(MAXPHASE * SCALE_UPDATE);
1202         else
1203                 time_offset = ltemp * SCALE_UPDATE;
1204 
1205         /*
1206          * Select whether the frequency is to be controlled and in which
1207          * mode (PLL or FLL). Clamp to the operating range. Ugly
1208          * multiply/divide should be replaced someday.
1209          */
1210         if (time_status & STA_FREQHOLD || time_reftime == 0)
1211                 time_reftime = hrestime.tv_sec;
1212 
1213         mtemp = hrestime.tv_sec - time_reftime;
1214         time_reftime = hrestime.tv_sec;
1215 
1216         if (time_status & STA_FLL) {
1217                 if (mtemp >= MINSEC) {
1218                         ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1219                             SCALE_UPDATE));
1220                         if (ltemp)
1221                                 time_freq += ltemp / SCALE_KH;
1222                 }
1223         } else {
1224                 if (mtemp < MAXSEC) {
1225                         ltemp *= mtemp;
1226                         if (ltemp)
1227                                 time_freq += (int)(((int64_t)ltemp *
1228                                     SCALE_USEC) / SCALE_KF)
1229                                     / (1 << (time_constant * 2));
1230                 }
1231         }
1232         if (time_freq > time_tolerance)
1233                 time_freq = time_tolerance;
1234         else if (time_freq < -time_tolerance)
1235                 time_freq = -time_tolerance;
1236 
1237         s = hr_clock_lock();
1238         tod_needsync = 1;
1239         hr_clock_unlock(s);
1240 }
1241 
1242 /*
1243  * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1244  *
1245  * This routine is called at each PPS interrupt in order to discipline
1246  * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1247  * and leaves it in a handy spot for the clock() routine. It
1248  * integrates successive PPS phase differences and calculates the
1249  * frequency offset. This is used in clock() to discipline the CPU
1250  * clock oscillator so that intrinsic frequency error is cancelled out.
1251  * The code requires the caller to capture the time and hardware counter
1252  * value at the on-time PPS signal transition.
1253  *
1254  * Note that, on some Unix systems, this routine runs at an interrupt
1255  * priority level higher than the timer interrupt routine clock().
1256  * Therefore, the variables used are distinct from the clock()
1257  * variables, except for certain exceptions: The PPS frequency pps_freq
1258  * and phase pps_offset variables are determined by this routine and
1259  * updated atomically. The time_tolerance variable can be considered a
1260  * constant, since it is infrequently changed, and then only when the
1261  * PPS signal is disabled. The watchdog counter pps_valid is updated
1262  * once per second by clock() and is atomically cleared in this
1263  * routine.
1264  *
1265  * tvp is the time of the last tick; usec is a microsecond count since the
1266  * last tick.
1267  *
1268  * Note: In Solaris systems, the tick value is actually given by
1269  *       usec_per_tick.  This is called from the serial driver cdintr(),
1270  *       or equivalent, at a high PIL.  Because the kernel keeps a
1271  *       highresolution time, the following code can accept either
1272  *       the traditional argument pair, or the current highres timestamp
1273  *       in tvp and zero in usec.
1274  */
1275 void
1276 ddi_hardpps(struct timeval *tvp, int usec)
1277 {
1278         int u_usec, v_usec, bigtick;
1279         time_t cal_sec;
1280         int cal_usec;
1281 
1282         /*
1283          * An occasional glitch can be produced when the PPS interrupt
1284          * occurs in the clock() routine before the time variable is
1285          * updated. Here the offset is discarded when the difference
1286          * between it and the last one is greater than tick/2, but not
1287          * if the interval since the first discard exceeds 30 s.
1288          */
1289         time_status |= STA_PPSSIGNAL;
1290         time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1291         pps_valid = 0;
1292         u_usec = -tvp->tv_usec;
1293         if (u_usec < -(MICROSEC/2))
1294                 u_usec += MICROSEC;
1295         v_usec = pps_offset - u_usec;
1296         if (v_usec < 0)
1297                 v_usec = -v_usec;
1298         if (v_usec > (usec_per_tick >> 1)) {
1299                 if (pps_glitch > MAXGLITCH) {
1300                         pps_glitch = 0;
1301                         pps_tf[2] = u_usec;
1302                         pps_tf[1] = u_usec;
1303                 } else {
1304                         pps_glitch++;
1305                         u_usec = pps_offset;
1306                 }
1307         } else
1308                 pps_glitch = 0;
1309 
1310         /*
1311          * A three-stage median filter is used to help deglitch the pps
1312          * time. The median sample becomes the time offset estimate; the
1313          * difference between the other two samples becomes the time
1314          * dispersion (jitter) estimate.
1315          */
1316         pps_tf[2] = pps_tf[1];
1317         pps_tf[1] = pps_tf[0];
1318         pps_tf[0] = u_usec;
1319         if (pps_tf[0] > pps_tf[1]) {
1320                 if (pps_tf[1] > pps_tf[2]) {
1321                         pps_offset = pps_tf[1];         /* 0 1 2 */
1322                         v_usec = pps_tf[0] - pps_tf[2];
1323                 } else if (pps_tf[2] > pps_tf[0]) {
1324                         pps_offset = pps_tf[0];         /* 2 0 1 */
1325                         v_usec = pps_tf[2] - pps_tf[1];
1326                 } else {
1327                         pps_offset = pps_tf[2];         /* 0 2 1 */
1328                         v_usec = pps_tf[0] - pps_tf[1];
1329                 }
1330         } else {
1331                 if (pps_tf[1] < pps_tf[2]) {
1332                         pps_offset = pps_tf[1];         /* 2 1 0 */
1333                         v_usec = pps_tf[2] - pps_tf[0];
1334                 } else  if (pps_tf[2] < pps_tf[0]) {
1335                         pps_offset = pps_tf[0];         /* 1 0 2 */
1336                         v_usec = pps_tf[1] - pps_tf[2];
1337                 } else {
1338                         pps_offset = pps_tf[2];         /* 1 2 0 */
1339                         v_usec = pps_tf[1] - pps_tf[0];
1340                 }
1341         }
1342         if (v_usec > MAXTIME)
1343                 pps_jitcnt++;
1344         v_usec = (v_usec << PPS_AVG) - pps_jitter;
1345         pps_jitter += v_usec / (1 << PPS_AVG);
1346         if (pps_jitter > (MAXTIME >> 1))
1347                 time_status |= STA_PPSJITTER;
1348 
1349         /*
1350          * During the calibration interval adjust the starting time when
1351          * the tick overflows. At the end of the interval compute the
1352          * duration of the interval and the difference of the hardware
1353          * counters at the beginning and end of the interval. This code
1354          * is deliciously complicated by the fact valid differences may
1355          * exceed the value of tick when using long calibration
1356          * intervals and small ticks. Note that the counter can be
1357          * greater than tick if caught at just the wrong instant, but
1358          * the values returned and used here are correct.
1359          */
1360         bigtick = (int)usec_per_tick * SCALE_USEC;
1361         pps_usec -= pps_freq;
1362         if (pps_usec >= bigtick)
1363                 pps_usec -= bigtick;
1364         if (pps_usec < 0)
1365                 pps_usec += bigtick;
1366         pps_time.tv_sec++;
1367         pps_count++;
1368         if (pps_count < (1 << pps_shift))
1369                 return;
1370         pps_count = 0;
1371         pps_calcnt++;
1372         u_usec = usec * SCALE_USEC;
1373         v_usec = pps_usec - u_usec;
1374         if (v_usec >= bigtick >> 1)
1375                 v_usec -= bigtick;
1376         if (v_usec < -(bigtick >> 1))
1377                 v_usec += bigtick;
1378         if (v_usec < 0)
1379                 v_usec = -(-v_usec >> pps_shift);
1380         else
1381                 v_usec = v_usec >> pps_shift;
1382         pps_usec = u_usec;
1383         cal_sec = tvp->tv_sec;
1384         cal_usec = tvp->tv_usec;
1385         cal_sec -= pps_time.tv_sec;
1386         cal_usec -= pps_time.tv_usec;
1387         if (cal_usec < 0) {
1388                 cal_usec += MICROSEC;
1389                 cal_sec--;
1390         }
1391         pps_time = *tvp;
1392 
1393         /*
1394          * Check for lost interrupts, noise, excessive jitter and
1395          * excessive frequency error. The number of timer ticks during
1396          * the interval may vary +-1 tick. Add to this a margin of one
1397          * tick for the PPS signal jitter and maximum frequency
1398          * deviation. If the limits are exceeded, the calibration
1399          * interval is reset to the minimum and we start over.
1400          */
1401         u_usec = (int)usec_per_tick << 1;
1402         if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1403             (cal_sec == 0 && cal_usec < u_usec)) ||
1404             v_usec > time_tolerance || v_usec < -time_tolerance) {
1405                 pps_errcnt++;
1406                 pps_shift = PPS_SHIFT;
1407                 pps_intcnt = 0;
1408                 time_status |= STA_PPSERROR;
1409                 return;
1410         }
1411 
1412         /*
1413          * A three-stage median filter is used to help deglitch the pps
1414          * frequency. The median sample becomes the frequency offset
1415          * estimate; the difference between the other two samples
1416          * becomes the frequency dispersion (stability) estimate.
1417          */
1418         pps_ff[2] = pps_ff[1];
1419         pps_ff[1] = pps_ff[0];
1420         pps_ff[0] = v_usec;
1421         if (pps_ff[0] > pps_ff[1]) {
1422                 if (pps_ff[1] > pps_ff[2]) {
1423                         u_usec = pps_ff[1];             /* 0 1 2 */
1424                         v_usec = pps_ff[0] - pps_ff[2];
1425                 } else if (pps_ff[2] > pps_ff[0]) {
1426                         u_usec = pps_ff[0];             /* 2 0 1 */
1427                         v_usec = pps_ff[2] - pps_ff[1];
1428                 } else {
1429                         u_usec = pps_ff[2];             /* 0 2 1 */
1430                         v_usec = pps_ff[0] - pps_ff[1];
1431                 }
1432         } else {
1433                 if (pps_ff[1] < pps_ff[2]) {
1434                         u_usec = pps_ff[1];             /* 2 1 0 */
1435                         v_usec = pps_ff[2] - pps_ff[0];
1436                 } else  if (pps_ff[2] < pps_ff[0]) {
1437                         u_usec = pps_ff[0];             /* 1 0 2 */
1438                         v_usec = pps_ff[1] - pps_ff[2];
1439                 } else {
1440                         u_usec = pps_ff[2];             /* 1 2 0 */
1441                         v_usec = pps_ff[1] - pps_ff[0];
1442                 }
1443         }
1444 
1445         /*
1446          * Here the frequency dispersion (stability) is updated. If it
1447          * is less than one-fourth the maximum (MAXFREQ), the frequency
1448          * offset is updated as well, but clamped to the tolerance. It
1449          * will be processed later by the clock() routine.
1450          */
1451         v_usec = (v_usec >> 1) - pps_stabil;
1452         if (v_usec < 0)
1453                 pps_stabil -= -v_usec >> PPS_AVG;
1454         else
1455                 pps_stabil += v_usec >> PPS_AVG;
1456         if (pps_stabil > MAXFREQ >> 2) {
1457                 pps_stbcnt++;
1458                 time_status |= STA_PPSWANDER;
1459                 return;
1460         }
1461         if (time_status & STA_PPSFREQ) {
1462                 if (u_usec < 0) {
1463                         pps_freq -= -u_usec >> PPS_AVG;
1464                         if (pps_freq < -time_tolerance)
1465                                 pps_freq = -time_tolerance;
1466                         u_usec = -u_usec;
1467                 } else {
1468                         pps_freq += u_usec >> PPS_AVG;
1469                         if (pps_freq > time_tolerance)
1470                                 pps_freq = time_tolerance;
1471                 }
1472         }
1473 
1474         /*
1475          * Here the calibration interval is adjusted. If the maximum
1476          * time difference is greater than tick / 4, reduce the interval
1477          * by half. If this is not the case for four consecutive
1478          * intervals, double the interval.
1479          */
1480         if (u_usec << pps_shift > bigtick >> 2) {
1481                 pps_intcnt = 0;
1482                 if (pps_shift > PPS_SHIFT)
1483                         pps_shift--;
1484         } else if (pps_intcnt >= 4) {
1485                 pps_intcnt = 0;
1486                 if (pps_shift < PPS_SHIFTMAX)
1487                         pps_shift++;
1488         } else
1489                 pps_intcnt++;
1490 
1491         /*
1492          * If recovering from kmdb, then make sure the tod chip gets resynced.
1493          * If we took an early exit above, then we don't yet have a stable
1494          * calibration signal to lock onto, so don't mark the tod for sync
1495          * until we get all the way here.
1496          */
1497         {
1498                 int s = hr_clock_lock();
1499 
1500                 tod_needsync = 1;
1501                 hr_clock_unlock(s);
1502         }
1503 }
1504 
1505 /*
1506  * Handle clock tick processing for a thread.
1507  * Check for timer action, enforce CPU rlimit, do profiling etc.
1508  */
1509 void
1510 clock_tick(kthread_t *t, int pending)
1511 {
1512         struct proc *pp;
1513         klwp_id_t    lwp;
1514         struct as *as;
1515         clock_t ticks;
1516         int     poke = 0;               /* notify another CPU */
1517         int     user_mode;
1518         size_t   rss;
1519         int i, total_usec, usec;
1520         rctl_qty_t secs;
1521 
1522         ASSERT(pending > 0);
1523 
1524         /* Must be operating on a lwp/thread */
1525         if ((lwp = ttolwp(t)) == NULL) {
1526                 panic("clock_tick: no lwp");
1527                 /*NOTREACHED*/
1528         }
1529 
1530         for (i = 0; i < pending; i++) {
1531                 CL_TICK(t);     /* Class specific tick processing */
1532                 DTRACE_SCHED1(tick, kthread_t *, t);
1533         }
1534 
1535         pp = ttoproc(t);
1536 
1537         /* pp->p_lock makes sure that the thread does not exit */
1538         ASSERT(MUTEX_HELD(&pp->p_lock));
1539 
1540         user_mode = (lwp->lwp_state == LWP_USER);
1541 
1542         ticks = (pp->p_utime + pp->p_stime) % hz;
1543         /*
1544          * Update process times. Should use high res clock and state
1545          * changes instead of statistical sampling method. XXX
1546          */
1547         if (user_mode) {
1548                 pp->p_utime += pending;
1549         } else {
1550                 pp->p_stime += pending;
1551         }
1552 
1553         pp->p_ttime += pending;
1554         as = pp->p_as;
1555 
1556         /*
1557          * Update user profiling statistics. Get the pc from the
1558          * lwp when the AST happens.
1559          */
1560         if (pp->p_prof.pr_scale) {
1561                 atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
1562                 if (user_mode) {
1563                         poke = 1;
1564                         aston(t);
1565                 }
1566         }
1567 
1568         /*
1569          * If CPU was in user state, process lwp-virtual time
1570          * interval timer. The value passed to itimerdecr() has to be
1571          * in microseconds and has to be less than one second. Hence
1572          * this loop.
1573          */
1574         total_usec = usec_per_tick * pending;
1575         while (total_usec > 0) {
1576                 usec = MIN(total_usec, (MICROSEC - 1));
1577                 if (user_mode &&
1578                     timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1579                     itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
1580                         poke = 1;
1581                         sigtoproc(pp, t, SIGVTALRM);
1582                 }
1583                 total_usec -= usec;
1584         }
1585 
1586         /*
1587          * If CPU was in user state, process lwp-profile
1588          * interval timer.
1589          */
1590         total_usec = usec_per_tick * pending;
1591         while (total_usec > 0) {
1592                 usec = MIN(total_usec, (MICROSEC - 1));
1593                 if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1594                     itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
1595                         poke = 1;
1596                         sigtoproc(pp, t, SIGPROF);
1597                 }
1598                 total_usec -= usec;
1599         }
1600 
1601         /*
1602          * Enforce CPU resource controls:
1603          *   (a) process.max-cpu-time resource control
1604          *
1605          * Perform the check only if we have accumulated more a second.
1606          */
1607         if ((ticks + pending) >= hz) {
1608                 (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1609                     (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
1610         }
1611 
1612         /*
1613          *   (b) task.max-cpu-time resource control
1614          *
1615          * If we have accumulated enough ticks, increment the task CPU
1616          * time usage and test for the resource limit. This minimizes the
1617          * number of calls to the rct_test(). The task CPU time mutex
1618          * is highly contentious as many processes can be sharing a task.
1619          */
1620         if (pp->p_ttime >= clock_tick_proc_max) {
1621                 secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
1622                 pp->p_ttime = 0;
1623                 if (secs) {
1624                         (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
1625                             pp, secs, RCA_UNSAFE_SIGINFO);
1626                 }
1627         }
1628 
1629         /*
1630          * Update memory usage for the currently running process.
1631          */
1632         rss = rm_asrss(as);
1633         PTOU(pp)->u_mem += rss;
1634         if (rss > PTOU(pp)->u_mem_max)
1635                 PTOU(pp)->u_mem_max = rss;
1636 
1637         /*
1638          * Notify the CPU the thread is running on.
1639          */
1640         if (poke && t->t_cpu != CPU)
1641                 poke_cpu(t->t_cpu->cpu_id);
1642 }
1643 
1644 void
1645 profil_tick(uintptr_t upc)
1646 {
1647         int ticks;
1648         proc_t *p = ttoproc(curthread);
1649         klwp_t *lwp = ttolwp(curthread);
1650         struct prof *pr = &p->p_prof;
1651 
1652         do {
1653                 ticks = lwp->lwp_oweupc;
1654         } while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1655 
1656         mutex_enter(&p->p_pflock);
1657         if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1658                 /*
1659                  * Old-style profiling
1660                  */
1661                 uint16_t *slot = pr->pr_base;
1662                 uint16_t old, new;
1663                 if (pr->pr_scale != 2) {
1664                         uintptr_t delta = upc - pr->pr_off;
1665                         uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1666                             (((delta & 0xffff) * pr->pr_scale) >> 16);
1667                         if (byteoff >= (uintptr_t)pr->pr_size) {
1668                                 mutex_exit(&p->p_pflock);
1669                                 return;
1670                         }
1671                         slot += byteoff / sizeof (uint16_t);
1672                 }
1673                 if (fuword16(slot, &old) < 0 ||
1674                     (new = old + ticks) > SHRT_MAX ||
1675                     suword16(slot, new) < 0) {
1676                         pr->pr_scale = 0;
1677                 }
1678         } else if (pr->pr_scale == 1) {
1679                 /*
1680                  * PC Sampling
1681                  */
1682                 model_t model = lwp_getdatamodel(lwp);
1683                 int result;
1684 #ifdef __lint
1685                 model = model;
1686 #endif
1687                 while (ticks-- > 0) {
1688                         if (pr->pr_samples == pr->pr_size) {
1689                                 /* buffer full, turn off sampling */
1690                                 pr->pr_scale = 0;
1691                                 break;
1692                         }
1693                         switch (SIZEOF_PTR(model)) {
1694                         case sizeof (uint32_t):
1695                                 result = suword32(pr->pr_base, (uint32_t)upc);
1696                                 break;
1697 #ifdef _LP64
1698                         case sizeof (uint64_t):
1699                                 result = suword64(pr->pr_base, (uint64_t)upc);
1700                                 break;
1701 #endif
1702                         default:
1703                                 cmn_err(CE_WARN, "profil_tick: unexpected "
1704                                     "data model");
1705                                 result = -1;
1706                                 break;
1707                         }
1708                         if (result != 0) {
1709                                 pr->pr_scale = 0;
1710                                 break;
1711                         }
1712                         pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1713                         pr->pr_samples++;
1714                 }
1715         }
1716         mutex_exit(&p->p_pflock);
1717 }
1718 
1719 static void
1720 delay_wakeup(void *arg)
1721 {
1722         kthread_t       *t = arg;
1723 
1724         mutex_enter(&t->t_delay_lock);
1725         cv_signal(&t->t_delay_cv);
1726         mutex_exit(&t->t_delay_lock);
1727 }
1728 
1729 /*
1730  * The delay(9F) man page indicates that it can only be called from user or
1731  * kernel context - detect and diagnose bad calls. The following macro will
1732  * produce a limited number of messages identifying bad callers.  This is done
1733  * in a macro so that caller() is meaningful. When a bad caller is identified,
1734  * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
1735  */
1736 #define DELAY_CONTEXT_CHECK()   {                                       \
1737         uint32_t        m;                                              \
1738         char            *f;                                             \
1739         ulong_t         off;                                            \
1740                                                                         \
1741         m = delay_from_interrupt_msg;                                   \
1742         if (delay_from_interrupt_diagnose && servicing_interrupt() &&   \
1743             !panicstr && !devinfo_freeze &&                             \
1744             atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) {     \
1745                 f = modgetsymname((uintptr_t)caller(), &off);               \
1746                 cmn_err(CE_WARN, "delay(9F) called from "               \
1747                     "interrupt context: %s`%s",                         \
1748                     mod_containing_pc(caller()), f ? f : "...");        \
1749         }                                                               \
1750 }
1751 
1752 /*
1753  * delay_common: common delay code.
1754  */
1755 static void
1756 delay_common(clock_t ticks)
1757 {
1758         kthread_t       *t = curthread;
1759         clock_t         deadline;
1760         clock_t         timeleft;
1761         callout_id_t    id;
1762 
1763         /* If timeouts aren't running all we can do is spin. */
1764         if (panicstr || devinfo_freeze) {
1765                 /* Convert delay(9F) call into drv_usecwait(9F) call. */
1766                 if (ticks > 0)
1767                         drv_usecwait(TICK_TO_USEC(ticks));
1768                 return;
1769         }
1770 
1771         deadline = ddi_get_lbolt() + ticks;
1772         while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
1773                 mutex_enter(&t->t_delay_lock);
1774                 id = timeout_default(delay_wakeup, t, timeleft);
1775                 cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1776                 mutex_exit(&t->t_delay_lock);
1777                 (void) untimeout_default(id, 0);
1778         }
1779 }
1780 
1781 /*
1782  * Delay specified number of clock ticks.
1783  */
1784 void
1785 delay(clock_t ticks)
1786 {
1787         DELAY_CONTEXT_CHECK();
1788 
1789         delay_common(ticks);
1790 }
1791 
1792 /*
1793  * Delay a random number of clock ticks between 1 and ticks.
1794  */
1795 void
1796 delay_random(clock_t ticks)
1797 {
1798         int     r;
1799 
1800         DELAY_CONTEXT_CHECK();
1801 
1802         (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
1803         if (ticks == 0)
1804                 ticks = 1;
1805         ticks = (r % ticks) + 1;
1806         delay_common(ticks);
1807 }
1808 
1809 /*
1810  * Like delay, but interruptible by a signal.
1811  */
1812 int
1813 delay_sig(clock_t ticks)
1814 {
1815         kthread_t       *t = curthread;
1816         clock_t         deadline;
1817         clock_t         rc;
1818 
1819         /* If timeouts aren't running all we can do is spin. */
1820         if (panicstr || devinfo_freeze) {
1821                 if (ticks > 0)
1822                         drv_usecwait(TICK_TO_USEC(ticks));
1823                 return (0);
1824         }
1825 
1826         deadline = ddi_get_lbolt() + ticks;
1827         mutex_enter(&t->t_delay_lock);
1828         do {
1829                 rc = cv_timedwait_sig(&t->t_delay_cv,
1830                     &t->t_delay_lock, deadline);
1831                 /* loop until past deadline or signaled */
1832         } while (rc > 0);
1833         mutex_exit(&t->t_delay_lock);
1834         if (rc == 0)
1835                 return (EINTR);
1836         return (0);
1837 }
1838 
1839 
1840 #define SECONDS_PER_DAY 86400
1841 
1842 /*
1843  * Initialize the system time based on the TOD chip.  approx is used as
1844  * an approximation of time (e.g. from the filesystem) in the event that
1845  * the TOD chip has been cleared or is unresponsive.  An approx of -1
1846  * means the filesystem doesn't keep time.
1847  */
1848 void
1849 clkset(time_t approx)
1850 {
1851         timestruc_t ts;
1852         int spl;
1853         int set_clock = 0;
1854 
1855         mutex_enter(&tod_lock);
1856         ts = tod_get();
1857 
1858         if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1859                 /*
1860                  * If the TOD chip is reporting some time after 1971,
1861                  * then it probably didn't lose power or become otherwise
1862                  * cleared in the recent past;  check to assure that
1863                  * the time coming from the filesystem isn't in the future
1864                  * according to the TOD chip.
1865                  */
1866                 if (approx != -1 && approx > ts.tv_sec) {
1867                         cmn_err(CE_WARN, "Last shutdown is later "
1868                             "than time on time-of-day chip; check date.");
1869                 }
1870         } else {
1871                 /*
1872                  * If the TOD chip isn't giving correct time, set it to the
1873                  * greater of i) approx and ii) 1987. That way if approx
1874                  * is negative or is earlier than 1987, we set the clock
1875                  * back to a time when Oliver North, ALF and Dire Straits
1876                  * were all on the collective brain:  1987.
1877                  */
1878                 timestruc_t tmp;
1879                 time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1880                 ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
1881                 ts.tv_nsec = 0;
1882 
1883                 /*
1884                  * Attempt to write the new time to the TOD chip.  Set spl high
1885                  * to avoid getting preempted between the tod_set and tod_get.
1886                  */
1887                 spl = splhi();
1888                 tod_set(ts);
1889                 tmp = tod_get();
1890                 splx(spl);
1891 
1892                 if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1893                         tod_broken = 1;
1894                         dosynctodr = 0;
1895                         cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
1896                 } else {
1897                         cmn_err(CE_WARN, "Time-of-day chip had "
1898                             "incorrect date; check and reset.");
1899                 }
1900                 set_clock = 1;
1901         }
1902 
1903         if (!boot_time) {
1904                 boot_time = ts.tv_sec;
1905                 set_clock = 1;
1906         }
1907 
1908         if (set_clock)
1909                 set_hrestime(&ts);
1910 
1911         mutex_exit(&tod_lock);
1912 }
1913 
1914 int     timechanged;    /* for testing if the system time has been reset */
1915 
1916 void
1917 set_hrestime(timestruc_t *ts)
1918 {
1919         int spl = hr_clock_lock();
1920         hrestime = *ts;
1921         membar_enter(); /* hrestime must be visible before timechanged++ */
1922         timedelta = 0;
1923         timechanged++;
1924         hr_clock_unlock(spl);
1925         callout_hrestime();
1926 }
1927 
1928 static uint_t deadman_seconds;
1929 static uint32_t deadman_panics;
1930 static int deadman_enabled = 0;
1931 static int deadman_panic_timers = 1;
1932 
1933 static void
1934 deadman(void)
1935 {
1936         if (panicstr) {
1937                 /*
1938                  * During panic, other CPUs besides the panic
1939                  * master continue to handle cyclics and some other
1940                  * interrupts.  The code below is intended to be
1941                  * single threaded, so any CPU other than the master
1942                  * must keep out.
1943                  */
1944                 if (CPU->cpu_id != panic_cpu.cpu_id)
1945                         return;
1946 
1947                 if (!deadman_panic_timers)
1948                         return; /* allow all timers to be manually disabled */
1949 
1950                 /*
1951                  * If we are generating a crash dump or syncing filesystems and
1952                  * the corresponding timer is set, decrement it and re-enter
1953                  * the panic code to abort it and advance to the next state.
1954                  * The panic states and triggers are explained in panic.c.
1955                  */
1956                 if (panic_dump) {
1957                         if (dump_timeleft && (--dump_timeleft == 0)) {
1958                                 panic("panic dump timeout");
1959                                 /*NOTREACHED*/
1960                         }
1961                 } else if (panic_sync) {
1962                         if (sync_timeleft && (--sync_timeleft == 0)) {
1963                                 panic("panic sync timeout");
1964                                 /*NOTREACHED*/
1965                         }
1966                 }
1967 
1968                 return;
1969         }
1970 
1971         if (deadman_counter != CPU->cpu_deadman_counter) {
1972                 CPU->cpu_deadman_counter = deadman_counter;
1973                 CPU->cpu_deadman_countdown = deadman_seconds;
1974                 return;
1975         }
1976 
1977         if (--CPU->cpu_deadman_countdown > 0)
1978                 return;
1979 
1980         /*
1981          * Regardless of whether or not we actually bring the system down,
1982          * bump the deadman_panics variable.
1983          *
1984          * N.B. deadman_panics is incremented once for each CPU that
1985          * passes through here.  It's expected that all the CPUs will
1986          * detect this condition within one second of each other, so
1987          * when deadman_enabled is off, deadman_panics will
1988          * typically be a multiple of the total number of CPUs in
1989          * the system.
1990          */
1991         atomic_add_32(&deadman_panics, 1);
1992 
1993         if (!deadman_enabled) {
1994                 CPU->cpu_deadman_countdown = deadman_seconds;
1995                 return;
1996         }
1997 
1998         /*
1999          * If we're here, we want to bring the system down.
2000          */
2001         panic("deadman: timed out after %d seconds of clock "
2002             "inactivity", deadman_seconds);
2003         /*NOTREACHED*/
2004 }
2005 
2006 /*ARGSUSED*/
2007 static void
2008 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
2009 {
2010         cpu->cpu_deadman_counter = 0;
2011         cpu->cpu_deadman_countdown = deadman_seconds;
2012 
2013         hdlr->cyh_func = (cyc_func_t)deadman;
2014         hdlr->cyh_level = CY_HIGH_LEVEL;
2015         hdlr->cyh_arg = NULL;
2016 
2017         /*
2018          * Stagger the CPUs so that they don't all run deadman() at
2019          * the same time.  Simplest reason to do this is to make it
2020          * more likely that only one CPU will panic in case of a
2021          * timeout.  This is (strictly speaking) an aesthetic, not a
2022          * technical consideration.
2023          */
2024         when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
2025         when->cyt_interval = NANOSEC;
2026 }
2027 
2028 
2029 void
2030 deadman_init(void)
2031 {
2032         cyc_omni_handler_t hdlr;
2033 
2034         if (deadman_seconds == 0)
2035                 deadman_seconds = snoop_interval / MICROSEC;
2036 
2037         if (snooping)
2038                 deadman_enabled = 1;
2039 
2040         hdlr.cyo_online = deadman_online;
2041         hdlr.cyo_offline = NULL;
2042         hdlr.cyo_arg = NULL;
2043 
2044         mutex_enter(&cpu_lock);
2045         deadman_cyclic = cyclic_add_omni(&hdlr);
2046         mutex_exit(&cpu_lock);
2047 }
2048 
2049 /*
2050  * tod_fault() is for updating tod validate mechanism state:
2051  * (1) TOD_NOFAULT: for resetting the state to 'normal'.
2052  *     currently used for debugging only
2053  * (2) The following four cases detected by tod validate mechanism:
2054  *       TOD_REVERSED: current tod value is less than previous value.
2055  *       TOD_STALLED: current tod value hasn't advanced.
2056  *       TOD_JUMPED: current tod value advanced too far from previous value.
2057  *       TOD_RATECHANGED: the ratio between average tod delta and
2058  *       average tick delta has changed.
2059  * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
2060  *     a virtual TOD provided by a hypervisor.
2061  */
2062 enum tod_fault_type
2063 tod_fault(enum tod_fault_type ftype, int off)
2064 {
2065         ASSERT(MUTEX_HELD(&tod_lock));
2066 
2067         if (tod_faulted != ftype) {
2068                 switch (ftype) {
2069                 case TOD_NOFAULT:
2070                         plat_tod_fault(TOD_NOFAULT);
2071                         cmn_err(CE_NOTE, "Restarted tracking "
2072                             "Time of Day clock.");
2073                         tod_faulted = ftype;
2074                         break;
2075                 case TOD_REVERSED:
2076                 case TOD_JUMPED:
2077                         if (tod_faulted == TOD_NOFAULT) {
2078                                 plat_tod_fault(ftype);
2079                                 cmn_err(CE_WARN, "Time of Day clock error: "
2080                                     "reason [%s by 0x%x]. -- "
2081                                     " Stopped tracking Time Of Day clock.",
2082                                     tod_fault_table[ftype], off);
2083                                 tod_faulted = ftype;
2084                         }
2085                         break;
2086                 case TOD_STALLED:
2087                 case TOD_RATECHANGED:
2088                         if (tod_faulted == TOD_NOFAULT) {
2089                                 plat_tod_fault(ftype);
2090                                 cmn_err(CE_WARN, "Time of Day clock error: "
2091                                     "reason [%s]. -- "
2092                                     " Stopped tracking Time Of Day clock.",
2093                                     tod_fault_table[ftype]);
2094                                 tod_faulted = ftype;
2095                         }
2096                         break;
2097                 case TOD_RDONLY:
2098                         if (tod_faulted == TOD_NOFAULT) {
2099                                 plat_tod_fault(ftype);
2100                                 cmn_err(CE_NOTE, "!Time of Day clock is "
2101                                     "Read-Only; set of Date/Time will not "
2102                                     "persist across reboot.");
2103                                 tod_faulted = ftype;
2104                         }
2105                         break;
2106                 default:
2107                         break;
2108                 }
2109         }
2110         return (tod_faulted);
2111 }
2112 
2113 /*
2114  * Two functions that allow tod_status_flag to be manipulated by functions
2115  * external to this file.
2116  */
2117 
2118 void
2119 tod_status_set(int tod_flag)
2120 {
2121         tod_status_flag |= tod_flag;
2122 }
2123 
2124 void
2125 tod_status_clear(int tod_flag)
2126 {
2127         tod_status_flag &= ~tod_flag;
2128 }
2129 
2130 /*
2131  * Record a timestamp and the value passed to tod_set().  The next call to
2132  * tod_validate() can use these values, prev_set_tick and prev_set_tod,
2133  * when checking the timestruc_t returned by tod_get().  Ordinarily,
2134  * tod_validate() will use prev_tick and prev_tod for this task but these
2135  * become obsolete, and will be re-assigned with the prev_set_* values,
2136  * in the case when the TOD is re-written.
2137  */
2138 void
2139 tod_set_prev(timestruc_t ts)
2140 {
2141         if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2142             tod_validate_deferred) {
2143                 return;
2144         }
2145         prev_set_tick = gethrtime();
2146         /*
2147          * A negative value will be set to zero in utc_to_tod() so we fake
2148          * a zero here in such a case.  This would need to change if the
2149          * behavior of utc_to_tod() changes.
2150          */
2151         prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
2152 }
2153 
2154 /*
2155  * tod_validate() is used for checking values returned by tod_get().
2156  * Four error cases can be detected by this routine:
2157  *   TOD_REVERSED: current tod value is less than previous.
2158  *   TOD_STALLED: current tod value hasn't advanced.
2159  *   TOD_JUMPED: current tod value advanced too far from previous value.
2160  *   TOD_RATECHANGED: the ratio between average tod delta and
2161  *   average tick delta has changed.
2162  */
2163 time_t
2164 tod_validate(time_t tod)
2165 {
2166         time_t diff_tod;
2167         hrtime_t diff_tick;
2168 
2169         long dtick;
2170         int dtick_delta;
2171 
2172         int off = 0;
2173         enum tod_fault_type tod_bad = TOD_NOFAULT;
2174 
2175         static int firsttime = 1;
2176 
2177         static time_t prev_tod = 0;
2178         static hrtime_t prev_tick = 0;
2179         static long dtick_avg = TOD_REF_FREQ;
2180 
2181         int cpr_resume_done = 0;
2182         int dr_resume_done = 0;
2183 
2184         hrtime_t tick = gethrtime();
2185 
2186         ASSERT(MUTEX_HELD(&tod_lock));
2187 
2188         /*
2189          * tod_validate_enable is patchable via /etc/system.
2190          * If TOD is already faulted, or if TOD validation is deferred,
2191          * there is nothing to do.
2192          */
2193         if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2194             tod_validate_deferred) {
2195                 return (tod);
2196         }
2197 
2198         /*
2199          * If this is the first time through, we just need to save the tod
2200          * we were called with and hrtime so we can use them next time to
2201          * validate tod_get().
2202          */
2203         if (firsttime) {
2204                 firsttime = 0;
2205                 prev_tod = tod;
2206                 prev_tick = tick;
2207                 return (tod);
2208         }
2209 
2210         /*
2211          * Handle any flags that have been turned on by tod_status_set().
2212          * In the case where a tod_set() is done and then a subsequent
2213          * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
2214          * true), we treat the TOD_GET_FAILED with precedence by switching
2215          * off the flag, returning tod and leaving TOD_SET_DONE asserted
2216          * until such time as tod_get() completes successfully.
2217          */
2218         if (tod_status_flag & TOD_GET_FAILED) {
2219                 /*
2220                  * tod_get() has encountered an issue, possibly transitory,
2221                  * when reading TOD.  We'll just return the incoming tod
2222                  * value (which is actually hrestime.tv_sec in this case)
2223                  * and when we get a genuine tod, following a successful
2224                  * tod_get(), we can validate using prev_tod and prev_tick.
2225                  */
2226                 tod_status_flag &= ~TOD_GET_FAILED;
2227                 return (tod);
2228         } else if (tod_status_flag & TOD_SET_DONE) {
2229                 /*
2230                  * TOD has been modified.  Just before the TOD was written,
2231                  * tod_set_prev() saved tod and hrtime; we can now use
2232                  * those values, prev_set_tod and prev_set_tick, to validate
2233                  * the incoming tod that's just been read.
2234                  */
2235                 prev_tod = prev_set_tod;
2236                 prev_tick = prev_set_tick;
2237                 dtick_avg = TOD_REF_FREQ;
2238                 tod_status_flag &= ~TOD_SET_DONE;
2239                 /*
2240                  * If a tod_set() preceded a cpr_suspend() without an
2241                  * intervening tod_validate(), we need to ensure that a
2242                  * TOD_JUMPED condition is ignored.
2243                  * Note this isn't a concern in the case of DR as we've
2244                  * just reassigned dtick_avg, above.
2245                  */
2246                 if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2247                         cpr_resume_done = 1;
2248                         tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2249                 }
2250         } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2251                 /*
2252                  * The system's coming back from a checkpoint resume.
2253                  */
2254                 cpr_resume_done = 1;
2255                 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2256                 /*
2257                  * We need to handle the possibility of a CPR suspend
2258                  * operation having been initiated whilst a DR event was
2259                  * in-flight.
2260                  */
2261                 if (tod_status_flag & TOD_DR_RESUME_DONE) {
2262                         dr_resume_done = 1;
2263                         tod_status_flag &= ~TOD_DR_RESUME_DONE;
2264                 }
2265         } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
2266                 /*
2267                  * A Dynamic Reconfiguration event has taken place.
2268                  */
2269                 dr_resume_done = 1;
2270                 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2271         }
2272 
2273         /* test hook */
2274         switch (tod_unit_test) {
2275         case 1: /* for testing jumping tod */
2276                 tod += tod_test_injector;
2277                 tod_unit_test = 0;
2278                 break;
2279         case 2: /* for testing stuck tod bit */
2280                 tod |= 1 << tod_test_injector;
2281                 tod_unit_test = 0;
2282                 break;
2283         case 3: /* for testing stalled tod */
2284                 tod = prev_tod;
2285                 tod_unit_test = 0;
2286                 break;
2287         case 4: /* reset tod fault status */
2288                 (void) tod_fault(TOD_NOFAULT, 0);
2289                 tod_unit_test = 0;
2290                 break;
2291         default:
2292                 break;
2293         }
2294 
2295         diff_tod = tod - prev_tod;
2296         diff_tick = tick - prev_tick;
2297 
2298         ASSERT(diff_tick >= 0);
2299 
2300         if (diff_tod < 0) {
2301                 /* ERROR - tod reversed */
2302                 tod_bad = TOD_REVERSED;
2303                 off = (int)(prev_tod - tod);
2304         } else if (diff_tod == 0) {
2305                 /* tod did not advance */
2306                 if (diff_tick > TOD_STALL_THRESHOLD) {
2307                         /* ERROR - tod stalled */
2308                         tod_bad = TOD_STALLED;
2309                 } else {
2310                         /*
2311                          * Make sure we don't update prev_tick
2312                          * so that diff_tick is calculated since
2313                          * the first diff_tod == 0
2314                          */
2315                         return (tod);
2316                 }
2317         } else {
2318                 /* calculate dtick */
2319                 dtick = diff_tick / diff_tod;
2320 
2321                 /* update dtick averages */
2322                 dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2323 
2324                 /*
2325                  * Calculate dtick_delta as
2326                  * variation from reference freq in quartiles
2327                  */
2328                 dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2329                     (TOD_REF_FREQ >> 2);
2330 
2331                 /*
2332                  * Even with a perfectly functioning TOD device,
2333                  * when the number of elapsed seconds is low the
2334                  * algorithm can calculate a rate that is beyond
2335                  * tolerance, causing an error.  The algorithm is
2336                  * inaccurate when elapsed time is low (less than
2337                  * 5 seconds).
2338                  */
2339                 if (diff_tod > 4) {
2340                         if (dtick < TOD_JUMP_THRESHOLD) {
2341                                 /*
2342                                  * If we've just done a CPR resume, we detect
2343                                  * a jump in the TOD but, actually, what's
2344                                  * happened is that the TOD has been increasing
2345                                  * whilst the system was suspended and the tick
2346                                  * count hasn't kept up.  We consider the first
2347                                  * occurrence of this after a resume as normal
2348                                  * and ignore it; otherwise, in a non-resume
2349                                  * case, we regard it as a TOD problem.
2350                                  */
2351                                 if (!cpr_resume_done) {
2352                                         /* ERROR - tod jumped */
2353                                         tod_bad = TOD_JUMPED;
2354                                         off = (int)diff_tod;
2355                                 }
2356                         }
2357                         if (dtick_delta) {
2358                                 /*
2359                                  * If we've just done a DR resume, dtick_avg
2360                                  * can go a bit askew so we reset it and carry
2361                                  * on; otherwise, the TOD is in error.
2362                                  */
2363                                 if (dr_resume_done) {
2364                                         dtick_avg = TOD_REF_FREQ;
2365                                 } else {
2366                                         /* ERROR - change in clock rate */
2367                                         tod_bad = TOD_RATECHANGED;
2368                                 }
2369                         }
2370                 }
2371         }
2372 
2373         if (tod_bad != TOD_NOFAULT) {
2374                 (void) tod_fault(tod_bad, off);
2375 
2376                 /*
2377                  * Disable dosynctodr since we are going to fault
2378                  * the TOD chip anyway here
2379                  */
2380                 dosynctodr = 0;
2381 
2382                 /*
2383                  * Set tod to the correct value from hrestime
2384                  */
2385                 tod = hrestime.tv_sec;
2386         }
2387 
2388         prev_tod = tod;
2389         prev_tick = tick;
2390         return (tod);
2391 }
2392 
2393 static void
2394 calcloadavg(int nrun, uint64_t *hp_ave)
2395 {
2396         static int64_t f[3] = { 135, 27, 9 };
2397         uint_t i;
2398         int64_t q, r;
2399 
2400         /*
2401          * Compute load average over the last 1, 5, and 15 minutes
2402          * (60, 300, and 900 seconds).  The constants in f[3] are for
2403          * exponential decay:
2404          * (1 - exp(-1/60)) << 13 = 135,
2405          * (1 - exp(-1/300)) << 13 = 27,
2406          * (1 - exp(-1/900)) << 13 = 9.
2407          */
2408 
2409         /*
2410          * a little hoop-jumping to avoid integer overflow
2411          */
2412         for (i = 0; i < 3; i++) {
2413                 q = (hp_ave[i]  >> 16) << 7;
2414                 r = (hp_ave[i]  & 0xffff) << 7;
2415                 hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2416         }
2417 }
2418 
2419 /*
2420  * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
2421  * calculate the value of lbolt according to the current mode. In the event
2422  * driven mode (the default), lbolt is calculated by dividing the current hires
2423  * time by the number of nanoseconds per clock tick. In the cyclic driven mode
2424  * an internal variable is incremented at each firing of the lbolt cyclic
2425  * and returned by lbolt_cyclic_driven().
2426  *
2427  * The system will transition from event to cyclic driven mode when the number
2428  * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
2429  * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
2430  * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
2431  * causing enough activity to cross the thresholds.
2432  */
2433 int64_t
2434 lbolt_bootstrap(void)
2435 {
2436         return (0);
2437 }
2438 
2439 /* ARGSUSED */
2440 uint_t
2441 lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
2442 {
2443         hrtime_t ts, exp;
2444         int ret;
2445 
2446         ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
2447 
2448         kpreempt_disable();
2449 
2450         ts = gethrtime();
2451         lb_info->lbi_internal = (ts/nsec_per_tick);
2452 
2453         /*
2454          * Align the next expiration to a clock tick boundary.
2455          */
2456         exp = ts + nsec_per_tick - 1;
2457         exp = (exp/nsec_per_tick) * nsec_per_tick;
2458 
2459         ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
2460         ASSERT(ret);
2461 
2462         lbolt_hybrid = lbolt_cyclic_driven;
2463         lb_info->lbi_cyc_deactivate = B_FALSE;
2464         lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2465 
2466         kpreempt_enable();
2467 
2468         ret = atomic_dec_32_nv(&lb_info->lbi_token);
2469         ASSERT(ret == 0);
2470 
2471         return (1);
2472 }
2473 
2474 int64_t
2475 lbolt_event_driven(void)
2476 {
2477         hrtime_t ts;
2478         int64_t lb;
2479         int ret, cpu = CPU->cpu_seqid;
2480 
2481         ts = gethrtime();
2482         ASSERT(ts > 0);
2483 
2484         ASSERT(nsec_per_tick > 0);
2485         lb = (ts/nsec_per_tick);
2486 
2487         /*
2488          * Switch to cyclic mode if the number of calls to this routine
2489          * has reached the threshold within the interval.
2490          */
2491         if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
2492 
2493                 if (--lb_cpu[cpu].lbc_counter == 0) {
2494                         /*
2495                          * Reached the threshold within the interval, reset
2496                          * the usage statistics.
2497                          */
2498                         lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2499                         lb_cpu[cpu].lbc_cnt_start = lb;
2500 
2501                         /*
2502                          * Make sure only one thread reprograms the
2503                          * lbolt cyclic and changes the mode.
2504                          */
2505                         if (panicstr == NULL &&
2506                             atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2507 
2508                                 if (lbolt_hybrid == lbolt_cyclic_driven) {
2509                                         ret = atomic_dec_32_nv(
2510                                             &lb_info->lbi_token);
2511                                         ASSERT(ret == 0);
2512                                 } else {
2513                                         lbolt_softint_post();
2514                                 }
2515                         }
2516                 }
2517         } else {
2518                 /*
2519                  * Exceeded the interval, reset the usage statistics.
2520                  */
2521                 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2522                 lb_cpu[cpu].lbc_cnt_start = lb;
2523         }
2524 
2525         ASSERT(lb >= lb_info->lbi_debug_time);
2526 
2527         return (lb - lb_info->lbi_debug_time);
2528 }
2529 
2530 int64_t
2531 lbolt_cyclic_driven(void)
2532 {
2533         int64_t lb = lb_info->lbi_internal;
2534         int cpu;
2535 
2536         /*
2537          * If a CPU has already prevented the lbolt cyclic from deactivating
2538          * itself, don't bother tracking the usage. Otherwise check if we're
2539          * within the interval and how the per CPU counter is doing.
2540          */
2541         if (lb_info->lbi_cyc_deactivate) {
2542                 cpu = CPU->cpu_seqid;
2543                 if ((lb - lb_cpu[cpu].lbc_cnt_start) <
2544                     lb_info->lbi_thresh_interval) {
2545 
2546                         if (lb_cpu[cpu].lbc_counter == 0)
2547                                 /*
2548                                  * Reached the threshold within the interval,
2549                                  * prevent the lbolt cyclic from turning itself
2550                                  * off.
2551                                  */
2552                                 lb_info->lbi_cyc_deactivate = B_FALSE;
2553                         else
2554                                 lb_cpu[cpu].lbc_counter--;
2555                 } else {
2556                         /*
2557                          * Only reset the usage statistics when we have
2558                          * exceeded the interval.
2559                          */
2560                         lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2561                         lb_cpu[cpu].lbc_cnt_start = lb;
2562                 }
2563         }
2564 
2565         ASSERT(lb >= lb_info->lbi_debug_time);
2566 
2567         return (lb - lb_info->lbi_debug_time);
2568 }
2569 
2570 /*
2571  * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
2572  * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
2573  * It is inactive by default, and will be activated when switching from event
2574  * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
2575  * by lbolt_cyclic_driven().
2576  */
2577 static void
2578 lbolt_cyclic(void)
2579 {
2580         int ret;
2581 
2582         lb_info->lbi_internal++;
2583 
2584         if (!lbolt_cyc_only) {
2585 
2586                 if (lb_info->lbi_cyc_deactivate) {
2587                         /*
2588                          * Switching from cyclic to event driven mode.
2589                          */
2590                         if (panicstr == NULL &&
2591                             atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2592 
2593                                 if (lbolt_hybrid == lbolt_event_driven) {
2594                                         ret = atomic_dec_32_nv(
2595                                             &lb_info->lbi_token);
2596                                         ASSERT(ret == 0);
2597                                         return;
2598                                 }
2599 
2600                                 kpreempt_disable();
2601 
2602                                 lbolt_hybrid = lbolt_event_driven;
2603                                 ret = cyclic_reprogram(
2604                                     lb_info->id.lbi_cyclic_id,
2605                                     CY_INFINITY);
2606                                 ASSERT(ret);
2607 
2608                                 kpreempt_enable();
2609 
2610                                 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2611                                 ASSERT(ret == 0);
2612                         }
2613                 }
2614 
2615                 /*
2616                  * The lbolt cyclic should not try to deactivate itself before
2617                  * the sampling period has elapsed.
2618                  */
2619                 if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
2620                     lb_info->lbi_thresh_interval) {
2621                         lb_info->lbi_cyc_deactivate = B_TRUE;
2622                         lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2623                 }
2624         }
2625 }
2626 
2627 /*
2628  * Since the lbolt service was historically cyclic driven, it must be 'stopped'
2629  * when the system drops into the kernel debugger. lbolt_debug_entry() is
2630  * called by the KDI system claim callbacks to record a hires timestamp at
2631  * debug enter time. lbolt_debug_return() is called by the sistem release
2632  * callbacks to account for the time spent in the debugger. The value is then
2633  * accumulated in the lb_info structure and used by lbolt_event_driven() and
2634  * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
2635  */
2636 void
2637 lbolt_debug_entry(void)
2638 {
2639         if (lbolt_hybrid != lbolt_bootstrap) {
2640                 ASSERT(lb_info != NULL);
2641                 lb_info->lbi_debug_ts = gethrtime();
2642         }
2643 }
2644 
2645 /*
2646  * Calculate the time spent in the debugger and add it to the lbolt info
2647  * structure. We also update the internal lbolt value in case we were in
2648  * cyclic driven mode going in.
2649  */
2650 void
2651 lbolt_debug_return(void)
2652 {
2653         hrtime_t ts;
2654 
2655         if (lbolt_hybrid != lbolt_bootstrap) {
2656                 ASSERT(lb_info != NULL);
2657                 ASSERT(nsec_per_tick > 0);
2658 
2659                 ts = gethrtime();
2660                 lb_info->lbi_internal = (ts/nsec_per_tick);
2661                 lb_info->lbi_debug_time +=
2662                     ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
2663 
2664                 lb_info->lbi_debug_ts = 0;
2665         }
2666 }