Print this page
OS-2366 ddi_periodic_add(9F) is entirely rubbish (MORE)
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/clock.c
+++ new/usr/src/uts/common/os/clock.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 22 /* All Rights Reserved */
23 23
24 24 /*
25 25 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
26 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
27 27 */
28 28
29 29 #include <sys/param.h>
30 30 #include <sys/t_lock.h>
31 31 #include <sys/types.h>
32 32 #include <sys/tuneable.h>
33 33 #include <sys/sysmacros.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/cpuvar.h>
36 36 #include <sys/lgrp.h>
37 37 #include <sys/user.h>
38 38 #include <sys/proc.h>
39 39 #include <sys/callo.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/var.h>
42 42 #include <sys/cmn_err.h>
43 43 #include <sys/swap.h>
44 44 #include <sys/vmsystm.h>
45 45 #include <sys/class.h>
46 46 #include <sys/time.h>
47 47 #include <sys/debug.h>
48 48 #include <sys/vtrace.h>
49 49 #include <sys/spl.h>
50 50 #include <sys/atomic.h>
51 51 #include <sys/dumphdr.h>
52 52 #include <sys/archsystm.h>
53 53 #include <sys/fs/swapnode.h>
54 54 #include <sys/panic.h>
55 55 #include <sys/disp.h>
56 56 #include <sys/msacct.h>
|
↓ open down ↓ |
56 lines elided |
↑ open up ↑ |
57 57 #include <sys/mem_cage.h>
58 58
59 59 #include <vm/page.h>
60 60 #include <vm/anon.h>
61 61 #include <vm/rm.h>
62 62 #include <sys/cyclic.h>
63 63 #include <sys/cpupart.h>
64 64 #include <sys/rctl.h>
65 65 #include <sys/task.h>
66 66 #include <sys/sdt.h>
67 -#include <sys/ddi_timer.h>
67 +#include <sys/ddi_periodic.h>
68 68 #include <sys/random.h>
69 69 #include <sys/modctl.h>
70 70 #include <sys/zone.h>
71 71
72 72 /*
73 73 * for NTP support
74 74 */
75 75 #include <sys/timex.h>
76 76 #include <sys/inttypes.h>
77 77
78 78 #include <sys/sunddi.h>
79 79 #include <sys/clock_impl.h>
80 80
81 81 /*
82 82 * clock() is called straight from the clock cyclic; see clock_init().
83 83 *
84 84 * Functions:
85 85 * reprime clock
86 86 * maintain date
87 87 * jab the scheduler
88 88 */
89 89
90 90 extern kcondvar_t fsflush_cv;
91 91 extern sysinfo_t sysinfo;
92 92 extern vminfo_t vminfo;
93 93 extern int idleswtch; /* flag set while idle in pswtch() */
94 94 extern hrtime_t volatile devinfo_freeze;
95 95
96 96 /*
97 97 * high-precision avenrun values. These are needed to make the
98 98 * regular avenrun values accurate.
99 99 */
100 100 static uint64_t hp_avenrun[3];
101 101 int avenrun[3]; /* FSCALED average run queue lengths */
102 102 time_t time; /* time in seconds since 1970 - for compatibility only */
103 103
104 104 static struct loadavg_s loadavg;
105 105 /*
106 106 * Phase/frequency-lock loop (PLL/FLL) definitions
107 107 *
108 108 * The following variables are read and set by the ntp_adjtime() system
109 109 * call.
110 110 *
111 111 * time_state shows the state of the system clock, with values defined
112 112 * in the timex.h header file.
113 113 *
114 114 * time_status shows the status of the system clock, with bits defined
115 115 * in the timex.h header file.
116 116 *
117 117 * time_offset is used by the PLL/FLL to adjust the system time in small
118 118 * increments.
119 119 *
120 120 * time_constant determines the bandwidth or "stiffness" of the PLL.
121 121 *
122 122 * time_tolerance determines maximum frequency error or tolerance of the
123 123 * CPU clock oscillator and is a property of the architecture; however,
124 124 * in principle it could change as result of the presence of external
125 125 * discipline signals, for instance.
126 126 *
127 127 * time_precision is usually equal to the kernel tick variable; however,
128 128 * in cases where a precision clock counter or external clock is
129 129 * available, the resolution can be much less than this and depend on
130 130 * whether the external clock is working or not.
131 131 *
132 132 * time_maxerror is initialized by a ntp_adjtime() call and increased by
133 133 * the kernel once each second to reflect the maximum error bound
134 134 * growth.
135 135 *
136 136 * time_esterror is set and read by the ntp_adjtime() call, but
137 137 * otherwise not used by the kernel.
138 138 */
139 139 int32_t time_state = TIME_OK; /* clock state */
140 140 int32_t time_status = STA_UNSYNC; /* clock status bits */
141 141 int32_t time_offset = 0; /* time offset (us) */
142 142 int32_t time_constant = 0; /* pll time constant */
143 143 int32_t time_tolerance = MAXFREQ; /* frequency tolerance (scaled ppm) */
144 144 int32_t time_precision = 1; /* clock precision (us) */
145 145 int32_t time_maxerror = MAXPHASE; /* maximum error (us) */
146 146 int32_t time_esterror = MAXPHASE; /* estimated error (us) */
147 147
148 148 /*
149 149 * The following variables establish the state of the PLL/FLL and the
150 150 * residual time and frequency offset of the local clock. The scale
151 151 * factors are defined in the timex.h header file.
152 152 *
153 153 * time_phase and time_freq are the phase increment and the frequency
154 154 * increment, respectively, of the kernel time variable.
155 155 *
156 156 * time_freq is set via ntp_adjtime() from a value stored in a file when
157 157 * the synchronization daemon is first started. Its value is retrieved
158 158 * via ntp_adjtime() and written to the file about once per hour by the
159 159 * daemon.
160 160 *
161 161 * time_adj is the adjustment added to the value of tick at each timer
162 162 * interrupt and is recomputed from time_phase and time_freq at each
163 163 * seconds rollover.
164 164 *
165 165 * time_reftime is the second's portion of the system time at the last
166 166 * call to ntp_adjtime(). It is used to adjust the time_freq variable
167 167 * and to increase the time_maxerror as the time since last update
168 168 * increases.
169 169 */
170 170 int32_t time_phase = 0; /* phase offset (scaled us) */
171 171 int32_t time_freq = 0; /* frequency offset (scaled ppm) */
172 172 int32_t time_adj = 0; /* tick adjust (scaled 1 / hz) */
173 173 int32_t time_reftime = 0; /* time at last adjustment (s) */
174 174
175 175 /*
176 176 * The scale factors of the following variables are defined in the
177 177 * timex.h header file.
178 178 *
179 179 * pps_time contains the time at each calibration interval, as read by
180 180 * microtime(). pps_count counts the seconds of the calibration
181 181 * interval, the duration of which is nominally pps_shift in powers of
182 182 * two.
183 183 *
184 184 * pps_offset is the time offset produced by the time median filter
185 185 * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
186 186 * this filter.
187 187 *
188 188 * pps_freq is the frequency offset produced by the frequency median
189 189 * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
190 190 * by this filter.
191 191 *
192 192 * pps_usec is latched from a high resolution counter or external clock
193 193 * at pps_time. Here we want the hardware counter contents only, not the
194 194 * contents plus the time_tv.usec as usual.
195 195 *
196 196 * pps_valid counts the number of seconds since the last PPS update. It
197 197 * is used as a watchdog timer to disable the PPS discipline should the
198 198 * PPS signal be lost.
199 199 *
200 200 * pps_glitch counts the number of seconds since the beginning of an
201 201 * offset burst more than tick/2 from current nominal offset. It is used
202 202 * mainly to suppress error bursts due to priority conflicts between the
203 203 * PPS interrupt and timer interrupt.
204 204 *
205 205 * pps_intcnt counts the calibration intervals for use in the interval-
206 206 * adaptation algorithm. It's just too complicated for words.
207 207 */
208 208 struct timeval pps_time; /* kernel time at last interval */
209 209 int32_t pps_tf[] = {0, 0, 0}; /* pps time offset median filter (us) */
210 210 int32_t pps_offset = 0; /* pps time offset (us) */
211 211 int32_t pps_jitter = MAXTIME; /* time dispersion (jitter) (us) */
212 212 int32_t pps_ff[] = {0, 0, 0}; /* pps frequency offset median filter */
213 213 int32_t pps_freq = 0; /* frequency offset (scaled ppm) */
214 214 int32_t pps_stabil = MAXFREQ; /* frequency dispersion (scaled ppm) */
215 215 int32_t pps_usec = 0; /* microsec counter at last interval */
216 216 int32_t pps_valid = PPS_VALID; /* pps signal watchdog counter */
217 217 int32_t pps_glitch = 0; /* pps signal glitch counter */
218 218 int32_t pps_count = 0; /* calibration interval counter (s) */
219 219 int32_t pps_shift = PPS_SHIFT; /* interval duration (s) (shift) */
220 220 int32_t pps_intcnt = 0; /* intervals at current duration */
221 221
222 222 /*
223 223 * PPS signal quality monitors
224 224 *
225 225 * pps_jitcnt counts the seconds that have been discarded because the
226 226 * jitter measured by the time median filter exceeds the limit MAXTIME
227 227 * (100 us).
228 228 *
229 229 * pps_calcnt counts the frequency calibration intervals, which are
230 230 * variable from 4 s to 256 s.
231 231 *
232 232 * pps_errcnt counts the calibration intervals which have been discarded
233 233 * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
234 234 * calibration interval jitter exceeds two ticks.
235 235 *
236 236 * pps_stbcnt counts the calibration intervals that have been discarded
237 237 * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
238 238 */
239 239 int32_t pps_jitcnt = 0; /* jitter limit exceeded */
240 240 int32_t pps_calcnt = 0; /* calibration intervals */
241 241 int32_t pps_errcnt = 0; /* calibration errors */
242 242 int32_t pps_stbcnt = 0; /* stability limit exceeded */
243 243
244 244 kcondvar_t lbolt_cv;
245 245
246 246 /*
247 247 * Hybrid lbolt implementation:
248 248 *
249 249 * The service historically provided by the lbolt and lbolt64 variables has
250 250 * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
251 251 * original symbols removed from the system. The once clock driven variables are
252 252 * now implemented in an event driven fashion, backed by gethrtime() coarsed to
253 253 * the appropriate clock resolution. The default event driven implementation is
254 254 * complemented by a cyclic driven one, active only during periods of intense
255 255 * activity around the DDI lbolt routines, when a lbolt specific cyclic is
256 256 * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
257 257 * rely on the original low cost of consulting a memory position.
258 258 *
259 259 * The implementation uses the number of calls to these routines and the
260 260 * frequency of these to determine when to transition from event to cyclic
261 261 * driven and vice-versa. These values are kept on a per CPU basis for
262 262 * scalability reasons and to prevent CPUs from constantly invalidating a single
263 263 * cache line when modifying a global variable. The transition from event to
264 264 * cyclic mode happens once the thresholds are crossed, and activity on any CPU
265 265 * can cause such transition.
266 266 *
267 267 * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
268 268 * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
269 269 * lbolt_cyclic_driven() according to the current mode. When the thresholds
270 270 * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
271 271 * fire at a nsec_per_tick interval and increment an internal variable at
272 272 * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
273 273 * will simply return the value of such variable. lbolt_cyclic() will attempt
274 274 * to shut itself off at each threshold interval (sampling period for calls
275 275 * to the DDI lbolt routines), and return to the event driven mode, but will
276 276 * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
277 277 *
278 278 * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
279 279 * for the cyclic subsystem to be intialized.
280 280 *
281 281 */
282 282 int64_t lbolt_bootstrap(void);
283 283 int64_t lbolt_event_driven(void);
284 284 int64_t lbolt_cyclic_driven(void);
285 285 int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
286 286 uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
287 287
288 288 /*
289 289 * lbolt's cyclic, installed by clock_init().
290 290 */
291 291 static void lbolt_cyclic(void);
292 292
293 293 /*
294 294 * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
295 295 * from switching back to event driven, once it reaches cyclic mode.
296 296 */
297 297 static boolean_t lbolt_cyc_only = B_FALSE;
298 298
299 299 /*
300 300 * Cache aligned, per CPU structure with lbolt usage statistics.
301 301 */
302 302 static lbolt_cpu_t *lb_cpu;
303 303
304 304 /*
305 305 * Single, cache aligned, structure with all the information required by
306 306 * the lbolt implementation.
307 307 */
308 308 lbolt_info_t *lb_info;
|
↓ open down ↓ |
231 lines elided |
↑ open up ↑ |
309 309
310 310
311 311 int one_sec = 1; /* turned on once every second */
312 312 static int fsflushcnt; /* counter for t_fsflushr */
313 313 int dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
314 314 int tod_needsync = 0; /* need to sync tod chip with software time */
315 315 static int tod_broken = 0; /* clock chip doesn't work */
316 316 time_t boot_time = 0; /* Boot time in seconds since 1970 */
317 317 cyclic_id_t clock_cyclic; /* clock()'s cyclic_id */
318 318 cyclic_id_t deadman_cyclic; /* deadman()'s cyclic_id */
319 -cyclic_id_t ddi_timer_cyclic; /* cyclic_timer()'s cyclic_id */
320 319
321 320 extern void clock_tick_schedule(int);
322 321
323 322 static int lgrp_ticks; /* counter to schedule lgrp load calcs */
324 323
325 324 /*
326 325 * for tod fault detection
327 326 */
328 327 #define TOD_REF_FREQ ((longlong_t)(NANOSEC))
329 328 #define TOD_STALL_THRESHOLD (TOD_REF_FREQ * 3 / 2)
330 329 #define TOD_JUMP_THRESHOLD (TOD_REF_FREQ / 2)
331 330 #define TOD_FILTER_N 4
332 331 #define TOD_FILTER_SETTLE (4 * TOD_FILTER_N)
333 332 static int tod_faulted = TOD_NOFAULT;
334 333
335 334 static int tod_status_flag = 0; /* used by tod_validate() */
336 335
337 336 static hrtime_t prev_set_tick = 0; /* gethrtime() prior to tod_set() */
338 337 static time_t prev_set_tod = 0; /* tv_sec value passed to tod_set() */
339 338
340 339 /* patchable via /etc/system */
341 340 int tod_validate_enable = 1;
342 341
343 342 /* Diagnose/Limit messages about delay(9F) called from interrupt context */
344 343 int delay_from_interrupt_diagnose = 0;
345 344 volatile uint32_t delay_from_interrupt_msg = 20;
346 345
347 346 /*
348 347 * On non-SPARC systems, TOD validation must be deferred until gethrtime
349 348 * returns non-zero values (after mach_clkinit's execution).
350 349 * On SPARC systems, it must be deferred until after hrtime_base
351 350 * and hres_last_tick are set (in the first invocation of hres_tick).
352 351 * Since in both cases the prerequisites occur before the invocation of
353 352 * tod_get() in clock(), the deferment is lifted there.
354 353 */
355 354 static boolean_t tod_validate_deferred = B_TRUE;
356 355
357 356 /*
358 357 * tod_fault_table[] must be aligned with
359 358 * enum tod_fault_type in systm.h
360 359 */
361 360 static char *tod_fault_table[] = {
362 361 "Reversed", /* TOD_REVERSED */
363 362 "Stalled", /* TOD_STALLED */
364 363 "Jumped", /* TOD_JUMPED */
365 364 "Changed in Clock Rate", /* TOD_RATECHANGED */
366 365 "Is Read-Only" /* TOD_RDONLY */
367 366 /*
368 367 * no strings needed for TOD_NOFAULT
369 368 */
370 369 };
371 370
372 371 /*
373 372 * test hook for tod broken detection in tod_validate
374 373 */
375 374 int tod_unit_test = 0;
376 375 time_t tod_test_injector;
377 376
378 377 #define CLOCK_ADJ_HIST_SIZE 4
379 378
380 379 static int adj_hist_entry;
381 380
382 381 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
383 382
384 383 static void calcloadavg(int, uint64_t *);
385 384 static int genloadavg(struct loadavg_s *);
386 385 static void loadavg_update();
387 386
388 387 void (*cmm_clock_callout)() = NULL;
389 388 void (*cpucaps_clock_callout)() = NULL;
390 389
391 390 extern clock_t clock_tick_proc_max;
392 391
393 392 static int64_t deadman_counter = 0;
394 393
395 394 static void
396 395 clock(void)
397 396 {
398 397 kthread_t *t;
399 398 uint_t nrunnable;
400 399 uint_t w_io;
401 400 cpu_t *cp;
402 401 cpupart_t *cpupart;
403 402 extern void set_freemem();
404 403 void (*funcp)();
405 404 int32_t ltemp;
406 405 int64_t lltemp;
407 406 int s;
408 407 int do_lgrp_load;
409 408 int i;
410 409 clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
411 410
412 411 if (panicstr)
413 412 return;
414 413
415 414 /*
416 415 * Make sure that 'freemem' do not drift too far from the truth
417 416 */
418 417 set_freemem();
419 418
420 419
421 420 /*
422 421 * Before the section which is repeated is executed, we do
423 422 * the time delta processing which occurs every clock tick
424 423 *
425 424 * There is additional processing which happens every time
426 425 * the nanosecond counter rolls over which is described
427 426 * below - see the section which begins with : if (one_sec)
428 427 *
429 428 * This section marks the beginning of the precision-kernel
430 429 * code fragment.
431 430 *
432 431 * First, compute the phase adjustment. If the low-order bits
433 432 * (time_phase) of the update overflow, bump the higher order
434 433 * bits (time_update).
435 434 */
436 435 time_phase += time_adj;
437 436 if (time_phase <= -FINEUSEC) {
438 437 ltemp = -time_phase / SCALE_PHASE;
439 438 time_phase += ltemp * SCALE_PHASE;
440 439 s = hr_clock_lock();
441 440 timedelta -= ltemp * (NANOSEC/MICROSEC);
442 441 hr_clock_unlock(s);
443 442 } else if (time_phase >= FINEUSEC) {
444 443 ltemp = time_phase / SCALE_PHASE;
445 444 time_phase -= ltemp * SCALE_PHASE;
446 445 s = hr_clock_lock();
447 446 timedelta += ltemp * (NANOSEC/MICROSEC);
448 447 hr_clock_unlock(s);
449 448 }
450 449
451 450 /*
452 451 * End of precision-kernel code fragment which is processed
453 452 * every timer interrupt.
454 453 *
455 454 * Continue with the interrupt processing as scheduled.
456 455 */
457 456 /*
458 457 * Count the number of runnable threads and the number waiting
459 458 * for some form of I/O to complete -- gets added to
460 459 * sysinfo.waiting. To know the state of the system, must add
461 460 * wait counts from all CPUs. Also add up the per-partition
462 461 * statistics.
463 462 */
464 463 w_io = 0;
465 464 nrunnable = 0;
466 465
467 466 /*
468 467 * keep track of when to update lgrp/part loads
469 468 */
470 469
471 470 do_lgrp_load = 0;
472 471 if (lgrp_ticks++ >= hz / 10) {
473 472 lgrp_ticks = 0;
474 473 do_lgrp_load = 1;
475 474 }
476 475
477 476 if (one_sec) {
478 477 loadavg_update();
479 478 deadman_counter++;
480 479 }
481 480
482 481 /*
483 482 * First count the threads waiting on kpreempt queues in each
484 483 * CPU partition.
485 484 */
486 485
487 486 cpupart = cp_list_head;
488 487 do {
489 488 uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
490 489
491 490 cpupart->cp_updates++;
492 491 nrunnable += cpupart_nrunnable;
493 492 cpupart->cp_nrunnable_cum += cpupart_nrunnable;
494 493 if (one_sec) {
495 494 cpupart->cp_nrunning = 0;
496 495 cpupart->cp_nrunnable = cpupart_nrunnable;
497 496 }
498 497 } while ((cpupart = cpupart->cp_next) != cp_list_head);
499 498
500 499
501 500 /* Now count the per-CPU statistics. */
502 501 cp = cpu_list;
503 502 do {
504 503 uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
505 504
506 505 nrunnable += cpu_nrunnable;
507 506 cpupart = cp->cpu_part;
508 507 cpupart->cp_nrunnable_cum += cpu_nrunnable;
509 508 if (one_sec) {
510 509 cpupart->cp_nrunnable += cpu_nrunnable;
511 510 /*
512 511 * Update user, system, and idle cpu times.
513 512 */
514 513 cpupart->cp_nrunning++;
515 514 /*
516 515 * w_io is used to update sysinfo.waiting during
517 516 * one_second processing below. Only gather w_io
518 517 * information when we walk the list of cpus if we're
519 518 * going to perform one_second processing.
520 519 */
521 520 w_io += CPU_STATS(cp, sys.iowait);
522 521 }
523 522
524 523 if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
525 524 int i, load, change;
526 525 hrtime_t intracct, intrused;
527 526 const hrtime_t maxnsec = 1000000000;
528 527 const int precision = 100;
529 528
530 529 /*
531 530 * Estimate interrupt load on this cpu each second.
532 531 * Computes cpu_intrload as %utilization (0-99).
533 532 */
534 533
535 534 /* add up interrupt time from all micro states */
536 535 for (intracct = 0, i = 0; i < NCMSTATES; i++)
537 536 intracct += cp->cpu_intracct[i];
538 537 scalehrtime(&intracct);
539 538
540 539 /* compute nsec used in the past second */
541 540 intrused = intracct - cp->cpu_intrlast;
542 541 cp->cpu_intrlast = intracct;
543 542
544 543 /* limit the value for safety (and the first pass) */
545 544 if (intrused >= maxnsec)
546 545 intrused = maxnsec - 1;
547 546
548 547 /* calculate %time in interrupt */
549 548 load = (precision * intrused) / maxnsec;
550 549 ASSERT(load >= 0 && load < precision);
551 550 change = cp->cpu_intrload - load;
552 551
553 552 /* jump to new max, or decay the old max */
554 553 if (change < 0)
555 554 cp->cpu_intrload = load;
556 555 else if (change > 0)
557 556 cp->cpu_intrload -= (change + 3) / 4;
558 557
559 558 DTRACE_PROBE3(cpu_intrload,
560 559 cpu_t *, cp,
561 560 hrtime_t, intracct,
562 561 hrtime_t, intrused);
563 562 }
564 563
565 564 if (do_lgrp_load &&
566 565 (cp->cpu_flags & CPU_EXISTS)) {
567 566 /*
568 567 * When updating the lgroup's load average,
569 568 * account for the thread running on the CPU.
570 569 * If the CPU is the current one, then we need
571 570 * to account for the underlying thread which
572 571 * got the clock interrupt not the thread that is
573 572 * handling the interrupt and caculating the load
574 573 * average
575 574 */
576 575 t = cp->cpu_thread;
577 576 if (CPU == cp)
578 577 t = t->t_intr;
579 578
580 579 /*
581 580 * Account for the load average for this thread if
582 581 * it isn't the idle thread or it is on the interrupt
583 582 * stack and not the current CPU handling the clock
584 583 * interrupt
585 584 */
586 585 if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
587 586 CPU_ON_INTR(cp))) {
588 587 if (t->t_lpl == cp->cpu_lpl) {
589 588 /* local thread */
590 589 cpu_nrunnable++;
591 590 } else {
592 591 /*
593 592 * This is a remote thread, charge it
594 593 * against its home lgroup. Note that
595 594 * we notice that a thread is remote
596 595 * only if it's currently executing.
597 596 * This is a reasonable approximation,
598 597 * since queued remote threads are rare.
599 598 * Note also that if we didn't charge
600 599 * it to its home lgroup, remote
601 600 * execution would often make a system
602 601 * appear balanced even though it was
603 602 * not, and thread placement/migration
604 603 * would often not be done correctly.
605 604 */
606 605 lgrp_loadavg(t->t_lpl,
607 606 LGRP_LOADAVG_IN_THREAD_MAX, 0);
608 607 }
609 608 }
610 609 lgrp_loadavg(cp->cpu_lpl,
611 610 cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
612 611 }
613 612 } while ((cp = cp->cpu_next) != cpu_list);
614 613
615 614 clock_tick_schedule(one_sec);
616 615
617 616 /*
618 617 * Check for a callout that needs be called from the clock
619 618 * thread to support the membership protocol in a clustered
620 619 * system. Copy the function pointer so that we can reset
621 620 * this to NULL if needed.
622 621 */
623 622 if ((funcp = cmm_clock_callout) != NULL)
624 623 (*funcp)();
625 624
626 625 if ((funcp = cpucaps_clock_callout) != NULL)
627 626 (*funcp)();
628 627
629 628 /*
630 629 * Wakeup the cageout thread waiters once per second.
631 630 */
632 631 if (one_sec)
633 632 kcage_tick();
634 633
635 634 if (one_sec) {
636 635
637 636 int drift, absdrift;
638 637 timestruc_t tod;
639 638 int s;
640 639
641 640 /*
642 641 * Beginning of precision-kernel code fragment executed
643 642 * every second.
644 643 *
645 644 * On rollover of the second the phase adjustment to be
646 645 * used for the next second is calculated. Also, the
647 646 * maximum error is increased by the tolerance. If the
648 647 * PPS frequency discipline code is present, the phase is
649 648 * increased to compensate for the CPU clock oscillator
650 649 * frequency error.
651 650 *
652 651 * On a 32-bit machine and given parameters in the timex.h
653 652 * header file, the maximum phase adjustment is +-512 ms
654 653 * and maximum frequency offset is (a tad less than)
655 654 * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
656 655 */
657 656 time_maxerror += time_tolerance / SCALE_USEC;
658 657
659 658 /*
660 659 * Leap second processing. If in leap-insert state at
661 660 * the end of the day, the system clock is set back one
662 661 * second; if in leap-delete state, the system clock is
663 662 * set ahead one second. The microtime() routine or
664 663 * external clock driver will insure that reported time
665 664 * is always monotonic. The ugly divides should be
666 665 * replaced.
667 666 */
668 667 switch (time_state) {
669 668
670 669 case TIME_OK:
671 670 if (time_status & STA_INS)
672 671 time_state = TIME_INS;
673 672 else if (time_status & STA_DEL)
674 673 time_state = TIME_DEL;
675 674 break;
676 675
677 676 case TIME_INS:
678 677 if (hrestime.tv_sec % 86400 == 0) {
679 678 s = hr_clock_lock();
680 679 hrestime.tv_sec--;
681 680 hr_clock_unlock(s);
682 681 time_state = TIME_OOP;
683 682 }
684 683 break;
685 684
686 685 case TIME_DEL:
687 686 if ((hrestime.tv_sec + 1) % 86400 == 0) {
688 687 s = hr_clock_lock();
689 688 hrestime.tv_sec++;
690 689 hr_clock_unlock(s);
691 690 time_state = TIME_WAIT;
692 691 }
693 692 break;
694 693
695 694 case TIME_OOP:
696 695 time_state = TIME_WAIT;
697 696 break;
698 697
699 698 case TIME_WAIT:
700 699 if (!(time_status & (STA_INS | STA_DEL)))
701 700 time_state = TIME_OK;
702 701 default:
703 702 break;
704 703 }
705 704
706 705 /*
707 706 * Compute the phase adjustment for the next second. In
708 707 * PLL mode, the offset is reduced by a fixed factor
709 708 * times the time constant. In FLL mode the offset is
710 709 * used directly. In either mode, the maximum phase
711 710 * adjustment for each second is clamped so as to spread
712 711 * the adjustment over not more than the number of
713 712 * seconds between updates.
714 713 */
715 714 if (time_offset == 0)
716 715 time_adj = 0;
717 716 else if (time_offset < 0) {
718 717 lltemp = -time_offset;
719 718 if (!(time_status & STA_FLL)) {
720 719 if ((1 << time_constant) >= SCALE_KG)
721 720 lltemp *= (1 << time_constant) /
722 721 SCALE_KG;
723 722 else
724 723 lltemp = (lltemp / SCALE_KG) >>
725 724 time_constant;
726 725 }
727 726 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
728 727 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
729 728 time_offset += lltemp;
730 729 time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
731 730 } else {
732 731 lltemp = time_offset;
733 732 if (!(time_status & STA_FLL)) {
734 733 if ((1 << time_constant) >= SCALE_KG)
735 734 lltemp *= (1 << time_constant) /
736 735 SCALE_KG;
737 736 else
738 737 lltemp = (lltemp / SCALE_KG) >>
739 738 time_constant;
740 739 }
741 740 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
742 741 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
743 742 time_offset -= lltemp;
744 743 time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
745 744 }
746 745
747 746 /*
748 747 * Compute the frequency estimate and additional phase
749 748 * adjustment due to frequency error for the next
750 749 * second. When the PPS signal is engaged, gnaw on the
751 750 * watchdog counter and update the frequency computed by
752 751 * the pll and the PPS signal.
753 752 */
754 753 pps_valid++;
755 754 if (pps_valid == PPS_VALID) {
756 755 pps_jitter = MAXTIME;
757 756 pps_stabil = MAXFREQ;
758 757 time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
759 758 STA_PPSWANDER | STA_PPSERROR);
760 759 }
761 760 lltemp = time_freq + pps_freq;
762 761
763 762 if (lltemp)
764 763 time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
765 764
766 765 /*
767 766 * End of precision kernel-code fragment
768 767 *
769 768 * The section below should be modified if we are planning
770 769 * to use NTP for synchronization.
771 770 *
772 771 * Note: the clock synchronization code now assumes
773 772 * the following:
774 773 * - if dosynctodr is 1, then compute the drift between
775 774 * the tod chip and software time and adjust one or
776 775 * the other depending on the circumstances
777 776 *
778 777 * - if dosynctodr is 0, then the tod chip is independent
779 778 * of the software clock and should not be adjusted,
780 779 * but allowed to free run. this allows NTP to sync.
781 780 * hrestime without any interference from the tod chip.
782 781 */
783 782
784 783 tod_validate_deferred = B_FALSE;
785 784 mutex_enter(&tod_lock);
786 785 tod = tod_get();
787 786 drift = tod.tv_sec - hrestime.tv_sec;
788 787 absdrift = (drift >= 0) ? drift : -drift;
789 788 if (tod_needsync || absdrift > 1) {
790 789 int s;
791 790 if (absdrift > 2) {
792 791 if (!tod_broken && tod_faulted == TOD_NOFAULT) {
793 792 s = hr_clock_lock();
794 793 hrestime = tod;
795 794 membar_enter(); /* hrestime visible */
796 795 timedelta = 0;
797 796 timechanged++;
798 797 tod_needsync = 0;
799 798 hr_clock_unlock(s);
800 799 callout_hrestime();
801 800
802 801 }
803 802 } else {
804 803 if (tod_needsync || !dosynctodr) {
805 804 gethrestime(&tod);
806 805 tod_set(tod);
807 806 s = hr_clock_lock();
808 807 if (timedelta == 0)
809 808 tod_needsync = 0;
810 809 hr_clock_unlock(s);
811 810 } else {
812 811 /*
813 812 * If the drift is 2 seconds on the
814 813 * money, then the TOD is adjusting
815 814 * the clock; record that.
816 815 */
817 816 clock_adj_hist[adj_hist_entry++ %
818 817 CLOCK_ADJ_HIST_SIZE] = now;
819 818 s = hr_clock_lock();
820 819 timedelta = (int64_t)drift*NANOSEC;
821 820 hr_clock_unlock(s);
822 821 }
823 822 }
824 823 }
825 824 one_sec = 0;
826 825 time = gethrestime_sec(); /* for crusty old kmem readers */
827 826 mutex_exit(&tod_lock);
828 827
829 828 /*
830 829 * Some drivers still depend on this... XXX
831 830 */
832 831 cv_broadcast(&lbolt_cv);
833 832
834 833 vminfo.freemem += freemem;
835 834 {
836 835 pgcnt_t maxswap, resv, free;
837 836 pgcnt_t avail =
838 837 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
839 838
840 839 maxswap = k_anoninfo.ani_mem_resv +
841 840 k_anoninfo.ani_max +avail;
842 841 /* Update ani_free */
843 842 set_anoninfo();
844 843 free = k_anoninfo.ani_free + avail;
845 844 resv = k_anoninfo.ani_phys_resv +
846 845 k_anoninfo.ani_mem_resv;
847 846
848 847 vminfo.swap_resv += resv;
849 848 /* number of reserved and allocated pages */
850 849 #ifdef DEBUG
851 850 if (maxswap < free)
852 851 cmn_err(CE_WARN, "clock: maxswap < free");
853 852 if (maxswap < resv)
854 853 cmn_err(CE_WARN, "clock: maxswap < resv");
855 854 #endif
856 855 vminfo.swap_alloc += maxswap - free;
857 856 vminfo.swap_avail += maxswap - resv;
858 857 vminfo.swap_free += free;
859 858 }
860 859 vminfo.updates++;
861 860 if (nrunnable) {
862 861 sysinfo.runque += nrunnable;
863 862 sysinfo.runocc++;
864 863 }
865 864 if (nswapped) {
866 865 sysinfo.swpque += nswapped;
867 866 sysinfo.swpocc++;
868 867 }
869 868 sysinfo.waiting += w_io;
870 869 sysinfo.updates++;
871 870
872 871 /*
873 872 * Wake up fsflush to write out DELWRI
874 873 * buffers, dirty pages and other cached
875 874 * administrative data, e.g. inodes.
876 875 */
877 876 if (--fsflushcnt <= 0) {
878 877 fsflushcnt = tune.t_fsflushr;
879 878 cv_signal(&fsflush_cv);
880 879 }
881 880
882 881 vmmeter();
883 882 calcloadavg(genloadavg(&loadavg), hp_avenrun);
884 883 for (i = 0; i < 3; i++)
885 884 /*
886 885 * At the moment avenrun[] can only hold 31
887 886 * bits of load average as it is a signed
888 887 * int in the API. We need to ensure that
889 888 * hp_avenrun[i] >> (16 - FSHIFT) will not be
890 889 * too large. If it is, we put the largest value
891 890 * that we can use into avenrun[i]. This is
892 891 * kludgey, but about all we can do until we
893 892 * avenrun[] is declared as an array of uint64[]
894 893 */
895 894 if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
896 895 avenrun[i] = (int32_t)(hp_avenrun[i] >>
897 896 (16 - FSHIFT));
898 897 else
899 898 avenrun[i] = 0x7fffffff;
900 899
901 900 cpupart = cp_list_head;
902 901 do {
903 902 calcloadavg(genloadavg(&cpupart->cp_loadavg),
904 903 cpupart->cp_hp_avenrun);
905 904 } while ((cpupart = cpupart->cp_next) != cp_list_head);
906 905
907 906 /*
908 907 * Wake up the swapper thread if necessary.
909 908 */
910 909 if (runin ||
911 910 (runout && (avefree < desfree || wake_sched_sec))) {
912 911 t = &t0;
913 912 thread_lock(t);
914 913 if (t->t_state == TS_STOPPED) {
915 914 runin = runout = 0;
916 915 wake_sched_sec = 0;
917 916 t->t_whystop = 0;
918 917 t->t_whatstop = 0;
919 918 t->t_schedflag &= ~TS_ALLSTART;
920 919 THREAD_TRANSITION(t);
921 920 setfrontdq(t);
922 921 }
923 922 thread_unlock(t);
924 923 }
925 924 }
926 925
927 926 /*
928 927 * Wake up the swapper if any high priority swapped-out threads
929 928 * became runable during the last tick.
930 929 */
931 930 if (wake_sched) {
932 931 t = &t0;
933 932 thread_lock(t);
934 933 if (t->t_state == TS_STOPPED) {
935 934 runin = runout = 0;
936 935 wake_sched = 0;
937 936 t->t_whystop = 0;
938 937 t->t_whatstop = 0;
939 938 t->t_schedflag &= ~TS_ALLSTART;
940 939 THREAD_TRANSITION(t);
941 940 setfrontdq(t);
942 941 }
943 942 thread_unlock(t);
944 943 }
945 944 }
946 945
947 946 void
948 947 clock_init(void)
949 948 {
950 949 cyc_handler_t clk_hdlr, lbolt_hdlr;
951 950 cyc_time_t clk_when, lbolt_when;
952 951 int i, sz;
953 952 intptr_t buf;
954 953
955 954 /*
956 955 * Setup handler and timer for the clock cyclic.
957 956 */
958 957 clk_hdlr.cyh_func = (cyc_func_t)clock;
959 958 clk_hdlr.cyh_level = CY_LOCK_LEVEL;
960 959 clk_hdlr.cyh_arg = NULL;
961 960
962 961 clk_when.cyt_when = 0;
963 962 clk_when.cyt_interval = nsec_per_tick;
964 963
965 964 /*
966 965 * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
967 966 * interval to satisfy performance needs of the DDI lbolt consumers.
968 967 * It is off by default.
969 968 */
970 969 lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
971 970 lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
972 971 lbolt_hdlr.cyh_arg = NULL;
973 972
974 973 lbolt_when.cyt_interval = nsec_per_tick;
975 974
976 975 /*
977 976 * Allocate cache line aligned space for the per CPU lbolt data and
978 977 * lbolt info structures, and initialize them with their default
979 978 * values. Note that these structures are also cache line sized.
980 979 */
981 980 sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
982 981 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
983 982 lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
984 983
985 984 if (hz != HZ_DEFAULT)
986 985 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
987 986 hz/HZ_DEFAULT;
988 987 else
989 988 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
990 989
991 990 lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
992 991
993 992 sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
994 993 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
995 994 lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
996 995
997 996 for (i = 0; i < max_ncpus; i++)
998 997 lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
999 998
1000 999 /*
1001 1000 * Install the softint used to switch between event and cyclic driven
1002 1001 * lbolt. We use a soft interrupt to make sure the context of the
1003 1002 * cyclic reprogram call is safe.
1004 1003 */
1005 1004 lbolt_softint_add();
1006 1005
1007 1006 /*
1008 1007 * Since the hybrid lbolt implementation is based on a hardware counter
1009 1008 * that is reset at every hardware reboot and that we'd like to have
1010 1009 * the lbolt value starting at zero after both a hardware and a fast
1011 1010 * reboot, we calculate the number of clock ticks the system's been up
1012 1011 * and store it in the lbi_debug_time field of the lbolt info structure.
1013 1012 * The value of this field will be subtracted from lbolt before
1014 1013 * returning it.
1015 1014 */
1016 1015 lb_info->lbi_internal = lb_info->lbi_debug_time =
1017 1016 (gethrtime()/nsec_per_tick);
1018 1017
1019 1018 /*
1020 1019 * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
1021 1020 * and lbolt_debug_{enter,return} use this value as an indication that
1022 1021 * the initializaion above hasn't been completed. Setting lbolt_hybrid
1023 1022 * to either lbolt_{cyclic,event}_driven here signals those code paths
1024 1023 * that the lbolt related structures can be used.
1025 1024 */
1026 1025 if (lbolt_cyc_only) {
1027 1026 lbolt_when.cyt_when = 0;
1028 1027 lbolt_hybrid = lbolt_cyclic_driven;
1029 1028 } else {
1030 1029 lbolt_when.cyt_when = CY_INFINITY;
1031 1030 lbolt_hybrid = lbolt_event_driven;
1032 1031 }
1033 1032
1034 1033 /*
1035 1034 * Grab cpu_lock and install all three cyclics.
1036 1035 */
1037 1036 mutex_enter(&cpu_lock);
1038 1037
1039 1038 clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
1040 1039 lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
1041 1040
1042 1041 mutex_exit(&cpu_lock);
1043 1042 }
1044 1043
1045 1044 /*
1046 1045 * Called before calcloadavg to get 10-sec moving loadavg together
1047 1046 */
1048 1047
1049 1048 static int
1050 1049 genloadavg(struct loadavg_s *avgs)
1051 1050 {
1052 1051 int avg;
1053 1052 int spos; /* starting position */
1054 1053 int cpos; /* moving current position */
1055 1054 int i;
1056 1055 int slen;
1057 1056 hrtime_t hr_avg;
1058 1057
1059 1058 /* 10-second snapshot, calculate first positon */
1060 1059 if (avgs->lg_len == 0) {
1061 1060 return (0);
1062 1061 }
1063 1062 slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1064 1063
1065 1064 spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1066 1065 S_LOADAVG_SZ + (avgs->lg_cur - 1);
1067 1066 for (i = hr_avg = 0; i < slen; i++) {
1068 1067 cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1069 1068 hr_avg += avgs->lg_loads[cpos];
1070 1069 }
1071 1070
1072 1071 hr_avg = hr_avg / slen;
1073 1072 avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1074 1073
1075 1074 return (avg);
1076 1075 }
1077 1076
1078 1077 /*
1079 1078 * Run every second from clock () to update the loadavg count available to the
1080 1079 * system and cpu-partitions.
1081 1080 *
1082 1081 * This works by sampling the previous usr, sys, wait time elapsed,
1083 1082 * computing a delta, and adding that delta to the elapsed usr, sys,
1084 1083 * wait increase.
1085 1084 */
1086 1085
1087 1086 static void
1088 1087 loadavg_update()
1089 1088 {
1090 1089 cpu_t *cp;
1091 1090 cpupart_t *cpupart;
1092 1091 hrtime_t cpu_total;
1093 1092 int prev;
1094 1093
1095 1094 cp = cpu_list;
1096 1095 loadavg.lg_total = 0;
1097 1096
1098 1097 /*
1099 1098 * first pass totals up per-cpu statistics for system and cpu
1100 1099 * partitions
1101 1100 */
1102 1101
1103 1102 do {
1104 1103 struct loadavg_s *lavg;
1105 1104
1106 1105 lavg = &cp->cpu_loadavg;
1107 1106
1108 1107 cpu_total = cp->cpu_acct[CMS_USER] +
1109 1108 cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1110 1109 /* compute delta against last total */
1111 1110 scalehrtime(&cpu_total);
1112 1111 prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1113 1112 S_LOADAVG_SZ + (lavg->lg_cur - 1);
1114 1113 if (lavg->lg_loads[prev] <= 0) {
1115 1114 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1116 1115 cpu_total = 0;
1117 1116 } else {
1118 1117 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1119 1118 cpu_total = cpu_total - lavg->lg_loads[prev];
1120 1119 if (cpu_total < 0)
1121 1120 cpu_total = 0;
1122 1121 }
1123 1122
1124 1123 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1125 1124 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1126 1125 lavg->lg_len + 1 : S_LOADAVG_SZ;
1127 1126
1128 1127 loadavg.lg_total += cpu_total;
1129 1128 cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1130 1129
1131 1130 } while ((cp = cp->cpu_next) != cpu_list);
1132 1131
1133 1132 loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1134 1133 loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1135 1134 loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1136 1135 loadavg.lg_len + 1 : S_LOADAVG_SZ;
1137 1136 /*
1138 1137 * Second pass updates counts
1139 1138 */
1140 1139 cpupart = cp_list_head;
1141 1140
1142 1141 do {
1143 1142 struct loadavg_s *lavg;
1144 1143
1145 1144 lavg = &cpupart->cp_loadavg;
1146 1145 lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1147 1146 lavg->lg_total = 0;
1148 1147 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1149 1148 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1150 1149 lavg->lg_len + 1 : S_LOADAVG_SZ;
1151 1150
1152 1151 } while ((cpupart = cpupart->cp_next) != cp_list_head);
1153 1152
1154 1153 /*
1155 1154 * Third pass totals up per-zone statistics.
1156 1155 */
1157 1156 zone_loadavg_update();
1158 1157 }
1159 1158
1160 1159 /*
1161 1160 * clock_update() - local clock update
1162 1161 *
1163 1162 * This routine is called by ntp_adjtime() to update the local clock
1164 1163 * phase and frequency. The implementation is of an
1165 1164 * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1166 1165 * routine computes new time and frequency offset estimates for each
1167 1166 * call. The PPS signal itself determines the new time offset,
1168 1167 * instead of the calling argument. Presumably, calls to
1169 1168 * ntp_adjtime() occur only when the caller believes the local clock
1170 1169 * is valid within some bound (+-128 ms with NTP). If the caller's
1171 1170 * time is far different than the PPS time, an argument will ensue,
1172 1171 * and it's not clear who will lose.
1173 1172 *
1174 1173 * For uncompensated quartz crystal oscillatores and nominal update
1175 1174 * intervals less than 1024 s, operation should be in phase-lock mode
1176 1175 * (STA_FLL = 0), where the loop is disciplined to phase. For update
1177 1176 * intervals greater than this, operation should be in frequency-lock
1178 1177 * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1179 1178 *
1180 1179 * Note: mutex(&tod_lock) is in effect.
1181 1180 */
1182 1181 void
1183 1182 clock_update(int offset)
1184 1183 {
1185 1184 int ltemp, mtemp, s;
1186 1185
1187 1186 ASSERT(MUTEX_HELD(&tod_lock));
1188 1187
1189 1188 if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1190 1189 return;
1191 1190 ltemp = offset;
1192 1191 if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1193 1192 ltemp = pps_offset;
1194 1193
1195 1194 /*
1196 1195 * Scale the phase adjustment and clamp to the operating range.
1197 1196 */
1198 1197 if (ltemp > MAXPHASE)
1199 1198 time_offset = MAXPHASE * SCALE_UPDATE;
1200 1199 else if (ltemp < -MAXPHASE)
1201 1200 time_offset = -(MAXPHASE * SCALE_UPDATE);
1202 1201 else
1203 1202 time_offset = ltemp * SCALE_UPDATE;
1204 1203
1205 1204 /*
1206 1205 * Select whether the frequency is to be controlled and in which
1207 1206 * mode (PLL or FLL). Clamp to the operating range. Ugly
1208 1207 * multiply/divide should be replaced someday.
1209 1208 */
1210 1209 if (time_status & STA_FREQHOLD || time_reftime == 0)
1211 1210 time_reftime = hrestime.tv_sec;
1212 1211
1213 1212 mtemp = hrestime.tv_sec - time_reftime;
1214 1213 time_reftime = hrestime.tv_sec;
1215 1214
1216 1215 if (time_status & STA_FLL) {
1217 1216 if (mtemp >= MINSEC) {
1218 1217 ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1219 1218 SCALE_UPDATE));
1220 1219 if (ltemp)
1221 1220 time_freq += ltemp / SCALE_KH;
1222 1221 }
1223 1222 } else {
1224 1223 if (mtemp < MAXSEC) {
1225 1224 ltemp *= mtemp;
1226 1225 if (ltemp)
1227 1226 time_freq += (int)(((int64_t)ltemp *
1228 1227 SCALE_USEC) / SCALE_KF)
1229 1228 / (1 << (time_constant * 2));
1230 1229 }
1231 1230 }
1232 1231 if (time_freq > time_tolerance)
1233 1232 time_freq = time_tolerance;
1234 1233 else if (time_freq < -time_tolerance)
1235 1234 time_freq = -time_tolerance;
1236 1235
1237 1236 s = hr_clock_lock();
1238 1237 tod_needsync = 1;
1239 1238 hr_clock_unlock(s);
1240 1239 }
1241 1240
1242 1241 /*
1243 1242 * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1244 1243 *
1245 1244 * This routine is called at each PPS interrupt in order to discipline
1246 1245 * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1247 1246 * and leaves it in a handy spot for the clock() routine. It
1248 1247 * integrates successive PPS phase differences and calculates the
1249 1248 * frequency offset. This is used in clock() to discipline the CPU
1250 1249 * clock oscillator so that intrinsic frequency error is cancelled out.
1251 1250 * The code requires the caller to capture the time and hardware counter
1252 1251 * value at the on-time PPS signal transition.
1253 1252 *
1254 1253 * Note that, on some Unix systems, this routine runs at an interrupt
1255 1254 * priority level higher than the timer interrupt routine clock().
1256 1255 * Therefore, the variables used are distinct from the clock()
1257 1256 * variables, except for certain exceptions: The PPS frequency pps_freq
1258 1257 * and phase pps_offset variables are determined by this routine and
1259 1258 * updated atomically. The time_tolerance variable can be considered a
1260 1259 * constant, since it is infrequently changed, and then only when the
1261 1260 * PPS signal is disabled. The watchdog counter pps_valid is updated
1262 1261 * once per second by clock() and is atomically cleared in this
1263 1262 * routine.
1264 1263 *
1265 1264 * tvp is the time of the last tick; usec is a microsecond count since the
1266 1265 * last tick.
1267 1266 *
1268 1267 * Note: In Solaris systems, the tick value is actually given by
1269 1268 * usec_per_tick. This is called from the serial driver cdintr(),
1270 1269 * or equivalent, at a high PIL. Because the kernel keeps a
1271 1270 * highresolution time, the following code can accept either
1272 1271 * the traditional argument pair, or the current highres timestamp
1273 1272 * in tvp and zero in usec.
1274 1273 */
1275 1274 void
1276 1275 ddi_hardpps(struct timeval *tvp, int usec)
1277 1276 {
1278 1277 int u_usec, v_usec, bigtick;
1279 1278 time_t cal_sec;
1280 1279 int cal_usec;
1281 1280
1282 1281 /*
1283 1282 * An occasional glitch can be produced when the PPS interrupt
1284 1283 * occurs in the clock() routine before the time variable is
1285 1284 * updated. Here the offset is discarded when the difference
1286 1285 * between it and the last one is greater than tick/2, but not
1287 1286 * if the interval since the first discard exceeds 30 s.
1288 1287 */
1289 1288 time_status |= STA_PPSSIGNAL;
1290 1289 time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1291 1290 pps_valid = 0;
1292 1291 u_usec = -tvp->tv_usec;
1293 1292 if (u_usec < -(MICROSEC/2))
1294 1293 u_usec += MICROSEC;
1295 1294 v_usec = pps_offset - u_usec;
1296 1295 if (v_usec < 0)
1297 1296 v_usec = -v_usec;
1298 1297 if (v_usec > (usec_per_tick >> 1)) {
1299 1298 if (pps_glitch > MAXGLITCH) {
1300 1299 pps_glitch = 0;
1301 1300 pps_tf[2] = u_usec;
1302 1301 pps_tf[1] = u_usec;
1303 1302 } else {
1304 1303 pps_glitch++;
1305 1304 u_usec = pps_offset;
1306 1305 }
1307 1306 } else
1308 1307 pps_glitch = 0;
1309 1308
1310 1309 /*
1311 1310 * A three-stage median filter is used to help deglitch the pps
1312 1311 * time. The median sample becomes the time offset estimate; the
1313 1312 * difference between the other two samples becomes the time
1314 1313 * dispersion (jitter) estimate.
1315 1314 */
1316 1315 pps_tf[2] = pps_tf[1];
1317 1316 pps_tf[1] = pps_tf[0];
1318 1317 pps_tf[0] = u_usec;
1319 1318 if (pps_tf[0] > pps_tf[1]) {
1320 1319 if (pps_tf[1] > pps_tf[2]) {
1321 1320 pps_offset = pps_tf[1]; /* 0 1 2 */
1322 1321 v_usec = pps_tf[0] - pps_tf[2];
1323 1322 } else if (pps_tf[2] > pps_tf[0]) {
1324 1323 pps_offset = pps_tf[0]; /* 2 0 1 */
1325 1324 v_usec = pps_tf[2] - pps_tf[1];
1326 1325 } else {
1327 1326 pps_offset = pps_tf[2]; /* 0 2 1 */
1328 1327 v_usec = pps_tf[0] - pps_tf[1];
1329 1328 }
1330 1329 } else {
1331 1330 if (pps_tf[1] < pps_tf[2]) {
1332 1331 pps_offset = pps_tf[1]; /* 2 1 0 */
1333 1332 v_usec = pps_tf[2] - pps_tf[0];
1334 1333 } else if (pps_tf[2] < pps_tf[0]) {
1335 1334 pps_offset = pps_tf[0]; /* 1 0 2 */
1336 1335 v_usec = pps_tf[1] - pps_tf[2];
1337 1336 } else {
1338 1337 pps_offset = pps_tf[2]; /* 1 2 0 */
1339 1338 v_usec = pps_tf[1] - pps_tf[0];
1340 1339 }
1341 1340 }
1342 1341 if (v_usec > MAXTIME)
1343 1342 pps_jitcnt++;
1344 1343 v_usec = (v_usec << PPS_AVG) - pps_jitter;
1345 1344 pps_jitter += v_usec / (1 << PPS_AVG);
1346 1345 if (pps_jitter > (MAXTIME >> 1))
1347 1346 time_status |= STA_PPSJITTER;
1348 1347
1349 1348 /*
1350 1349 * During the calibration interval adjust the starting time when
1351 1350 * the tick overflows. At the end of the interval compute the
1352 1351 * duration of the interval and the difference of the hardware
1353 1352 * counters at the beginning and end of the interval. This code
1354 1353 * is deliciously complicated by the fact valid differences may
1355 1354 * exceed the value of tick when using long calibration
1356 1355 * intervals and small ticks. Note that the counter can be
1357 1356 * greater than tick if caught at just the wrong instant, but
1358 1357 * the values returned and used here are correct.
1359 1358 */
1360 1359 bigtick = (int)usec_per_tick * SCALE_USEC;
1361 1360 pps_usec -= pps_freq;
1362 1361 if (pps_usec >= bigtick)
1363 1362 pps_usec -= bigtick;
1364 1363 if (pps_usec < 0)
1365 1364 pps_usec += bigtick;
1366 1365 pps_time.tv_sec++;
1367 1366 pps_count++;
1368 1367 if (pps_count < (1 << pps_shift))
1369 1368 return;
1370 1369 pps_count = 0;
1371 1370 pps_calcnt++;
1372 1371 u_usec = usec * SCALE_USEC;
1373 1372 v_usec = pps_usec - u_usec;
1374 1373 if (v_usec >= bigtick >> 1)
1375 1374 v_usec -= bigtick;
1376 1375 if (v_usec < -(bigtick >> 1))
1377 1376 v_usec += bigtick;
1378 1377 if (v_usec < 0)
1379 1378 v_usec = -(-v_usec >> pps_shift);
1380 1379 else
1381 1380 v_usec = v_usec >> pps_shift;
1382 1381 pps_usec = u_usec;
1383 1382 cal_sec = tvp->tv_sec;
1384 1383 cal_usec = tvp->tv_usec;
1385 1384 cal_sec -= pps_time.tv_sec;
1386 1385 cal_usec -= pps_time.tv_usec;
1387 1386 if (cal_usec < 0) {
1388 1387 cal_usec += MICROSEC;
1389 1388 cal_sec--;
1390 1389 }
1391 1390 pps_time = *tvp;
1392 1391
1393 1392 /*
1394 1393 * Check for lost interrupts, noise, excessive jitter and
1395 1394 * excessive frequency error. The number of timer ticks during
1396 1395 * the interval may vary +-1 tick. Add to this a margin of one
1397 1396 * tick for the PPS signal jitter and maximum frequency
1398 1397 * deviation. If the limits are exceeded, the calibration
1399 1398 * interval is reset to the minimum and we start over.
1400 1399 */
1401 1400 u_usec = (int)usec_per_tick << 1;
1402 1401 if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1403 1402 (cal_sec == 0 && cal_usec < u_usec)) ||
1404 1403 v_usec > time_tolerance || v_usec < -time_tolerance) {
1405 1404 pps_errcnt++;
1406 1405 pps_shift = PPS_SHIFT;
1407 1406 pps_intcnt = 0;
1408 1407 time_status |= STA_PPSERROR;
1409 1408 return;
1410 1409 }
1411 1410
1412 1411 /*
1413 1412 * A three-stage median filter is used to help deglitch the pps
1414 1413 * frequency. The median sample becomes the frequency offset
1415 1414 * estimate; the difference between the other two samples
1416 1415 * becomes the frequency dispersion (stability) estimate.
1417 1416 */
1418 1417 pps_ff[2] = pps_ff[1];
1419 1418 pps_ff[1] = pps_ff[0];
1420 1419 pps_ff[0] = v_usec;
1421 1420 if (pps_ff[0] > pps_ff[1]) {
1422 1421 if (pps_ff[1] > pps_ff[2]) {
1423 1422 u_usec = pps_ff[1]; /* 0 1 2 */
1424 1423 v_usec = pps_ff[0] - pps_ff[2];
1425 1424 } else if (pps_ff[2] > pps_ff[0]) {
1426 1425 u_usec = pps_ff[0]; /* 2 0 1 */
1427 1426 v_usec = pps_ff[2] - pps_ff[1];
1428 1427 } else {
1429 1428 u_usec = pps_ff[2]; /* 0 2 1 */
1430 1429 v_usec = pps_ff[0] - pps_ff[1];
1431 1430 }
1432 1431 } else {
1433 1432 if (pps_ff[1] < pps_ff[2]) {
1434 1433 u_usec = pps_ff[1]; /* 2 1 0 */
1435 1434 v_usec = pps_ff[2] - pps_ff[0];
1436 1435 } else if (pps_ff[2] < pps_ff[0]) {
1437 1436 u_usec = pps_ff[0]; /* 1 0 2 */
1438 1437 v_usec = pps_ff[1] - pps_ff[2];
1439 1438 } else {
1440 1439 u_usec = pps_ff[2]; /* 1 2 0 */
1441 1440 v_usec = pps_ff[1] - pps_ff[0];
1442 1441 }
1443 1442 }
1444 1443
1445 1444 /*
1446 1445 * Here the frequency dispersion (stability) is updated. If it
1447 1446 * is less than one-fourth the maximum (MAXFREQ), the frequency
1448 1447 * offset is updated as well, but clamped to the tolerance. It
1449 1448 * will be processed later by the clock() routine.
1450 1449 */
1451 1450 v_usec = (v_usec >> 1) - pps_stabil;
1452 1451 if (v_usec < 0)
1453 1452 pps_stabil -= -v_usec >> PPS_AVG;
1454 1453 else
1455 1454 pps_stabil += v_usec >> PPS_AVG;
1456 1455 if (pps_stabil > MAXFREQ >> 2) {
1457 1456 pps_stbcnt++;
1458 1457 time_status |= STA_PPSWANDER;
1459 1458 return;
1460 1459 }
1461 1460 if (time_status & STA_PPSFREQ) {
1462 1461 if (u_usec < 0) {
1463 1462 pps_freq -= -u_usec >> PPS_AVG;
1464 1463 if (pps_freq < -time_tolerance)
1465 1464 pps_freq = -time_tolerance;
1466 1465 u_usec = -u_usec;
1467 1466 } else {
1468 1467 pps_freq += u_usec >> PPS_AVG;
1469 1468 if (pps_freq > time_tolerance)
1470 1469 pps_freq = time_tolerance;
1471 1470 }
1472 1471 }
1473 1472
1474 1473 /*
1475 1474 * Here the calibration interval is adjusted. If the maximum
1476 1475 * time difference is greater than tick / 4, reduce the interval
1477 1476 * by half. If this is not the case for four consecutive
1478 1477 * intervals, double the interval.
1479 1478 */
1480 1479 if (u_usec << pps_shift > bigtick >> 2) {
1481 1480 pps_intcnt = 0;
1482 1481 if (pps_shift > PPS_SHIFT)
1483 1482 pps_shift--;
1484 1483 } else if (pps_intcnt >= 4) {
1485 1484 pps_intcnt = 0;
1486 1485 if (pps_shift < PPS_SHIFTMAX)
1487 1486 pps_shift++;
1488 1487 } else
1489 1488 pps_intcnt++;
1490 1489
1491 1490 /*
1492 1491 * If recovering from kmdb, then make sure the tod chip gets resynced.
1493 1492 * If we took an early exit above, then we don't yet have a stable
1494 1493 * calibration signal to lock onto, so don't mark the tod for sync
1495 1494 * until we get all the way here.
1496 1495 */
1497 1496 {
1498 1497 int s = hr_clock_lock();
1499 1498
1500 1499 tod_needsync = 1;
1501 1500 hr_clock_unlock(s);
1502 1501 }
1503 1502 }
1504 1503
1505 1504 /*
1506 1505 * Handle clock tick processing for a thread.
1507 1506 * Check for timer action, enforce CPU rlimit, do profiling etc.
1508 1507 */
1509 1508 void
1510 1509 clock_tick(kthread_t *t, int pending)
1511 1510 {
1512 1511 struct proc *pp;
1513 1512 klwp_id_t lwp;
1514 1513 struct as *as;
1515 1514 clock_t ticks;
1516 1515 int poke = 0; /* notify another CPU */
1517 1516 int user_mode;
1518 1517 size_t rss;
1519 1518 int i, total_usec, usec;
1520 1519 rctl_qty_t secs;
1521 1520
1522 1521 ASSERT(pending > 0);
1523 1522
1524 1523 /* Must be operating on a lwp/thread */
1525 1524 if ((lwp = ttolwp(t)) == NULL) {
1526 1525 panic("clock_tick: no lwp");
1527 1526 /*NOTREACHED*/
1528 1527 }
1529 1528
1530 1529 for (i = 0; i < pending; i++) {
1531 1530 CL_TICK(t); /* Class specific tick processing */
1532 1531 DTRACE_SCHED1(tick, kthread_t *, t);
1533 1532 }
1534 1533
1535 1534 pp = ttoproc(t);
1536 1535
1537 1536 /* pp->p_lock makes sure that the thread does not exit */
1538 1537 ASSERT(MUTEX_HELD(&pp->p_lock));
1539 1538
1540 1539 user_mode = (lwp->lwp_state == LWP_USER);
1541 1540
1542 1541 ticks = (pp->p_utime + pp->p_stime) % hz;
1543 1542 /*
1544 1543 * Update process times. Should use high res clock and state
1545 1544 * changes instead of statistical sampling method. XXX
1546 1545 */
1547 1546 if (user_mode) {
1548 1547 pp->p_utime += pending;
1549 1548 } else {
1550 1549 pp->p_stime += pending;
1551 1550 }
1552 1551
1553 1552 pp->p_ttime += pending;
1554 1553 as = pp->p_as;
1555 1554
1556 1555 /*
1557 1556 * Update user profiling statistics. Get the pc from the
1558 1557 * lwp when the AST happens.
1559 1558 */
1560 1559 if (pp->p_prof.pr_scale) {
1561 1560 atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
1562 1561 if (user_mode) {
1563 1562 poke = 1;
1564 1563 aston(t);
1565 1564 }
1566 1565 }
1567 1566
1568 1567 /*
1569 1568 * If CPU was in user state, process lwp-virtual time
1570 1569 * interval timer. The value passed to itimerdecr() has to be
1571 1570 * in microseconds and has to be less than one second. Hence
1572 1571 * this loop.
1573 1572 */
1574 1573 total_usec = usec_per_tick * pending;
1575 1574 while (total_usec > 0) {
1576 1575 usec = MIN(total_usec, (MICROSEC - 1));
1577 1576 if (user_mode &&
1578 1577 timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1579 1578 itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
1580 1579 poke = 1;
1581 1580 sigtoproc(pp, t, SIGVTALRM);
1582 1581 }
1583 1582 total_usec -= usec;
1584 1583 }
1585 1584
1586 1585 /*
1587 1586 * If CPU was in user state, process lwp-profile
1588 1587 * interval timer.
1589 1588 */
1590 1589 total_usec = usec_per_tick * pending;
1591 1590 while (total_usec > 0) {
1592 1591 usec = MIN(total_usec, (MICROSEC - 1));
1593 1592 if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1594 1593 itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
1595 1594 poke = 1;
1596 1595 sigtoproc(pp, t, SIGPROF);
1597 1596 }
1598 1597 total_usec -= usec;
1599 1598 }
1600 1599
1601 1600 /*
1602 1601 * Enforce CPU resource controls:
1603 1602 * (a) process.max-cpu-time resource control
1604 1603 *
1605 1604 * Perform the check only if we have accumulated more a second.
1606 1605 */
1607 1606 if ((ticks + pending) >= hz) {
1608 1607 (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1609 1608 (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
1610 1609 }
1611 1610
1612 1611 /*
1613 1612 * (b) task.max-cpu-time resource control
1614 1613 *
1615 1614 * If we have accumulated enough ticks, increment the task CPU
1616 1615 * time usage and test for the resource limit. This minimizes the
1617 1616 * number of calls to the rct_test(). The task CPU time mutex
1618 1617 * is highly contentious as many processes can be sharing a task.
1619 1618 */
1620 1619 if (pp->p_ttime >= clock_tick_proc_max) {
1621 1620 secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
1622 1621 pp->p_ttime = 0;
1623 1622 if (secs) {
1624 1623 (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
1625 1624 pp, secs, RCA_UNSAFE_SIGINFO);
1626 1625 }
1627 1626 }
1628 1627
1629 1628 /*
1630 1629 * Update memory usage for the currently running process.
1631 1630 */
1632 1631 rss = rm_asrss(as);
1633 1632 PTOU(pp)->u_mem += rss;
1634 1633 if (rss > PTOU(pp)->u_mem_max)
1635 1634 PTOU(pp)->u_mem_max = rss;
1636 1635
1637 1636 /*
1638 1637 * Notify the CPU the thread is running on.
1639 1638 */
1640 1639 if (poke && t->t_cpu != CPU)
1641 1640 poke_cpu(t->t_cpu->cpu_id);
1642 1641 }
1643 1642
1644 1643 void
1645 1644 profil_tick(uintptr_t upc)
1646 1645 {
1647 1646 int ticks;
1648 1647 proc_t *p = ttoproc(curthread);
1649 1648 klwp_t *lwp = ttolwp(curthread);
1650 1649 struct prof *pr = &p->p_prof;
1651 1650
1652 1651 do {
1653 1652 ticks = lwp->lwp_oweupc;
1654 1653 } while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1655 1654
1656 1655 mutex_enter(&p->p_pflock);
1657 1656 if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1658 1657 /*
1659 1658 * Old-style profiling
1660 1659 */
1661 1660 uint16_t *slot = pr->pr_base;
1662 1661 uint16_t old, new;
1663 1662 if (pr->pr_scale != 2) {
1664 1663 uintptr_t delta = upc - pr->pr_off;
1665 1664 uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1666 1665 (((delta & 0xffff) * pr->pr_scale) >> 16);
1667 1666 if (byteoff >= (uintptr_t)pr->pr_size) {
1668 1667 mutex_exit(&p->p_pflock);
1669 1668 return;
1670 1669 }
1671 1670 slot += byteoff / sizeof (uint16_t);
1672 1671 }
1673 1672 if (fuword16(slot, &old) < 0 ||
1674 1673 (new = old + ticks) > SHRT_MAX ||
1675 1674 suword16(slot, new) < 0) {
1676 1675 pr->pr_scale = 0;
1677 1676 }
1678 1677 } else if (pr->pr_scale == 1) {
1679 1678 /*
1680 1679 * PC Sampling
1681 1680 */
1682 1681 model_t model = lwp_getdatamodel(lwp);
1683 1682 int result;
1684 1683 #ifdef __lint
1685 1684 model = model;
1686 1685 #endif
1687 1686 while (ticks-- > 0) {
1688 1687 if (pr->pr_samples == pr->pr_size) {
1689 1688 /* buffer full, turn off sampling */
1690 1689 pr->pr_scale = 0;
1691 1690 break;
1692 1691 }
1693 1692 switch (SIZEOF_PTR(model)) {
1694 1693 case sizeof (uint32_t):
1695 1694 result = suword32(pr->pr_base, (uint32_t)upc);
1696 1695 break;
1697 1696 #ifdef _LP64
1698 1697 case sizeof (uint64_t):
1699 1698 result = suword64(pr->pr_base, (uint64_t)upc);
1700 1699 break;
1701 1700 #endif
1702 1701 default:
1703 1702 cmn_err(CE_WARN, "profil_tick: unexpected "
1704 1703 "data model");
1705 1704 result = -1;
1706 1705 break;
1707 1706 }
1708 1707 if (result != 0) {
1709 1708 pr->pr_scale = 0;
1710 1709 break;
1711 1710 }
1712 1711 pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1713 1712 pr->pr_samples++;
1714 1713 }
1715 1714 }
1716 1715 mutex_exit(&p->p_pflock);
1717 1716 }
1718 1717
1719 1718 static void
1720 1719 delay_wakeup(void *arg)
1721 1720 {
1722 1721 kthread_t *t = arg;
1723 1722
1724 1723 mutex_enter(&t->t_delay_lock);
1725 1724 cv_signal(&t->t_delay_cv);
1726 1725 mutex_exit(&t->t_delay_lock);
1727 1726 }
1728 1727
1729 1728 /*
1730 1729 * The delay(9F) man page indicates that it can only be called from user or
1731 1730 * kernel context - detect and diagnose bad calls. The following macro will
1732 1731 * produce a limited number of messages identifying bad callers. This is done
1733 1732 * in a macro so that caller() is meaningful. When a bad caller is identified,
1734 1733 * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
1735 1734 */
1736 1735 #define DELAY_CONTEXT_CHECK() { \
1737 1736 uint32_t m; \
1738 1737 char *f; \
1739 1738 ulong_t off; \
1740 1739 \
1741 1740 m = delay_from_interrupt_msg; \
1742 1741 if (delay_from_interrupt_diagnose && servicing_interrupt() && \
1743 1742 !panicstr && !devinfo_freeze && \
1744 1743 atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) { \
1745 1744 f = modgetsymname((uintptr_t)caller(), &off); \
1746 1745 cmn_err(CE_WARN, "delay(9F) called from " \
1747 1746 "interrupt context: %s`%s", \
1748 1747 mod_containing_pc(caller()), f ? f : "..."); \
1749 1748 } \
1750 1749 }
1751 1750
1752 1751 /*
1753 1752 * delay_common: common delay code.
1754 1753 */
1755 1754 static void
1756 1755 delay_common(clock_t ticks)
1757 1756 {
1758 1757 kthread_t *t = curthread;
1759 1758 clock_t deadline;
1760 1759 clock_t timeleft;
1761 1760 callout_id_t id;
1762 1761
1763 1762 /* If timeouts aren't running all we can do is spin. */
1764 1763 if (panicstr || devinfo_freeze) {
1765 1764 /* Convert delay(9F) call into drv_usecwait(9F) call. */
1766 1765 if (ticks > 0)
1767 1766 drv_usecwait(TICK_TO_USEC(ticks));
1768 1767 return;
1769 1768 }
1770 1769
1771 1770 deadline = ddi_get_lbolt() + ticks;
1772 1771 while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
1773 1772 mutex_enter(&t->t_delay_lock);
1774 1773 id = timeout_default(delay_wakeup, t, timeleft);
1775 1774 cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1776 1775 mutex_exit(&t->t_delay_lock);
1777 1776 (void) untimeout_default(id, 0);
1778 1777 }
1779 1778 }
1780 1779
1781 1780 /*
1782 1781 * Delay specified number of clock ticks.
1783 1782 */
1784 1783 void
1785 1784 delay(clock_t ticks)
1786 1785 {
1787 1786 DELAY_CONTEXT_CHECK();
1788 1787
1789 1788 delay_common(ticks);
1790 1789 }
1791 1790
1792 1791 /*
1793 1792 * Delay a random number of clock ticks between 1 and ticks.
1794 1793 */
1795 1794 void
1796 1795 delay_random(clock_t ticks)
1797 1796 {
1798 1797 int r;
1799 1798
1800 1799 DELAY_CONTEXT_CHECK();
1801 1800
1802 1801 (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
1803 1802 if (ticks == 0)
1804 1803 ticks = 1;
1805 1804 ticks = (r % ticks) + 1;
1806 1805 delay_common(ticks);
1807 1806 }
1808 1807
1809 1808 /*
1810 1809 * Like delay, but interruptible by a signal.
1811 1810 */
1812 1811 int
1813 1812 delay_sig(clock_t ticks)
1814 1813 {
1815 1814 kthread_t *t = curthread;
1816 1815 clock_t deadline;
1817 1816 clock_t rc;
1818 1817
1819 1818 /* If timeouts aren't running all we can do is spin. */
1820 1819 if (panicstr || devinfo_freeze) {
1821 1820 if (ticks > 0)
1822 1821 drv_usecwait(TICK_TO_USEC(ticks));
1823 1822 return (0);
1824 1823 }
1825 1824
1826 1825 deadline = ddi_get_lbolt() + ticks;
1827 1826 mutex_enter(&t->t_delay_lock);
1828 1827 do {
1829 1828 rc = cv_timedwait_sig(&t->t_delay_cv,
1830 1829 &t->t_delay_lock, deadline);
1831 1830 /* loop until past deadline or signaled */
1832 1831 } while (rc > 0);
1833 1832 mutex_exit(&t->t_delay_lock);
1834 1833 if (rc == 0)
1835 1834 return (EINTR);
1836 1835 return (0);
1837 1836 }
1838 1837
1839 1838
1840 1839 #define SECONDS_PER_DAY 86400
1841 1840
1842 1841 /*
1843 1842 * Initialize the system time based on the TOD chip. approx is used as
1844 1843 * an approximation of time (e.g. from the filesystem) in the event that
1845 1844 * the TOD chip has been cleared or is unresponsive. An approx of -1
1846 1845 * means the filesystem doesn't keep time.
1847 1846 */
1848 1847 void
1849 1848 clkset(time_t approx)
1850 1849 {
1851 1850 timestruc_t ts;
1852 1851 int spl;
1853 1852 int set_clock = 0;
1854 1853
1855 1854 mutex_enter(&tod_lock);
1856 1855 ts = tod_get();
1857 1856
1858 1857 if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1859 1858 /*
1860 1859 * If the TOD chip is reporting some time after 1971,
1861 1860 * then it probably didn't lose power or become otherwise
1862 1861 * cleared in the recent past; check to assure that
1863 1862 * the time coming from the filesystem isn't in the future
1864 1863 * according to the TOD chip.
1865 1864 */
1866 1865 if (approx != -1 && approx > ts.tv_sec) {
1867 1866 cmn_err(CE_WARN, "Last shutdown is later "
1868 1867 "than time on time-of-day chip; check date.");
1869 1868 }
1870 1869 } else {
1871 1870 /*
1872 1871 * If the TOD chip isn't giving correct time, set it to the
1873 1872 * greater of i) approx and ii) 1987. That way if approx
1874 1873 * is negative or is earlier than 1987, we set the clock
1875 1874 * back to a time when Oliver North, ALF and Dire Straits
1876 1875 * were all on the collective brain: 1987.
1877 1876 */
1878 1877 timestruc_t tmp;
1879 1878 time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1880 1879 ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
1881 1880 ts.tv_nsec = 0;
1882 1881
1883 1882 /*
1884 1883 * Attempt to write the new time to the TOD chip. Set spl high
1885 1884 * to avoid getting preempted between the tod_set and tod_get.
1886 1885 */
1887 1886 spl = splhi();
1888 1887 tod_set(ts);
1889 1888 tmp = tod_get();
1890 1889 splx(spl);
1891 1890
1892 1891 if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1893 1892 tod_broken = 1;
1894 1893 dosynctodr = 0;
1895 1894 cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
1896 1895 } else {
1897 1896 cmn_err(CE_WARN, "Time-of-day chip had "
1898 1897 "incorrect date; check and reset.");
1899 1898 }
1900 1899 set_clock = 1;
1901 1900 }
1902 1901
1903 1902 if (!boot_time) {
1904 1903 boot_time = ts.tv_sec;
1905 1904 set_clock = 1;
1906 1905 }
1907 1906
1908 1907 if (set_clock)
1909 1908 set_hrestime(&ts);
1910 1909
1911 1910 mutex_exit(&tod_lock);
1912 1911 }
1913 1912
1914 1913 int timechanged; /* for testing if the system time has been reset */
1915 1914
1916 1915 void
1917 1916 set_hrestime(timestruc_t *ts)
1918 1917 {
1919 1918 int spl = hr_clock_lock();
1920 1919 hrestime = *ts;
1921 1920 membar_enter(); /* hrestime must be visible before timechanged++ */
1922 1921 timedelta = 0;
1923 1922 timechanged++;
1924 1923 hr_clock_unlock(spl);
1925 1924 callout_hrestime();
1926 1925 }
1927 1926
1928 1927 static uint_t deadman_seconds;
1929 1928 static uint32_t deadman_panics;
1930 1929 static int deadman_enabled = 0;
1931 1930 static int deadman_panic_timers = 1;
1932 1931
1933 1932 static void
1934 1933 deadman(void)
1935 1934 {
1936 1935 if (panicstr) {
1937 1936 /*
1938 1937 * During panic, other CPUs besides the panic
1939 1938 * master continue to handle cyclics and some other
1940 1939 * interrupts. The code below is intended to be
1941 1940 * single threaded, so any CPU other than the master
1942 1941 * must keep out.
1943 1942 */
1944 1943 if (CPU->cpu_id != panic_cpu.cpu_id)
1945 1944 return;
1946 1945
1947 1946 if (!deadman_panic_timers)
1948 1947 return; /* allow all timers to be manually disabled */
1949 1948
1950 1949 /*
1951 1950 * If we are generating a crash dump or syncing filesystems and
1952 1951 * the corresponding timer is set, decrement it and re-enter
1953 1952 * the panic code to abort it and advance to the next state.
1954 1953 * The panic states and triggers are explained in panic.c.
1955 1954 */
1956 1955 if (panic_dump) {
1957 1956 if (dump_timeleft && (--dump_timeleft == 0)) {
1958 1957 panic("panic dump timeout");
1959 1958 /*NOTREACHED*/
1960 1959 }
1961 1960 } else if (panic_sync) {
1962 1961 if (sync_timeleft && (--sync_timeleft == 0)) {
1963 1962 panic("panic sync timeout");
1964 1963 /*NOTREACHED*/
1965 1964 }
1966 1965 }
1967 1966
1968 1967 return;
1969 1968 }
1970 1969
1971 1970 if (deadman_counter != CPU->cpu_deadman_counter) {
1972 1971 CPU->cpu_deadman_counter = deadman_counter;
1973 1972 CPU->cpu_deadman_countdown = deadman_seconds;
1974 1973 return;
1975 1974 }
1976 1975
1977 1976 if (--CPU->cpu_deadman_countdown > 0)
1978 1977 return;
1979 1978
1980 1979 /*
1981 1980 * Regardless of whether or not we actually bring the system down,
1982 1981 * bump the deadman_panics variable.
1983 1982 *
1984 1983 * N.B. deadman_panics is incremented once for each CPU that
1985 1984 * passes through here. It's expected that all the CPUs will
1986 1985 * detect this condition within one second of each other, so
1987 1986 * when deadman_enabled is off, deadman_panics will
1988 1987 * typically be a multiple of the total number of CPUs in
1989 1988 * the system.
1990 1989 */
1991 1990 atomic_add_32(&deadman_panics, 1);
1992 1991
1993 1992 if (!deadman_enabled) {
1994 1993 CPU->cpu_deadman_countdown = deadman_seconds;
1995 1994 return;
1996 1995 }
1997 1996
1998 1997 /*
1999 1998 * If we're here, we want to bring the system down.
2000 1999 */
2001 2000 panic("deadman: timed out after %d seconds of clock "
2002 2001 "inactivity", deadman_seconds);
2003 2002 /*NOTREACHED*/
2004 2003 }
2005 2004
2006 2005 /*ARGSUSED*/
2007 2006 static void
2008 2007 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
2009 2008 {
2010 2009 cpu->cpu_deadman_counter = 0;
2011 2010 cpu->cpu_deadman_countdown = deadman_seconds;
2012 2011
2013 2012 hdlr->cyh_func = (cyc_func_t)deadman;
2014 2013 hdlr->cyh_level = CY_HIGH_LEVEL;
2015 2014 hdlr->cyh_arg = NULL;
2016 2015
2017 2016 /*
2018 2017 * Stagger the CPUs so that they don't all run deadman() at
2019 2018 * the same time. Simplest reason to do this is to make it
2020 2019 * more likely that only one CPU will panic in case of a
2021 2020 * timeout. This is (strictly speaking) an aesthetic, not a
2022 2021 * technical consideration.
2023 2022 */
2024 2023 when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
2025 2024 when->cyt_interval = NANOSEC;
2026 2025 }
2027 2026
2028 2027
2029 2028 void
2030 2029 deadman_init(void)
2031 2030 {
2032 2031 cyc_omni_handler_t hdlr;
2033 2032
2034 2033 if (deadman_seconds == 0)
2035 2034 deadman_seconds = snoop_interval / MICROSEC;
2036 2035
2037 2036 if (snooping)
2038 2037 deadman_enabled = 1;
2039 2038
2040 2039 hdlr.cyo_online = deadman_online;
2041 2040 hdlr.cyo_offline = NULL;
2042 2041 hdlr.cyo_arg = NULL;
2043 2042
2044 2043 mutex_enter(&cpu_lock);
2045 2044 deadman_cyclic = cyclic_add_omni(&hdlr);
2046 2045 mutex_exit(&cpu_lock);
2047 2046 }
2048 2047
2049 2048 /*
2050 2049 * tod_fault() is for updating tod validate mechanism state:
2051 2050 * (1) TOD_NOFAULT: for resetting the state to 'normal'.
2052 2051 * currently used for debugging only
2053 2052 * (2) The following four cases detected by tod validate mechanism:
2054 2053 * TOD_REVERSED: current tod value is less than previous value.
2055 2054 * TOD_STALLED: current tod value hasn't advanced.
2056 2055 * TOD_JUMPED: current tod value advanced too far from previous value.
2057 2056 * TOD_RATECHANGED: the ratio between average tod delta and
2058 2057 * average tick delta has changed.
2059 2058 * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
2060 2059 * a virtual TOD provided by a hypervisor.
2061 2060 */
2062 2061 enum tod_fault_type
2063 2062 tod_fault(enum tod_fault_type ftype, int off)
2064 2063 {
2065 2064 ASSERT(MUTEX_HELD(&tod_lock));
2066 2065
2067 2066 if (tod_faulted != ftype) {
2068 2067 switch (ftype) {
2069 2068 case TOD_NOFAULT:
2070 2069 plat_tod_fault(TOD_NOFAULT);
2071 2070 cmn_err(CE_NOTE, "Restarted tracking "
2072 2071 "Time of Day clock.");
2073 2072 tod_faulted = ftype;
2074 2073 break;
2075 2074 case TOD_REVERSED:
2076 2075 case TOD_JUMPED:
2077 2076 if (tod_faulted == TOD_NOFAULT) {
2078 2077 plat_tod_fault(ftype);
2079 2078 cmn_err(CE_WARN, "Time of Day clock error: "
2080 2079 "reason [%s by 0x%x]. -- "
2081 2080 " Stopped tracking Time Of Day clock.",
2082 2081 tod_fault_table[ftype], off);
2083 2082 tod_faulted = ftype;
2084 2083 }
2085 2084 break;
2086 2085 case TOD_STALLED:
2087 2086 case TOD_RATECHANGED:
2088 2087 if (tod_faulted == TOD_NOFAULT) {
2089 2088 plat_tod_fault(ftype);
2090 2089 cmn_err(CE_WARN, "Time of Day clock error: "
2091 2090 "reason [%s]. -- "
2092 2091 " Stopped tracking Time Of Day clock.",
2093 2092 tod_fault_table[ftype]);
2094 2093 tod_faulted = ftype;
2095 2094 }
2096 2095 break;
2097 2096 case TOD_RDONLY:
2098 2097 if (tod_faulted == TOD_NOFAULT) {
2099 2098 plat_tod_fault(ftype);
2100 2099 cmn_err(CE_NOTE, "!Time of Day clock is "
2101 2100 "Read-Only; set of Date/Time will not "
2102 2101 "persist across reboot.");
2103 2102 tod_faulted = ftype;
2104 2103 }
2105 2104 break;
2106 2105 default:
2107 2106 break;
2108 2107 }
2109 2108 }
2110 2109 return (tod_faulted);
2111 2110 }
2112 2111
2113 2112 /*
2114 2113 * Two functions that allow tod_status_flag to be manipulated by functions
2115 2114 * external to this file.
2116 2115 */
2117 2116
2118 2117 void
2119 2118 tod_status_set(int tod_flag)
2120 2119 {
2121 2120 tod_status_flag |= tod_flag;
2122 2121 }
2123 2122
2124 2123 void
2125 2124 tod_status_clear(int tod_flag)
2126 2125 {
2127 2126 tod_status_flag &= ~tod_flag;
2128 2127 }
2129 2128
2130 2129 /*
2131 2130 * Record a timestamp and the value passed to tod_set(). The next call to
2132 2131 * tod_validate() can use these values, prev_set_tick and prev_set_tod,
2133 2132 * when checking the timestruc_t returned by tod_get(). Ordinarily,
2134 2133 * tod_validate() will use prev_tick and prev_tod for this task but these
2135 2134 * become obsolete, and will be re-assigned with the prev_set_* values,
2136 2135 * in the case when the TOD is re-written.
2137 2136 */
2138 2137 void
2139 2138 tod_set_prev(timestruc_t ts)
2140 2139 {
2141 2140 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2142 2141 tod_validate_deferred) {
2143 2142 return;
2144 2143 }
2145 2144 prev_set_tick = gethrtime();
2146 2145 /*
2147 2146 * A negative value will be set to zero in utc_to_tod() so we fake
2148 2147 * a zero here in such a case. This would need to change if the
2149 2148 * behavior of utc_to_tod() changes.
2150 2149 */
2151 2150 prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
2152 2151 }
2153 2152
2154 2153 /*
2155 2154 * tod_validate() is used for checking values returned by tod_get().
2156 2155 * Four error cases can be detected by this routine:
2157 2156 * TOD_REVERSED: current tod value is less than previous.
2158 2157 * TOD_STALLED: current tod value hasn't advanced.
2159 2158 * TOD_JUMPED: current tod value advanced too far from previous value.
2160 2159 * TOD_RATECHANGED: the ratio between average tod delta and
2161 2160 * average tick delta has changed.
2162 2161 */
2163 2162 time_t
2164 2163 tod_validate(time_t tod)
2165 2164 {
2166 2165 time_t diff_tod;
2167 2166 hrtime_t diff_tick;
2168 2167
2169 2168 long dtick;
2170 2169 int dtick_delta;
2171 2170
2172 2171 int off = 0;
2173 2172 enum tod_fault_type tod_bad = TOD_NOFAULT;
2174 2173
2175 2174 static int firsttime = 1;
2176 2175
2177 2176 static time_t prev_tod = 0;
2178 2177 static hrtime_t prev_tick = 0;
2179 2178 static long dtick_avg = TOD_REF_FREQ;
2180 2179
2181 2180 int cpr_resume_done = 0;
2182 2181 int dr_resume_done = 0;
2183 2182
2184 2183 hrtime_t tick = gethrtime();
2185 2184
2186 2185 ASSERT(MUTEX_HELD(&tod_lock));
2187 2186
2188 2187 /*
2189 2188 * tod_validate_enable is patchable via /etc/system.
2190 2189 * If TOD is already faulted, or if TOD validation is deferred,
2191 2190 * there is nothing to do.
2192 2191 */
2193 2192 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2194 2193 tod_validate_deferred) {
2195 2194 return (tod);
2196 2195 }
2197 2196
2198 2197 /*
2199 2198 * If this is the first time through, we just need to save the tod
2200 2199 * we were called with and hrtime so we can use them next time to
2201 2200 * validate tod_get().
2202 2201 */
2203 2202 if (firsttime) {
2204 2203 firsttime = 0;
2205 2204 prev_tod = tod;
2206 2205 prev_tick = tick;
2207 2206 return (tod);
2208 2207 }
2209 2208
2210 2209 /*
2211 2210 * Handle any flags that have been turned on by tod_status_set().
2212 2211 * In the case where a tod_set() is done and then a subsequent
2213 2212 * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
2214 2213 * true), we treat the TOD_GET_FAILED with precedence by switching
2215 2214 * off the flag, returning tod and leaving TOD_SET_DONE asserted
2216 2215 * until such time as tod_get() completes successfully.
2217 2216 */
2218 2217 if (tod_status_flag & TOD_GET_FAILED) {
2219 2218 /*
2220 2219 * tod_get() has encountered an issue, possibly transitory,
2221 2220 * when reading TOD. We'll just return the incoming tod
2222 2221 * value (which is actually hrestime.tv_sec in this case)
2223 2222 * and when we get a genuine tod, following a successful
2224 2223 * tod_get(), we can validate using prev_tod and prev_tick.
2225 2224 */
2226 2225 tod_status_flag &= ~TOD_GET_FAILED;
2227 2226 return (tod);
2228 2227 } else if (tod_status_flag & TOD_SET_DONE) {
2229 2228 /*
2230 2229 * TOD has been modified. Just before the TOD was written,
2231 2230 * tod_set_prev() saved tod and hrtime; we can now use
2232 2231 * those values, prev_set_tod and prev_set_tick, to validate
2233 2232 * the incoming tod that's just been read.
2234 2233 */
2235 2234 prev_tod = prev_set_tod;
2236 2235 prev_tick = prev_set_tick;
2237 2236 dtick_avg = TOD_REF_FREQ;
2238 2237 tod_status_flag &= ~TOD_SET_DONE;
2239 2238 /*
2240 2239 * If a tod_set() preceded a cpr_suspend() without an
2241 2240 * intervening tod_validate(), we need to ensure that a
2242 2241 * TOD_JUMPED condition is ignored.
2243 2242 * Note this isn't a concern in the case of DR as we've
2244 2243 * just reassigned dtick_avg, above.
2245 2244 */
2246 2245 if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2247 2246 cpr_resume_done = 1;
2248 2247 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2249 2248 }
2250 2249 } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2251 2250 /*
2252 2251 * The system's coming back from a checkpoint resume.
2253 2252 */
2254 2253 cpr_resume_done = 1;
2255 2254 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2256 2255 /*
2257 2256 * We need to handle the possibility of a CPR suspend
2258 2257 * operation having been initiated whilst a DR event was
2259 2258 * in-flight.
2260 2259 */
2261 2260 if (tod_status_flag & TOD_DR_RESUME_DONE) {
2262 2261 dr_resume_done = 1;
2263 2262 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2264 2263 }
2265 2264 } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
2266 2265 /*
2267 2266 * A Dynamic Reconfiguration event has taken place.
2268 2267 */
2269 2268 dr_resume_done = 1;
2270 2269 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2271 2270 }
2272 2271
2273 2272 /* test hook */
2274 2273 switch (tod_unit_test) {
2275 2274 case 1: /* for testing jumping tod */
2276 2275 tod += tod_test_injector;
2277 2276 tod_unit_test = 0;
2278 2277 break;
2279 2278 case 2: /* for testing stuck tod bit */
2280 2279 tod |= 1 << tod_test_injector;
2281 2280 tod_unit_test = 0;
2282 2281 break;
2283 2282 case 3: /* for testing stalled tod */
2284 2283 tod = prev_tod;
2285 2284 tod_unit_test = 0;
2286 2285 break;
2287 2286 case 4: /* reset tod fault status */
2288 2287 (void) tod_fault(TOD_NOFAULT, 0);
2289 2288 tod_unit_test = 0;
2290 2289 break;
2291 2290 default:
2292 2291 break;
2293 2292 }
2294 2293
2295 2294 diff_tod = tod - prev_tod;
2296 2295 diff_tick = tick - prev_tick;
2297 2296
2298 2297 ASSERT(diff_tick >= 0);
2299 2298
2300 2299 if (diff_tod < 0) {
2301 2300 /* ERROR - tod reversed */
2302 2301 tod_bad = TOD_REVERSED;
2303 2302 off = (int)(prev_tod - tod);
2304 2303 } else if (diff_tod == 0) {
2305 2304 /* tod did not advance */
2306 2305 if (diff_tick > TOD_STALL_THRESHOLD) {
2307 2306 /* ERROR - tod stalled */
2308 2307 tod_bad = TOD_STALLED;
2309 2308 } else {
2310 2309 /*
2311 2310 * Make sure we don't update prev_tick
2312 2311 * so that diff_tick is calculated since
2313 2312 * the first diff_tod == 0
2314 2313 */
2315 2314 return (tod);
2316 2315 }
2317 2316 } else {
2318 2317 /* calculate dtick */
2319 2318 dtick = diff_tick / diff_tod;
2320 2319
2321 2320 /* update dtick averages */
2322 2321 dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2323 2322
2324 2323 /*
2325 2324 * Calculate dtick_delta as
2326 2325 * variation from reference freq in quartiles
2327 2326 */
2328 2327 dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2329 2328 (TOD_REF_FREQ >> 2);
2330 2329
2331 2330 /*
2332 2331 * Even with a perfectly functioning TOD device,
2333 2332 * when the number of elapsed seconds is low the
2334 2333 * algorithm can calculate a rate that is beyond
2335 2334 * tolerance, causing an error. The algorithm is
2336 2335 * inaccurate when elapsed time is low (less than
2337 2336 * 5 seconds).
2338 2337 */
2339 2338 if (diff_tod > 4) {
2340 2339 if (dtick < TOD_JUMP_THRESHOLD) {
2341 2340 /*
2342 2341 * If we've just done a CPR resume, we detect
2343 2342 * a jump in the TOD but, actually, what's
2344 2343 * happened is that the TOD has been increasing
2345 2344 * whilst the system was suspended and the tick
2346 2345 * count hasn't kept up. We consider the first
2347 2346 * occurrence of this after a resume as normal
2348 2347 * and ignore it; otherwise, in a non-resume
2349 2348 * case, we regard it as a TOD problem.
2350 2349 */
2351 2350 if (!cpr_resume_done) {
2352 2351 /* ERROR - tod jumped */
2353 2352 tod_bad = TOD_JUMPED;
2354 2353 off = (int)diff_tod;
2355 2354 }
2356 2355 }
2357 2356 if (dtick_delta) {
2358 2357 /*
2359 2358 * If we've just done a DR resume, dtick_avg
2360 2359 * can go a bit askew so we reset it and carry
2361 2360 * on; otherwise, the TOD is in error.
2362 2361 */
2363 2362 if (dr_resume_done) {
2364 2363 dtick_avg = TOD_REF_FREQ;
2365 2364 } else {
2366 2365 /* ERROR - change in clock rate */
2367 2366 tod_bad = TOD_RATECHANGED;
2368 2367 }
2369 2368 }
2370 2369 }
2371 2370 }
2372 2371
2373 2372 if (tod_bad != TOD_NOFAULT) {
2374 2373 (void) tod_fault(tod_bad, off);
2375 2374
2376 2375 /*
2377 2376 * Disable dosynctodr since we are going to fault
2378 2377 * the TOD chip anyway here
2379 2378 */
2380 2379 dosynctodr = 0;
2381 2380
2382 2381 /*
2383 2382 * Set tod to the correct value from hrestime
2384 2383 */
2385 2384 tod = hrestime.tv_sec;
2386 2385 }
2387 2386
2388 2387 prev_tod = tod;
2389 2388 prev_tick = tick;
2390 2389 return (tod);
2391 2390 }
2392 2391
2393 2392 static void
2394 2393 calcloadavg(int nrun, uint64_t *hp_ave)
2395 2394 {
2396 2395 static int64_t f[3] = { 135, 27, 9 };
2397 2396 uint_t i;
2398 2397 int64_t q, r;
2399 2398
2400 2399 /*
2401 2400 * Compute load average over the last 1, 5, and 15 minutes
2402 2401 * (60, 300, and 900 seconds). The constants in f[3] are for
2403 2402 * exponential decay:
2404 2403 * (1 - exp(-1/60)) << 13 = 135,
2405 2404 * (1 - exp(-1/300)) << 13 = 27,
2406 2405 * (1 - exp(-1/900)) << 13 = 9.
2407 2406 */
2408 2407
2409 2408 /*
2410 2409 * a little hoop-jumping to avoid integer overflow
2411 2410 */
2412 2411 for (i = 0; i < 3; i++) {
2413 2412 q = (hp_ave[i] >> 16) << 7;
2414 2413 r = (hp_ave[i] & 0xffff) << 7;
2415 2414 hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2416 2415 }
2417 2416 }
2418 2417
2419 2418 /*
2420 2419 * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
2421 2420 * calculate the value of lbolt according to the current mode. In the event
2422 2421 * driven mode (the default), lbolt is calculated by dividing the current hires
2423 2422 * time by the number of nanoseconds per clock tick. In the cyclic driven mode
2424 2423 * an internal variable is incremented at each firing of the lbolt cyclic
2425 2424 * and returned by lbolt_cyclic_driven().
2426 2425 *
2427 2426 * The system will transition from event to cyclic driven mode when the number
2428 2427 * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
2429 2428 * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
2430 2429 * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
2431 2430 * causing enough activity to cross the thresholds.
2432 2431 */
2433 2432 int64_t
2434 2433 lbolt_bootstrap(void)
2435 2434 {
2436 2435 return (0);
2437 2436 }
2438 2437
2439 2438 /* ARGSUSED */
2440 2439 uint_t
2441 2440 lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
2442 2441 {
2443 2442 hrtime_t ts, exp;
2444 2443 int ret;
2445 2444
2446 2445 ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
2447 2446
2448 2447 kpreempt_disable();
2449 2448
2450 2449 ts = gethrtime();
2451 2450 lb_info->lbi_internal = (ts/nsec_per_tick);
2452 2451
2453 2452 /*
2454 2453 * Align the next expiration to a clock tick boundary.
2455 2454 */
2456 2455 exp = ts + nsec_per_tick - 1;
2457 2456 exp = (exp/nsec_per_tick) * nsec_per_tick;
2458 2457
2459 2458 ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
2460 2459 ASSERT(ret);
2461 2460
2462 2461 lbolt_hybrid = lbolt_cyclic_driven;
2463 2462 lb_info->lbi_cyc_deactivate = B_FALSE;
2464 2463 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2465 2464
2466 2465 kpreempt_enable();
2467 2466
2468 2467 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2469 2468 ASSERT(ret == 0);
2470 2469
2471 2470 return (1);
2472 2471 }
2473 2472
2474 2473 int64_t
2475 2474 lbolt_event_driven(void)
2476 2475 {
2477 2476 hrtime_t ts;
2478 2477 int64_t lb;
2479 2478 int ret, cpu = CPU->cpu_seqid;
2480 2479
2481 2480 ts = gethrtime();
2482 2481 ASSERT(ts > 0);
2483 2482
2484 2483 ASSERT(nsec_per_tick > 0);
2485 2484 lb = (ts/nsec_per_tick);
2486 2485
2487 2486 /*
2488 2487 * Switch to cyclic mode if the number of calls to this routine
2489 2488 * has reached the threshold within the interval.
2490 2489 */
2491 2490 if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
2492 2491
2493 2492 if (--lb_cpu[cpu].lbc_counter == 0) {
2494 2493 /*
2495 2494 * Reached the threshold within the interval, reset
2496 2495 * the usage statistics.
2497 2496 */
2498 2497 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2499 2498 lb_cpu[cpu].lbc_cnt_start = lb;
2500 2499
2501 2500 /*
2502 2501 * Make sure only one thread reprograms the
2503 2502 * lbolt cyclic and changes the mode.
2504 2503 */
2505 2504 if (panicstr == NULL &&
2506 2505 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2507 2506
2508 2507 if (lbolt_hybrid == lbolt_cyclic_driven) {
2509 2508 ret = atomic_dec_32_nv(
2510 2509 &lb_info->lbi_token);
2511 2510 ASSERT(ret == 0);
2512 2511 } else {
2513 2512 lbolt_softint_post();
2514 2513 }
2515 2514 }
2516 2515 }
2517 2516 } else {
2518 2517 /*
2519 2518 * Exceeded the interval, reset the usage statistics.
2520 2519 */
2521 2520 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2522 2521 lb_cpu[cpu].lbc_cnt_start = lb;
2523 2522 }
2524 2523
2525 2524 ASSERT(lb >= lb_info->lbi_debug_time);
2526 2525
2527 2526 return (lb - lb_info->lbi_debug_time);
2528 2527 }
2529 2528
2530 2529 int64_t
2531 2530 lbolt_cyclic_driven(void)
2532 2531 {
2533 2532 int64_t lb = lb_info->lbi_internal;
2534 2533 int cpu;
2535 2534
2536 2535 /*
2537 2536 * If a CPU has already prevented the lbolt cyclic from deactivating
2538 2537 * itself, don't bother tracking the usage. Otherwise check if we're
2539 2538 * within the interval and how the per CPU counter is doing.
2540 2539 */
2541 2540 if (lb_info->lbi_cyc_deactivate) {
2542 2541 cpu = CPU->cpu_seqid;
2543 2542 if ((lb - lb_cpu[cpu].lbc_cnt_start) <
2544 2543 lb_info->lbi_thresh_interval) {
2545 2544
2546 2545 if (lb_cpu[cpu].lbc_counter == 0)
2547 2546 /*
2548 2547 * Reached the threshold within the interval,
2549 2548 * prevent the lbolt cyclic from turning itself
2550 2549 * off.
2551 2550 */
2552 2551 lb_info->lbi_cyc_deactivate = B_FALSE;
2553 2552 else
2554 2553 lb_cpu[cpu].lbc_counter--;
2555 2554 } else {
2556 2555 /*
2557 2556 * Only reset the usage statistics when we have
2558 2557 * exceeded the interval.
2559 2558 */
2560 2559 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2561 2560 lb_cpu[cpu].lbc_cnt_start = lb;
2562 2561 }
2563 2562 }
2564 2563
2565 2564 ASSERT(lb >= lb_info->lbi_debug_time);
2566 2565
2567 2566 return (lb - lb_info->lbi_debug_time);
2568 2567 }
2569 2568
2570 2569 /*
2571 2570 * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
2572 2571 * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
2573 2572 * It is inactive by default, and will be activated when switching from event
2574 2573 * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
2575 2574 * by lbolt_cyclic_driven().
2576 2575 */
2577 2576 static void
2578 2577 lbolt_cyclic(void)
2579 2578 {
2580 2579 int ret;
2581 2580
2582 2581 lb_info->lbi_internal++;
2583 2582
2584 2583 if (!lbolt_cyc_only) {
2585 2584
2586 2585 if (lb_info->lbi_cyc_deactivate) {
2587 2586 /*
2588 2587 * Switching from cyclic to event driven mode.
2589 2588 */
2590 2589 if (panicstr == NULL &&
2591 2590 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2592 2591
2593 2592 if (lbolt_hybrid == lbolt_event_driven) {
2594 2593 ret = atomic_dec_32_nv(
2595 2594 &lb_info->lbi_token);
2596 2595 ASSERT(ret == 0);
2597 2596 return;
2598 2597 }
2599 2598
2600 2599 kpreempt_disable();
2601 2600
2602 2601 lbolt_hybrid = lbolt_event_driven;
2603 2602 ret = cyclic_reprogram(
2604 2603 lb_info->id.lbi_cyclic_id,
2605 2604 CY_INFINITY);
2606 2605 ASSERT(ret);
2607 2606
2608 2607 kpreempt_enable();
2609 2608
2610 2609 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2611 2610 ASSERT(ret == 0);
2612 2611 }
2613 2612 }
2614 2613
2615 2614 /*
2616 2615 * The lbolt cyclic should not try to deactivate itself before
2617 2616 * the sampling period has elapsed.
2618 2617 */
2619 2618 if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
2620 2619 lb_info->lbi_thresh_interval) {
2621 2620 lb_info->lbi_cyc_deactivate = B_TRUE;
2622 2621 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2623 2622 }
2624 2623 }
2625 2624 }
2626 2625
2627 2626 /*
2628 2627 * Since the lbolt service was historically cyclic driven, it must be 'stopped'
2629 2628 * when the system drops into the kernel debugger. lbolt_debug_entry() is
2630 2629 * called by the KDI system claim callbacks to record a hires timestamp at
2631 2630 * debug enter time. lbolt_debug_return() is called by the sistem release
2632 2631 * callbacks to account for the time spent in the debugger. The value is then
2633 2632 * accumulated in the lb_info structure and used by lbolt_event_driven() and
2634 2633 * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
2635 2634 */
2636 2635 void
2637 2636 lbolt_debug_entry(void)
2638 2637 {
2639 2638 if (lbolt_hybrid != lbolt_bootstrap) {
2640 2639 ASSERT(lb_info != NULL);
2641 2640 lb_info->lbi_debug_ts = gethrtime();
2642 2641 }
2643 2642 }
2644 2643
2645 2644 /*
2646 2645 * Calculate the time spent in the debugger and add it to the lbolt info
2647 2646 * structure. We also update the internal lbolt value in case we were in
2648 2647 * cyclic driven mode going in.
2649 2648 */
2650 2649 void
2651 2650 lbolt_debug_return(void)
2652 2651 {
2653 2652 hrtime_t ts;
2654 2653
2655 2654 if (lbolt_hybrid != lbolt_bootstrap) {
2656 2655 ASSERT(lb_info != NULL);
2657 2656 ASSERT(nsec_per_tick > 0);
2658 2657
2659 2658 ts = gethrtime();
2660 2659 lb_info->lbi_internal = (ts/nsec_per_tick);
2661 2660 lb_info->lbi_debug_time +=
2662 2661 ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
2663 2662
2664 2663 lb_info->lbi_debug_ts = 0;
2665 2664 }
2666 2665 }
|
↓ open down ↓ |
2337 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX