Print this page
OS-2366 ddi_periodic_add(9F) is entirely rubbish
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/clock.c
+++ new/usr/src/uts/common/os/clock.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 22 /* All Rights Reserved */
23 23
24 24 /*
25 25 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
26 - * Copyright (c) 2011, Joyent, Inc. All rights reserved.
26 + * Copyright (c) 2013, Joyent, Inc. All rights reserved.
27 27 */
28 28
29 29 #include <sys/param.h>
30 30 #include <sys/t_lock.h>
31 31 #include <sys/types.h>
32 32 #include <sys/tuneable.h>
33 33 #include <sys/sysmacros.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/cpuvar.h>
36 36 #include <sys/lgrp.h>
37 37 #include <sys/user.h>
38 38 #include <sys/proc.h>
39 39 #include <sys/callo.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/var.h>
42 42 #include <sys/cmn_err.h>
43 43 #include <sys/swap.h>
44 44 #include <sys/vmsystm.h>
45 45 #include <sys/class.h>
46 46 #include <sys/time.h>
47 47 #include <sys/debug.h>
48 48 #include <sys/vtrace.h>
49 49 #include <sys/spl.h>
50 50 #include <sys/atomic.h>
51 51 #include <sys/dumphdr.h>
52 52 #include <sys/archsystm.h>
53 53 #include <sys/fs/swapnode.h>
54 54 #include <sys/panic.h>
55 55 #include <sys/disp.h>
56 56 #include <sys/msacct.h>
57 57 #include <sys/mem_cage.h>
58 58
59 59 #include <vm/page.h>
60 60 #include <vm/anon.h>
61 61 #include <vm/rm.h>
62 62 #include <sys/cyclic.h>
63 63 #include <sys/cpupart.h>
64 64 #include <sys/rctl.h>
65 65 #include <sys/task.h>
66 66 #include <sys/sdt.h>
67 67 #include <sys/ddi_timer.h>
68 68 #include <sys/random.h>
69 69 #include <sys/modctl.h>
70 70 #include <sys/zone.h>
71 71
72 72 /*
73 73 * for NTP support
74 74 */
75 75 #include <sys/timex.h>
76 76 #include <sys/inttypes.h>
77 77
78 78 #include <sys/sunddi.h>
79 79 #include <sys/clock_impl.h>
80 80
81 81 /*
82 82 * clock() is called straight from the clock cyclic; see clock_init().
83 83 *
84 84 * Functions:
85 85 * reprime clock
86 86 * maintain date
87 87 * jab the scheduler
88 88 */
89 89
90 90 extern kcondvar_t fsflush_cv;
91 91 extern sysinfo_t sysinfo;
92 92 extern vminfo_t vminfo;
93 93 extern int idleswtch; /* flag set while idle in pswtch() */
94 94 extern hrtime_t volatile devinfo_freeze;
95 95
96 96 /*
97 97 * high-precision avenrun values. These are needed to make the
98 98 * regular avenrun values accurate.
99 99 */
100 100 static uint64_t hp_avenrun[3];
101 101 int avenrun[3]; /* FSCALED average run queue lengths */
102 102 time_t time; /* time in seconds since 1970 - for compatibility only */
103 103
104 104 static struct loadavg_s loadavg;
105 105 /*
106 106 * Phase/frequency-lock loop (PLL/FLL) definitions
107 107 *
108 108 * The following variables are read and set by the ntp_adjtime() system
109 109 * call.
110 110 *
111 111 * time_state shows the state of the system clock, with values defined
112 112 * in the timex.h header file.
113 113 *
114 114 * time_status shows the status of the system clock, with bits defined
115 115 * in the timex.h header file.
116 116 *
117 117 * time_offset is used by the PLL/FLL to adjust the system time in small
118 118 * increments.
119 119 *
120 120 * time_constant determines the bandwidth or "stiffness" of the PLL.
121 121 *
122 122 * time_tolerance determines maximum frequency error or tolerance of the
123 123 * CPU clock oscillator and is a property of the architecture; however,
124 124 * in principle it could change as result of the presence of external
125 125 * discipline signals, for instance.
126 126 *
127 127 * time_precision is usually equal to the kernel tick variable; however,
128 128 * in cases where a precision clock counter or external clock is
129 129 * available, the resolution can be much less than this and depend on
130 130 * whether the external clock is working or not.
131 131 *
132 132 * time_maxerror is initialized by a ntp_adjtime() call and increased by
133 133 * the kernel once each second to reflect the maximum error bound
134 134 * growth.
135 135 *
136 136 * time_esterror is set and read by the ntp_adjtime() call, but
137 137 * otherwise not used by the kernel.
138 138 */
139 139 int32_t time_state = TIME_OK; /* clock state */
140 140 int32_t time_status = STA_UNSYNC; /* clock status bits */
141 141 int32_t time_offset = 0; /* time offset (us) */
142 142 int32_t time_constant = 0; /* pll time constant */
143 143 int32_t time_tolerance = MAXFREQ; /* frequency tolerance (scaled ppm) */
144 144 int32_t time_precision = 1; /* clock precision (us) */
145 145 int32_t time_maxerror = MAXPHASE; /* maximum error (us) */
146 146 int32_t time_esterror = MAXPHASE; /* estimated error (us) */
147 147
148 148 /*
149 149 * The following variables establish the state of the PLL/FLL and the
150 150 * residual time and frequency offset of the local clock. The scale
151 151 * factors are defined in the timex.h header file.
152 152 *
153 153 * time_phase and time_freq are the phase increment and the frequency
154 154 * increment, respectively, of the kernel time variable.
155 155 *
156 156 * time_freq is set via ntp_adjtime() from a value stored in a file when
157 157 * the synchronization daemon is first started. Its value is retrieved
158 158 * via ntp_adjtime() and written to the file about once per hour by the
159 159 * daemon.
160 160 *
161 161 * time_adj is the adjustment added to the value of tick at each timer
162 162 * interrupt and is recomputed from time_phase and time_freq at each
163 163 * seconds rollover.
164 164 *
165 165 * time_reftime is the second's portion of the system time at the last
166 166 * call to ntp_adjtime(). It is used to adjust the time_freq variable
167 167 * and to increase the time_maxerror as the time since last update
168 168 * increases.
169 169 */
170 170 int32_t time_phase = 0; /* phase offset (scaled us) */
171 171 int32_t time_freq = 0; /* frequency offset (scaled ppm) */
172 172 int32_t time_adj = 0; /* tick adjust (scaled 1 / hz) */
173 173 int32_t time_reftime = 0; /* time at last adjustment (s) */
174 174
175 175 /*
176 176 * The scale factors of the following variables are defined in the
177 177 * timex.h header file.
178 178 *
179 179 * pps_time contains the time at each calibration interval, as read by
180 180 * microtime(). pps_count counts the seconds of the calibration
181 181 * interval, the duration of which is nominally pps_shift in powers of
182 182 * two.
183 183 *
184 184 * pps_offset is the time offset produced by the time median filter
185 185 * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
186 186 * this filter.
187 187 *
188 188 * pps_freq is the frequency offset produced by the frequency median
189 189 * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
190 190 * by this filter.
191 191 *
192 192 * pps_usec is latched from a high resolution counter or external clock
193 193 * at pps_time. Here we want the hardware counter contents only, not the
194 194 * contents plus the time_tv.usec as usual.
195 195 *
196 196 * pps_valid counts the number of seconds since the last PPS update. It
197 197 * is used as a watchdog timer to disable the PPS discipline should the
198 198 * PPS signal be lost.
199 199 *
200 200 * pps_glitch counts the number of seconds since the beginning of an
201 201 * offset burst more than tick/2 from current nominal offset. It is used
202 202 * mainly to suppress error bursts due to priority conflicts between the
203 203 * PPS interrupt and timer interrupt.
204 204 *
205 205 * pps_intcnt counts the calibration intervals for use in the interval-
206 206 * adaptation algorithm. It's just too complicated for words.
207 207 */
208 208 struct timeval pps_time; /* kernel time at last interval */
209 209 int32_t pps_tf[] = {0, 0, 0}; /* pps time offset median filter (us) */
210 210 int32_t pps_offset = 0; /* pps time offset (us) */
211 211 int32_t pps_jitter = MAXTIME; /* time dispersion (jitter) (us) */
212 212 int32_t pps_ff[] = {0, 0, 0}; /* pps frequency offset median filter */
213 213 int32_t pps_freq = 0; /* frequency offset (scaled ppm) */
214 214 int32_t pps_stabil = MAXFREQ; /* frequency dispersion (scaled ppm) */
215 215 int32_t pps_usec = 0; /* microsec counter at last interval */
216 216 int32_t pps_valid = PPS_VALID; /* pps signal watchdog counter */
217 217 int32_t pps_glitch = 0; /* pps signal glitch counter */
218 218 int32_t pps_count = 0; /* calibration interval counter (s) */
219 219 int32_t pps_shift = PPS_SHIFT; /* interval duration (s) (shift) */
220 220 int32_t pps_intcnt = 0; /* intervals at current duration */
221 221
222 222 /*
223 223 * PPS signal quality monitors
224 224 *
225 225 * pps_jitcnt counts the seconds that have been discarded because the
226 226 * jitter measured by the time median filter exceeds the limit MAXTIME
227 227 * (100 us).
228 228 *
229 229 * pps_calcnt counts the frequency calibration intervals, which are
230 230 * variable from 4 s to 256 s.
231 231 *
232 232 * pps_errcnt counts the calibration intervals which have been discarded
233 233 * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
234 234 * calibration interval jitter exceeds two ticks.
235 235 *
236 236 * pps_stbcnt counts the calibration intervals that have been discarded
237 237 * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
238 238 */
239 239 int32_t pps_jitcnt = 0; /* jitter limit exceeded */
240 240 int32_t pps_calcnt = 0; /* calibration intervals */
241 241 int32_t pps_errcnt = 0; /* calibration errors */
242 242 int32_t pps_stbcnt = 0; /* stability limit exceeded */
243 243
244 244 kcondvar_t lbolt_cv;
245 245
246 246 /*
247 247 * Hybrid lbolt implementation:
248 248 *
249 249 * The service historically provided by the lbolt and lbolt64 variables has
250 250 * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
251 251 * original symbols removed from the system. The once clock driven variables are
252 252 * now implemented in an event driven fashion, backed by gethrtime() coarsed to
253 253 * the appropriate clock resolution. The default event driven implementation is
254 254 * complemented by a cyclic driven one, active only during periods of intense
255 255 * activity around the DDI lbolt routines, when a lbolt specific cyclic is
256 256 * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
257 257 * rely on the original low cost of consulting a memory position.
258 258 *
259 259 * The implementation uses the number of calls to these routines and the
260 260 * frequency of these to determine when to transition from event to cyclic
261 261 * driven and vice-versa. These values are kept on a per CPU basis for
262 262 * scalability reasons and to prevent CPUs from constantly invalidating a single
263 263 * cache line when modifying a global variable. The transition from event to
264 264 * cyclic mode happens once the thresholds are crossed, and activity on any CPU
265 265 * can cause such transition.
266 266 *
267 267 * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
268 268 * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
269 269 * lbolt_cyclic_driven() according to the current mode. When the thresholds
270 270 * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
271 271 * fire at a nsec_per_tick interval and increment an internal variable at
272 272 * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
273 273 * will simply return the value of such variable. lbolt_cyclic() will attempt
274 274 * to shut itself off at each threshold interval (sampling period for calls
275 275 * to the DDI lbolt routines), and return to the event driven mode, but will
276 276 * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
277 277 *
278 278 * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
279 279 * for the cyclic subsystem to be intialized.
280 280 *
281 281 */
282 282 int64_t lbolt_bootstrap(void);
283 283 int64_t lbolt_event_driven(void);
284 284 int64_t lbolt_cyclic_driven(void);
285 285 int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
286 286 uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);
287 287
288 288 /*
289 289 * lbolt's cyclic, installed by clock_init().
290 290 */
291 291 static void lbolt_cyclic(void);
292 292
293 293 /*
294 294 * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
295 295 * from switching back to event driven, once it reaches cyclic mode.
296 296 */
297 297 static boolean_t lbolt_cyc_only = B_FALSE;
298 298
299 299 /*
300 300 * Cache aligned, per CPU structure with lbolt usage statistics.
301 301 */
302 302 static lbolt_cpu_t *lb_cpu;
303 303
304 304 /*
305 305 * Single, cache aligned, structure with all the information required by
306 306 * the lbolt implementation.
307 307 */
308 308 lbolt_info_t *lb_info;
309 309
310 310
311 311 int one_sec = 1; /* turned on once every second */
312 312 static int fsflushcnt; /* counter for t_fsflushr */
313 313 int dosynctodr = 1; /* patchable; enable/disable sync to TOD chip */
314 314 int tod_needsync = 0; /* need to sync tod chip with software time */
315 315 static int tod_broken = 0; /* clock chip doesn't work */
316 316 time_t boot_time = 0; /* Boot time in seconds since 1970 */
317 317 cyclic_id_t clock_cyclic; /* clock()'s cyclic_id */
318 318 cyclic_id_t deadman_cyclic; /* deadman()'s cyclic_id */
319 319 cyclic_id_t ddi_timer_cyclic; /* cyclic_timer()'s cyclic_id */
320 320
321 321 extern void clock_tick_schedule(int);
322 322
323 323 static int lgrp_ticks; /* counter to schedule lgrp load calcs */
324 324
325 325 /*
326 326 * for tod fault detection
327 327 */
328 328 #define TOD_REF_FREQ ((longlong_t)(NANOSEC))
329 329 #define TOD_STALL_THRESHOLD (TOD_REF_FREQ * 3 / 2)
330 330 #define TOD_JUMP_THRESHOLD (TOD_REF_FREQ / 2)
331 331 #define TOD_FILTER_N 4
332 332 #define TOD_FILTER_SETTLE (4 * TOD_FILTER_N)
333 333 static int tod_faulted = TOD_NOFAULT;
334 334
335 335 static int tod_status_flag = 0; /* used by tod_validate() */
336 336
337 337 static hrtime_t prev_set_tick = 0; /* gethrtime() prior to tod_set() */
338 338 static time_t prev_set_tod = 0; /* tv_sec value passed to tod_set() */
339 339
340 340 /* patchable via /etc/system */
341 341 int tod_validate_enable = 1;
342 342
343 343 /* Diagnose/Limit messages about delay(9F) called from interrupt context */
344 344 int delay_from_interrupt_diagnose = 0;
345 345 volatile uint32_t delay_from_interrupt_msg = 20;
346 346
347 347 /*
348 348 * On non-SPARC systems, TOD validation must be deferred until gethrtime
349 349 * returns non-zero values (after mach_clkinit's execution).
350 350 * On SPARC systems, it must be deferred until after hrtime_base
351 351 * and hres_last_tick are set (in the first invocation of hres_tick).
352 352 * Since in both cases the prerequisites occur before the invocation of
353 353 * tod_get() in clock(), the deferment is lifted there.
354 354 */
355 355 static boolean_t tod_validate_deferred = B_TRUE;
356 356
357 357 /*
358 358 * tod_fault_table[] must be aligned with
359 359 * enum tod_fault_type in systm.h
360 360 */
361 361 static char *tod_fault_table[] = {
362 362 "Reversed", /* TOD_REVERSED */
363 363 "Stalled", /* TOD_STALLED */
364 364 "Jumped", /* TOD_JUMPED */
365 365 "Changed in Clock Rate", /* TOD_RATECHANGED */
366 366 "Is Read-Only" /* TOD_RDONLY */
367 367 /*
368 368 * no strings needed for TOD_NOFAULT
369 369 */
370 370 };
371 371
372 372 /*
373 373 * test hook for tod broken detection in tod_validate
374 374 */
375 375 int tod_unit_test = 0;
376 376 time_t tod_test_injector;
377 377
378 378 #define CLOCK_ADJ_HIST_SIZE 4
379 379
380 380 static int adj_hist_entry;
381 381
382 382 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
383 383
384 384 static void calcloadavg(int, uint64_t *);
385 385 static int genloadavg(struct loadavg_s *);
386 386 static void loadavg_update();
387 387
388 388 void (*cmm_clock_callout)() = NULL;
389 389 void (*cpucaps_clock_callout)() = NULL;
390 390
391 391 extern clock_t clock_tick_proc_max;
392 392
393 393 static int64_t deadman_counter = 0;
394 394
395 395 static void
396 396 clock(void)
397 397 {
398 398 kthread_t *t;
399 399 uint_t nrunnable;
400 400 uint_t w_io;
401 401 cpu_t *cp;
402 402 cpupart_t *cpupart;
403 403 extern void set_freemem();
404 404 void (*funcp)();
405 405 int32_t ltemp;
406 406 int64_t lltemp;
407 407 int s;
408 408 int do_lgrp_load;
409 409 int i;
410 410 clock_t now = LBOLT_NO_ACCOUNT; /* current tick */
411 411
412 412 if (panicstr)
413 413 return;
414 414
415 415 /*
416 416 * Make sure that 'freemem' do not drift too far from the truth
417 417 */
418 418 set_freemem();
419 419
420 420
421 421 /*
422 422 * Before the section which is repeated is executed, we do
423 423 * the time delta processing which occurs every clock tick
424 424 *
425 425 * There is additional processing which happens every time
426 426 * the nanosecond counter rolls over which is described
427 427 * below - see the section which begins with : if (one_sec)
428 428 *
429 429 * This section marks the beginning of the precision-kernel
430 430 * code fragment.
431 431 *
432 432 * First, compute the phase adjustment. If the low-order bits
433 433 * (time_phase) of the update overflow, bump the higher order
434 434 * bits (time_update).
435 435 */
436 436 time_phase += time_adj;
437 437 if (time_phase <= -FINEUSEC) {
438 438 ltemp = -time_phase / SCALE_PHASE;
439 439 time_phase += ltemp * SCALE_PHASE;
440 440 s = hr_clock_lock();
441 441 timedelta -= ltemp * (NANOSEC/MICROSEC);
442 442 hr_clock_unlock(s);
443 443 } else if (time_phase >= FINEUSEC) {
444 444 ltemp = time_phase / SCALE_PHASE;
445 445 time_phase -= ltemp * SCALE_PHASE;
446 446 s = hr_clock_lock();
447 447 timedelta += ltemp * (NANOSEC/MICROSEC);
448 448 hr_clock_unlock(s);
449 449 }
450 450
451 451 /*
452 452 * End of precision-kernel code fragment which is processed
453 453 * every timer interrupt.
454 454 *
455 455 * Continue with the interrupt processing as scheduled.
456 456 */
457 457 /*
458 458 * Count the number of runnable threads and the number waiting
459 459 * for some form of I/O to complete -- gets added to
460 460 * sysinfo.waiting. To know the state of the system, must add
461 461 * wait counts from all CPUs. Also add up the per-partition
462 462 * statistics.
463 463 */
464 464 w_io = 0;
465 465 nrunnable = 0;
466 466
467 467 /*
468 468 * keep track of when to update lgrp/part loads
469 469 */
470 470
471 471 do_lgrp_load = 0;
472 472 if (lgrp_ticks++ >= hz / 10) {
473 473 lgrp_ticks = 0;
474 474 do_lgrp_load = 1;
475 475 }
476 476
477 477 if (one_sec) {
478 478 loadavg_update();
479 479 deadman_counter++;
480 480 }
481 481
482 482 /*
483 483 * First count the threads waiting on kpreempt queues in each
484 484 * CPU partition.
485 485 */
486 486
487 487 cpupart = cp_list_head;
488 488 do {
489 489 uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
490 490
491 491 cpupart->cp_updates++;
492 492 nrunnable += cpupart_nrunnable;
493 493 cpupart->cp_nrunnable_cum += cpupart_nrunnable;
494 494 if (one_sec) {
495 495 cpupart->cp_nrunning = 0;
496 496 cpupart->cp_nrunnable = cpupart_nrunnable;
497 497 }
498 498 } while ((cpupart = cpupart->cp_next) != cp_list_head);
499 499
500 500
501 501 /* Now count the per-CPU statistics. */
502 502 cp = cpu_list;
503 503 do {
504 504 uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
505 505
506 506 nrunnable += cpu_nrunnable;
507 507 cpupart = cp->cpu_part;
508 508 cpupart->cp_nrunnable_cum += cpu_nrunnable;
509 509 if (one_sec) {
510 510 cpupart->cp_nrunnable += cpu_nrunnable;
511 511 /*
512 512 * Update user, system, and idle cpu times.
513 513 */
514 514 cpupart->cp_nrunning++;
515 515 /*
516 516 * w_io is used to update sysinfo.waiting during
517 517 * one_second processing below. Only gather w_io
518 518 * information when we walk the list of cpus if we're
519 519 * going to perform one_second processing.
520 520 */
521 521 w_io += CPU_STATS(cp, sys.iowait);
522 522 }
523 523
524 524 if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
525 525 int i, load, change;
526 526 hrtime_t intracct, intrused;
527 527 const hrtime_t maxnsec = 1000000000;
528 528 const int precision = 100;
529 529
530 530 /*
531 531 * Estimate interrupt load on this cpu each second.
532 532 * Computes cpu_intrload as %utilization (0-99).
533 533 */
534 534
535 535 /* add up interrupt time from all micro states */
536 536 for (intracct = 0, i = 0; i < NCMSTATES; i++)
537 537 intracct += cp->cpu_intracct[i];
538 538 scalehrtime(&intracct);
539 539
540 540 /* compute nsec used in the past second */
541 541 intrused = intracct - cp->cpu_intrlast;
542 542 cp->cpu_intrlast = intracct;
543 543
544 544 /* limit the value for safety (and the first pass) */
545 545 if (intrused >= maxnsec)
546 546 intrused = maxnsec - 1;
547 547
548 548 /* calculate %time in interrupt */
549 549 load = (precision * intrused) / maxnsec;
550 550 ASSERT(load >= 0 && load < precision);
551 551 change = cp->cpu_intrload - load;
552 552
553 553 /* jump to new max, or decay the old max */
554 554 if (change < 0)
555 555 cp->cpu_intrload = load;
556 556 else if (change > 0)
557 557 cp->cpu_intrload -= (change + 3) / 4;
558 558
559 559 DTRACE_PROBE3(cpu_intrload,
560 560 cpu_t *, cp,
561 561 hrtime_t, intracct,
562 562 hrtime_t, intrused);
563 563 }
564 564
565 565 if (do_lgrp_load &&
566 566 (cp->cpu_flags & CPU_EXISTS)) {
567 567 /*
568 568 * When updating the lgroup's load average,
569 569 * account for the thread running on the CPU.
570 570 * If the CPU is the current one, then we need
571 571 * to account for the underlying thread which
572 572 * got the clock interrupt not the thread that is
573 573 * handling the interrupt and caculating the load
574 574 * average
575 575 */
576 576 t = cp->cpu_thread;
577 577 if (CPU == cp)
578 578 t = t->t_intr;
579 579
580 580 /*
581 581 * Account for the load average for this thread if
582 582 * it isn't the idle thread or it is on the interrupt
583 583 * stack and not the current CPU handling the clock
584 584 * interrupt
585 585 */
586 586 if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
587 587 CPU_ON_INTR(cp))) {
588 588 if (t->t_lpl == cp->cpu_lpl) {
589 589 /* local thread */
590 590 cpu_nrunnable++;
591 591 } else {
592 592 /*
593 593 * This is a remote thread, charge it
594 594 * against its home lgroup. Note that
595 595 * we notice that a thread is remote
596 596 * only if it's currently executing.
597 597 * This is a reasonable approximation,
598 598 * since queued remote threads are rare.
599 599 * Note also that if we didn't charge
600 600 * it to its home lgroup, remote
601 601 * execution would often make a system
602 602 * appear balanced even though it was
603 603 * not, and thread placement/migration
604 604 * would often not be done correctly.
605 605 */
606 606 lgrp_loadavg(t->t_lpl,
607 607 LGRP_LOADAVG_IN_THREAD_MAX, 0);
608 608 }
609 609 }
610 610 lgrp_loadavg(cp->cpu_lpl,
611 611 cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
612 612 }
613 613 } while ((cp = cp->cpu_next) != cpu_list);
614 614
615 615 clock_tick_schedule(one_sec);
616 616
617 617 /*
618 618 * Check for a callout that needs be called from the clock
619 619 * thread to support the membership protocol in a clustered
620 620 * system. Copy the function pointer so that we can reset
621 621 * this to NULL if needed.
622 622 */
623 623 if ((funcp = cmm_clock_callout) != NULL)
624 624 (*funcp)();
625 625
626 626 if ((funcp = cpucaps_clock_callout) != NULL)
627 627 (*funcp)();
628 628
629 629 /*
630 630 * Wakeup the cageout thread waiters once per second.
631 631 */
632 632 if (one_sec)
633 633 kcage_tick();
634 634
635 635 if (one_sec) {
636 636
637 637 int drift, absdrift;
638 638 timestruc_t tod;
639 639 int s;
640 640
641 641 /*
642 642 * Beginning of precision-kernel code fragment executed
643 643 * every second.
644 644 *
645 645 * On rollover of the second the phase adjustment to be
646 646 * used for the next second is calculated. Also, the
647 647 * maximum error is increased by the tolerance. If the
648 648 * PPS frequency discipline code is present, the phase is
649 649 * increased to compensate for the CPU clock oscillator
650 650 * frequency error.
651 651 *
652 652 * On a 32-bit machine and given parameters in the timex.h
653 653 * header file, the maximum phase adjustment is +-512 ms
654 654 * and maximum frequency offset is (a tad less than)
655 655 * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
656 656 */
657 657 time_maxerror += time_tolerance / SCALE_USEC;
658 658
659 659 /*
660 660 * Leap second processing. If in leap-insert state at
661 661 * the end of the day, the system clock is set back one
662 662 * second; if in leap-delete state, the system clock is
663 663 * set ahead one second. The microtime() routine or
664 664 * external clock driver will insure that reported time
665 665 * is always monotonic. The ugly divides should be
666 666 * replaced.
667 667 */
668 668 switch (time_state) {
669 669
670 670 case TIME_OK:
671 671 if (time_status & STA_INS)
672 672 time_state = TIME_INS;
673 673 else if (time_status & STA_DEL)
674 674 time_state = TIME_DEL;
675 675 break;
676 676
677 677 case TIME_INS:
678 678 if (hrestime.tv_sec % 86400 == 0) {
679 679 s = hr_clock_lock();
680 680 hrestime.tv_sec--;
681 681 hr_clock_unlock(s);
682 682 time_state = TIME_OOP;
683 683 }
684 684 break;
685 685
686 686 case TIME_DEL:
687 687 if ((hrestime.tv_sec + 1) % 86400 == 0) {
688 688 s = hr_clock_lock();
689 689 hrestime.tv_sec++;
690 690 hr_clock_unlock(s);
691 691 time_state = TIME_WAIT;
692 692 }
693 693 break;
694 694
695 695 case TIME_OOP:
696 696 time_state = TIME_WAIT;
697 697 break;
698 698
699 699 case TIME_WAIT:
700 700 if (!(time_status & (STA_INS | STA_DEL)))
701 701 time_state = TIME_OK;
702 702 default:
703 703 break;
704 704 }
705 705
706 706 /*
707 707 * Compute the phase adjustment for the next second. In
708 708 * PLL mode, the offset is reduced by a fixed factor
709 709 * times the time constant. In FLL mode the offset is
710 710 * used directly. In either mode, the maximum phase
711 711 * adjustment for each second is clamped so as to spread
712 712 * the adjustment over not more than the number of
713 713 * seconds between updates.
714 714 */
715 715 if (time_offset == 0)
716 716 time_adj = 0;
717 717 else if (time_offset < 0) {
718 718 lltemp = -time_offset;
719 719 if (!(time_status & STA_FLL)) {
720 720 if ((1 << time_constant) >= SCALE_KG)
721 721 lltemp *= (1 << time_constant) /
722 722 SCALE_KG;
723 723 else
724 724 lltemp = (lltemp / SCALE_KG) >>
725 725 time_constant;
726 726 }
727 727 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
728 728 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
729 729 time_offset += lltemp;
730 730 time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
731 731 } else {
732 732 lltemp = time_offset;
733 733 if (!(time_status & STA_FLL)) {
734 734 if ((1 << time_constant) >= SCALE_KG)
735 735 lltemp *= (1 << time_constant) /
736 736 SCALE_KG;
737 737 else
738 738 lltemp = (lltemp / SCALE_KG) >>
739 739 time_constant;
740 740 }
741 741 if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
742 742 lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
743 743 time_offset -= lltemp;
744 744 time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
745 745 }
746 746
747 747 /*
748 748 * Compute the frequency estimate and additional phase
749 749 * adjustment due to frequency error for the next
750 750 * second. When the PPS signal is engaged, gnaw on the
751 751 * watchdog counter and update the frequency computed by
752 752 * the pll and the PPS signal.
753 753 */
754 754 pps_valid++;
755 755 if (pps_valid == PPS_VALID) {
756 756 pps_jitter = MAXTIME;
757 757 pps_stabil = MAXFREQ;
758 758 time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
759 759 STA_PPSWANDER | STA_PPSERROR);
760 760 }
761 761 lltemp = time_freq + pps_freq;
762 762
763 763 if (lltemp)
764 764 time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
765 765
766 766 /*
767 767 * End of precision kernel-code fragment
768 768 *
769 769 * The section below should be modified if we are planning
770 770 * to use NTP for synchronization.
771 771 *
772 772 * Note: the clock synchronization code now assumes
773 773 * the following:
774 774 * - if dosynctodr is 1, then compute the drift between
775 775 * the tod chip and software time and adjust one or
776 776 * the other depending on the circumstances
777 777 *
778 778 * - if dosynctodr is 0, then the tod chip is independent
779 779 * of the software clock and should not be adjusted,
780 780 * but allowed to free run. this allows NTP to sync.
781 781 * hrestime without any interference from the tod chip.
782 782 */
783 783
784 784 tod_validate_deferred = B_FALSE;
785 785 mutex_enter(&tod_lock);
786 786 tod = tod_get();
787 787 drift = tod.tv_sec - hrestime.tv_sec;
788 788 absdrift = (drift >= 0) ? drift : -drift;
789 789 if (tod_needsync || absdrift > 1) {
790 790 int s;
791 791 if (absdrift > 2) {
792 792 if (!tod_broken && tod_faulted == TOD_NOFAULT) {
793 793 s = hr_clock_lock();
794 794 hrestime = tod;
795 795 membar_enter(); /* hrestime visible */
796 796 timedelta = 0;
797 797 timechanged++;
798 798 tod_needsync = 0;
799 799 hr_clock_unlock(s);
800 800 callout_hrestime();
801 801
802 802 }
803 803 } else {
804 804 if (tod_needsync || !dosynctodr) {
805 805 gethrestime(&tod);
806 806 tod_set(tod);
807 807 s = hr_clock_lock();
808 808 if (timedelta == 0)
809 809 tod_needsync = 0;
810 810 hr_clock_unlock(s);
811 811 } else {
812 812 /*
813 813 * If the drift is 2 seconds on the
814 814 * money, then the TOD is adjusting
815 815 * the clock; record that.
816 816 */
817 817 clock_adj_hist[adj_hist_entry++ %
818 818 CLOCK_ADJ_HIST_SIZE] = now;
819 819 s = hr_clock_lock();
820 820 timedelta = (int64_t)drift*NANOSEC;
821 821 hr_clock_unlock(s);
822 822 }
823 823 }
824 824 }
825 825 one_sec = 0;
826 826 time = gethrestime_sec(); /* for crusty old kmem readers */
827 827 mutex_exit(&tod_lock);
828 828
829 829 /*
830 830 * Some drivers still depend on this... XXX
831 831 */
832 832 cv_broadcast(&lbolt_cv);
833 833
834 834 vminfo.freemem += freemem;
835 835 {
836 836 pgcnt_t maxswap, resv, free;
837 837 pgcnt_t avail =
838 838 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
839 839
840 840 maxswap = k_anoninfo.ani_mem_resv +
841 841 k_anoninfo.ani_max +avail;
842 842 /* Update ani_free */
843 843 set_anoninfo();
844 844 free = k_anoninfo.ani_free + avail;
845 845 resv = k_anoninfo.ani_phys_resv +
846 846 k_anoninfo.ani_mem_resv;
847 847
848 848 vminfo.swap_resv += resv;
849 849 /* number of reserved and allocated pages */
850 850 #ifdef DEBUG
851 851 if (maxswap < free)
852 852 cmn_err(CE_WARN, "clock: maxswap < free");
853 853 if (maxswap < resv)
854 854 cmn_err(CE_WARN, "clock: maxswap < resv");
855 855 #endif
856 856 vminfo.swap_alloc += maxswap - free;
857 857 vminfo.swap_avail += maxswap - resv;
858 858 vminfo.swap_free += free;
859 859 }
860 860 vminfo.updates++;
861 861 if (nrunnable) {
862 862 sysinfo.runque += nrunnable;
863 863 sysinfo.runocc++;
864 864 }
865 865 if (nswapped) {
866 866 sysinfo.swpque += nswapped;
867 867 sysinfo.swpocc++;
868 868 }
869 869 sysinfo.waiting += w_io;
870 870 sysinfo.updates++;
871 871
872 872 /*
873 873 * Wake up fsflush to write out DELWRI
874 874 * buffers, dirty pages and other cached
875 875 * administrative data, e.g. inodes.
876 876 */
877 877 if (--fsflushcnt <= 0) {
878 878 fsflushcnt = tune.t_fsflushr;
879 879 cv_signal(&fsflush_cv);
880 880 }
881 881
882 882 vmmeter();
883 883 calcloadavg(genloadavg(&loadavg), hp_avenrun);
884 884 for (i = 0; i < 3; i++)
885 885 /*
886 886 * At the moment avenrun[] can only hold 31
887 887 * bits of load average as it is a signed
888 888 * int in the API. We need to ensure that
889 889 * hp_avenrun[i] >> (16 - FSHIFT) will not be
890 890 * too large. If it is, we put the largest value
891 891 * that we can use into avenrun[i]. This is
892 892 * kludgey, but about all we can do until we
893 893 * avenrun[] is declared as an array of uint64[]
894 894 */
895 895 if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
896 896 avenrun[i] = (int32_t)(hp_avenrun[i] >>
897 897 (16 - FSHIFT));
898 898 else
899 899 avenrun[i] = 0x7fffffff;
900 900
901 901 cpupart = cp_list_head;
902 902 do {
903 903 calcloadavg(genloadavg(&cpupart->cp_loadavg),
904 904 cpupart->cp_hp_avenrun);
905 905 } while ((cpupart = cpupart->cp_next) != cp_list_head);
906 906
907 907 /*
908 908 * Wake up the swapper thread if necessary.
909 909 */
910 910 if (runin ||
911 911 (runout && (avefree < desfree || wake_sched_sec))) {
912 912 t = &t0;
913 913 thread_lock(t);
914 914 if (t->t_state == TS_STOPPED) {
915 915 runin = runout = 0;
916 916 wake_sched_sec = 0;
917 917 t->t_whystop = 0;
918 918 t->t_whatstop = 0;
919 919 t->t_schedflag &= ~TS_ALLSTART;
920 920 THREAD_TRANSITION(t);
921 921 setfrontdq(t);
922 922 }
923 923 thread_unlock(t);
924 924 }
925 925 }
926 926
927 927 /*
928 928 * Wake up the swapper if any high priority swapped-out threads
929 929 * became runable during the last tick.
930 930 */
931 931 if (wake_sched) {
932 932 t = &t0;
933 933 thread_lock(t);
934 934 if (t->t_state == TS_STOPPED) {
935 935 runin = runout = 0;
936 936 wake_sched = 0;
937 937 t->t_whystop = 0;
938 938 t->t_whatstop = 0;
939 939 t->t_schedflag &= ~TS_ALLSTART;
|
↓ open down ↓ |
903 lines elided |
↑ open up ↑ |
940 940 THREAD_TRANSITION(t);
941 941 setfrontdq(t);
942 942 }
943 943 thread_unlock(t);
944 944 }
945 945 }
946 946
947 947 void
948 948 clock_init(void)
949 949 {
950 - cyc_handler_t clk_hdlr, timer_hdlr, lbolt_hdlr;
950 + cyc_handler_t clk_hdlr, lbolt_hdlr;
951 951 cyc_time_t clk_when, lbolt_when;
952 952 int i, sz;
953 953 intptr_t buf;
954 954
955 955 /*
956 956 * Setup handler and timer for the clock cyclic.
957 957 */
958 958 clk_hdlr.cyh_func = (cyc_func_t)clock;
959 959 clk_hdlr.cyh_level = CY_LOCK_LEVEL;
960 960 clk_hdlr.cyh_arg = NULL;
961 961
962 962 clk_when.cyt_when = 0;
963 963 clk_when.cyt_interval = nsec_per_tick;
964 964
965 965 /*
966 - * cyclic_timer is dedicated to the ddi interface, which
967 - * uses the same clock resolution as the system one.
968 - */
969 - timer_hdlr.cyh_func = (cyc_func_t)cyclic_timer;
970 - timer_hdlr.cyh_level = CY_LOCK_LEVEL;
971 - timer_hdlr.cyh_arg = NULL;
972 -
973 - /*
974 966 * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
975 967 * interval to satisfy performance needs of the DDI lbolt consumers.
976 968 * It is off by default.
977 969 */
978 970 lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
979 971 lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
980 972 lbolt_hdlr.cyh_arg = NULL;
981 973
982 974 lbolt_when.cyt_interval = nsec_per_tick;
983 975
984 976 /*
985 977 * Allocate cache line aligned space for the per CPU lbolt data and
986 978 * lbolt info structures, and initialize them with their default
987 979 * values. Note that these structures are also cache line sized.
988 980 */
989 981 sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
990 982 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
991 983 lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
992 984
993 985 if (hz != HZ_DEFAULT)
994 986 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
995 987 hz/HZ_DEFAULT;
996 988 else
997 989 lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;
998 990
999 991 lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;
1000 992
1001 993 sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
1002 994 buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
1003 995 lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);
1004 996
1005 997 for (i = 0; i < max_ncpus; i++)
1006 998 lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;
1007 999
1008 1000 /*
1009 1001 * Install the softint used to switch between event and cyclic driven
1010 1002 * lbolt. We use a soft interrupt to make sure the context of the
1011 1003 * cyclic reprogram call is safe.
1012 1004 */
1013 1005 lbolt_softint_add();
1014 1006
1015 1007 /*
1016 1008 * Since the hybrid lbolt implementation is based on a hardware counter
1017 1009 * that is reset at every hardware reboot and that we'd like to have
1018 1010 * the lbolt value starting at zero after both a hardware and a fast
1019 1011 * reboot, we calculate the number of clock ticks the system's been up
1020 1012 * and store it in the lbi_debug_time field of the lbolt info structure.
1021 1013 * The value of this field will be subtracted from lbolt before
1022 1014 * returning it.
1023 1015 */
1024 1016 lb_info->lbi_internal = lb_info->lbi_debug_time =
1025 1017 (gethrtime()/nsec_per_tick);
1026 1018
1027 1019 /*
1028 1020 * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
1029 1021 * and lbolt_debug_{enter,return} use this value as an indication that
1030 1022 * the initializaion above hasn't been completed. Setting lbolt_hybrid
1031 1023 * to either lbolt_{cyclic,event}_driven here signals those code paths
1032 1024 * that the lbolt related structures can be used.
1033 1025 */
1034 1026 if (lbolt_cyc_only) {
1035 1027 lbolt_when.cyt_when = 0;
1036 1028 lbolt_hybrid = lbolt_cyclic_driven;
1037 1029 } else {
|
↓ open down ↓ |
54 lines elided |
↑ open up ↑ |
1038 1030 lbolt_when.cyt_when = CY_INFINITY;
1039 1031 lbolt_hybrid = lbolt_event_driven;
1040 1032 }
1041 1033
1042 1034 /*
1043 1035 * Grab cpu_lock and install all three cyclics.
1044 1036 */
1045 1037 mutex_enter(&cpu_lock);
1046 1038
1047 1039 clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
1048 - ddi_timer_cyclic = cyclic_add(&timer_hdlr, &clk_when);
1049 1040 lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);
1050 1041
1051 1042 mutex_exit(&cpu_lock);
1052 1043 }
1053 1044
1054 1045 /*
1055 1046 * Called before calcloadavg to get 10-sec moving loadavg together
1056 1047 */
1057 1048
1058 1049 static int
1059 1050 genloadavg(struct loadavg_s *avgs)
1060 1051 {
1061 1052 int avg;
1062 1053 int spos; /* starting position */
1063 1054 int cpos; /* moving current position */
1064 1055 int i;
1065 1056 int slen;
1066 1057 hrtime_t hr_avg;
1067 1058
1068 1059 /* 10-second snapshot, calculate first positon */
1069 1060 if (avgs->lg_len == 0) {
1070 1061 return (0);
1071 1062 }
1072 1063 slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1073 1064
1074 1065 spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1075 1066 S_LOADAVG_SZ + (avgs->lg_cur - 1);
1076 1067 for (i = hr_avg = 0; i < slen; i++) {
1077 1068 cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1078 1069 hr_avg += avgs->lg_loads[cpos];
1079 1070 }
1080 1071
1081 1072 hr_avg = hr_avg / slen;
1082 1073 avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1083 1074
1084 1075 return (avg);
1085 1076 }
1086 1077
1087 1078 /*
1088 1079 * Run every second from clock () to update the loadavg count available to the
1089 1080 * system and cpu-partitions.
1090 1081 *
1091 1082 * This works by sampling the previous usr, sys, wait time elapsed,
1092 1083 * computing a delta, and adding that delta to the elapsed usr, sys,
1093 1084 * wait increase.
1094 1085 */
1095 1086
1096 1087 static void
1097 1088 loadavg_update()
1098 1089 {
1099 1090 cpu_t *cp;
1100 1091 cpupart_t *cpupart;
1101 1092 hrtime_t cpu_total;
1102 1093 int prev;
1103 1094
1104 1095 cp = cpu_list;
1105 1096 loadavg.lg_total = 0;
1106 1097
1107 1098 /*
1108 1099 * first pass totals up per-cpu statistics for system and cpu
1109 1100 * partitions
1110 1101 */
1111 1102
1112 1103 do {
1113 1104 struct loadavg_s *lavg;
1114 1105
1115 1106 lavg = &cp->cpu_loadavg;
1116 1107
1117 1108 cpu_total = cp->cpu_acct[CMS_USER] +
1118 1109 cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1119 1110 /* compute delta against last total */
1120 1111 scalehrtime(&cpu_total);
1121 1112 prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1122 1113 S_LOADAVG_SZ + (lavg->lg_cur - 1);
1123 1114 if (lavg->lg_loads[prev] <= 0) {
1124 1115 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1125 1116 cpu_total = 0;
1126 1117 } else {
1127 1118 lavg->lg_loads[lavg->lg_cur] = cpu_total;
1128 1119 cpu_total = cpu_total - lavg->lg_loads[prev];
1129 1120 if (cpu_total < 0)
1130 1121 cpu_total = 0;
1131 1122 }
1132 1123
1133 1124 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1134 1125 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1135 1126 lavg->lg_len + 1 : S_LOADAVG_SZ;
1136 1127
1137 1128 loadavg.lg_total += cpu_total;
1138 1129 cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1139 1130
1140 1131 } while ((cp = cp->cpu_next) != cpu_list);
1141 1132
1142 1133 loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1143 1134 loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1144 1135 loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1145 1136 loadavg.lg_len + 1 : S_LOADAVG_SZ;
1146 1137 /*
1147 1138 * Second pass updates counts
1148 1139 */
1149 1140 cpupart = cp_list_head;
1150 1141
1151 1142 do {
1152 1143 struct loadavg_s *lavg;
1153 1144
1154 1145 lavg = &cpupart->cp_loadavg;
1155 1146 lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1156 1147 lavg->lg_total = 0;
1157 1148 lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1158 1149 lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1159 1150 lavg->lg_len + 1 : S_LOADAVG_SZ;
1160 1151
1161 1152 } while ((cpupart = cpupart->cp_next) != cp_list_head);
1162 1153
1163 1154 /*
1164 1155 * Third pass totals up per-zone statistics.
1165 1156 */
1166 1157 zone_loadavg_update();
1167 1158 }
1168 1159
1169 1160 /*
1170 1161 * clock_update() - local clock update
1171 1162 *
1172 1163 * This routine is called by ntp_adjtime() to update the local clock
1173 1164 * phase and frequency. The implementation is of an
1174 1165 * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1175 1166 * routine computes new time and frequency offset estimates for each
1176 1167 * call. The PPS signal itself determines the new time offset,
1177 1168 * instead of the calling argument. Presumably, calls to
1178 1169 * ntp_adjtime() occur only when the caller believes the local clock
1179 1170 * is valid within some bound (+-128 ms with NTP). If the caller's
1180 1171 * time is far different than the PPS time, an argument will ensue,
1181 1172 * and it's not clear who will lose.
1182 1173 *
1183 1174 * For uncompensated quartz crystal oscillatores and nominal update
1184 1175 * intervals less than 1024 s, operation should be in phase-lock mode
1185 1176 * (STA_FLL = 0), where the loop is disciplined to phase. For update
1186 1177 * intervals greater than this, operation should be in frequency-lock
1187 1178 * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1188 1179 *
1189 1180 * Note: mutex(&tod_lock) is in effect.
1190 1181 */
1191 1182 void
1192 1183 clock_update(int offset)
1193 1184 {
1194 1185 int ltemp, mtemp, s;
1195 1186
1196 1187 ASSERT(MUTEX_HELD(&tod_lock));
1197 1188
1198 1189 if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1199 1190 return;
1200 1191 ltemp = offset;
1201 1192 if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1202 1193 ltemp = pps_offset;
1203 1194
1204 1195 /*
1205 1196 * Scale the phase adjustment and clamp to the operating range.
1206 1197 */
1207 1198 if (ltemp > MAXPHASE)
1208 1199 time_offset = MAXPHASE * SCALE_UPDATE;
1209 1200 else if (ltemp < -MAXPHASE)
1210 1201 time_offset = -(MAXPHASE * SCALE_UPDATE);
1211 1202 else
1212 1203 time_offset = ltemp * SCALE_UPDATE;
1213 1204
1214 1205 /*
1215 1206 * Select whether the frequency is to be controlled and in which
1216 1207 * mode (PLL or FLL). Clamp to the operating range. Ugly
1217 1208 * multiply/divide should be replaced someday.
1218 1209 */
1219 1210 if (time_status & STA_FREQHOLD || time_reftime == 0)
1220 1211 time_reftime = hrestime.tv_sec;
1221 1212
1222 1213 mtemp = hrestime.tv_sec - time_reftime;
1223 1214 time_reftime = hrestime.tv_sec;
1224 1215
1225 1216 if (time_status & STA_FLL) {
1226 1217 if (mtemp >= MINSEC) {
1227 1218 ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1228 1219 SCALE_UPDATE));
1229 1220 if (ltemp)
1230 1221 time_freq += ltemp / SCALE_KH;
1231 1222 }
1232 1223 } else {
1233 1224 if (mtemp < MAXSEC) {
1234 1225 ltemp *= mtemp;
1235 1226 if (ltemp)
1236 1227 time_freq += (int)(((int64_t)ltemp *
1237 1228 SCALE_USEC) / SCALE_KF)
1238 1229 / (1 << (time_constant * 2));
1239 1230 }
1240 1231 }
1241 1232 if (time_freq > time_tolerance)
1242 1233 time_freq = time_tolerance;
1243 1234 else if (time_freq < -time_tolerance)
1244 1235 time_freq = -time_tolerance;
1245 1236
1246 1237 s = hr_clock_lock();
1247 1238 tod_needsync = 1;
1248 1239 hr_clock_unlock(s);
1249 1240 }
1250 1241
1251 1242 /*
1252 1243 * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1253 1244 *
1254 1245 * This routine is called at each PPS interrupt in order to discipline
1255 1246 * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1256 1247 * and leaves it in a handy spot for the clock() routine. It
1257 1248 * integrates successive PPS phase differences and calculates the
1258 1249 * frequency offset. This is used in clock() to discipline the CPU
1259 1250 * clock oscillator so that intrinsic frequency error is cancelled out.
1260 1251 * The code requires the caller to capture the time and hardware counter
1261 1252 * value at the on-time PPS signal transition.
1262 1253 *
1263 1254 * Note that, on some Unix systems, this routine runs at an interrupt
1264 1255 * priority level higher than the timer interrupt routine clock().
1265 1256 * Therefore, the variables used are distinct from the clock()
1266 1257 * variables, except for certain exceptions: The PPS frequency pps_freq
1267 1258 * and phase pps_offset variables are determined by this routine and
1268 1259 * updated atomically. The time_tolerance variable can be considered a
1269 1260 * constant, since it is infrequently changed, and then only when the
1270 1261 * PPS signal is disabled. The watchdog counter pps_valid is updated
1271 1262 * once per second by clock() and is atomically cleared in this
1272 1263 * routine.
1273 1264 *
1274 1265 * tvp is the time of the last tick; usec is a microsecond count since the
1275 1266 * last tick.
1276 1267 *
1277 1268 * Note: In Solaris systems, the tick value is actually given by
1278 1269 * usec_per_tick. This is called from the serial driver cdintr(),
1279 1270 * or equivalent, at a high PIL. Because the kernel keeps a
1280 1271 * highresolution time, the following code can accept either
1281 1272 * the traditional argument pair, or the current highres timestamp
1282 1273 * in tvp and zero in usec.
1283 1274 */
1284 1275 void
1285 1276 ddi_hardpps(struct timeval *tvp, int usec)
1286 1277 {
1287 1278 int u_usec, v_usec, bigtick;
1288 1279 time_t cal_sec;
1289 1280 int cal_usec;
1290 1281
1291 1282 /*
1292 1283 * An occasional glitch can be produced when the PPS interrupt
1293 1284 * occurs in the clock() routine before the time variable is
1294 1285 * updated. Here the offset is discarded when the difference
1295 1286 * between it and the last one is greater than tick/2, but not
1296 1287 * if the interval since the first discard exceeds 30 s.
1297 1288 */
1298 1289 time_status |= STA_PPSSIGNAL;
1299 1290 time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1300 1291 pps_valid = 0;
1301 1292 u_usec = -tvp->tv_usec;
1302 1293 if (u_usec < -(MICROSEC/2))
1303 1294 u_usec += MICROSEC;
1304 1295 v_usec = pps_offset - u_usec;
1305 1296 if (v_usec < 0)
1306 1297 v_usec = -v_usec;
1307 1298 if (v_usec > (usec_per_tick >> 1)) {
1308 1299 if (pps_glitch > MAXGLITCH) {
1309 1300 pps_glitch = 0;
1310 1301 pps_tf[2] = u_usec;
1311 1302 pps_tf[1] = u_usec;
1312 1303 } else {
1313 1304 pps_glitch++;
1314 1305 u_usec = pps_offset;
1315 1306 }
1316 1307 } else
1317 1308 pps_glitch = 0;
1318 1309
1319 1310 /*
1320 1311 * A three-stage median filter is used to help deglitch the pps
1321 1312 * time. The median sample becomes the time offset estimate; the
1322 1313 * difference between the other two samples becomes the time
1323 1314 * dispersion (jitter) estimate.
1324 1315 */
1325 1316 pps_tf[2] = pps_tf[1];
1326 1317 pps_tf[1] = pps_tf[0];
1327 1318 pps_tf[0] = u_usec;
1328 1319 if (pps_tf[0] > pps_tf[1]) {
1329 1320 if (pps_tf[1] > pps_tf[2]) {
1330 1321 pps_offset = pps_tf[1]; /* 0 1 2 */
1331 1322 v_usec = pps_tf[0] - pps_tf[2];
1332 1323 } else if (pps_tf[2] > pps_tf[0]) {
1333 1324 pps_offset = pps_tf[0]; /* 2 0 1 */
1334 1325 v_usec = pps_tf[2] - pps_tf[1];
1335 1326 } else {
1336 1327 pps_offset = pps_tf[2]; /* 0 2 1 */
1337 1328 v_usec = pps_tf[0] - pps_tf[1];
1338 1329 }
1339 1330 } else {
1340 1331 if (pps_tf[1] < pps_tf[2]) {
1341 1332 pps_offset = pps_tf[1]; /* 2 1 0 */
1342 1333 v_usec = pps_tf[2] - pps_tf[0];
1343 1334 } else if (pps_tf[2] < pps_tf[0]) {
1344 1335 pps_offset = pps_tf[0]; /* 1 0 2 */
1345 1336 v_usec = pps_tf[1] - pps_tf[2];
1346 1337 } else {
1347 1338 pps_offset = pps_tf[2]; /* 1 2 0 */
1348 1339 v_usec = pps_tf[1] - pps_tf[0];
1349 1340 }
1350 1341 }
1351 1342 if (v_usec > MAXTIME)
1352 1343 pps_jitcnt++;
1353 1344 v_usec = (v_usec << PPS_AVG) - pps_jitter;
1354 1345 pps_jitter += v_usec / (1 << PPS_AVG);
1355 1346 if (pps_jitter > (MAXTIME >> 1))
1356 1347 time_status |= STA_PPSJITTER;
1357 1348
1358 1349 /*
1359 1350 * During the calibration interval adjust the starting time when
1360 1351 * the tick overflows. At the end of the interval compute the
1361 1352 * duration of the interval and the difference of the hardware
1362 1353 * counters at the beginning and end of the interval. This code
1363 1354 * is deliciously complicated by the fact valid differences may
1364 1355 * exceed the value of tick when using long calibration
1365 1356 * intervals and small ticks. Note that the counter can be
1366 1357 * greater than tick if caught at just the wrong instant, but
1367 1358 * the values returned and used here are correct.
1368 1359 */
1369 1360 bigtick = (int)usec_per_tick * SCALE_USEC;
1370 1361 pps_usec -= pps_freq;
1371 1362 if (pps_usec >= bigtick)
1372 1363 pps_usec -= bigtick;
1373 1364 if (pps_usec < 0)
1374 1365 pps_usec += bigtick;
1375 1366 pps_time.tv_sec++;
1376 1367 pps_count++;
1377 1368 if (pps_count < (1 << pps_shift))
1378 1369 return;
1379 1370 pps_count = 0;
1380 1371 pps_calcnt++;
1381 1372 u_usec = usec * SCALE_USEC;
1382 1373 v_usec = pps_usec - u_usec;
1383 1374 if (v_usec >= bigtick >> 1)
1384 1375 v_usec -= bigtick;
1385 1376 if (v_usec < -(bigtick >> 1))
1386 1377 v_usec += bigtick;
1387 1378 if (v_usec < 0)
1388 1379 v_usec = -(-v_usec >> pps_shift);
1389 1380 else
1390 1381 v_usec = v_usec >> pps_shift;
1391 1382 pps_usec = u_usec;
1392 1383 cal_sec = tvp->tv_sec;
1393 1384 cal_usec = tvp->tv_usec;
1394 1385 cal_sec -= pps_time.tv_sec;
1395 1386 cal_usec -= pps_time.tv_usec;
1396 1387 if (cal_usec < 0) {
1397 1388 cal_usec += MICROSEC;
1398 1389 cal_sec--;
1399 1390 }
1400 1391 pps_time = *tvp;
1401 1392
1402 1393 /*
1403 1394 * Check for lost interrupts, noise, excessive jitter and
1404 1395 * excessive frequency error. The number of timer ticks during
1405 1396 * the interval may vary +-1 tick. Add to this a margin of one
1406 1397 * tick for the PPS signal jitter and maximum frequency
1407 1398 * deviation. If the limits are exceeded, the calibration
1408 1399 * interval is reset to the minimum and we start over.
1409 1400 */
1410 1401 u_usec = (int)usec_per_tick << 1;
1411 1402 if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1412 1403 (cal_sec == 0 && cal_usec < u_usec)) ||
1413 1404 v_usec > time_tolerance || v_usec < -time_tolerance) {
1414 1405 pps_errcnt++;
1415 1406 pps_shift = PPS_SHIFT;
1416 1407 pps_intcnt = 0;
1417 1408 time_status |= STA_PPSERROR;
1418 1409 return;
1419 1410 }
1420 1411
1421 1412 /*
1422 1413 * A three-stage median filter is used to help deglitch the pps
1423 1414 * frequency. The median sample becomes the frequency offset
1424 1415 * estimate; the difference between the other two samples
1425 1416 * becomes the frequency dispersion (stability) estimate.
1426 1417 */
1427 1418 pps_ff[2] = pps_ff[1];
1428 1419 pps_ff[1] = pps_ff[0];
1429 1420 pps_ff[0] = v_usec;
1430 1421 if (pps_ff[0] > pps_ff[1]) {
1431 1422 if (pps_ff[1] > pps_ff[2]) {
1432 1423 u_usec = pps_ff[1]; /* 0 1 2 */
1433 1424 v_usec = pps_ff[0] - pps_ff[2];
1434 1425 } else if (pps_ff[2] > pps_ff[0]) {
1435 1426 u_usec = pps_ff[0]; /* 2 0 1 */
1436 1427 v_usec = pps_ff[2] - pps_ff[1];
1437 1428 } else {
1438 1429 u_usec = pps_ff[2]; /* 0 2 1 */
1439 1430 v_usec = pps_ff[0] - pps_ff[1];
1440 1431 }
1441 1432 } else {
1442 1433 if (pps_ff[1] < pps_ff[2]) {
1443 1434 u_usec = pps_ff[1]; /* 2 1 0 */
1444 1435 v_usec = pps_ff[2] - pps_ff[0];
1445 1436 } else if (pps_ff[2] < pps_ff[0]) {
1446 1437 u_usec = pps_ff[0]; /* 1 0 2 */
1447 1438 v_usec = pps_ff[1] - pps_ff[2];
1448 1439 } else {
1449 1440 u_usec = pps_ff[2]; /* 1 2 0 */
1450 1441 v_usec = pps_ff[1] - pps_ff[0];
1451 1442 }
1452 1443 }
1453 1444
1454 1445 /*
1455 1446 * Here the frequency dispersion (stability) is updated. If it
1456 1447 * is less than one-fourth the maximum (MAXFREQ), the frequency
1457 1448 * offset is updated as well, but clamped to the tolerance. It
1458 1449 * will be processed later by the clock() routine.
1459 1450 */
1460 1451 v_usec = (v_usec >> 1) - pps_stabil;
1461 1452 if (v_usec < 0)
1462 1453 pps_stabil -= -v_usec >> PPS_AVG;
1463 1454 else
1464 1455 pps_stabil += v_usec >> PPS_AVG;
1465 1456 if (pps_stabil > MAXFREQ >> 2) {
1466 1457 pps_stbcnt++;
1467 1458 time_status |= STA_PPSWANDER;
1468 1459 return;
1469 1460 }
1470 1461 if (time_status & STA_PPSFREQ) {
1471 1462 if (u_usec < 0) {
1472 1463 pps_freq -= -u_usec >> PPS_AVG;
1473 1464 if (pps_freq < -time_tolerance)
1474 1465 pps_freq = -time_tolerance;
1475 1466 u_usec = -u_usec;
1476 1467 } else {
1477 1468 pps_freq += u_usec >> PPS_AVG;
1478 1469 if (pps_freq > time_tolerance)
1479 1470 pps_freq = time_tolerance;
1480 1471 }
1481 1472 }
1482 1473
1483 1474 /*
1484 1475 * Here the calibration interval is adjusted. If the maximum
1485 1476 * time difference is greater than tick / 4, reduce the interval
1486 1477 * by half. If this is not the case for four consecutive
1487 1478 * intervals, double the interval.
1488 1479 */
1489 1480 if (u_usec << pps_shift > bigtick >> 2) {
1490 1481 pps_intcnt = 0;
1491 1482 if (pps_shift > PPS_SHIFT)
1492 1483 pps_shift--;
1493 1484 } else if (pps_intcnt >= 4) {
1494 1485 pps_intcnt = 0;
1495 1486 if (pps_shift < PPS_SHIFTMAX)
1496 1487 pps_shift++;
1497 1488 } else
1498 1489 pps_intcnt++;
1499 1490
1500 1491 /*
1501 1492 * If recovering from kmdb, then make sure the tod chip gets resynced.
1502 1493 * If we took an early exit above, then we don't yet have a stable
1503 1494 * calibration signal to lock onto, so don't mark the tod for sync
1504 1495 * until we get all the way here.
1505 1496 */
1506 1497 {
1507 1498 int s = hr_clock_lock();
1508 1499
1509 1500 tod_needsync = 1;
1510 1501 hr_clock_unlock(s);
1511 1502 }
1512 1503 }
1513 1504
1514 1505 /*
1515 1506 * Handle clock tick processing for a thread.
1516 1507 * Check for timer action, enforce CPU rlimit, do profiling etc.
1517 1508 */
1518 1509 void
1519 1510 clock_tick(kthread_t *t, int pending)
1520 1511 {
1521 1512 struct proc *pp;
1522 1513 klwp_id_t lwp;
1523 1514 struct as *as;
1524 1515 clock_t ticks;
1525 1516 int poke = 0; /* notify another CPU */
1526 1517 int user_mode;
1527 1518 size_t rss;
1528 1519 int i, total_usec, usec;
1529 1520 rctl_qty_t secs;
1530 1521
1531 1522 ASSERT(pending > 0);
1532 1523
1533 1524 /* Must be operating on a lwp/thread */
1534 1525 if ((lwp = ttolwp(t)) == NULL) {
1535 1526 panic("clock_tick: no lwp");
1536 1527 /*NOTREACHED*/
1537 1528 }
1538 1529
1539 1530 for (i = 0; i < pending; i++) {
1540 1531 CL_TICK(t); /* Class specific tick processing */
1541 1532 DTRACE_SCHED1(tick, kthread_t *, t);
1542 1533 }
1543 1534
1544 1535 pp = ttoproc(t);
1545 1536
1546 1537 /* pp->p_lock makes sure that the thread does not exit */
1547 1538 ASSERT(MUTEX_HELD(&pp->p_lock));
1548 1539
1549 1540 user_mode = (lwp->lwp_state == LWP_USER);
1550 1541
1551 1542 ticks = (pp->p_utime + pp->p_stime) % hz;
1552 1543 /*
1553 1544 * Update process times. Should use high res clock and state
1554 1545 * changes instead of statistical sampling method. XXX
1555 1546 */
1556 1547 if (user_mode) {
1557 1548 pp->p_utime += pending;
1558 1549 } else {
1559 1550 pp->p_stime += pending;
1560 1551 }
1561 1552
1562 1553 pp->p_ttime += pending;
1563 1554 as = pp->p_as;
1564 1555
1565 1556 /*
1566 1557 * Update user profiling statistics. Get the pc from the
1567 1558 * lwp when the AST happens.
1568 1559 */
1569 1560 if (pp->p_prof.pr_scale) {
1570 1561 atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
1571 1562 if (user_mode) {
1572 1563 poke = 1;
1573 1564 aston(t);
1574 1565 }
1575 1566 }
1576 1567
1577 1568 /*
1578 1569 * If CPU was in user state, process lwp-virtual time
1579 1570 * interval timer. The value passed to itimerdecr() has to be
1580 1571 * in microseconds and has to be less than one second. Hence
1581 1572 * this loop.
1582 1573 */
1583 1574 total_usec = usec_per_tick * pending;
1584 1575 while (total_usec > 0) {
1585 1576 usec = MIN(total_usec, (MICROSEC - 1));
1586 1577 if (user_mode &&
1587 1578 timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1588 1579 itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
1589 1580 poke = 1;
1590 1581 sigtoproc(pp, t, SIGVTALRM);
1591 1582 }
1592 1583 total_usec -= usec;
1593 1584 }
1594 1585
1595 1586 /*
1596 1587 * If CPU was in user state, process lwp-profile
1597 1588 * interval timer.
1598 1589 */
1599 1590 total_usec = usec_per_tick * pending;
1600 1591 while (total_usec > 0) {
1601 1592 usec = MIN(total_usec, (MICROSEC - 1));
1602 1593 if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1603 1594 itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
1604 1595 poke = 1;
1605 1596 sigtoproc(pp, t, SIGPROF);
1606 1597 }
1607 1598 total_usec -= usec;
1608 1599 }
1609 1600
1610 1601 /*
1611 1602 * Enforce CPU resource controls:
1612 1603 * (a) process.max-cpu-time resource control
1613 1604 *
1614 1605 * Perform the check only if we have accumulated more a second.
1615 1606 */
1616 1607 if ((ticks + pending) >= hz) {
1617 1608 (void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1618 1609 (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
1619 1610 }
1620 1611
1621 1612 /*
1622 1613 * (b) task.max-cpu-time resource control
1623 1614 *
1624 1615 * If we have accumulated enough ticks, increment the task CPU
1625 1616 * time usage and test for the resource limit. This minimizes the
1626 1617 * number of calls to the rct_test(). The task CPU time mutex
1627 1618 * is highly contentious as many processes can be sharing a task.
1628 1619 */
1629 1620 if (pp->p_ttime >= clock_tick_proc_max) {
1630 1621 secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
1631 1622 pp->p_ttime = 0;
1632 1623 if (secs) {
1633 1624 (void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
1634 1625 pp, secs, RCA_UNSAFE_SIGINFO);
1635 1626 }
1636 1627 }
1637 1628
1638 1629 /*
1639 1630 * Update memory usage for the currently running process.
1640 1631 */
1641 1632 rss = rm_asrss(as);
1642 1633 PTOU(pp)->u_mem += rss;
1643 1634 if (rss > PTOU(pp)->u_mem_max)
1644 1635 PTOU(pp)->u_mem_max = rss;
1645 1636
1646 1637 /*
1647 1638 * Notify the CPU the thread is running on.
1648 1639 */
1649 1640 if (poke && t->t_cpu != CPU)
1650 1641 poke_cpu(t->t_cpu->cpu_id);
1651 1642 }
1652 1643
1653 1644 void
1654 1645 profil_tick(uintptr_t upc)
1655 1646 {
1656 1647 int ticks;
1657 1648 proc_t *p = ttoproc(curthread);
1658 1649 klwp_t *lwp = ttolwp(curthread);
1659 1650 struct prof *pr = &p->p_prof;
1660 1651
1661 1652 do {
1662 1653 ticks = lwp->lwp_oweupc;
1663 1654 } while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1664 1655
1665 1656 mutex_enter(&p->p_pflock);
1666 1657 if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1667 1658 /*
1668 1659 * Old-style profiling
1669 1660 */
1670 1661 uint16_t *slot = pr->pr_base;
1671 1662 uint16_t old, new;
1672 1663 if (pr->pr_scale != 2) {
1673 1664 uintptr_t delta = upc - pr->pr_off;
1674 1665 uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1675 1666 (((delta & 0xffff) * pr->pr_scale) >> 16);
1676 1667 if (byteoff >= (uintptr_t)pr->pr_size) {
1677 1668 mutex_exit(&p->p_pflock);
1678 1669 return;
1679 1670 }
1680 1671 slot += byteoff / sizeof (uint16_t);
1681 1672 }
1682 1673 if (fuword16(slot, &old) < 0 ||
1683 1674 (new = old + ticks) > SHRT_MAX ||
1684 1675 suword16(slot, new) < 0) {
1685 1676 pr->pr_scale = 0;
1686 1677 }
1687 1678 } else if (pr->pr_scale == 1) {
1688 1679 /*
1689 1680 * PC Sampling
1690 1681 */
1691 1682 model_t model = lwp_getdatamodel(lwp);
1692 1683 int result;
1693 1684 #ifdef __lint
1694 1685 model = model;
1695 1686 #endif
1696 1687 while (ticks-- > 0) {
1697 1688 if (pr->pr_samples == pr->pr_size) {
1698 1689 /* buffer full, turn off sampling */
1699 1690 pr->pr_scale = 0;
1700 1691 break;
1701 1692 }
1702 1693 switch (SIZEOF_PTR(model)) {
1703 1694 case sizeof (uint32_t):
1704 1695 result = suword32(pr->pr_base, (uint32_t)upc);
1705 1696 break;
1706 1697 #ifdef _LP64
1707 1698 case sizeof (uint64_t):
1708 1699 result = suword64(pr->pr_base, (uint64_t)upc);
1709 1700 break;
1710 1701 #endif
1711 1702 default:
1712 1703 cmn_err(CE_WARN, "profil_tick: unexpected "
1713 1704 "data model");
1714 1705 result = -1;
1715 1706 break;
1716 1707 }
1717 1708 if (result != 0) {
1718 1709 pr->pr_scale = 0;
1719 1710 break;
1720 1711 }
1721 1712 pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1722 1713 pr->pr_samples++;
1723 1714 }
1724 1715 }
1725 1716 mutex_exit(&p->p_pflock);
1726 1717 }
1727 1718
1728 1719 static void
1729 1720 delay_wakeup(void *arg)
1730 1721 {
1731 1722 kthread_t *t = arg;
1732 1723
1733 1724 mutex_enter(&t->t_delay_lock);
1734 1725 cv_signal(&t->t_delay_cv);
1735 1726 mutex_exit(&t->t_delay_lock);
1736 1727 }
1737 1728
1738 1729 /*
1739 1730 * The delay(9F) man page indicates that it can only be called from user or
1740 1731 * kernel context - detect and diagnose bad calls. The following macro will
1741 1732 * produce a limited number of messages identifying bad callers. This is done
1742 1733 * in a macro so that caller() is meaningful. When a bad caller is identified,
1743 1734 * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
1744 1735 */
1745 1736 #define DELAY_CONTEXT_CHECK() { \
1746 1737 uint32_t m; \
1747 1738 char *f; \
1748 1739 ulong_t off; \
1749 1740 \
1750 1741 m = delay_from_interrupt_msg; \
1751 1742 if (delay_from_interrupt_diagnose && servicing_interrupt() && \
1752 1743 !panicstr && !devinfo_freeze && \
1753 1744 atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) { \
1754 1745 f = modgetsymname((uintptr_t)caller(), &off); \
1755 1746 cmn_err(CE_WARN, "delay(9F) called from " \
1756 1747 "interrupt context: %s`%s", \
1757 1748 mod_containing_pc(caller()), f ? f : "..."); \
1758 1749 } \
1759 1750 }
1760 1751
1761 1752 /*
1762 1753 * delay_common: common delay code.
1763 1754 */
1764 1755 static void
1765 1756 delay_common(clock_t ticks)
1766 1757 {
1767 1758 kthread_t *t = curthread;
1768 1759 clock_t deadline;
1769 1760 clock_t timeleft;
1770 1761 callout_id_t id;
1771 1762
1772 1763 /* If timeouts aren't running all we can do is spin. */
1773 1764 if (panicstr || devinfo_freeze) {
1774 1765 /* Convert delay(9F) call into drv_usecwait(9F) call. */
1775 1766 if (ticks > 0)
1776 1767 drv_usecwait(TICK_TO_USEC(ticks));
1777 1768 return;
1778 1769 }
1779 1770
1780 1771 deadline = ddi_get_lbolt() + ticks;
1781 1772 while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
1782 1773 mutex_enter(&t->t_delay_lock);
1783 1774 id = timeout_default(delay_wakeup, t, timeleft);
1784 1775 cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1785 1776 mutex_exit(&t->t_delay_lock);
1786 1777 (void) untimeout_default(id, 0);
1787 1778 }
1788 1779 }
1789 1780
1790 1781 /*
1791 1782 * Delay specified number of clock ticks.
1792 1783 */
1793 1784 void
1794 1785 delay(clock_t ticks)
1795 1786 {
1796 1787 DELAY_CONTEXT_CHECK();
1797 1788
1798 1789 delay_common(ticks);
1799 1790 }
1800 1791
1801 1792 /*
1802 1793 * Delay a random number of clock ticks between 1 and ticks.
1803 1794 */
1804 1795 void
1805 1796 delay_random(clock_t ticks)
1806 1797 {
1807 1798 int r;
1808 1799
1809 1800 DELAY_CONTEXT_CHECK();
1810 1801
1811 1802 (void) random_get_pseudo_bytes((void *)&r, sizeof (r));
1812 1803 if (ticks == 0)
1813 1804 ticks = 1;
1814 1805 ticks = (r % ticks) + 1;
1815 1806 delay_common(ticks);
1816 1807 }
1817 1808
1818 1809 /*
1819 1810 * Like delay, but interruptible by a signal.
1820 1811 */
1821 1812 int
1822 1813 delay_sig(clock_t ticks)
1823 1814 {
1824 1815 kthread_t *t = curthread;
1825 1816 clock_t deadline;
1826 1817 clock_t rc;
1827 1818
1828 1819 /* If timeouts aren't running all we can do is spin. */
1829 1820 if (panicstr || devinfo_freeze) {
1830 1821 if (ticks > 0)
1831 1822 drv_usecwait(TICK_TO_USEC(ticks));
1832 1823 return (0);
1833 1824 }
1834 1825
1835 1826 deadline = ddi_get_lbolt() + ticks;
1836 1827 mutex_enter(&t->t_delay_lock);
1837 1828 do {
1838 1829 rc = cv_timedwait_sig(&t->t_delay_cv,
1839 1830 &t->t_delay_lock, deadline);
1840 1831 /* loop until past deadline or signaled */
1841 1832 } while (rc > 0);
1842 1833 mutex_exit(&t->t_delay_lock);
1843 1834 if (rc == 0)
1844 1835 return (EINTR);
1845 1836 return (0);
1846 1837 }
1847 1838
1848 1839
1849 1840 #define SECONDS_PER_DAY 86400
1850 1841
1851 1842 /*
1852 1843 * Initialize the system time based on the TOD chip. approx is used as
1853 1844 * an approximation of time (e.g. from the filesystem) in the event that
1854 1845 * the TOD chip has been cleared or is unresponsive. An approx of -1
1855 1846 * means the filesystem doesn't keep time.
1856 1847 */
1857 1848 void
1858 1849 clkset(time_t approx)
1859 1850 {
1860 1851 timestruc_t ts;
1861 1852 int spl;
1862 1853 int set_clock = 0;
1863 1854
1864 1855 mutex_enter(&tod_lock);
1865 1856 ts = tod_get();
1866 1857
1867 1858 if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1868 1859 /*
1869 1860 * If the TOD chip is reporting some time after 1971,
1870 1861 * then it probably didn't lose power or become otherwise
1871 1862 * cleared in the recent past; check to assure that
1872 1863 * the time coming from the filesystem isn't in the future
1873 1864 * according to the TOD chip.
1874 1865 */
1875 1866 if (approx != -1 && approx > ts.tv_sec) {
1876 1867 cmn_err(CE_WARN, "Last shutdown is later "
1877 1868 "than time on time-of-day chip; check date.");
1878 1869 }
1879 1870 } else {
1880 1871 /*
1881 1872 * If the TOD chip isn't giving correct time, set it to the
1882 1873 * greater of i) approx and ii) 1987. That way if approx
1883 1874 * is negative or is earlier than 1987, we set the clock
1884 1875 * back to a time when Oliver North, ALF and Dire Straits
1885 1876 * were all on the collective brain: 1987.
1886 1877 */
1887 1878 timestruc_t tmp;
1888 1879 time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1889 1880 ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
1890 1881 ts.tv_nsec = 0;
1891 1882
1892 1883 /*
1893 1884 * Attempt to write the new time to the TOD chip. Set spl high
1894 1885 * to avoid getting preempted between the tod_set and tod_get.
1895 1886 */
1896 1887 spl = splhi();
1897 1888 tod_set(ts);
1898 1889 tmp = tod_get();
1899 1890 splx(spl);
1900 1891
1901 1892 if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1902 1893 tod_broken = 1;
1903 1894 dosynctodr = 0;
1904 1895 cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
1905 1896 } else {
1906 1897 cmn_err(CE_WARN, "Time-of-day chip had "
1907 1898 "incorrect date; check and reset.");
1908 1899 }
1909 1900 set_clock = 1;
1910 1901 }
1911 1902
1912 1903 if (!boot_time) {
1913 1904 boot_time = ts.tv_sec;
1914 1905 set_clock = 1;
1915 1906 }
1916 1907
1917 1908 if (set_clock)
1918 1909 set_hrestime(&ts);
1919 1910
1920 1911 mutex_exit(&tod_lock);
1921 1912 }
1922 1913
1923 1914 int timechanged; /* for testing if the system time has been reset */
1924 1915
1925 1916 void
1926 1917 set_hrestime(timestruc_t *ts)
1927 1918 {
1928 1919 int spl = hr_clock_lock();
1929 1920 hrestime = *ts;
1930 1921 membar_enter(); /* hrestime must be visible before timechanged++ */
1931 1922 timedelta = 0;
1932 1923 timechanged++;
1933 1924 hr_clock_unlock(spl);
1934 1925 callout_hrestime();
1935 1926 }
1936 1927
1937 1928 static uint_t deadman_seconds;
1938 1929 static uint32_t deadman_panics;
1939 1930 static int deadman_enabled = 0;
1940 1931 static int deadman_panic_timers = 1;
1941 1932
1942 1933 static void
1943 1934 deadman(void)
1944 1935 {
1945 1936 if (panicstr) {
1946 1937 /*
1947 1938 * During panic, other CPUs besides the panic
1948 1939 * master continue to handle cyclics and some other
1949 1940 * interrupts. The code below is intended to be
1950 1941 * single threaded, so any CPU other than the master
1951 1942 * must keep out.
1952 1943 */
1953 1944 if (CPU->cpu_id != panic_cpu.cpu_id)
1954 1945 return;
1955 1946
1956 1947 if (!deadman_panic_timers)
1957 1948 return; /* allow all timers to be manually disabled */
1958 1949
1959 1950 /*
1960 1951 * If we are generating a crash dump or syncing filesystems and
1961 1952 * the corresponding timer is set, decrement it and re-enter
1962 1953 * the panic code to abort it and advance to the next state.
1963 1954 * The panic states and triggers are explained in panic.c.
1964 1955 */
1965 1956 if (panic_dump) {
1966 1957 if (dump_timeleft && (--dump_timeleft == 0)) {
1967 1958 panic("panic dump timeout");
1968 1959 /*NOTREACHED*/
1969 1960 }
1970 1961 } else if (panic_sync) {
1971 1962 if (sync_timeleft && (--sync_timeleft == 0)) {
1972 1963 panic("panic sync timeout");
1973 1964 /*NOTREACHED*/
1974 1965 }
1975 1966 }
1976 1967
1977 1968 return;
1978 1969 }
1979 1970
1980 1971 if (deadman_counter != CPU->cpu_deadman_counter) {
1981 1972 CPU->cpu_deadman_counter = deadman_counter;
1982 1973 CPU->cpu_deadman_countdown = deadman_seconds;
1983 1974 return;
1984 1975 }
1985 1976
1986 1977 if (--CPU->cpu_deadman_countdown > 0)
1987 1978 return;
1988 1979
1989 1980 /*
1990 1981 * Regardless of whether or not we actually bring the system down,
1991 1982 * bump the deadman_panics variable.
1992 1983 *
1993 1984 * N.B. deadman_panics is incremented once for each CPU that
1994 1985 * passes through here. It's expected that all the CPUs will
1995 1986 * detect this condition within one second of each other, so
1996 1987 * when deadman_enabled is off, deadman_panics will
1997 1988 * typically be a multiple of the total number of CPUs in
1998 1989 * the system.
1999 1990 */
2000 1991 atomic_add_32(&deadman_panics, 1);
2001 1992
2002 1993 if (!deadman_enabled) {
2003 1994 CPU->cpu_deadman_countdown = deadman_seconds;
2004 1995 return;
2005 1996 }
2006 1997
2007 1998 /*
2008 1999 * If we're here, we want to bring the system down.
2009 2000 */
2010 2001 panic("deadman: timed out after %d seconds of clock "
2011 2002 "inactivity", deadman_seconds);
2012 2003 /*NOTREACHED*/
2013 2004 }
2014 2005
2015 2006 /*ARGSUSED*/
2016 2007 static void
2017 2008 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
2018 2009 {
2019 2010 cpu->cpu_deadman_counter = 0;
2020 2011 cpu->cpu_deadman_countdown = deadman_seconds;
2021 2012
2022 2013 hdlr->cyh_func = (cyc_func_t)deadman;
2023 2014 hdlr->cyh_level = CY_HIGH_LEVEL;
2024 2015 hdlr->cyh_arg = NULL;
2025 2016
2026 2017 /*
2027 2018 * Stagger the CPUs so that they don't all run deadman() at
2028 2019 * the same time. Simplest reason to do this is to make it
2029 2020 * more likely that only one CPU will panic in case of a
2030 2021 * timeout. This is (strictly speaking) an aesthetic, not a
2031 2022 * technical consideration.
2032 2023 */
2033 2024 when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
2034 2025 when->cyt_interval = NANOSEC;
2035 2026 }
2036 2027
2037 2028
2038 2029 void
2039 2030 deadman_init(void)
2040 2031 {
2041 2032 cyc_omni_handler_t hdlr;
2042 2033
2043 2034 if (deadman_seconds == 0)
2044 2035 deadman_seconds = snoop_interval / MICROSEC;
2045 2036
2046 2037 if (snooping)
2047 2038 deadman_enabled = 1;
2048 2039
2049 2040 hdlr.cyo_online = deadman_online;
2050 2041 hdlr.cyo_offline = NULL;
2051 2042 hdlr.cyo_arg = NULL;
2052 2043
2053 2044 mutex_enter(&cpu_lock);
2054 2045 deadman_cyclic = cyclic_add_omni(&hdlr);
2055 2046 mutex_exit(&cpu_lock);
2056 2047 }
2057 2048
2058 2049 /*
2059 2050 * tod_fault() is for updating tod validate mechanism state:
2060 2051 * (1) TOD_NOFAULT: for resetting the state to 'normal'.
2061 2052 * currently used for debugging only
2062 2053 * (2) The following four cases detected by tod validate mechanism:
2063 2054 * TOD_REVERSED: current tod value is less than previous value.
2064 2055 * TOD_STALLED: current tod value hasn't advanced.
2065 2056 * TOD_JUMPED: current tod value advanced too far from previous value.
2066 2057 * TOD_RATECHANGED: the ratio between average tod delta and
2067 2058 * average tick delta has changed.
2068 2059 * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
2069 2060 * a virtual TOD provided by a hypervisor.
2070 2061 */
2071 2062 enum tod_fault_type
2072 2063 tod_fault(enum tod_fault_type ftype, int off)
2073 2064 {
2074 2065 ASSERT(MUTEX_HELD(&tod_lock));
2075 2066
2076 2067 if (tod_faulted != ftype) {
2077 2068 switch (ftype) {
2078 2069 case TOD_NOFAULT:
2079 2070 plat_tod_fault(TOD_NOFAULT);
2080 2071 cmn_err(CE_NOTE, "Restarted tracking "
2081 2072 "Time of Day clock.");
2082 2073 tod_faulted = ftype;
2083 2074 break;
2084 2075 case TOD_REVERSED:
2085 2076 case TOD_JUMPED:
2086 2077 if (tod_faulted == TOD_NOFAULT) {
2087 2078 plat_tod_fault(ftype);
2088 2079 cmn_err(CE_WARN, "Time of Day clock error: "
2089 2080 "reason [%s by 0x%x]. -- "
2090 2081 " Stopped tracking Time Of Day clock.",
2091 2082 tod_fault_table[ftype], off);
2092 2083 tod_faulted = ftype;
2093 2084 }
2094 2085 break;
2095 2086 case TOD_STALLED:
2096 2087 case TOD_RATECHANGED:
2097 2088 if (tod_faulted == TOD_NOFAULT) {
2098 2089 plat_tod_fault(ftype);
2099 2090 cmn_err(CE_WARN, "Time of Day clock error: "
2100 2091 "reason [%s]. -- "
2101 2092 " Stopped tracking Time Of Day clock.",
2102 2093 tod_fault_table[ftype]);
2103 2094 tod_faulted = ftype;
2104 2095 }
2105 2096 break;
2106 2097 case TOD_RDONLY:
2107 2098 if (tod_faulted == TOD_NOFAULT) {
2108 2099 plat_tod_fault(ftype);
2109 2100 cmn_err(CE_NOTE, "!Time of Day clock is "
2110 2101 "Read-Only; set of Date/Time will not "
2111 2102 "persist across reboot.");
2112 2103 tod_faulted = ftype;
2113 2104 }
2114 2105 break;
2115 2106 default:
2116 2107 break;
2117 2108 }
2118 2109 }
2119 2110 return (tod_faulted);
2120 2111 }
2121 2112
2122 2113 /*
2123 2114 * Two functions that allow tod_status_flag to be manipulated by functions
2124 2115 * external to this file.
2125 2116 */
2126 2117
2127 2118 void
2128 2119 tod_status_set(int tod_flag)
2129 2120 {
2130 2121 tod_status_flag |= tod_flag;
2131 2122 }
2132 2123
2133 2124 void
2134 2125 tod_status_clear(int tod_flag)
2135 2126 {
2136 2127 tod_status_flag &= ~tod_flag;
2137 2128 }
2138 2129
2139 2130 /*
2140 2131 * Record a timestamp and the value passed to tod_set(). The next call to
2141 2132 * tod_validate() can use these values, prev_set_tick and prev_set_tod,
2142 2133 * when checking the timestruc_t returned by tod_get(). Ordinarily,
2143 2134 * tod_validate() will use prev_tick and prev_tod for this task but these
2144 2135 * become obsolete, and will be re-assigned with the prev_set_* values,
2145 2136 * in the case when the TOD is re-written.
2146 2137 */
2147 2138 void
2148 2139 tod_set_prev(timestruc_t ts)
2149 2140 {
2150 2141 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2151 2142 tod_validate_deferred) {
2152 2143 return;
2153 2144 }
2154 2145 prev_set_tick = gethrtime();
2155 2146 /*
2156 2147 * A negative value will be set to zero in utc_to_tod() so we fake
2157 2148 * a zero here in such a case. This would need to change if the
2158 2149 * behavior of utc_to_tod() changes.
2159 2150 */
2160 2151 prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
2161 2152 }
2162 2153
2163 2154 /*
2164 2155 * tod_validate() is used for checking values returned by tod_get().
2165 2156 * Four error cases can be detected by this routine:
2166 2157 * TOD_REVERSED: current tod value is less than previous.
2167 2158 * TOD_STALLED: current tod value hasn't advanced.
2168 2159 * TOD_JUMPED: current tod value advanced too far from previous value.
2169 2160 * TOD_RATECHANGED: the ratio between average tod delta and
2170 2161 * average tick delta has changed.
2171 2162 */
2172 2163 time_t
2173 2164 tod_validate(time_t tod)
2174 2165 {
2175 2166 time_t diff_tod;
2176 2167 hrtime_t diff_tick;
2177 2168
2178 2169 long dtick;
2179 2170 int dtick_delta;
2180 2171
2181 2172 int off = 0;
2182 2173 enum tod_fault_type tod_bad = TOD_NOFAULT;
2183 2174
2184 2175 static int firsttime = 1;
2185 2176
2186 2177 static time_t prev_tod = 0;
2187 2178 static hrtime_t prev_tick = 0;
2188 2179 static long dtick_avg = TOD_REF_FREQ;
2189 2180
2190 2181 int cpr_resume_done = 0;
2191 2182 int dr_resume_done = 0;
2192 2183
2193 2184 hrtime_t tick = gethrtime();
2194 2185
2195 2186 ASSERT(MUTEX_HELD(&tod_lock));
2196 2187
2197 2188 /*
2198 2189 * tod_validate_enable is patchable via /etc/system.
2199 2190 * If TOD is already faulted, or if TOD validation is deferred,
2200 2191 * there is nothing to do.
2201 2192 */
2202 2193 if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2203 2194 tod_validate_deferred) {
2204 2195 return (tod);
2205 2196 }
2206 2197
2207 2198 /*
2208 2199 * If this is the first time through, we just need to save the tod
2209 2200 * we were called with and hrtime so we can use them next time to
2210 2201 * validate tod_get().
2211 2202 */
2212 2203 if (firsttime) {
2213 2204 firsttime = 0;
2214 2205 prev_tod = tod;
2215 2206 prev_tick = tick;
2216 2207 return (tod);
2217 2208 }
2218 2209
2219 2210 /*
2220 2211 * Handle any flags that have been turned on by tod_status_set().
2221 2212 * In the case where a tod_set() is done and then a subsequent
2222 2213 * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
2223 2214 * true), we treat the TOD_GET_FAILED with precedence by switching
2224 2215 * off the flag, returning tod and leaving TOD_SET_DONE asserted
2225 2216 * until such time as tod_get() completes successfully.
2226 2217 */
2227 2218 if (tod_status_flag & TOD_GET_FAILED) {
2228 2219 /*
2229 2220 * tod_get() has encountered an issue, possibly transitory,
2230 2221 * when reading TOD. We'll just return the incoming tod
2231 2222 * value (which is actually hrestime.tv_sec in this case)
2232 2223 * and when we get a genuine tod, following a successful
2233 2224 * tod_get(), we can validate using prev_tod and prev_tick.
2234 2225 */
2235 2226 tod_status_flag &= ~TOD_GET_FAILED;
2236 2227 return (tod);
2237 2228 } else if (tod_status_flag & TOD_SET_DONE) {
2238 2229 /*
2239 2230 * TOD has been modified. Just before the TOD was written,
2240 2231 * tod_set_prev() saved tod and hrtime; we can now use
2241 2232 * those values, prev_set_tod and prev_set_tick, to validate
2242 2233 * the incoming tod that's just been read.
2243 2234 */
2244 2235 prev_tod = prev_set_tod;
2245 2236 prev_tick = prev_set_tick;
2246 2237 dtick_avg = TOD_REF_FREQ;
2247 2238 tod_status_flag &= ~TOD_SET_DONE;
2248 2239 /*
2249 2240 * If a tod_set() preceded a cpr_suspend() without an
2250 2241 * intervening tod_validate(), we need to ensure that a
2251 2242 * TOD_JUMPED condition is ignored.
2252 2243 * Note this isn't a concern in the case of DR as we've
2253 2244 * just reassigned dtick_avg, above.
2254 2245 */
2255 2246 if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2256 2247 cpr_resume_done = 1;
2257 2248 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2258 2249 }
2259 2250 } else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
2260 2251 /*
2261 2252 * The system's coming back from a checkpoint resume.
2262 2253 */
2263 2254 cpr_resume_done = 1;
2264 2255 tod_status_flag &= ~TOD_CPR_RESUME_DONE;
2265 2256 /*
2266 2257 * We need to handle the possibility of a CPR suspend
2267 2258 * operation having been initiated whilst a DR event was
2268 2259 * in-flight.
2269 2260 */
2270 2261 if (tod_status_flag & TOD_DR_RESUME_DONE) {
2271 2262 dr_resume_done = 1;
2272 2263 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2273 2264 }
2274 2265 } else if (tod_status_flag & TOD_DR_RESUME_DONE) {
2275 2266 /*
2276 2267 * A Dynamic Reconfiguration event has taken place.
2277 2268 */
2278 2269 dr_resume_done = 1;
2279 2270 tod_status_flag &= ~TOD_DR_RESUME_DONE;
2280 2271 }
2281 2272
2282 2273 /* test hook */
2283 2274 switch (tod_unit_test) {
2284 2275 case 1: /* for testing jumping tod */
2285 2276 tod += tod_test_injector;
2286 2277 tod_unit_test = 0;
2287 2278 break;
2288 2279 case 2: /* for testing stuck tod bit */
2289 2280 tod |= 1 << tod_test_injector;
2290 2281 tod_unit_test = 0;
2291 2282 break;
2292 2283 case 3: /* for testing stalled tod */
2293 2284 tod = prev_tod;
2294 2285 tod_unit_test = 0;
2295 2286 break;
2296 2287 case 4: /* reset tod fault status */
2297 2288 (void) tod_fault(TOD_NOFAULT, 0);
2298 2289 tod_unit_test = 0;
2299 2290 break;
2300 2291 default:
2301 2292 break;
2302 2293 }
2303 2294
2304 2295 diff_tod = tod - prev_tod;
2305 2296 diff_tick = tick - prev_tick;
2306 2297
2307 2298 ASSERT(diff_tick >= 0);
2308 2299
2309 2300 if (diff_tod < 0) {
2310 2301 /* ERROR - tod reversed */
2311 2302 tod_bad = TOD_REVERSED;
2312 2303 off = (int)(prev_tod - tod);
2313 2304 } else if (diff_tod == 0) {
2314 2305 /* tod did not advance */
2315 2306 if (diff_tick > TOD_STALL_THRESHOLD) {
2316 2307 /* ERROR - tod stalled */
2317 2308 tod_bad = TOD_STALLED;
2318 2309 } else {
2319 2310 /*
2320 2311 * Make sure we don't update prev_tick
2321 2312 * so that diff_tick is calculated since
2322 2313 * the first diff_tod == 0
2323 2314 */
2324 2315 return (tod);
2325 2316 }
2326 2317 } else {
2327 2318 /* calculate dtick */
2328 2319 dtick = diff_tick / diff_tod;
2329 2320
2330 2321 /* update dtick averages */
2331 2322 dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2332 2323
2333 2324 /*
2334 2325 * Calculate dtick_delta as
2335 2326 * variation from reference freq in quartiles
2336 2327 */
2337 2328 dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2338 2329 (TOD_REF_FREQ >> 2);
2339 2330
2340 2331 /*
2341 2332 * Even with a perfectly functioning TOD device,
2342 2333 * when the number of elapsed seconds is low the
2343 2334 * algorithm can calculate a rate that is beyond
2344 2335 * tolerance, causing an error. The algorithm is
2345 2336 * inaccurate when elapsed time is low (less than
2346 2337 * 5 seconds).
2347 2338 */
2348 2339 if (diff_tod > 4) {
2349 2340 if (dtick < TOD_JUMP_THRESHOLD) {
2350 2341 /*
2351 2342 * If we've just done a CPR resume, we detect
2352 2343 * a jump in the TOD but, actually, what's
2353 2344 * happened is that the TOD has been increasing
2354 2345 * whilst the system was suspended and the tick
2355 2346 * count hasn't kept up. We consider the first
2356 2347 * occurrence of this after a resume as normal
2357 2348 * and ignore it; otherwise, in a non-resume
2358 2349 * case, we regard it as a TOD problem.
2359 2350 */
2360 2351 if (!cpr_resume_done) {
2361 2352 /* ERROR - tod jumped */
2362 2353 tod_bad = TOD_JUMPED;
2363 2354 off = (int)diff_tod;
2364 2355 }
2365 2356 }
2366 2357 if (dtick_delta) {
2367 2358 /*
2368 2359 * If we've just done a DR resume, dtick_avg
2369 2360 * can go a bit askew so we reset it and carry
2370 2361 * on; otherwise, the TOD is in error.
2371 2362 */
2372 2363 if (dr_resume_done) {
2373 2364 dtick_avg = TOD_REF_FREQ;
2374 2365 } else {
2375 2366 /* ERROR - change in clock rate */
2376 2367 tod_bad = TOD_RATECHANGED;
2377 2368 }
2378 2369 }
2379 2370 }
2380 2371 }
2381 2372
2382 2373 if (tod_bad != TOD_NOFAULT) {
2383 2374 (void) tod_fault(tod_bad, off);
2384 2375
2385 2376 /*
2386 2377 * Disable dosynctodr since we are going to fault
2387 2378 * the TOD chip anyway here
2388 2379 */
2389 2380 dosynctodr = 0;
2390 2381
2391 2382 /*
2392 2383 * Set tod to the correct value from hrestime
2393 2384 */
2394 2385 tod = hrestime.tv_sec;
2395 2386 }
2396 2387
2397 2388 prev_tod = tod;
2398 2389 prev_tick = tick;
2399 2390 return (tod);
2400 2391 }
2401 2392
2402 2393 static void
2403 2394 calcloadavg(int nrun, uint64_t *hp_ave)
2404 2395 {
2405 2396 static int64_t f[3] = { 135, 27, 9 };
2406 2397 uint_t i;
2407 2398 int64_t q, r;
2408 2399
2409 2400 /*
2410 2401 * Compute load average over the last 1, 5, and 15 minutes
2411 2402 * (60, 300, and 900 seconds). The constants in f[3] are for
2412 2403 * exponential decay:
2413 2404 * (1 - exp(-1/60)) << 13 = 135,
2414 2405 * (1 - exp(-1/300)) << 13 = 27,
2415 2406 * (1 - exp(-1/900)) << 13 = 9.
2416 2407 */
2417 2408
2418 2409 /*
2419 2410 * a little hoop-jumping to avoid integer overflow
2420 2411 */
2421 2412 for (i = 0; i < 3; i++) {
2422 2413 q = (hp_ave[i] >> 16) << 7;
2423 2414 r = (hp_ave[i] & 0xffff) << 7;
2424 2415 hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2425 2416 }
2426 2417 }
2427 2418
2428 2419 /*
2429 2420 * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
2430 2421 * calculate the value of lbolt according to the current mode. In the event
2431 2422 * driven mode (the default), lbolt is calculated by dividing the current hires
2432 2423 * time by the number of nanoseconds per clock tick. In the cyclic driven mode
2433 2424 * an internal variable is incremented at each firing of the lbolt cyclic
2434 2425 * and returned by lbolt_cyclic_driven().
2435 2426 *
2436 2427 * The system will transition from event to cyclic driven mode when the number
2437 2428 * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
2438 2429 * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
2439 2430 * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
2440 2431 * causing enough activity to cross the thresholds.
2441 2432 */
2442 2433 int64_t
2443 2434 lbolt_bootstrap(void)
2444 2435 {
2445 2436 return (0);
2446 2437 }
2447 2438
2448 2439 /* ARGSUSED */
2449 2440 uint_t
2450 2441 lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
2451 2442 {
2452 2443 hrtime_t ts, exp;
2453 2444 int ret;
2454 2445
2455 2446 ASSERT(lbolt_hybrid != lbolt_cyclic_driven);
2456 2447
2457 2448 kpreempt_disable();
2458 2449
2459 2450 ts = gethrtime();
2460 2451 lb_info->lbi_internal = (ts/nsec_per_tick);
2461 2452
2462 2453 /*
2463 2454 * Align the next expiration to a clock tick boundary.
2464 2455 */
2465 2456 exp = ts + nsec_per_tick - 1;
2466 2457 exp = (exp/nsec_per_tick) * nsec_per_tick;
2467 2458
2468 2459 ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
2469 2460 ASSERT(ret);
2470 2461
2471 2462 lbolt_hybrid = lbolt_cyclic_driven;
2472 2463 lb_info->lbi_cyc_deactivate = B_FALSE;
2473 2464 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2474 2465
2475 2466 kpreempt_enable();
2476 2467
2477 2468 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2478 2469 ASSERT(ret == 0);
2479 2470
2480 2471 return (1);
2481 2472 }
2482 2473
2483 2474 int64_t
2484 2475 lbolt_event_driven(void)
2485 2476 {
2486 2477 hrtime_t ts;
2487 2478 int64_t lb;
2488 2479 int ret, cpu = CPU->cpu_seqid;
2489 2480
2490 2481 ts = gethrtime();
2491 2482 ASSERT(ts > 0);
2492 2483
2493 2484 ASSERT(nsec_per_tick > 0);
2494 2485 lb = (ts/nsec_per_tick);
2495 2486
2496 2487 /*
2497 2488 * Switch to cyclic mode if the number of calls to this routine
2498 2489 * has reached the threshold within the interval.
2499 2490 */
2500 2491 if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {
2501 2492
2502 2493 if (--lb_cpu[cpu].lbc_counter == 0) {
2503 2494 /*
2504 2495 * Reached the threshold within the interval, reset
2505 2496 * the usage statistics.
2506 2497 */
2507 2498 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2508 2499 lb_cpu[cpu].lbc_cnt_start = lb;
2509 2500
2510 2501 /*
2511 2502 * Make sure only one thread reprograms the
2512 2503 * lbolt cyclic and changes the mode.
2513 2504 */
2514 2505 if (panicstr == NULL &&
2515 2506 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2516 2507
2517 2508 if (lbolt_hybrid == lbolt_cyclic_driven) {
2518 2509 ret = atomic_dec_32_nv(
2519 2510 &lb_info->lbi_token);
2520 2511 ASSERT(ret == 0);
2521 2512 } else {
2522 2513 lbolt_softint_post();
2523 2514 }
2524 2515 }
2525 2516 }
2526 2517 } else {
2527 2518 /*
2528 2519 * Exceeded the interval, reset the usage statistics.
2529 2520 */
2530 2521 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2531 2522 lb_cpu[cpu].lbc_cnt_start = lb;
2532 2523 }
2533 2524
2534 2525 ASSERT(lb >= lb_info->lbi_debug_time);
2535 2526
2536 2527 return (lb - lb_info->lbi_debug_time);
2537 2528 }
2538 2529
2539 2530 int64_t
2540 2531 lbolt_cyclic_driven(void)
2541 2532 {
2542 2533 int64_t lb = lb_info->lbi_internal;
2543 2534 int cpu;
2544 2535
2545 2536 /*
2546 2537 * If a CPU has already prevented the lbolt cyclic from deactivating
2547 2538 * itself, don't bother tracking the usage. Otherwise check if we're
2548 2539 * within the interval and how the per CPU counter is doing.
2549 2540 */
2550 2541 if (lb_info->lbi_cyc_deactivate) {
2551 2542 cpu = CPU->cpu_seqid;
2552 2543 if ((lb - lb_cpu[cpu].lbc_cnt_start) <
2553 2544 lb_info->lbi_thresh_interval) {
2554 2545
2555 2546 if (lb_cpu[cpu].lbc_counter == 0)
2556 2547 /*
2557 2548 * Reached the threshold within the interval,
2558 2549 * prevent the lbolt cyclic from turning itself
2559 2550 * off.
2560 2551 */
2561 2552 lb_info->lbi_cyc_deactivate = B_FALSE;
2562 2553 else
2563 2554 lb_cpu[cpu].lbc_counter--;
2564 2555 } else {
2565 2556 /*
2566 2557 * Only reset the usage statistics when we have
2567 2558 * exceeded the interval.
2568 2559 */
2569 2560 lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
2570 2561 lb_cpu[cpu].lbc_cnt_start = lb;
2571 2562 }
2572 2563 }
2573 2564
2574 2565 ASSERT(lb >= lb_info->lbi_debug_time);
2575 2566
2576 2567 return (lb - lb_info->lbi_debug_time);
2577 2568 }
2578 2569
2579 2570 /*
2580 2571 * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
2581 2572 * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
2582 2573 * It is inactive by default, and will be activated when switching from event
2583 2574 * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
2584 2575 * by lbolt_cyclic_driven().
2585 2576 */
2586 2577 static void
2587 2578 lbolt_cyclic(void)
2588 2579 {
2589 2580 int ret;
2590 2581
2591 2582 lb_info->lbi_internal++;
2592 2583
2593 2584 if (!lbolt_cyc_only) {
2594 2585
2595 2586 if (lb_info->lbi_cyc_deactivate) {
2596 2587 /*
2597 2588 * Switching from cyclic to event driven mode.
2598 2589 */
2599 2590 if (panicstr == NULL &&
2600 2591 atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {
2601 2592
2602 2593 if (lbolt_hybrid == lbolt_event_driven) {
2603 2594 ret = atomic_dec_32_nv(
2604 2595 &lb_info->lbi_token);
2605 2596 ASSERT(ret == 0);
2606 2597 return;
2607 2598 }
2608 2599
2609 2600 kpreempt_disable();
2610 2601
2611 2602 lbolt_hybrid = lbolt_event_driven;
2612 2603 ret = cyclic_reprogram(
2613 2604 lb_info->id.lbi_cyclic_id,
2614 2605 CY_INFINITY);
2615 2606 ASSERT(ret);
2616 2607
2617 2608 kpreempt_enable();
2618 2609
2619 2610 ret = atomic_dec_32_nv(&lb_info->lbi_token);
2620 2611 ASSERT(ret == 0);
2621 2612 }
2622 2613 }
2623 2614
2624 2615 /*
2625 2616 * The lbolt cyclic should not try to deactivate itself before
2626 2617 * the sampling period has elapsed.
2627 2618 */
2628 2619 if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
2629 2620 lb_info->lbi_thresh_interval) {
2630 2621 lb_info->lbi_cyc_deactivate = B_TRUE;
2631 2622 lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
2632 2623 }
2633 2624 }
2634 2625 }
2635 2626
2636 2627 /*
2637 2628 * Since the lbolt service was historically cyclic driven, it must be 'stopped'
2638 2629 * when the system drops into the kernel debugger. lbolt_debug_entry() is
2639 2630 * called by the KDI system claim callbacks to record a hires timestamp at
2640 2631 * debug enter time. lbolt_debug_return() is called by the sistem release
2641 2632 * callbacks to account for the time spent in the debugger. The value is then
2642 2633 * accumulated in the lb_info structure and used by lbolt_event_driven() and
2643 2634 * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
2644 2635 */
2645 2636 void
2646 2637 lbolt_debug_entry(void)
2647 2638 {
2648 2639 if (lbolt_hybrid != lbolt_bootstrap) {
2649 2640 ASSERT(lb_info != NULL);
2650 2641 lb_info->lbi_debug_ts = gethrtime();
2651 2642 }
2652 2643 }
2653 2644
2654 2645 /*
2655 2646 * Calculate the time spent in the debugger and add it to the lbolt info
2656 2647 * structure. We also update the internal lbolt value in case we were in
2657 2648 * cyclic driven mode going in.
2658 2649 */
2659 2650 void
2660 2651 lbolt_debug_return(void)
2661 2652 {
2662 2653 hrtime_t ts;
2663 2654
2664 2655 if (lbolt_hybrid != lbolt_bootstrap) {
2665 2656 ASSERT(lb_info != NULL);
2666 2657 ASSERT(nsec_per_tick > 0);
2667 2658
2668 2659 ts = gethrtime();
2669 2660 lb_info->lbi_internal = (ts/nsec_per_tick);
2670 2661 lb_info->lbi_debug_time +=
2671 2662 ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);
2672 2663
2673 2664 lb_info->lbi_debug_ts = 0;
2674 2665 }
2675 2666 }
|
↓ open down ↓ |
1617 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX