1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 /*
28 * Architecture-independent CPU control functions.
29 */
30
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/var.h>
34 #include <sys/thread.h>
35 #include <sys/cpuvar.h>
36 #include <sys/cpu_event.h>
37 #include <sys/kstat.h>
38 #include <sys/uadmin.h>
39 #include <sys/systm.h>
40 #include <sys/errno.h>
41 #include <sys/cmn_err.h>
42 #include <sys/procset.h>
43 #include <sys/processor.h>
44 #include <sys/debug.h>
45 #include <sys/cpupart.h>
46 #include <sys/lgrp.h>
47 #include <sys/pset.h>
48 #include <sys/pghw.h>
49 #include <sys/kmem.h>
50 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */
51 #include <sys/atomic.h>
52 #include <sys/callb.h>
53 #include <sys/vtrace.h>
54 #include <sys/cyclic.h>
55 #include <sys/bitmap.h>
56 #include <sys/nvpair.h>
57 #include <sys/pool_pset.h>
58 #include <sys/msacct.h>
59 #include <sys/time.h>
60 #include <sys/archsystm.h>
61 #include <sys/sdt.h>
62 #if defined(__x86) || defined(__amd64)
63 #include <sys/x86_archext.h>
64 #endif
65 #include <sys/callo.h>
66
67 extern int mp_cpu_start(cpu_t *);
68 extern int mp_cpu_stop(cpu_t *);
69 extern int mp_cpu_poweron(cpu_t *);
70 extern int mp_cpu_poweroff(cpu_t *);
71 extern int mp_cpu_configure(int);
72 extern int mp_cpu_unconfigure(int);
73 extern void mp_cpu_faulted_enter(cpu_t *);
74 extern void mp_cpu_faulted_exit(cpu_t *);
75
76 extern int cmp_cpu_to_chip(processorid_t cpuid);
77 #ifdef __sparcv9
78 extern char *cpu_fru_fmri(cpu_t *cp);
79 #endif
80
81 static void cpu_add_active_internal(cpu_t *cp);
82 static void cpu_remove_active(cpu_t *cp);
83 static void cpu_info_kstat_create(cpu_t *cp);
84 static void cpu_info_kstat_destroy(cpu_t *cp);
85 static void cpu_stats_kstat_create(cpu_t *cp);
86 static void cpu_stats_kstat_destroy(cpu_t *cp);
87
88 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
89 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
90 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
91 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
92
93 /*
94 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
95 * max_cpu_seqid_ever, and dispatch queue reallocations. The lock ordering with
96 * respect to related locks is:
97 *
98 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock()
99 *
100 * Warning: Certain sections of code do not use the cpu_lock when
101 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since
102 * all cpus are paused during modifications to this list, a solution
103 * to protect the list is too either disable kernel preemption while
104 * walking the list, *or* recheck the cpu_next pointer at each
105 * iteration in the loop. Note that in no cases can any cached
106 * copies of the cpu pointers be kept as they may become invalid.
107 */
108 kmutex_t cpu_lock;
109 cpu_t *cpu_list; /* list of all CPUs */
110 cpu_t *clock_cpu_list; /* used by clock to walk CPUs */
111 cpu_t *cpu_active; /* list of active CPUs */
112 cpuset_t cpu_active_set; /* cached set of active CPUs */
113 static cpuset_t cpu_available; /* set of available CPUs */
114 cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */
115
116 cpu_t **cpu_seq; /* ptrs to CPUs, indexed by seq_id */
117
118 /*
119 * max_ncpus keeps the max cpus the system can have. Initially
120 * it's NCPU, but since most archs scan the devtree for cpus
121 * fairly early on during boot, the real max can be known before
122 * ncpus is set (useful for early NCPU based allocations).
123 */
124 int max_ncpus = NCPU;
125 /*
126 * platforms that set max_ncpus to maxiumum number of cpus that can be
127 * dynamically added will set boot_max_ncpus to the number of cpus found
128 * at device tree scan time during boot.
129 */
130 int boot_max_ncpus = -1;
131 int boot_ncpus = -1;
132 /*
133 * Maximum possible CPU id. This can never be >= NCPU since NCPU is
134 * used to size arrays that are indexed by CPU id.
135 */
136 processorid_t max_cpuid = NCPU - 1;
137
138 /*
139 * Maximum cpu_seqid was given. This number can only grow and never shrink. It
140 * can be used to optimize NCPU loops to avoid going through CPUs which were
141 * never on-line.
142 */
143 processorid_t max_cpu_seqid_ever = 0;
144
145 int ncpus = 1;
146 int ncpus_online = 1;
147
148 /*
149 * CPU that we're trying to offline. Protected by cpu_lock.
150 */
151 cpu_t *cpu_inmotion;
152
153 /*
154 * Can be raised to suppress further weakbinding, which are instead
155 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock,
156 * while individual thread weakbinding synchronization is done under thread
157 * lock.
158 */
159 int weakbindingbarrier;
160
161 /*
162 * Variables used in pause_cpus().
163 */
164 static volatile char safe_list[NCPU];
165
166 static struct _cpu_pause_info {
167 int cp_spl; /* spl saved in pause_cpus() */
168 volatile int cp_go; /* Go signal sent after all ready */
169 int cp_count; /* # of CPUs to pause */
170 ksema_t cp_sem; /* synch pause_cpus & cpu_pause */
171 kthread_id_t cp_paused;
172 void *(*cp_func)(void *);
173 } cpu_pause_info;
174
175 static kmutex_t pause_free_mutex;
176 static kcondvar_t pause_free_cv;
177
178
179 static struct cpu_sys_stats_ks_data {
180 kstat_named_t cpu_ticks_idle;
181 kstat_named_t cpu_ticks_user;
182 kstat_named_t cpu_ticks_kernel;
183 kstat_named_t cpu_ticks_wait;
184 kstat_named_t cpu_nsec_idle;
185 kstat_named_t cpu_nsec_user;
186 kstat_named_t cpu_nsec_kernel;
187 kstat_named_t cpu_nsec_dtrace;
188 kstat_named_t cpu_nsec_intr;
189 kstat_named_t cpu_load_intr;
190 kstat_named_t wait_ticks_io;
191 kstat_named_t dtrace_probes;
192 kstat_named_t bread;
193 kstat_named_t bwrite;
194 kstat_named_t lread;
195 kstat_named_t lwrite;
196 kstat_named_t phread;
197 kstat_named_t phwrite;
198 kstat_named_t pswitch;
199 kstat_named_t trap;
200 kstat_named_t intr;
201 kstat_named_t syscall;
202 kstat_named_t sysread;
203 kstat_named_t syswrite;
204 kstat_named_t sysfork;
205 kstat_named_t sysvfork;
206 kstat_named_t sysexec;
207 kstat_named_t readch;
208 kstat_named_t writech;
209 kstat_named_t rcvint;
210 kstat_named_t xmtint;
211 kstat_named_t mdmint;
212 kstat_named_t rawch;
213 kstat_named_t canch;
214 kstat_named_t outch;
215 kstat_named_t msg;
216 kstat_named_t sema;
217 kstat_named_t namei;
218 kstat_named_t ufsiget;
219 kstat_named_t ufsdirblk;
220 kstat_named_t ufsipage;
221 kstat_named_t ufsinopage;
222 kstat_named_t procovf;
223 kstat_named_t intrthread;
224 kstat_named_t intrblk;
225 kstat_named_t intrunpin;
226 kstat_named_t idlethread;
227 kstat_named_t inv_swtch;
228 kstat_named_t nthreads;
229 kstat_named_t cpumigrate;
230 kstat_named_t xcalls;
231 kstat_named_t mutex_adenters;
232 kstat_named_t rw_rdfails;
233 kstat_named_t rw_wrfails;
234 kstat_named_t modload;
235 kstat_named_t modunload;
236 kstat_named_t bawrite;
237 kstat_named_t iowait;
238 } cpu_sys_stats_ks_data_template = {
239 { "cpu_ticks_idle", KSTAT_DATA_UINT64 },
240 { "cpu_ticks_user", KSTAT_DATA_UINT64 },
241 { "cpu_ticks_kernel", KSTAT_DATA_UINT64 },
242 { "cpu_ticks_wait", KSTAT_DATA_UINT64 },
243 { "cpu_nsec_idle", KSTAT_DATA_UINT64 },
244 { "cpu_nsec_user", KSTAT_DATA_UINT64 },
245 { "cpu_nsec_kernel", KSTAT_DATA_UINT64 },
246 { "cpu_nsec_dtrace", KSTAT_DATA_UINT64 },
247 { "cpu_nsec_intr", KSTAT_DATA_UINT64 },
248 { "cpu_load_intr", KSTAT_DATA_UINT64 },
249 { "wait_ticks_io", KSTAT_DATA_UINT64 },
250 { "dtrace_probes", KSTAT_DATA_UINT64 },
251 { "bread", KSTAT_DATA_UINT64 },
252 { "bwrite", KSTAT_DATA_UINT64 },
253 { "lread", KSTAT_DATA_UINT64 },
254 { "lwrite", KSTAT_DATA_UINT64 },
255 { "phread", KSTAT_DATA_UINT64 },
256 { "phwrite", KSTAT_DATA_UINT64 },
257 { "pswitch", KSTAT_DATA_UINT64 },
258 { "trap", KSTAT_DATA_UINT64 },
259 { "intr", KSTAT_DATA_UINT64 },
260 { "syscall", KSTAT_DATA_UINT64 },
261 { "sysread", KSTAT_DATA_UINT64 },
262 { "syswrite", KSTAT_DATA_UINT64 },
263 { "sysfork", KSTAT_DATA_UINT64 },
264 { "sysvfork", KSTAT_DATA_UINT64 },
265 { "sysexec", KSTAT_DATA_UINT64 },
266 { "readch", KSTAT_DATA_UINT64 },
267 { "writech", KSTAT_DATA_UINT64 },
268 { "rcvint", KSTAT_DATA_UINT64 },
269 { "xmtint", KSTAT_DATA_UINT64 },
270 { "mdmint", KSTAT_DATA_UINT64 },
271 { "rawch", KSTAT_DATA_UINT64 },
272 { "canch", KSTAT_DATA_UINT64 },
273 { "outch", KSTAT_DATA_UINT64 },
274 { "msg", KSTAT_DATA_UINT64 },
275 { "sema", KSTAT_DATA_UINT64 },
276 { "namei", KSTAT_DATA_UINT64 },
277 { "ufsiget", KSTAT_DATA_UINT64 },
278 { "ufsdirblk", KSTAT_DATA_UINT64 },
279 { "ufsipage", KSTAT_DATA_UINT64 },
280 { "ufsinopage", KSTAT_DATA_UINT64 },
281 { "procovf", KSTAT_DATA_UINT64 },
282 { "intrthread", KSTAT_DATA_UINT64 },
283 { "intrblk", KSTAT_DATA_UINT64 },
284 { "intrunpin", KSTAT_DATA_UINT64 },
285 { "idlethread", KSTAT_DATA_UINT64 },
286 { "inv_swtch", KSTAT_DATA_UINT64 },
287 { "nthreads", KSTAT_DATA_UINT64 },
288 { "cpumigrate", KSTAT_DATA_UINT64 },
289 { "xcalls", KSTAT_DATA_UINT64 },
290 { "mutex_adenters", KSTAT_DATA_UINT64 },
291 { "rw_rdfails", KSTAT_DATA_UINT64 },
292 { "rw_wrfails", KSTAT_DATA_UINT64 },
293 { "modload", KSTAT_DATA_UINT64 },
294 { "modunload", KSTAT_DATA_UINT64 },
295 { "bawrite", KSTAT_DATA_UINT64 },
296 { "iowait", KSTAT_DATA_UINT64 },
297 };
298
299 static struct cpu_vm_stats_ks_data {
300 kstat_named_t pgrec;
301 kstat_named_t pgfrec;
302 kstat_named_t pgin;
303 kstat_named_t pgpgin;
304 kstat_named_t pgout;
305 kstat_named_t pgpgout;
306 kstat_named_t swapin;
307 kstat_named_t pgswapin;
308 kstat_named_t swapout;
309 kstat_named_t pgswapout;
310 kstat_named_t zfod;
311 kstat_named_t dfree;
312 kstat_named_t scan;
313 kstat_named_t rev;
314 kstat_named_t hat_fault;
315 kstat_named_t as_fault;
316 kstat_named_t maj_fault;
317 kstat_named_t cow_fault;
318 kstat_named_t prot_fault;
319 kstat_named_t softlock;
320 kstat_named_t kernel_asflt;
321 kstat_named_t pgrrun;
322 kstat_named_t execpgin;
323 kstat_named_t execpgout;
324 kstat_named_t execfree;
325 kstat_named_t anonpgin;
326 kstat_named_t anonpgout;
327 kstat_named_t anonfree;
328 kstat_named_t fspgin;
329 kstat_named_t fspgout;
330 kstat_named_t fsfree;
331 } cpu_vm_stats_ks_data_template = {
332 { "pgrec", KSTAT_DATA_UINT64 },
333 { "pgfrec", KSTAT_DATA_UINT64 },
334 { "pgin", KSTAT_DATA_UINT64 },
335 { "pgpgin", KSTAT_DATA_UINT64 },
336 { "pgout", KSTAT_DATA_UINT64 },
337 { "pgpgout", KSTAT_DATA_UINT64 },
338 { "swapin", KSTAT_DATA_UINT64 },
339 { "pgswapin", KSTAT_DATA_UINT64 },
340 { "swapout", KSTAT_DATA_UINT64 },
341 { "pgswapout", KSTAT_DATA_UINT64 },
342 { "zfod", KSTAT_DATA_UINT64 },
343 { "dfree", KSTAT_DATA_UINT64 },
344 { "scan", KSTAT_DATA_UINT64 },
345 { "rev", KSTAT_DATA_UINT64 },
346 { "hat_fault", KSTAT_DATA_UINT64 },
347 { "as_fault", KSTAT_DATA_UINT64 },
348 { "maj_fault", KSTAT_DATA_UINT64 },
349 { "cow_fault", KSTAT_DATA_UINT64 },
350 { "prot_fault", KSTAT_DATA_UINT64 },
351 { "softlock", KSTAT_DATA_UINT64 },
352 { "kernel_asflt", KSTAT_DATA_UINT64 },
353 { "pgrrun", KSTAT_DATA_UINT64 },
354 { "execpgin", KSTAT_DATA_UINT64 },
355 { "execpgout", KSTAT_DATA_UINT64 },
356 { "execfree", KSTAT_DATA_UINT64 },
357 { "anonpgin", KSTAT_DATA_UINT64 },
358 { "anonpgout", KSTAT_DATA_UINT64 },
359 { "anonfree", KSTAT_DATA_UINT64 },
360 { "fspgin", KSTAT_DATA_UINT64 },
361 { "fspgout", KSTAT_DATA_UINT64 },
362 { "fsfree", KSTAT_DATA_UINT64 },
363 };
364
365 /*
366 * Force the specified thread to migrate to the appropriate processor.
367 * Called with thread lock held, returns with it dropped.
368 */
369 static void
370 force_thread_migrate(kthread_id_t tp)
371 {
372 ASSERT(THREAD_LOCK_HELD(tp));
373 if (tp == curthread) {
374 THREAD_TRANSITION(tp);
375 CL_SETRUN(tp);
376 thread_unlock_nopreempt(tp);
377 swtch();
378 } else {
379 if (tp->t_state == TS_ONPROC) {
380 cpu_surrender(tp);
381 } else if (tp->t_state == TS_RUN) {
382 (void) dispdeq(tp);
383 setbackdq(tp);
384 }
385 thread_unlock(tp);
386 }
387 }
388
389 /*
390 * Set affinity for a specified CPU.
391 *
392 * Specifying a cpu_id of CPU_CURRENT, allowed _only_ when setting affinity for
393 * curthread, will set affinity to the CPU on which the thread is currently
394 * running. For other cpu_id values, the caller must ensure that the
395 * referenced CPU remains valid, which can be done by holding cpu_lock across
396 * this call.
397 *
398 * CPU affinity is guaranteed after return of thread_affinity_set(). If a
399 * caller setting affinity to CPU_CURRENT requires that its thread not migrate
400 * CPUs prior to a successful return, it should take extra precautions (such as
401 * their own call to kpreempt_disable) to ensure that safety.
402 *
403 * CPU_BEST can be used to pick a "best" CPU to migrate to, including
404 * potentially the current CPU.
405 *
406 * A CPU affinity reference count is maintained by thread_affinity_set and
407 * thread_affinity_clear (incrementing and decrementing it, respectively),
408 * maintaining CPU affinity while the count is non-zero, and allowing regions
409 * of code which require affinity to be nested.
410 */
411 void
412 thread_affinity_set(kthread_id_t t, int cpu_id)
413 {
414 cpu_t *cp;
415
416 ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
417
418 if (cpu_id == CPU_CURRENT) {
419 VERIFY3P(t, ==, curthread);
420 kpreempt_disable();
421 cp = CPU;
422 } else if (cpu_id == CPU_BEST) {
423 VERIFY3P(t, ==, curthread);
424 kpreempt_disable();
425 cp = disp_choose_best_cpu();
426 } else {
427 /*
428 * We should be asserting that cpu_lock is held here, but
429 * the NCA code doesn't acquire it. The following assert
430 * should be uncommented when the NCA code is fixed.
431 *
432 * ASSERT(MUTEX_HELD(&cpu_lock));
433 */
434 VERIFY((cpu_id >= 0) && (cpu_id < NCPU));
435 cp = cpu[cpu_id];
436
437 /* user must provide a good cpu_id */
438 VERIFY(cp != NULL);
439 }
440
441 /*
442 * If there is already a hard affinity requested, and this affinity
443 * conflicts with that, panic.
444 */
445 thread_lock(t);
446 if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
447 panic("affinity_set: setting %p but already bound to %p",
448 (void *)cp, (void *)t->t_bound_cpu);
449 }
450 t->t_affinitycnt++;
451 t->t_bound_cpu = cp;
452
453 /*
454 * Make sure we're running on the right CPU.
455 */
456 if (cp != t->t_cpu || t != curthread) {
457 ASSERT(cpu_id != CPU_CURRENT);
458 force_thread_migrate(t); /* drops thread lock */
459 } else {
460 thread_unlock(t);
461 }
462
463 if (cpu_id == CPU_CURRENT || cpu_id == CPU_BEST)
464 kpreempt_enable();
465 }
466
467 /*
468 * Wrapper for backward compatibility.
469 */
470 void
471 affinity_set(int cpu_id)
472 {
473 thread_affinity_set(curthread, cpu_id);
474 }
475
476 /*
477 * Decrement the affinity reservation count and if it becomes zero,
478 * clear the CPU affinity for the current thread, or set it to the user's
479 * software binding request.
480 */
481 void
482 thread_affinity_clear(kthread_id_t t)
483 {
484 register processorid_t binding;
485
486 thread_lock(t);
487 if (--t->t_affinitycnt == 0) {
488 if ((binding = t->t_bind_cpu) == PBIND_NONE) {
489 /*
490 * Adjust disp_max_unbound_pri if necessary.
491 */
492 disp_adjust_unbound_pri(t);
493 t->t_bound_cpu = NULL;
494 if (t->t_cpu->cpu_part != t->t_cpupart) {
495 force_thread_migrate(t);
496 return;
497 }
498 } else {
499 t->t_bound_cpu = cpu[binding];
500 /*
501 * Make sure the thread is running on the bound CPU.
502 */
503 if (t->t_cpu != t->t_bound_cpu) {
504 force_thread_migrate(t);
505 return; /* already dropped lock */
506 }
507 }
508 }
509 thread_unlock(t);
510 }
511
512 /*
513 * Wrapper for backward compatibility.
514 */
515 void
516 affinity_clear(void)
517 {
518 thread_affinity_clear(curthread);
519 }
520
521 /*
522 * Weak cpu affinity. Bind to the "current" cpu for short periods
523 * of time during which the thread must not block (but may be preempted).
524 * Use this instead of kpreempt_disable() when it is only "no migration"
525 * rather than "no preemption" semantics that are required - disabling
526 * preemption holds higher priority threads off of cpu and if the
527 * operation that is protected is more than momentary this is not good
528 * for realtime etc.
529 *
530 * Weakly bound threads will not prevent a cpu from being offlined -
531 * we'll only run them on the cpu to which they are weakly bound but
532 * (because they do not block) we'll always be able to move them on to
533 * another cpu at offline time if we give them just a short moment to
534 * run during which they will unbind. To give a cpu a chance of offlining,
535 * however, we require a barrier to weak bindings that may be raised for a
536 * given cpu (offline/move code may set this and then wait a short time for
537 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
538 *
539 * There are few restrictions on the calling context of thread_nomigrate.
540 * The caller must not hold the thread lock. Calls may be nested.
541 *
542 * After weakbinding a thread must not perform actions that may block.
543 * In particular it must not call thread_affinity_set; calling that when
544 * already weakbound is nonsensical anyway.
545 *
546 * If curthread is prevented from migrating for other reasons
547 * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
548 * then the weak binding will succeed even if this cpu is the target of an
549 * offline/move request.
550 */
551 void
552 thread_nomigrate(void)
553 {
554 cpu_t *cp;
555 kthread_id_t t = curthread;
556
557 again:
558 kpreempt_disable();
559 cp = CPU;
560
561 /*
562 * A highlevel interrupt must not modify t_nomigrate or
563 * t_weakbound_cpu of the thread it has interrupted. A lowlevel
564 * interrupt thread cannot migrate and we can avoid the
565 * thread_lock call below by short-circuiting here. In either
566 * case we can just return since no migration is possible and
567 * the condition will persist (ie, when we test for these again
568 * in thread_allowmigrate they can't have changed). Migration
569 * is also impossible if we're at or above DISP_LEVEL pil.
570 */
571 if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
572 getpil() >= DISP_LEVEL) {
573 kpreempt_enable();
574 return;
575 }
576
577 /*
578 * We must be consistent with existing weak bindings. Since we
579 * may be interrupted between the increment of t_nomigrate and
580 * the store to t_weakbound_cpu below we cannot assume that
581 * t_weakbound_cpu will be set if t_nomigrate is. Note that we
582 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
583 * always the case.
584 */
585 if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
586 if (!panicstr)
587 panic("thread_nomigrate: binding to %p but already "
588 "bound to %p", (void *)cp,
589 (void *)t->t_weakbound_cpu);
590 }
591
592 /*
593 * At this point we have preemption disabled and we don't yet hold
594 * the thread lock. So it's possible that somebody else could
595 * set t_bind_cpu here and not be able to force us across to the
596 * new cpu (since we have preemption disabled).
597 */
598 thread_lock(curthread);
599
600 /*
601 * If further weak bindings are being (temporarily) suppressed then
602 * we'll settle for disabling kernel preemption (which assures
603 * no migration provided the thread does not block which it is
604 * not allowed to if using thread_nomigrate). We must remember
605 * this disposition so we can take appropriate action in
606 * thread_allowmigrate. If this is a nested call and the
607 * thread is already weakbound then fall through as normal.
608 * We remember the decision to settle for kpreempt_disable through
609 * negative nesting counting in t_nomigrate. Once a thread has had one
610 * weakbinding request satisfied in this way any further (nested)
611 * requests will continue to be satisfied in the same way,
612 * even if weak bindings have recommenced.
613 */
614 if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
615 --t->t_nomigrate;
616 thread_unlock(curthread);
617 return; /* with kpreempt_disable still active */
618 }
619
620 /*
621 * We hold thread_lock so t_bind_cpu cannot change. We could,
622 * however, be running on a different cpu to which we are t_bound_cpu
623 * to (as explained above). If we grant the weak binding request
624 * in that case then the dispatcher must favour our weak binding
625 * over our strong (in which case, just as when preemption is
626 * disabled, we can continue to run on a cpu other than the one to
627 * which we are strongbound; the difference in this case is that
628 * this thread can be preempted and so can appear on the dispatch
629 * queues of a cpu other than the one it is strongbound to).
630 *
631 * If the cpu we are running on does not appear to be a current
632 * offline target (we check cpu_inmotion to determine this - since
633 * we don't hold cpu_lock we may not see a recent store to that,
634 * so it's possible that we at times can grant a weak binding to a
635 * cpu that is an offline target, but that one request will not
636 * prevent the offline from succeeding) then we will always grant
637 * the weak binding request. This includes the case above where
638 * we grant a weakbinding not commensurate with our strong binding.
639 *
640 * If our cpu does appear to be an offline target then we're inclined
641 * not to grant the weakbinding request just yet - we'd prefer to
642 * migrate to another cpu and grant the request there. The
643 * exceptions are those cases where going through preemption code
644 * will not result in us changing cpu:
645 *
646 * . interrupts have already bypassed this case (see above)
647 * . we are already weakbound to this cpu (dispatcher code will
648 * always return us to the weakbound cpu)
649 * . preemption was disabled even before we disabled it above
650 * . we are strongbound to this cpu (if we're strongbound to
651 * another and not yet running there the trip through the
652 * dispatcher will move us to the strongbound cpu and we
653 * will grant the weak binding there)
654 */
655 if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
656 t->t_bound_cpu == cp) {
657 /*
658 * Don't be tempted to store to t_weakbound_cpu only on
659 * the first nested bind request - if we're interrupted
660 * after the increment of t_nomigrate and before the
661 * store to t_weakbound_cpu and the interrupt calls
662 * thread_nomigrate then the assertion in thread_allowmigrate
663 * would fail.
664 */
665 t->t_nomigrate++;
666 t->t_weakbound_cpu = cp;
667 membar_producer();
668 thread_unlock(curthread);
669 /*
670 * Now that we have dropped the thread_lock another thread
671 * can set our t_weakbound_cpu, and will try to migrate us
672 * to the strongbound cpu (which will not be prevented by
673 * preemption being disabled since we're about to enable
674 * preemption). We have granted the weakbinding to the current
675 * cpu, so again we are in the position that is is is possible
676 * that our weak and strong bindings differ. Again this
677 * is catered for by dispatcher code which will favour our
678 * weak binding.
679 */
680 kpreempt_enable();
681 } else {
682 /*
683 * Move to another cpu before granting the request by
684 * forcing this thread through preemption code. When we
685 * get to set{front,back}dq called from CL_PREEMPT()
686 * cpu_choose() will be used to select a cpu to queue
687 * us on - that will see cpu_inmotion and take
688 * steps to avoid returning us to this cpu.
689 */
690 cp->cpu_kprunrun = 1;
691 thread_unlock(curthread);
692 kpreempt_enable(); /* will call preempt() */
693 goto again;
694 }
695 }
696
697 void
698 thread_allowmigrate(void)
699 {
700 kthread_id_t t = curthread;
701
702 ASSERT(t->t_weakbound_cpu == CPU ||
703 (t->t_nomigrate < 0 && t->t_preempt > 0) ||
704 CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
705 getpil() >= DISP_LEVEL);
706
707 if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
708 getpil() >= DISP_LEVEL)
709 return;
710
711 if (t->t_nomigrate < 0) {
712 /*
713 * This thread was granted "weak binding" in the
714 * stronger form of kernel preemption disabling.
715 * Undo a level of nesting for both t_nomigrate
716 * and t_preempt.
717 */
718 ++t->t_nomigrate;
719 kpreempt_enable();
720 } else if (--t->t_nomigrate == 0) {
721 /*
722 * Time to drop the weak binding. We need to cater
723 * for the case where we're weakbound to a different
724 * cpu than that to which we're strongbound (a very
725 * temporary arrangement that must only persist until
726 * weak binding drops). We don't acquire thread_lock
727 * here so even as this code executes t_bound_cpu
728 * may be changing. So we disable preemption and
729 * a) in the case that t_bound_cpu changes while we
730 * have preemption disabled kprunrun will be set
731 * asynchronously, and b) if before disabling
732 * preemption we were already on a different cpu to
733 * our t_bound_cpu then we set kprunrun ourselves
734 * to force a trip through the dispatcher when
735 * preemption is enabled.
736 */
737 kpreempt_disable();
738 if (t->t_bound_cpu &&
739 t->t_weakbound_cpu != t->t_bound_cpu)
740 CPU->cpu_kprunrun = 1;
741 t->t_weakbound_cpu = NULL;
742 membar_producer();
743 kpreempt_enable();
744 }
745 }
746
747 /*
748 * weakbinding_stop can be used to temporarily cause weakbindings made
749 * with thread_nomigrate to be satisfied through the stronger action of
750 * kpreempt_disable. weakbinding_start recommences normal weakbinding.
751 */
752
753 void
754 weakbinding_stop(void)
755 {
756 ASSERT(MUTEX_HELD(&cpu_lock));
757 weakbindingbarrier = 1;
758 membar_producer(); /* make visible before subsequent thread_lock */
759 }
760
761 void
762 weakbinding_start(void)
763 {
764 ASSERT(MUTEX_HELD(&cpu_lock));
765 weakbindingbarrier = 0;
766 }
767
768 void
769 null_xcall(void)
770 {
771 }
772
773 /*
774 * This routine is called to place the CPUs in a safe place so that
775 * one of them can be taken off line or placed on line. What we are
776 * trying to do here is prevent a thread from traversing the list
777 * of active CPUs while we are changing it or from getting placed on
778 * the run queue of a CPU that has just gone off line. We do this by
779 * creating a thread with the highest possible prio for each CPU and
780 * having it call this routine. The advantage of this method is that
781 * we can eliminate all checks for CPU_ACTIVE in the disp routines.
782 * This makes disp faster at the expense of making p_online() slower
783 * which is a good trade off.
784 */
785 static void
786 cpu_pause(int index)
787 {
788 int s;
789 struct _cpu_pause_info *cpi = &cpu_pause_info;
790 volatile char *safe = &safe_list[index];
791 long lindex = index;
792
793 ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
794
795 while (*safe != PAUSE_DIE) {
796 *safe = PAUSE_READY;
797 membar_enter(); /* make sure stores are flushed */
798 sema_v(&cpi->cp_sem); /* signal requesting thread */
799
800 /*
801 * Wait here until all pause threads are running. That
802 * indicates that it's safe to do the spl. Until
803 * cpu_pause_info.cp_go is set, we don't want to spl
804 * because that might block clock interrupts needed
805 * to preempt threads on other CPUs.
806 */
807 while (cpi->cp_go == 0)
808 ;
809 /*
810 * Even though we are at the highest disp prio, we need
811 * to block out all interrupts below LOCK_LEVEL so that
812 * an intr doesn't come in, wake up a thread, and call
813 * setbackdq/setfrontdq.
814 */
815 s = splhigh();
816 /*
817 * if cp_func has been set then call it using index as the
818 * argument, currently only used by cpr_suspend_cpus().
819 * This function is used as the code to execute on the
820 * "paused" cpu's when a machine comes out of a sleep state
821 * and CPU's were powered off. (could also be used for
822 * hotplugging CPU's).
823 */
824 if (cpi->cp_func != NULL)
825 (*cpi->cp_func)((void *)lindex);
826
827 mach_cpu_pause(safe);
828
829 splx(s);
830 /*
831 * Waiting is at an end. Switch out of cpu_pause
832 * loop and resume useful work.
833 */
834 swtch();
835 }
836
837 mutex_enter(&pause_free_mutex);
838 *safe = PAUSE_DEAD;
839 cv_broadcast(&pause_free_cv);
840 mutex_exit(&pause_free_mutex);
841 }
842
843 /*
844 * Allow the cpus to start running again.
845 */
846 void
847 start_cpus()
848 {
849 int i;
850
851 ASSERT(MUTEX_HELD(&cpu_lock));
852 ASSERT(cpu_pause_info.cp_paused);
853 cpu_pause_info.cp_paused = NULL;
854 for (i = 0; i < NCPU; i++)
855 safe_list[i] = PAUSE_IDLE;
856 membar_enter(); /* make sure stores are flushed */
857 affinity_clear();
858 splx(cpu_pause_info.cp_spl);
859 kpreempt_enable();
860 }
861
862 /*
863 * Allocate a pause thread for a CPU.
864 */
865 static void
866 cpu_pause_alloc(cpu_t *cp)
867 {
868 kthread_id_t t;
869 long cpun = cp->cpu_id;
870
871 /*
872 * Note, v.v_nglobpris will not change value as long as I hold
873 * cpu_lock.
874 */
875 t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
876 0, &p0, TS_STOPPED, v.v_nglobpris - 1);
877 thread_lock(t);
878 t->t_bound_cpu = cp;
879 t->t_disp_queue = cp->cpu_disp;
880 t->t_affinitycnt = 1;
881 t->t_preempt = 1;
882 thread_unlock(t);
883 cp->cpu_pause_thread = t;
884 /*
885 * Registering a thread in the callback table is usually done
886 * in the initialization code of the thread. In this
887 * case, we do it right after thread creation because the
888 * thread itself may never run, and we need to register the
889 * fact that it is safe for cpr suspend.
890 */
891 CALLB_CPR_INIT_SAFE(t, "cpu_pause");
892 }
893
894 /*
895 * Free a pause thread for a CPU.
896 */
897 static void
898 cpu_pause_free(cpu_t *cp)
899 {
900 kthread_id_t t;
901 int cpun = cp->cpu_id;
902
903 ASSERT(MUTEX_HELD(&cpu_lock));
904 /*
905 * We have to get the thread and tell it to die.
906 */
907 if ((t = cp->cpu_pause_thread) == NULL) {
908 ASSERT(safe_list[cpun] == PAUSE_IDLE);
909 return;
910 }
911 thread_lock(t);
912 t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */
913 t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */
914 t->t_pri = v.v_nglobpris - 1;
915 ASSERT(safe_list[cpun] == PAUSE_IDLE);
916 safe_list[cpun] = PAUSE_DIE;
917 THREAD_TRANSITION(t);
918 setbackdq(t);
919 thread_unlock_nopreempt(t);
920
921 /*
922 * If we don't wait for the thread to actually die, it may try to
923 * run on the wrong cpu as part of an actual call to pause_cpus().
924 */
925 mutex_enter(&pause_free_mutex);
926 while (safe_list[cpun] != PAUSE_DEAD) {
927 cv_wait(&pause_free_cv, &pause_free_mutex);
928 }
929 mutex_exit(&pause_free_mutex);
930 safe_list[cpun] = PAUSE_IDLE;
931
932 cp->cpu_pause_thread = NULL;
933 }
934
935 /*
936 * Initialize basic structures for pausing CPUs.
937 */
938 void
939 cpu_pause_init()
940 {
941 sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
942 /*
943 * Create initial CPU pause thread.
944 */
945 cpu_pause_alloc(CPU);
946 }
947
948 /*
949 * Start the threads used to pause another CPU.
950 */
951 static int
952 cpu_pause_start(processorid_t cpu_id)
953 {
954 int i;
955 int cpu_count = 0;
956
957 for (i = 0; i < NCPU; i++) {
958 cpu_t *cp;
959 kthread_id_t t;
960
961 cp = cpu[i];
962 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
963 safe_list[i] = PAUSE_WAIT;
964 continue;
965 }
966
967 /*
968 * Skip CPU if it is quiesced or not yet started.
969 */
970 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
971 safe_list[i] = PAUSE_WAIT;
972 continue;
973 }
974
975 /*
976 * Start this CPU's pause thread.
977 */
978 t = cp->cpu_pause_thread;
979 thread_lock(t);
980 /*
981 * Reset the priority, since nglobpris may have
982 * changed since the thread was created, if someone
983 * has loaded the RT (or some other) scheduling
984 * class.
985 */
986 t->t_pri = v.v_nglobpris - 1;
987 THREAD_TRANSITION(t);
988 setbackdq(t);
989 thread_unlock_nopreempt(t);
990 ++cpu_count;
991 }
992 return (cpu_count);
993 }
994
995
996 /*
997 * Pause all of the CPUs except the one we are on by creating a high
998 * priority thread bound to those CPUs.
999 *
1000 * Note that one must be extremely careful regarding code
1001 * executed while CPUs are paused. Since a CPU may be paused
1002 * while a thread scheduling on that CPU is holding an adaptive
1003 * lock, code executed with CPUs paused must not acquire adaptive
1004 * (or low-level spin) locks. Also, such code must not block,
1005 * since the thread that is supposed to initiate the wakeup may
1006 * never run.
1007 *
1008 * With a few exceptions, the restrictions on code executed with CPUs
1009 * paused match those for code executed at high-level interrupt
1010 * context.
1011 */
1012 void
1013 pause_cpus(cpu_t *off_cp, void *(*func)(void *))
1014 {
1015 processorid_t cpu_id;
1016 int i;
1017 struct _cpu_pause_info *cpi = &cpu_pause_info;
1018
1019 ASSERT(MUTEX_HELD(&cpu_lock));
1020 ASSERT(cpi->cp_paused == NULL);
1021 cpi->cp_count = 0;
1022 cpi->cp_go = 0;
1023 for (i = 0; i < NCPU; i++)
1024 safe_list[i] = PAUSE_IDLE;
1025 kpreempt_disable();
1026
1027 cpi->cp_func = func;
1028
1029 /*
1030 * If running on the cpu that is going offline, get off it.
1031 * This is so that it won't be necessary to rechoose a CPU
1032 * when done.
1033 */
1034 if (CPU == off_cp)
1035 cpu_id = off_cp->cpu_next_part->cpu_id;
1036 else
1037 cpu_id = CPU->cpu_id;
1038 affinity_set(cpu_id);
1039
1040 /*
1041 * Start the pause threads and record how many were started
1042 */
1043 cpi->cp_count = cpu_pause_start(cpu_id);
1044
1045 /*
1046 * Now wait for all CPUs to be running the pause thread.
1047 */
1048 while (cpi->cp_count > 0) {
1049 /*
1050 * Spin reading the count without grabbing the disp
1051 * lock to make sure we don't prevent the pause
1052 * threads from getting the lock.
1053 */
1054 while (sema_held(&cpi->cp_sem))
1055 ;
1056 if (sema_tryp(&cpi->cp_sem))
1057 --cpi->cp_count;
1058 }
1059 cpi->cp_go = 1; /* all have reached cpu_pause */
1060
1061 /*
1062 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1063 * to PAUSE_WAIT.)
1064 */
1065 for (i = 0; i < NCPU; i++) {
1066 while (safe_list[i] != PAUSE_WAIT)
1067 ;
1068 }
1069 cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */
1070 cpi->cp_paused = curthread;
1071 }
1072
1073 /*
1074 * Check whether the current thread has CPUs paused
1075 */
1076 int
1077 cpus_paused(void)
1078 {
1079 if (cpu_pause_info.cp_paused != NULL) {
1080 ASSERT(cpu_pause_info.cp_paused == curthread);
1081 return (1);
1082 }
1083 return (0);
1084 }
1085
1086 static cpu_t *
1087 cpu_get_all(processorid_t cpun)
1088 {
1089 ASSERT(MUTEX_HELD(&cpu_lock));
1090
1091 if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1092 return (NULL);
1093 return (cpu[cpun]);
1094 }
1095
1096 /*
1097 * Check whether cpun is a valid processor id and whether it should be
1098 * visible from the current zone. If it is, return a pointer to the
1099 * associated CPU structure.
1100 */
1101 cpu_t *
1102 cpu_get(processorid_t cpun)
1103 {
1104 cpu_t *c;
1105
1106 ASSERT(MUTEX_HELD(&cpu_lock));
1107 c = cpu_get_all(cpun);
1108 if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1109 zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1110 return (NULL);
1111 return (c);
1112 }
1113
1114 /*
1115 * The following functions should be used to check CPU states in the kernel.
1116 * They should be invoked with cpu_lock held. Kernel subsystems interested
1117 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1118 * states. Those are for user-land (and system call) use only.
1119 */
1120
1121 /*
1122 * Determine whether the CPU is online and handling interrupts.
1123 */
1124 int
1125 cpu_is_online(cpu_t *cpu)
1126 {
1127 ASSERT(MUTEX_HELD(&cpu_lock));
1128 return (cpu_flagged_online(cpu->cpu_flags));
1129 }
1130
1131 /*
1132 * Determine whether the CPU is offline (this includes spare and faulted).
1133 */
1134 int
1135 cpu_is_offline(cpu_t *cpu)
1136 {
1137 ASSERT(MUTEX_HELD(&cpu_lock));
1138 return (cpu_flagged_offline(cpu->cpu_flags));
1139 }
1140
1141 /*
1142 * Determine whether the CPU is powered off.
1143 */
1144 int
1145 cpu_is_poweredoff(cpu_t *cpu)
1146 {
1147 ASSERT(MUTEX_HELD(&cpu_lock));
1148 return (cpu_flagged_poweredoff(cpu->cpu_flags));
1149 }
1150
1151 /*
1152 * Determine whether the CPU is handling interrupts.
1153 */
1154 int
1155 cpu_is_nointr(cpu_t *cpu)
1156 {
1157 ASSERT(MUTEX_HELD(&cpu_lock));
1158 return (cpu_flagged_nointr(cpu->cpu_flags));
1159 }
1160
1161 /*
1162 * Determine whether the CPU is active (scheduling threads).
1163 */
1164 int
1165 cpu_is_active(cpu_t *cpu)
1166 {
1167 ASSERT(MUTEX_HELD(&cpu_lock));
1168 return (cpu_flagged_active(cpu->cpu_flags));
1169 }
1170
1171 /*
1172 * Same as above, but these require cpu_flags instead of cpu_t pointers.
1173 */
1174 int
1175 cpu_flagged_online(cpu_flag_t cpu_flags)
1176 {
1177 return (cpu_flagged_active(cpu_flags) &&
1178 (cpu_flags & CPU_ENABLE));
1179 }
1180
1181 int
1182 cpu_flagged_offline(cpu_flag_t cpu_flags)
1183 {
1184 return (((cpu_flags & CPU_POWEROFF) == 0) &&
1185 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1186 }
1187
1188 int
1189 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1190 {
1191 return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1192 }
1193
1194 int
1195 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1196 {
1197 return (cpu_flagged_active(cpu_flags) &&
1198 (cpu_flags & CPU_ENABLE) == 0);
1199 }
1200
1201 int
1202 cpu_flagged_active(cpu_flag_t cpu_flags)
1203 {
1204 return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1205 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1206 }
1207
1208 /*
1209 * Bring the indicated CPU online.
1210 */
1211 int
1212 cpu_online(cpu_t *cp)
1213 {
1214 int error = 0;
1215
1216 /*
1217 * Handle on-line request.
1218 * This code must put the new CPU on the active list before
1219 * starting it because it will not be paused, and will start
1220 * using the active list immediately. The real start occurs
1221 * when the CPU_QUIESCED flag is turned off.
1222 */
1223
1224 ASSERT(MUTEX_HELD(&cpu_lock));
1225
1226 /*
1227 * Put all the cpus into a known safe place.
1228 * No mutexes can be entered while CPUs are paused.
1229 */
1230 error = mp_cpu_start(cp); /* arch-dep hook */
1231 if (error == 0) {
1232 pg_cpupart_in(cp, cp->cpu_part);
1233 pause_cpus(NULL, NULL);
1234 cpu_add_active_internal(cp);
1235 if (cp->cpu_flags & CPU_FAULTED) {
1236 cp->cpu_flags &= ~CPU_FAULTED;
1237 mp_cpu_faulted_exit(cp);
1238 }
1239 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1240 CPU_SPARE);
1241 CPU_NEW_GENERATION(cp);
1242 start_cpus();
1243 cpu_stats_kstat_create(cp);
1244 cpu_create_intrstat(cp);
1245 lgrp_kstat_create(cp);
1246 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1247 cpu_intr_enable(cp); /* arch-dep hook */
1248 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1249 cpu_set_state(cp);
1250 cyclic_online(cp);
1251 /*
1252 * This has to be called only after cyclic_online(). This
1253 * function uses cyclics.
1254 */
1255 callout_cpu_online(cp);
1256 poke_cpu(cp->cpu_id);
1257 }
1258
1259 return (error);
1260 }
1261
1262 /*
1263 * Take the indicated CPU offline.
1264 */
1265 int
1266 cpu_offline(cpu_t *cp, int flags)
1267 {
1268 cpupart_t *pp;
1269 int error = 0;
1270 cpu_t *ncp;
1271 int intr_enable;
1272 int cyclic_off = 0;
1273 int callout_off = 0;
1274 int loop_count;
1275 int no_quiesce = 0;
1276 int (*bound_func)(struct cpu *, int);
1277 kthread_t *t;
1278 lpl_t *cpu_lpl;
1279 proc_t *p;
1280 int lgrp_diff_lpl;
1281 boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1282
1283 ASSERT(MUTEX_HELD(&cpu_lock));
1284
1285 /*
1286 * If we're going from faulted or spare to offline, just
1287 * clear these flags and update CPU state.
1288 */
1289 if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1290 if (cp->cpu_flags & CPU_FAULTED) {
1291 cp->cpu_flags &= ~CPU_FAULTED;
1292 mp_cpu_faulted_exit(cp);
1293 }
1294 cp->cpu_flags &= ~CPU_SPARE;
1295 cpu_set_state(cp);
1296 return (0);
1297 }
1298
1299 /*
1300 * Handle off-line request.
1301 */
1302 pp = cp->cpu_part;
1303 /*
1304 * Don't offline last online CPU in partition
1305 */
1306 if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1307 return (EBUSY);
1308 /*
1309 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1310 * if we were asked to.
1311 */
1312 error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1313 if (error != 0)
1314 return (error);
1315 /*
1316 * We shouldn't be bound to this CPU ourselves.
1317 */
1318 if (curthread->t_bound_cpu == cp)
1319 return (EBUSY);
1320
1321 /*
1322 * Tell interested parties that this CPU is going offline.
1323 */
1324 CPU_NEW_GENERATION(cp);
1325 cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1326
1327 /*
1328 * Tell the PG subsystem that the CPU is leaving the partition
1329 */
1330 pg_cpupart_out(cp, pp);
1331
1332 /*
1333 * Take the CPU out of interrupt participation so we won't find
1334 * bound kernel threads. If the architecture cannot completely
1335 * shut off interrupts on the CPU, don't quiesce it, but don't
1336 * run anything but interrupt thread... this is indicated by
1337 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1338 * off.
1339 */
1340 intr_enable = cp->cpu_flags & CPU_ENABLE;
1341 if (intr_enable)
1342 no_quiesce = cpu_intr_disable(cp);
1343
1344 /*
1345 * Record that we are aiming to offline this cpu. This acts as
1346 * a barrier to further weak binding requests in thread_nomigrate
1347 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1348 * lean away from this cpu. Further strong bindings are already
1349 * avoided since we hold cpu_lock. Since threads that are set
1350 * runnable around now and others coming off the target cpu are
1351 * directed away from the target, existing strong and weak bindings
1352 * (especially the latter) to the target cpu stand maximum chance of
1353 * being able to unbind during the short delay loop below (if other
1354 * unbound threads compete they may not see cpu in time to unbind
1355 * even if they would do so immediately.
1356 */
1357 cpu_inmotion = cp;
1358 membar_enter();
1359
1360 /*
1361 * Check for kernel threads (strong or weak) bound to that CPU.
1362 * Strongly bound threads may not unbind, and we'll have to return
1363 * EBUSY. Weakly bound threads should always disappear - we've
1364 * stopped more weak binding with cpu_inmotion and existing
1365 * bindings will drain imminently (they may not block). Nonetheless
1366 * we will wait for a fixed period for all bound threads to disappear.
1367 * Inactive interrupt threads are OK (they'll be in TS_FREE
1368 * state). If test finds some bound threads, wait a few ticks
1369 * to give short-lived threads (such as interrupts) chance to
1370 * complete. Note that if no_quiesce is set, i.e. this cpu
1371 * is required to service interrupts, then we take the route
1372 * that permits interrupt threads to be active (or bypassed).
1373 */
1374 bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1375
1376 again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1377 if (loop_count >= 5) {
1378 error = EBUSY; /* some threads still bound */
1379 break;
1380 }
1381
1382 /*
1383 * If some threads were assigned, give them
1384 * a chance to complete or move.
1385 *
1386 * This assumes that the clock_thread is not bound
1387 * to any CPU, because the clock_thread is needed to
1388 * do the delay(hz/100).
1389 *
1390 * Note: we still hold the cpu_lock while waiting for
1391 * the next clock tick. This is OK since it isn't
1392 * needed for anything else except processor_bind(2),
1393 * and system initialization. If we drop the lock,
1394 * we would risk another p_online disabling the last
1395 * processor.
1396 */
1397 delay(hz/100);
1398 }
1399
1400 if (error == 0 && callout_off == 0) {
1401 callout_cpu_offline(cp);
1402 callout_off = 1;
1403 }
1404
1405 if (error == 0 && cyclic_off == 0) {
1406 if (!cyclic_offline(cp)) {
1407 /*
1408 * We must have bound cyclics...
1409 */
1410 error = EBUSY;
1411 goto out;
1412 }
1413 cyclic_off = 1;
1414 }
1415
1416 /*
1417 * Call mp_cpu_stop() to perform any special operations
1418 * needed for this machine architecture to offline a CPU.
1419 */
1420 if (error == 0)
1421 error = mp_cpu_stop(cp); /* arch-dep hook */
1422
1423 /*
1424 * If that all worked, take the CPU offline and decrement
1425 * ncpus_online.
1426 */
1427 if (error == 0) {
1428 /*
1429 * Put all the cpus into a known safe place.
1430 * No mutexes can be entered while CPUs are paused.
1431 */
1432 pause_cpus(cp, NULL);
1433 /*
1434 * Repeat the operation, if necessary, to make sure that
1435 * all outstanding low-level interrupts run to completion
1436 * before we set the CPU_QUIESCED flag. It's also possible
1437 * that a thread has weak bound to the cpu despite our raising
1438 * cpu_inmotion above since it may have loaded that
1439 * value before the barrier became visible (this would have
1440 * to be the thread that was on the target cpu at the time
1441 * we raised the barrier).
1442 */
1443 if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1444 (*bound_func)(cp, 1)) {
1445 start_cpus();
1446 (void) mp_cpu_start(cp);
1447 goto again;
1448 }
1449 ncp = cp->cpu_next_part;
1450 cpu_lpl = cp->cpu_lpl;
1451 ASSERT(cpu_lpl != NULL);
1452
1453 /*
1454 * Remove the CPU from the list of active CPUs.
1455 */
1456 cpu_remove_active(cp);
1457
1458 /*
1459 * Walk the active process list and look for threads
1460 * whose home lgroup needs to be updated, or
1461 * the last CPU they run on is the one being offlined now.
1462 */
1463
1464 ASSERT(curthread->t_cpu != cp);
1465 for (p = practive; p != NULL; p = p->p_next) {
1466
1467 t = p->p_tlist;
1468
1469 if (t == NULL)
1470 continue;
1471
1472 lgrp_diff_lpl = 0;
1473
1474 do {
1475 ASSERT(t->t_lpl != NULL);
1476 /*
1477 * Taking last CPU in lpl offline
1478 * Rehome thread if it is in this lpl
1479 * Otherwise, update the count of how many
1480 * threads are in this CPU's lgroup but have
1481 * a different lpl.
1482 */
1483
1484 if (cpu_lpl->lpl_ncpu == 0) {
1485 if (t->t_lpl == cpu_lpl)
1486 lgrp_move_thread(t,
1487 lgrp_choose(t,
1488 t->t_cpupart), 0);
1489 else if (t->t_lpl->lpl_lgrpid ==
1490 cpu_lpl->lpl_lgrpid)
1491 lgrp_diff_lpl++;
1492 }
1493 ASSERT(t->t_lpl->lpl_ncpu > 0);
1494
1495 /*
1496 * Update CPU last ran on if it was this CPU
1497 */
1498 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1499 t->t_cpu = disp_lowpri_cpu(ncp, t,
1500 t->t_pri);
1501 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1502 t->t_weakbound_cpu == cp);
1503
1504 t = t->t_forw;
1505 } while (t != p->p_tlist);
1506
1507 /*
1508 * Didn't find any threads in the same lgroup as this
1509 * CPU with a different lpl, so remove the lgroup from
1510 * the process lgroup bitmask.
1511 */
1512
1513 if (lgrp_diff_lpl == 0)
1514 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1515 }
1516
1517 /*
1518 * Walk thread list looking for threads that need to be
1519 * rehomed, since there are some threads that are not in
1520 * their process's p_tlist.
1521 */
1522
1523 t = curthread;
1524 do {
1525 ASSERT(t != NULL && t->t_lpl != NULL);
1526
1527 /*
1528 * Rehome threads with same lpl as this CPU when this
1529 * is the last CPU in the lpl.
1530 */
1531
1532 if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1533 lgrp_move_thread(t,
1534 lgrp_choose(t, t->t_cpupart), 1);
1535
1536 ASSERT(t->t_lpl->lpl_ncpu > 0);
1537
1538 /*
1539 * Update CPU last ran on if it was this CPU
1540 */
1541
1542 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1543 t->t_cpu = disp_lowpri_cpu(ncp, t, t->t_pri);
1544
1545 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1546 t->t_weakbound_cpu == cp);
1547 t = t->t_next;
1548
1549 } while (t != curthread);
1550 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1551 cp->cpu_flags |= CPU_OFFLINE;
1552 disp_cpu_inactive(cp);
1553 if (!no_quiesce)
1554 cp->cpu_flags |= CPU_QUIESCED;
1555 ncpus_online--;
1556 cpu_set_state(cp);
1557 cpu_inmotion = NULL;
1558 start_cpus();
1559 cpu_stats_kstat_destroy(cp);
1560 cpu_delete_intrstat(cp);
1561 lgrp_kstat_destroy(cp);
1562 }
1563
1564 out:
1565 cpu_inmotion = NULL;
1566
1567 /*
1568 * If we failed, re-enable interrupts.
1569 * Do this even if cpu_intr_disable returned an error, because
1570 * it may have partially disabled interrupts.
1571 */
1572 if (error && intr_enable)
1573 cpu_intr_enable(cp);
1574
1575 /*
1576 * If we failed, but managed to offline the cyclic subsystem on this
1577 * CPU, bring it back online.
1578 */
1579 if (error && cyclic_off)
1580 cyclic_online(cp);
1581
1582 /*
1583 * If we failed, but managed to offline callouts on this CPU,
1584 * bring it back online.
1585 */
1586 if (error && callout_off)
1587 callout_cpu_online(cp);
1588
1589 /*
1590 * If we failed, tell the PG subsystem that the CPU is back
1591 */
1592 pg_cpupart_in(cp, pp);
1593
1594 /*
1595 * If we failed, we need to notify everyone that this CPU is back on.
1596 */
1597 if (error != 0) {
1598 CPU_NEW_GENERATION(cp);
1599 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1600 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1601 }
1602
1603 return (error);
1604 }
1605
1606 /*
1607 * Mark the indicated CPU as faulted, taking it offline.
1608 */
1609 int
1610 cpu_faulted(cpu_t *cp, int flags)
1611 {
1612 int error = 0;
1613
1614 ASSERT(MUTEX_HELD(&cpu_lock));
1615 ASSERT(!cpu_is_poweredoff(cp));
1616
1617 if (cpu_is_offline(cp)) {
1618 cp->cpu_flags &= ~CPU_SPARE;
1619 cp->cpu_flags |= CPU_FAULTED;
1620 mp_cpu_faulted_enter(cp);
1621 cpu_set_state(cp);
1622 return (0);
1623 }
1624
1625 if ((error = cpu_offline(cp, flags)) == 0) {
1626 cp->cpu_flags |= CPU_FAULTED;
1627 mp_cpu_faulted_enter(cp);
1628 cpu_set_state(cp);
1629 }
1630
1631 return (error);
1632 }
1633
1634 /*
1635 * Mark the indicated CPU as a spare, taking it offline.
1636 */
1637 int
1638 cpu_spare(cpu_t *cp, int flags)
1639 {
1640 int error = 0;
1641
1642 ASSERT(MUTEX_HELD(&cpu_lock));
1643 ASSERT(!cpu_is_poweredoff(cp));
1644
1645 if (cpu_is_offline(cp)) {
1646 if (cp->cpu_flags & CPU_FAULTED) {
1647 cp->cpu_flags &= ~CPU_FAULTED;
1648 mp_cpu_faulted_exit(cp);
1649 }
1650 cp->cpu_flags |= CPU_SPARE;
1651 cpu_set_state(cp);
1652 return (0);
1653 }
1654
1655 if ((error = cpu_offline(cp, flags)) == 0) {
1656 cp->cpu_flags |= CPU_SPARE;
1657 cpu_set_state(cp);
1658 }
1659
1660 return (error);
1661 }
1662
1663 /*
1664 * Take the indicated CPU from poweroff to offline.
1665 */
1666 int
1667 cpu_poweron(cpu_t *cp)
1668 {
1669 int error = ENOTSUP;
1670
1671 ASSERT(MUTEX_HELD(&cpu_lock));
1672 ASSERT(cpu_is_poweredoff(cp));
1673
1674 error = mp_cpu_poweron(cp); /* arch-dep hook */
1675 if (error == 0)
1676 cpu_set_state(cp);
1677
1678 return (error);
1679 }
1680
1681 /*
1682 * Take the indicated CPU from any inactive state to powered off.
1683 */
1684 int
1685 cpu_poweroff(cpu_t *cp)
1686 {
1687 int error = ENOTSUP;
1688
1689 ASSERT(MUTEX_HELD(&cpu_lock));
1690 ASSERT(cpu_is_offline(cp));
1691
1692 if (!(cp->cpu_flags & CPU_QUIESCED))
1693 return (EBUSY); /* not completely idle */
1694
1695 error = mp_cpu_poweroff(cp); /* arch-dep hook */
1696 if (error == 0)
1697 cpu_set_state(cp);
1698
1699 return (error);
1700 }
1701
1702 /*
1703 * Initialize the Sequential CPU id lookup table
1704 */
1705 void
1706 cpu_seq_tbl_init()
1707 {
1708 cpu_t **tbl;
1709
1710 tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1711 tbl[0] = CPU;
1712
1713 cpu_seq = tbl;
1714 }
1715
1716 /*
1717 * Initialize the CPU lists for the first CPU.
1718 */
1719 void
1720 cpu_list_init(cpu_t *cp)
1721 {
1722 cp->cpu_next = cp;
1723 cp->cpu_prev = cp;
1724 cpu_list = cp;
1725 clock_cpu_list = cp;
1726
1727 cp->cpu_next_onln = cp;
1728 cp->cpu_prev_onln = cp;
1729 cpu_active = cp;
1730
1731 cp->cpu_seqid = 0;
1732 CPUSET_ADD(cpu_seqid_inuse, 0);
1733
1734 /*
1735 * Bootstrap cpu_seq using cpu_list
1736 * The cpu_seq[] table will be dynamically allocated
1737 * when kmem later becomes available (but before going MP)
1738 */
1739 cpu_seq = &cpu_list;
1740
1741 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1742 cp_default.cp_cpulist = cp;
1743 cp_default.cp_ncpus = 1;
1744 cp->cpu_next_part = cp;
1745 cp->cpu_prev_part = cp;
1746 cp->cpu_part = &cp_default;
1747
1748 CPUSET_ADD(cpu_available, cp->cpu_id);
1749 CPUSET_ADD(cpu_active_set, cp->cpu_id);
1750 }
1751
1752 /*
1753 * Insert a CPU into the list of available CPUs.
1754 */
1755 void
1756 cpu_add_unit(cpu_t *cp)
1757 {
1758 int seqid;
1759
1760 ASSERT(MUTEX_HELD(&cpu_lock));
1761 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1762
1763 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1764
1765 /*
1766 * Note: most users of the cpu_list will grab the
1767 * cpu_lock to insure that it isn't modified. However,
1768 * certain users can't or won't do that. To allow this
1769 * we pause the other cpus. Users who walk the list
1770 * without cpu_lock, must disable kernel preemption
1771 * to insure that the list isn't modified underneath
1772 * them. Also, any cached pointers to cpu structures
1773 * must be revalidated by checking to see if the
1774 * cpu_next pointer points to itself. This check must
1775 * be done with the cpu_lock held or kernel preemption
1776 * disabled. This check relies upon the fact that
1777 * old cpu structures are not free'ed or cleared after
1778 * then are removed from the cpu_list.
1779 *
1780 * Note that the clock code walks the cpu list dereferencing
1781 * the cpu_part pointer, so we need to initialize it before
1782 * adding the cpu to the list.
1783 */
1784 cp->cpu_part = &cp_default;
1785 pause_cpus(NULL, NULL);
1786 cp->cpu_next = cpu_list;
1787 cp->cpu_prev = cpu_list->cpu_prev;
1788 cpu_list->cpu_prev->cpu_next = cp;
1789 cpu_list->cpu_prev = cp;
1790 start_cpus();
1791
1792 for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1793 continue;
1794 CPUSET_ADD(cpu_seqid_inuse, seqid);
1795 cp->cpu_seqid = seqid;
1796
1797 if (seqid > max_cpu_seqid_ever)
1798 max_cpu_seqid_ever = seqid;
1799
1800 ASSERT(ncpus < max_ncpus);
1801 ncpus++;
1802 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1803 cpu[cp->cpu_id] = cp;
1804 CPUSET_ADD(cpu_available, cp->cpu_id);
1805 cpu_seq[cp->cpu_seqid] = cp;
1806
1807 /*
1808 * allocate a pause thread for this CPU.
1809 */
1810 cpu_pause_alloc(cp);
1811
1812 /*
1813 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1814 * link them into a list of just that CPU.
1815 * This is so that disp_lowpri_cpu will work for thread_create in
1816 * pause_cpus() when called from the startup thread in a new CPU.
1817 */
1818 cp->cpu_next_onln = cp;
1819 cp->cpu_prev_onln = cp;
1820 cpu_info_kstat_create(cp);
1821 cp->cpu_next_part = cp;
1822 cp->cpu_prev_part = cp;
1823
1824 init_cpu_mstate(cp, CMS_SYSTEM);
1825
1826 pool_pset_mod = gethrtime();
1827 }
1828
1829 /*
1830 * Do the opposite of cpu_add_unit().
1831 */
1832 void
1833 cpu_del_unit(int cpuid)
1834 {
1835 struct cpu *cp, *cpnext;
1836
1837 ASSERT(MUTEX_HELD(&cpu_lock));
1838 cp = cpu[cpuid];
1839 ASSERT(cp != NULL);
1840
1841 ASSERT(cp->cpu_next_onln == cp);
1842 ASSERT(cp->cpu_prev_onln == cp);
1843 ASSERT(cp->cpu_next_part == cp);
1844 ASSERT(cp->cpu_prev_part == cp);
1845
1846 /*
1847 * Tear down the CPU's physical ID cache, and update any
1848 * processor groups
1849 */
1850 pg_cpu_fini(cp, NULL);
1851 pghw_physid_destroy(cp);
1852
1853 /*
1854 * Destroy kstat stuff.
1855 */
1856 cpu_info_kstat_destroy(cp);
1857 term_cpu_mstate(cp);
1858 /*
1859 * Free up pause thread.
1860 */
1861 cpu_pause_free(cp);
1862 CPUSET_DEL(cpu_available, cp->cpu_id);
1863 cpu[cp->cpu_id] = NULL;
1864 cpu_seq[cp->cpu_seqid] = NULL;
1865
1866 /*
1867 * The clock thread and mutex_vector_enter cannot hold the
1868 * cpu_lock while traversing the cpu list, therefore we pause
1869 * all other threads by pausing the other cpus. These, and any
1870 * other routines holding cpu pointers while possibly sleeping
1871 * must be sure to call kpreempt_disable before processing the
1872 * list and be sure to check that the cpu has not been deleted
1873 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1874 * to keep the deleted cpu structure around.
1875 *
1876 * Note that this MUST be done AFTER cpu_available
1877 * has been updated so that we don't waste time
1878 * trying to pause the cpu we're trying to delete.
1879 */
1880 pause_cpus(NULL, NULL);
1881
1882 cpnext = cp->cpu_next;
1883 cp->cpu_prev->cpu_next = cp->cpu_next;
1884 cp->cpu_next->cpu_prev = cp->cpu_prev;
1885 if (cp == cpu_list)
1886 cpu_list = cpnext;
1887
1888 /*
1889 * Signals that the cpu has been deleted (see above).
1890 */
1891 cp->cpu_next = NULL;
1892 cp->cpu_prev = NULL;
1893
1894 start_cpus();
1895
1896 CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1897 ncpus--;
1898 lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1899
1900 pool_pset_mod = gethrtime();
1901 }
1902
1903 /*
1904 * Add a CPU to the list of active CPUs.
1905 * This routine must not get any locks, because other CPUs are paused.
1906 */
1907 static void
1908 cpu_add_active_internal(cpu_t *cp)
1909 {
1910 cpupart_t *pp = cp->cpu_part;
1911
1912 ASSERT(MUTEX_HELD(&cpu_lock));
1913 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1914
1915 ncpus_online++;
1916 cpu_set_state(cp);
1917 cp->cpu_next_onln = cpu_active;
1918 cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1919 cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1920 cpu_active->cpu_prev_onln = cp;
1921 CPUSET_ADD(cpu_active_set, cp->cpu_id);
1922
1923 if (pp->cp_cpulist) {
1924 cp->cpu_next_part = pp->cp_cpulist;
1925 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1926 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1927 pp->cp_cpulist->cpu_prev_part = cp;
1928 } else {
1929 ASSERT(pp->cp_ncpus == 0);
1930 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1931 }
1932 pp->cp_ncpus++;
1933 if (pp->cp_ncpus == 1) {
1934 cp_numparts_nonempty++;
1935 ASSERT(cp_numparts_nonempty != 0);
1936 }
1937
1938 pg_cpu_active(cp);
1939 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1940
1941 bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1942 }
1943
1944 /*
1945 * Add a CPU to the list of active CPUs.
1946 * This is called from machine-dependent layers when a new CPU is started.
1947 */
1948 void
1949 cpu_add_active(cpu_t *cp)
1950 {
1951 pg_cpupart_in(cp, cp->cpu_part);
1952
1953 pause_cpus(NULL, NULL);
1954 cpu_add_active_internal(cp);
1955 start_cpus();
1956
1957 cpu_stats_kstat_create(cp);
1958 cpu_create_intrstat(cp);
1959 lgrp_kstat_create(cp);
1960 cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1961 }
1962
1963
1964 /*
1965 * Remove a CPU from the list of active CPUs.
1966 * This routine must not get any locks, because other CPUs are paused.
1967 */
1968 /* ARGSUSED */
1969 static void
1970 cpu_remove_active(cpu_t *cp)
1971 {
1972 cpupart_t *pp = cp->cpu_part;
1973
1974 ASSERT(MUTEX_HELD(&cpu_lock));
1975 ASSERT(cp->cpu_next_onln != cp); /* not the last one */
1976 ASSERT(cp->cpu_prev_onln != cp); /* not the last one */
1977
1978 pg_cpu_inactive(cp);
1979
1980 lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1981
1982 if (cp == clock_cpu_list)
1983 clock_cpu_list = cp->cpu_next_onln;
1984
1985 cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1986 cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1987 if (cpu_active == cp) {
1988 cpu_active = cp->cpu_next_onln;
1989 }
1990 cp->cpu_next_onln = cp;
1991 cp->cpu_prev_onln = cp;
1992 CPUSET_DEL(cpu_active_set, cp->cpu_id);
1993
1994 cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1995 cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1996 if (pp->cp_cpulist == cp) {
1997 pp->cp_cpulist = cp->cpu_next_part;
1998 ASSERT(pp->cp_cpulist != cp);
1999 }
2000 cp->cpu_next_part = cp;
2001 cp->cpu_prev_part = cp;
2002 pp->cp_ncpus--;
2003 if (pp->cp_ncpus == 0) {
2004 cp_numparts_nonempty--;
2005 ASSERT(cp_numparts_nonempty != 0);
2006 }
2007 }
2008
2009 /*
2010 * Routine used to setup a newly inserted CPU in preparation for starting
2011 * it running code.
2012 */
2013 int
2014 cpu_configure(int cpuid)
2015 {
2016 int retval = 0;
2017
2018 ASSERT(MUTEX_HELD(&cpu_lock));
2019
2020 /*
2021 * Some structures are statically allocated based upon
2022 * the maximum number of cpus the system supports. Do not
2023 * try to add anything beyond this limit.
2024 */
2025 if (cpuid < 0 || cpuid >= NCPU) {
2026 return (EINVAL);
2027 }
2028
2029 if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2030 return (EALREADY);
2031 }
2032
2033 if ((retval = mp_cpu_configure(cpuid)) != 0) {
2034 return (retval);
2035 }
2036
2037 cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2038 cpu_set_state(cpu[cpuid]);
2039 retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2040 if (retval != 0)
2041 (void) mp_cpu_unconfigure(cpuid);
2042
2043 return (retval);
2044 }
2045
2046 /*
2047 * Routine used to cleanup a CPU that has been powered off. This will
2048 * destroy all per-cpu information related to this cpu.
2049 */
2050 int
2051 cpu_unconfigure(int cpuid)
2052 {
2053 int error;
2054
2055 ASSERT(MUTEX_HELD(&cpu_lock));
2056
2057 if (cpu[cpuid] == NULL) {
2058 return (ENODEV);
2059 }
2060
2061 if (cpu[cpuid]->cpu_flags == 0) {
2062 return (EALREADY);
2063 }
2064
2065 if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2066 return (EBUSY);
2067 }
2068
2069 if (cpu[cpuid]->cpu_props != NULL) {
2070 (void) nvlist_free(cpu[cpuid]->cpu_props);
2071 cpu[cpuid]->cpu_props = NULL;
2072 }
2073
2074 error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2075
2076 if (error != 0)
2077 return (error);
2078
2079 return (mp_cpu_unconfigure(cpuid));
2080 }
2081
2082 /*
2083 * Routines for registering and de-registering cpu_setup callback functions.
2084 *
2085 * Caller's context
2086 * These routines must not be called from a driver's attach(9E) or
2087 * detach(9E) entry point.
2088 *
2089 * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2090 */
2091
2092 /*
2093 * Ideally, these would be dynamically allocated and put into a linked
2094 * list; however that is not feasible because the registration routine
2095 * has to be available before the kmem allocator is working (in fact,
2096 * it is called by the kmem allocator init code). In any case, there
2097 * are quite a few extra entries for future users.
2098 */
2099 #define NCPU_SETUPS 20
2100
2101 struct cpu_setup {
2102 cpu_setup_func_t *func;
2103 void *arg;
2104 } cpu_setups[NCPU_SETUPS];
2105
2106 void
2107 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2108 {
2109 int i;
2110
2111 ASSERT(MUTEX_HELD(&cpu_lock));
2112
2113 for (i = 0; i < NCPU_SETUPS; i++)
2114 if (cpu_setups[i].func == NULL)
2115 break;
2116 if (i >= NCPU_SETUPS)
2117 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2118
2119 cpu_setups[i].func = func;
2120 cpu_setups[i].arg = arg;
2121 }
2122
2123 void
2124 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2125 {
2126 int i;
2127
2128 ASSERT(MUTEX_HELD(&cpu_lock));
2129
2130 for (i = 0; i < NCPU_SETUPS; i++)
2131 if ((cpu_setups[i].func == func) &&
2132 (cpu_setups[i].arg == arg))
2133 break;
2134 if (i >= NCPU_SETUPS)
2135 cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2136 "deregister");
2137
2138 cpu_setups[i].func = NULL;
2139 cpu_setups[i].arg = 0;
2140 }
2141
2142 /*
2143 * Call any state change hooks for this CPU, ignore any errors.
2144 */
2145 void
2146 cpu_state_change_notify(int id, cpu_setup_t what)
2147 {
2148 int i;
2149
2150 ASSERT(MUTEX_HELD(&cpu_lock));
2151
2152 for (i = 0; i < NCPU_SETUPS; i++) {
2153 if (cpu_setups[i].func != NULL) {
2154 cpu_setups[i].func(what, id, cpu_setups[i].arg);
2155 }
2156 }
2157 }
2158
2159 /*
2160 * Call any state change hooks for this CPU, undo it if error found.
2161 */
2162 static int
2163 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2164 {
2165 int i;
2166 int retval = 0;
2167
2168 ASSERT(MUTEX_HELD(&cpu_lock));
2169
2170 for (i = 0; i < NCPU_SETUPS; i++) {
2171 if (cpu_setups[i].func != NULL) {
2172 retval = cpu_setups[i].func(what, id,
2173 cpu_setups[i].arg);
2174 if (retval) {
2175 for (i--; i >= 0; i--) {
2176 if (cpu_setups[i].func != NULL)
2177 cpu_setups[i].func(undo,
2178 id, cpu_setups[i].arg);
2179 }
2180 break;
2181 }
2182 }
2183 }
2184 return (retval);
2185 }
2186
2187 /*
2188 * Export information about this CPU via the kstat mechanism.
2189 */
2190 static struct {
2191 kstat_named_t ci_state;
2192 kstat_named_t ci_state_begin;
2193 kstat_named_t ci_cpu_type;
2194 kstat_named_t ci_fpu_type;
2195 kstat_named_t ci_clock_MHz;
2196 kstat_named_t ci_chip_id;
2197 kstat_named_t ci_implementation;
2198 kstat_named_t ci_brandstr;
2199 kstat_named_t ci_core_id;
2200 kstat_named_t ci_curr_clock_Hz;
2201 kstat_named_t ci_supp_freq_Hz;
2202 kstat_named_t ci_pg_id;
2203 #if defined(__sparcv9)
2204 kstat_named_t ci_device_ID;
2205 kstat_named_t ci_cpu_fru;
2206 #endif
2207 #if defined(__x86)
2208 kstat_named_t ci_vendorstr;
2209 kstat_named_t ci_family;
2210 kstat_named_t ci_model;
2211 kstat_named_t ci_step;
2212 kstat_named_t ci_clogid;
2213 kstat_named_t ci_pkg_core_id;
2214 kstat_named_t ci_ncpuperchip;
2215 kstat_named_t ci_ncoreperchip;
2216 kstat_named_t ci_max_cstates;
2217 kstat_named_t ci_curr_cstate;
2218 kstat_named_t ci_cacheid;
2219 kstat_named_t ci_sktstr;
2220 #endif
2221 } cpu_info_template = {
2222 { "state", KSTAT_DATA_CHAR },
2223 { "state_begin", KSTAT_DATA_LONG },
2224 { "cpu_type", KSTAT_DATA_CHAR },
2225 { "fpu_type", KSTAT_DATA_CHAR },
2226 { "clock_MHz", KSTAT_DATA_LONG },
2227 { "chip_id", KSTAT_DATA_LONG },
2228 { "implementation", KSTAT_DATA_STRING },
2229 { "brand", KSTAT_DATA_STRING },
2230 { "core_id", KSTAT_DATA_LONG },
2231 { "current_clock_Hz", KSTAT_DATA_UINT64 },
2232 { "supported_frequencies_Hz", KSTAT_DATA_STRING },
2233 { "pg_id", KSTAT_DATA_LONG },
2234 #if defined(__sparcv9)
2235 { "device_ID", KSTAT_DATA_UINT64 },
2236 { "cpu_fru", KSTAT_DATA_STRING },
2237 #endif
2238 #if defined(__x86)
2239 { "vendor_id", KSTAT_DATA_STRING },
2240 { "family", KSTAT_DATA_INT32 },
2241 { "model", KSTAT_DATA_INT32 },
2242 { "stepping", KSTAT_DATA_INT32 },
2243 { "clog_id", KSTAT_DATA_INT32 },
2244 { "pkg_core_id", KSTAT_DATA_LONG },
2245 { "ncpu_per_chip", KSTAT_DATA_INT32 },
2246 { "ncore_per_chip", KSTAT_DATA_INT32 },
2247 { "supported_max_cstates", KSTAT_DATA_INT32 },
2248 { "current_cstate", KSTAT_DATA_INT32 },
2249 { "cache_id", KSTAT_DATA_INT32 },
2250 { "socket_type", KSTAT_DATA_STRING },
2251 #endif
2252 };
2253
2254 static kmutex_t cpu_info_template_lock;
2255
2256 static int
2257 cpu_info_kstat_update(kstat_t *ksp, int rw)
2258 {
2259 cpu_t *cp = ksp->ks_private;
2260 const char *pi_state;
2261
2262 if (rw == KSTAT_WRITE)
2263 return (EACCES);
2264
2265 #if defined(__x86)
2266 /* Is the cpu still initialising itself? */
2267 if (cpuid_checkpass(cp, 1) == 0)
2268 return (ENXIO);
2269 #endif
2270 switch (cp->cpu_type_info.pi_state) {
2271 case P_ONLINE:
2272 pi_state = PS_ONLINE;
2273 break;
2274 case P_POWEROFF:
2275 pi_state = PS_POWEROFF;
2276 break;
2277 case P_NOINTR:
2278 pi_state = PS_NOINTR;
2279 break;
2280 case P_FAULTED:
2281 pi_state = PS_FAULTED;
2282 break;
2283 case P_SPARE:
2284 pi_state = PS_SPARE;
2285 break;
2286 case P_OFFLINE:
2287 pi_state = PS_OFFLINE;
2288 break;
2289 default:
2290 pi_state = "unknown";
2291 }
2292 (void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2293 cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2294 (void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2295 cp->cpu_type_info.pi_processor_type, 15);
2296 (void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2297 cp->cpu_type_info.pi_fputypes, 15);
2298 cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2299 cpu_info_template.ci_chip_id.value.l =
2300 pg_plat_hw_instance_id(cp, PGHW_CHIP);
2301 kstat_named_setstr(&cpu_info_template.ci_implementation,
2302 cp->cpu_idstr);
2303 kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2304 cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2305 cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2306 cp->cpu_curr_clock;
2307 cpu_info_template.ci_pg_id.value.l =
2308 cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2309 cp->cpu_pg->cmt_lineage->pg_id : -1;
2310 kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2311 cp->cpu_supp_freqs);
2312 #if defined(__sparcv9)
2313 cpu_info_template.ci_device_ID.value.ui64 =
2314 cpunodes[cp->cpu_id].device_id;
2315 kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2316 #endif
2317 #if defined(__x86)
2318 kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2319 cpuid_getvendorstr(cp));
2320 cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2321 cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2322 cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2323 cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2324 cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2325 cpu_info_template.ci_ncoreperchip.value.l =
2326 cpuid_get_ncore_per_chip(cp);
2327 cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2328 cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2329 cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2330 cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2331 kstat_named_setstr(&cpu_info_template.ci_sktstr,
2332 cpuid_getsocketstr(cp));
2333 #endif
2334
2335 return (0);
2336 }
2337
2338 static void
2339 cpu_info_kstat_create(cpu_t *cp)
2340 {
2341 zoneid_t zoneid;
2342
2343 ASSERT(MUTEX_HELD(&cpu_lock));
2344
2345 if (pool_pset_enabled())
2346 zoneid = GLOBAL_ZONEID;
2347 else
2348 zoneid = ALL_ZONES;
2349 if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2350 NULL, "misc", KSTAT_TYPE_NAMED,
2351 sizeof (cpu_info_template) / sizeof (kstat_named_t),
2352 KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2353 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2354 #if defined(__sparcv9)
2355 cp->cpu_info_kstat->ks_data_size +=
2356 strlen(cpu_fru_fmri(cp)) + 1;
2357 #endif
2358 #if defined(__x86)
2359 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2360 #endif
2361 if (cp->cpu_supp_freqs != NULL)
2362 cp->cpu_info_kstat->ks_data_size +=
2363 strlen(cp->cpu_supp_freqs) + 1;
2364 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2365 cp->cpu_info_kstat->ks_data = &cpu_info_template;
2366 cp->cpu_info_kstat->ks_private = cp;
2367 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2368 kstat_install(cp->cpu_info_kstat);
2369 }
2370 }
2371
2372 static void
2373 cpu_info_kstat_destroy(cpu_t *cp)
2374 {
2375 ASSERT(MUTEX_HELD(&cpu_lock));
2376
2377 kstat_delete(cp->cpu_info_kstat);
2378 cp->cpu_info_kstat = NULL;
2379 }
2380
2381 /*
2382 * Create and install kstats for the boot CPU.
2383 */
2384 void
2385 cpu_kstat_init(cpu_t *cp)
2386 {
2387 mutex_enter(&cpu_lock);
2388 cpu_info_kstat_create(cp);
2389 cpu_stats_kstat_create(cp);
2390 cpu_create_intrstat(cp);
2391 cpu_set_state(cp);
2392 mutex_exit(&cpu_lock);
2393 }
2394
2395 /*
2396 * Make visible to the zone that subset of the cpu information that would be
2397 * initialized when a cpu is configured (but still offline).
2398 */
2399 void
2400 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2401 {
2402 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2403
2404 ASSERT(MUTEX_HELD(&cpu_lock));
2405 ASSERT(pool_pset_enabled());
2406 ASSERT(cp != NULL);
2407
2408 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2409 zone->zone_ncpus++;
2410 ASSERT(zone->zone_ncpus <= ncpus);
2411 }
2412 if (cp->cpu_info_kstat != NULL)
2413 kstat_zone_add(cp->cpu_info_kstat, zoneid);
2414 }
2415
2416 /*
2417 * Make visible to the zone that subset of the cpu information that would be
2418 * initialized when a previously configured cpu is onlined.
2419 */
2420 void
2421 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2422 {
2423 kstat_t *ksp;
2424 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2425 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2426 processorid_t cpun;
2427
2428 ASSERT(MUTEX_HELD(&cpu_lock));
2429 ASSERT(pool_pset_enabled());
2430 ASSERT(cp != NULL);
2431 ASSERT(cpu_is_active(cp));
2432
2433 cpun = cp->cpu_id;
2434 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2435 zone->zone_ncpus_online++;
2436 ASSERT(zone->zone_ncpus_online <= ncpus_online);
2437 }
2438 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2439 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2440 != NULL) {
2441 kstat_zone_add(ksp, zoneid);
2442 kstat_rele(ksp);
2443 }
2444 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2445 kstat_zone_add(ksp, zoneid);
2446 kstat_rele(ksp);
2447 }
2448 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2449 kstat_zone_add(ksp, zoneid);
2450 kstat_rele(ksp);
2451 }
2452 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2453 NULL) {
2454 kstat_zone_add(ksp, zoneid);
2455 kstat_rele(ksp);
2456 }
2457 }
2458
2459 /*
2460 * Update relevant kstats such that cpu is now visible to processes
2461 * executing in specified zone.
2462 */
2463 void
2464 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2465 {
2466 cpu_visibility_configure(cp, zone);
2467 if (cpu_is_active(cp))
2468 cpu_visibility_online(cp, zone);
2469 }
2470
2471 /*
2472 * Make invisible to the zone that subset of the cpu information that would be
2473 * torn down when a previously offlined cpu is unconfigured.
2474 */
2475 void
2476 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2477 {
2478 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2479
2480 ASSERT(MUTEX_HELD(&cpu_lock));
2481 ASSERT(pool_pset_enabled());
2482 ASSERT(cp != NULL);
2483
2484 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2485 ASSERT(zone->zone_ncpus != 0);
2486 zone->zone_ncpus--;
2487 }
2488 if (cp->cpu_info_kstat)
2489 kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2490 }
2491
2492 /*
2493 * Make invisible to the zone that subset of the cpu information that would be
2494 * torn down when a cpu is offlined (but still configured).
2495 */
2496 void
2497 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2498 {
2499 kstat_t *ksp;
2500 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2501 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2502 processorid_t cpun;
2503
2504 ASSERT(MUTEX_HELD(&cpu_lock));
2505 ASSERT(pool_pset_enabled());
2506 ASSERT(cp != NULL);
2507 ASSERT(cpu_is_active(cp));
2508
2509 cpun = cp->cpu_id;
2510 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2511 ASSERT(zone->zone_ncpus_online != 0);
2512 zone->zone_ncpus_online--;
2513 }
2514
2515 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2516 NULL) {
2517 kstat_zone_remove(ksp, zoneid);
2518 kstat_rele(ksp);
2519 }
2520 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2521 kstat_zone_remove(ksp, zoneid);
2522 kstat_rele(ksp);
2523 }
2524 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2525 kstat_zone_remove(ksp, zoneid);
2526 kstat_rele(ksp);
2527 }
2528 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2529 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2530 != NULL) {
2531 kstat_zone_remove(ksp, zoneid);
2532 kstat_rele(ksp);
2533 }
2534 }
2535
2536 /*
2537 * Update relevant kstats such that cpu is no longer visible to processes
2538 * executing in specified zone.
2539 */
2540 void
2541 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2542 {
2543 if (cpu_is_active(cp))
2544 cpu_visibility_offline(cp, zone);
2545 cpu_visibility_unconfigure(cp, zone);
2546 }
2547
2548 /*
2549 * Bind a thread to a CPU as requested.
2550 */
2551 int
2552 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2553 int *error)
2554 {
2555 processorid_t binding;
2556 cpu_t *cp = NULL;
2557
2558 ASSERT(MUTEX_HELD(&cpu_lock));
2559 ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2560
2561 thread_lock(tp);
2562
2563 /*
2564 * Record old binding, but change the obind, which was initialized
2565 * to PBIND_NONE, only if this thread has a binding. This avoids
2566 * reporting PBIND_NONE for a process when some LWPs are bound.
2567 */
2568 binding = tp->t_bind_cpu;
2569 if (binding != PBIND_NONE)
2570 *obind = binding; /* record old binding */
2571
2572 switch (bind) {
2573 case PBIND_QUERY:
2574 /* Just return the old binding */
2575 thread_unlock(tp);
2576 return (0);
2577
2578 case PBIND_QUERY_TYPE:
2579 /* Return the binding type */
2580 *obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2581 thread_unlock(tp);
2582 return (0);
2583
2584 case PBIND_SOFT:
2585 /*
2586 * Set soft binding for this thread and return the actual
2587 * binding
2588 */
2589 TB_CPU_SOFT_SET(tp);
2590 thread_unlock(tp);
2591 return (0);
2592
2593 case PBIND_HARD:
2594 /*
2595 * Set hard binding for this thread and return the actual
2596 * binding
2597 */
2598 TB_CPU_HARD_SET(tp);
2599 thread_unlock(tp);
2600 return (0);
2601
2602 default:
2603 break;
2604 }
2605
2606 /*
2607 * If this thread/LWP cannot be bound because of permission
2608 * problems, just note that and return success so that the
2609 * other threads/LWPs will be bound. This is the way
2610 * processor_bind() is defined to work.
2611 *
2612 * Binding will get EPERM if the thread is of system class
2613 * or hasprocperm() fails.
2614 */
2615 if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2616 *error = EPERM;
2617 thread_unlock(tp);
2618 return (0);
2619 }
2620
2621 binding = bind;
2622 if (binding != PBIND_NONE) {
2623 cp = cpu_get((processorid_t)binding);
2624 /*
2625 * Make sure binding is valid and is in right partition.
2626 */
2627 if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2628 *error = EINVAL;
2629 thread_unlock(tp);
2630 return (0);
2631 }
2632 }
2633 tp->t_bind_cpu = binding; /* set new binding */
2634
2635 /*
2636 * If there is no system-set reason for affinity, set
2637 * the t_bound_cpu field to reflect the binding.
2638 */
2639 if (tp->t_affinitycnt == 0) {
2640 if (binding == PBIND_NONE) {
2641 /*
2642 * We may need to adjust disp_max_unbound_pri
2643 * since we're becoming unbound.
2644 */
2645 disp_adjust_unbound_pri(tp);
2646
2647 tp->t_bound_cpu = NULL; /* set new binding */
2648
2649 /*
2650 * Move thread to lgroup with strongest affinity
2651 * after unbinding
2652 */
2653 if (tp->t_lgrp_affinity)
2654 lgrp_move_thread(tp,
2655 lgrp_choose(tp, tp->t_cpupart), 1);
2656
2657 if (tp->t_state == TS_ONPROC &&
2658 tp->t_cpu->cpu_part != tp->t_cpupart)
2659 cpu_surrender(tp);
2660 } else {
2661 lpl_t *lpl;
2662
2663 tp->t_bound_cpu = cp;
2664 ASSERT(cp->cpu_lpl != NULL);
2665
2666 /*
2667 * Set home to lgroup with most affinity containing CPU
2668 * that thread is being bound or minimum bounding
2669 * lgroup if no affinities set
2670 */
2671 if (tp->t_lgrp_affinity)
2672 lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2673 LGRP_NONE, B_FALSE);
2674 else
2675 lpl = cp->cpu_lpl;
2676
2677 if (tp->t_lpl != lpl) {
2678 /* can't grab cpu_lock */
2679 lgrp_move_thread(tp, lpl, 1);
2680 }
2681
2682 /*
2683 * Make the thread switch to the bound CPU.
2684 * If the thread is runnable, we need to
2685 * requeue it even if t_cpu is already set
2686 * to the right CPU, since it may be on a
2687 * kpreempt queue and need to move to a local
2688 * queue. We could check t_disp_queue to
2689 * avoid unnecessary overhead if it's already
2690 * on the right queue, but since this isn't
2691 * a performance-critical operation it doesn't
2692 * seem worth the extra code and complexity.
2693 *
2694 * If the thread is weakbound to the cpu then it will
2695 * resist the new binding request until the weak
2696 * binding drops. The cpu_surrender or requeueing
2697 * below could be skipped in such cases (since it
2698 * will have no effect), but that would require
2699 * thread_allowmigrate to acquire thread_lock so
2700 * we'll take the very occasional hit here instead.
2701 */
2702 if (tp->t_state == TS_ONPROC) {
2703 cpu_surrender(tp);
2704 } else if (tp->t_state == TS_RUN) {
2705 cpu_t *ocp = tp->t_cpu;
2706
2707 (void) dispdeq(tp);
2708 setbackdq(tp);
2709 /*
2710 * Either on the bound CPU's disp queue now,
2711 * or swapped out or on the swap queue.
2712 */
2713 ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2714 tp->t_weakbound_cpu == ocp ||
2715 (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2716 != TS_LOAD);
2717 }
2718 }
2719 }
2720
2721 /*
2722 * Our binding has changed; set TP_CHANGEBIND.
2723 */
2724 tp->t_proc_flag |= TP_CHANGEBIND;
2725 aston(tp);
2726
2727 thread_unlock(tp);
2728
2729 return (0);
2730 }
2731
2732
2733 cpuset_t *
2734 cpuset_alloc(int kmflags)
2735 {
2736 return (kmem_alloc(sizeof (cpuset_t), kmflags));
2737 }
2738
2739 void
2740 cpuset_free(cpuset_t *s)
2741 {
2742 kmem_free(s, sizeof (cpuset_t));
2743 }
2744
2745 void
2746 cpuset_all(cpuset_t *s)
2747 {
2748 int i;
2749
2750 for (i = 0; i < CPUSET_WORDS; i++)
2751 s->cpub[i] = ~0UL;
2752 }
2753
2754 void
2755 cpuset_all_but(cpuset_t *s, const uint_t cpu)
2756 {
2757 cpuset_all(s);
2758 CPUSET_DEL(*s, cpu);
2759 }
2760
2761 void
2762 cpuset_only(cpuset_t *s, const uint_t cpu)
2763 {
2764 CPUSET_ZERO(*s);
2765 CPUSET_ADD(*s, cpu);
2766 }
2767
2768 long
2769 cpu_in_set(const cpuset_t *s, const uint_t cpu)
2770 {
2771 VERIFY(cpu < NCPU);
2772 return (BT_TEST(s->cpub, cpu));
2773 }
2774
2775 void
2776 cpuset_add(cpuset_t *s, const uint_t cpu)
2777 {
2778 VERIFY(cpu < NCPU);
2779 BT_SET(s->cpub, cpu);
2780 }
2781
2782 void
2783 cpuset_del(cpuset_t *s, const uint_t cpu)
2784 {
2785 VERIFY(cpu < NCPU);
2786 BT_CLEAR(s->cpub, cpu);
2787 }
2788
2789 int
2790 cpuset_isnull(const cpuset_t *s)
2791 {
2792 int i;
2793
2794 for (i = 0; i < CPUSET_WORDS; i++) {
2795 if (s->cpub[i] != 0)
2796 return (0);
2797 }
2798 return (1);
2799 }
2800
2801 int
2802 cpuset_isequal(const cpuset_t *s1, const cpuset_t *s2)
2803 {
2804 int i;
2805
2806 for (i = 0; i < CPUSET_WORDS; i++) {
2807 if (s1->cpub[i] != s2->cpub[i])
2808 return (0);
2809 }
2810 return (1);
2811 }
2812
2813 uint_t
2814 cpuset_find(const cpuset_t *s)
2815 {
2816
2817 uint_t i;
2818 uint_t cpu = (uint_t)-1;
2819
2820 /*
2821 * Find a cpu in the cpuset
2822 */
2823 for (i = 0; i < CPUSET_WORDS; i++) {
2824 cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2825 if (cpu != (uint_t)-1) {
2826 cpu += i * BT_NBIPUL;
2827 break;
2828 }
2829 }
2830 return (cpu);
2831 }
2832
2833 void
2834 cpuset_bounds(const cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2835 {
2836 int i, j;
2837 uint_t bit;
2838
2839 /*
2840 * First, find the smallest cpu id in the set.
2841 */
2842 for (i = 0; i < CPUSET_WORDS; i++) {
2843 if (s->cpub[i] != 0) {
2844 bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2845 ASSERT(bit != (uint_t)-1);
2846 *smallestid = bit + (i * BT_NBIPUL);
2847
2848 /*
2849 * Now find the largest cpu id in
2850 * the set and return immediately.
2851 * Done in an inner loop to avoid
2852 * having to break out of the first
2853 * loop.
2854 */
2855 for (j = CPUSET_WORDS - 1; j >= i; j--) {
2856 if (s->cpub[j] != 0) {
2857 bit = (uint_t)(highbit(s->cpub[j]) - 1);
2858 ASSERT(bit != (uint_t)-1);
2859 *largestid = bit + (j * BT_NBIPUL);
2860 ASSERT(*largestid >= *smallestid);
2861 return;
2862 }
2863 }
2864
2865 /*
2866 * If this code is reached, a
2867 * smallestid was found, but not a
2868 * largestid. The cpuset must have
2869 * been changed during the course
2870 * of this function call.
2871 */
2872 ASSERT(0);
2873 }
2874 }
2875 *smallestid = *largestid = CPUSET_NOTINSET;
2876 }
2877
2878 void
2879 cpuset_atomic_del(cpuset_t *s, const uint_t cpu)
2880 {
2881 VERIFY(cpu < NCPU);
2882 BT_ATOMIC_CLEAR(s->cpub, (cpu))
2883 }
2884
2885 void
2886 cpuset_atomic_add(cpuset_t *s, const uint_t cpu)
2887 {
2888 VERIFY(cpu < NCPU);
2889 BT_ATOMIC_SET(s->cpub, (cpu))
2890 }
2891
2892 long
2893 cpuset_atomic_xadd(cpuset_t *s, const uint_t cpu)
2894 {
2895 long res;
2896
2897 VERIFY(cpu < NCPU);
2898 BT_ATOMIC_SET_EXCL(s->cpub, cpu, res);
2899 return (res);
2900 }
2901
2902 long
2903 cpuset_atomic_xdel(cpuset_t *s, const uint_t cpu)
2904 {
2905 long res;
2906
2907 VERIFY(cpu < NCPU);
2908 BT_ATOMIC_CLEAR_EXCL(s->cpub, cpu, res);
2909 return (res);
2910 }
2911
2912 void
2913 cpuset_or(cpuset_t *dst, cpuset_t *src)
2914 {
2915 for (int i = 0; i < CPUSET_WORDS; i++) {
2916 dst->cpub[i] |= src->cpub[i];
2917 }
2918 }
2919
2920 void
2921 cpuset_xor(cpuset_t *dst, cpuset_t *src)
2922 {
2923 for (int i = 0; i < CPUSET_WORDS; i++) {
2924 dst->cpub[i] ^= src->cpub[i];
2925 }
2926 }
2927
2928 void
2929 cpuset_and(cpuset_t *dst, cpuset_t *src)
2930 {
2931 for (int i = 0; i < CPUSET_WORDS; i++) {
2932 dst->cpub[i] &= src->cpub[i];
2933 }
2934 }
2935
2936 void
2937 cpuset_zero(cpuset_t *dst)
2938 {
2939 for (int i = 0; i < CPUSET_WORDS; i++) {
2940 dst->cpub[i] = 0;
2941 }
2942 }
2943
2944
2945 /*
2946 * Unbind threads bound to specified CPU.
2947 *
2948 * If `unbind_all_threads' is true, unbind all user threads bound to a given
2949 * CPU. Otherwise unbind all soft-bound user threads.
2950 */
2951 int
2952 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2953 {
2954 processorid_t obind;
2955 kthread_t *tp;
2956 int ret = 0;
2957 proc_t *pp;
2958 int err, berr = 0;
2959
2960 ASSERT(MUTEX_HELD(&cpu_lock));
2961
2962 mutex_enter(&pidlock);
2963 for (pp = practive; pp != NULL; pp = pp->p_next) {
2964 mutex_enter(&pp->p_lock);
2965 tp = pp->p_tlist;
2966 /*
2967 * Skip zombies, kernel processes, and processes in
2968 * other zones, if called from a non-global zone.
2969 */
2970 if (tp == NULL || (pp->p_flag & SSYS) ||
2971 !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2972 mutex_exit(&pp->p_lock);
2973 continue;
2974 }
2975 do {
2976 if (tp->t_bind_cpu != cpu)
2977 continue;
2978 /*
2979 * Skip threads with hard binding when
2980 * `unbind_all_threads' is not specified.
2981 */
2982 if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2983 continue;
2984 err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2985 if (ret == 0)
2986 ret = err;
2987 } while ((tp = tp->t_forw) != pp->p_tlist);
2988 mutex_exit(&pp->p_lock);
2989 }
2990 mutex_exit(&pidlock);
2991 if (ret == 0)
2992 ret = berr;
2993 return (ret);
2994 }
2995
2996
2997 /*
2998 * Destroy all remaining bound threads on a cpu.
2999 */
3000 void
3001 cpu_destroy_bound_threads(cpu_t *cp)
3002 {
3003 extern id_t syscid;
3004 register kthread_id_t t, tlist, tnext;
3005
3006 /*
3007 * Destroy all remaining bound threads on the cpu. This
3008 * should include both the interrupt threads and the idle thread.
3009 * This requires some care, since we need to traverse the
3010 * thread list with the pidlock mutex locked, but thread_free
3011 * also locks the pidlock mutex. So, we collect the threads
3012 * we're going to reap in a list headed by "tlist", then we
3013 * unlock the pidlock mutex and traverse the tlist list,
3014 * doing thread_free's on the thread's. Simple, n'est pas?
3015 * Also, this depends on thread_free not mucking with the
3016 * t_next and t_prev links of the thread.
3017 */
3018
3019 if ((t = curthread) != NULL) {
3020
3021 tlist = NULL;
3022 mutex_enter(&pidlock);
3023 do {
3024 tnext = t->t_next;
3025 if (t->t_bound_cpu == cp) {
3026
3027 /*
3028 * We've found a bound thread, carefully unlink
3029 * it out of the thread list, and add it to
3030 * our "tlist". We "know" we don't have to
3031 * worry about unlinking curthread (the thread
3032 * that is executing this code).
3033 */
3034 t->t_next->t_prev = t->t_prev;
3035 t->t_prev->t_next = t->t_next;
3036 t->t_next = tlist;
3037 tlist = t;
3038 ASSERT(t->t_cid == syscid);
3039 /* wake up anyone blocked in thread_join */
3040 cv_broadcast(&t->t_joincv);
3041 /*
3042 * t_lwp set by interrupt threads and not
3043 * cleared.
3044 */
3045 t->t_lwp = NULL;
3046 /*
3047 * Pause and idle threads always have
3048 * t_state set to TS_ONPROC.
3049 */
3050 t->t_state = TS_FREE;
3051 t->t_prev = NULL; /* Just in case */
3052 }
3053
3054 } while ((t = tnext) != curthread);
3055
3056 mutex_exit(&pidlock);
3057
3058 mutex_sync();
3059 for (t = tlist; t != NULL; t = tnext) {
3060 tnext = t->t_next;
3061 thread_free(t);
3062 }
3063 }
3064 }
3065
3066 /*
3067 * Update the cpu_supp_freqs of this cpu. This information is returned
3068 * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
3069 * maintain the kstat data size.
3070 */
3071 void
3072 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
3073 {
3074 char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
3075 const char *lfreqs = clkstr;
3076 boolean_t kstat_exists = B_FALSE;
3077 kstat_t *ksp;
3078 size_t len;
3079
3080 /*
3081 * A NULL pointer means we only support one speed.
3082 */
3083 if (freqs == NULL)
3084 (void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
3085 cp->cpu_curr_clock);
3086 else
3087 lfreqs = freqs;
3088
3089 /*
3090 * Make sure the frequency doesn't change while a snapshot is
3091 * going on. Of course, we only need to worry about this if
3092 * the kstat exists.
3093 */
3094 if ((ksp = cp->cpu_info_kstat) != NULL) {
3095 mutex_enter(ksp->ks_lock);
3096 kstat_exists = B_TRUE;
3097 }
3098
3099 /*
3100 * Free any previously allocated string and if the kstat
3101 * already exists, then update its data size.
3102 */
3103 if (cp->cpu_supp_freqs != NULL) {
3104 len = strlen(cp->cpu_supp_freqs) + 1;
3105 kmem_free(cp->cpu_supp_freqs, len);
3106 if (kstat_exists)
3107 ksp->ks_data_size -= len;
3108 }
3109
3110 /*
3111 * Allocate the new string and set the pointer.
3112 */
3113 len = strlen(lfreqs) + 1;
3114 cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
3115 (void) strcpy(cp->cpu_supp_freqs, lfreqs);
3116
3117 /*
3118 * If the kstat already exists then update the data size and
3119 * free the lock.
3120 */
3121 if (kstat_exists) {
3122 ksp->ks_data_size += len;
3123 mutex_exit(ksp->ks_lock);
3124 }
3125 }
3126
3127 /*
3128 * Indicate the current CPU's clock freqency (in Hz).
3129 * The calling context must be such that CPU references are safe.
3130 */
3131 void
3132 cpu_set_curr_clock(uint64_t new_clk)
3133 {
3134 uint64_t old_clk;
3135
3136 old_clk = CPU->cpu_curr_clock;
3137 CPU->cpu_curr_clock = new_clk;
3138
3139 /*
3140 * The cpu-change-speed DTrace probe exports the frequency in Hz
3141 */
3142 DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3143 uint64_t, old_clk, uint64_t, new_clk);
3144 }
3145
3146 /*
3147 * processor_info(2) and p_online(2) status support functions
3148 * The constants returned by the cpu_get_state() and cpu_get_state_str() are
3149 * for use in communicating processor state information to userland. Kernel
3150 * subsystems should only be using the cpu_flags value directly. Subsystems
3151 * modifying cpu_flags should record the state change via a call to the
3152 * cpu_set_state().
3153 */
3154
3155 /*
3156 * Update the pi_state of this CPU. This function provides the CPU status for
3157 * the information returned by processor_info(2).
3158 */
3159 void
3160 cpu_set_state(cpu_t *cpu)
3161 {
3162 ASSERT(MUTEX_HELD(&cpu_lock));
3163 cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3164 cpu->cpu_state_begin = gethrestime_sec();
3165 pool_cpu_mod = gethrtime();
3166 }
3167
3168 /*
3169 * Return offline/online/other status for the indicated CPU. Use only for
3170 * communication with user applications; cpu_flags provides the in-kernel
3171 * interface.
3172 */
3173 int
3174 cpu_get_state(cpu_t *cpu)
3175 {
3176 ASSERT(MUTEX_HELD(&cpu_lock));
3177 if (cpu->cpu_flags & CPU_POWEROFF)
3178 return (P_POWEROFF);
3179 else if (cpu->cpu_flags & CPU_FAULTED)
3180 return (P_FAULTED);
3181 else if (cpu->cpu_flags & CPU_SPARE)
3182 return (P_SPARE);
3183 else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3184 return (P_OFFLINE);
3185 else if (cpu->cpu_flags & CPU_ENABLE)
3186 return (P_ONLINE);
3187 else
3188 return (P_NOINTR);
3189 }
3190
3191 /*
3192 * Return processor_info(2) state as a string.
3193 */
3194 const char *
3195 cpu_get_state_str(cpu_t *cpu)
3196 {
3197 const char *string;
3198
3199 switch (cpu_get_state(cpu)) {
3200 case P_ONLINE:
3201 string = PS_ONLINE;
3202 break;
3203 case P_POWEROFF:
3204 string = PS_POWEROFF;
3205 break;
3206 case P_NOINTR:
3207 string = PS_NOINTR;
3208 break;
3209 case P_SPARE:
3210 string = PS_SPARE;
3211 break;
3212 case P_FAULTED:
3213 string = PS_FAULTED;
3214 break;
3215 case P_OFFLINE:
3216 string = PS_OFFLINE;
3217 break;
3218 default:
3219 string = "unknown";
3220 break;
3221 }
3222 return (string);
3223 }
3224
3225 /*
3226 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3227 * kstats, respectively. This is done when a CPU is initialized or placed
3228 * online via p_online(2).
3229 */
3230 static void
3231 cpu_stats_kstat_create(cpu_t *cp)
3232 {
3233 int instance = cp->cpu_id;
3234 char *module = "cpu";
3235 char *class = "misc";
3236 kstat_t *ksp;
3237 zoneid_t zoneid;
3238
3239 ASSERT(MUTEX_HELD(&cpu_lock));
3240
3241 if (pool_pset_enabled())
3242 zoneid = GLOBAL_ZONEID;
3243 else
3244 zoneid = ALL_ZONES;
3245 /*
3246 * Create named kstats
3247 */
3248 #define CPU_STATS_KS_CREATE(name, tsize, update_func) \
3249 ksp = kstat_create_zone(module, instance, (name), class, \
3250 KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \
3251 zoneid); \
3252 if (ksp != NULL) { \
3253 ksp->ks_private = cp; \
3254 ksp->ks_update = (update_func); \
3255 kstat_install(ksp); \
3256 } else \
3257 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3258 module, instance, (name));
3259
3260 CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3261 cpu_sys_stats_ks_update);
3262 CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3263 cpu_vm_stats_ks_update);
3264
3265 /*
3266 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3267 */
3268 ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3269 "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3270 if (ksp != NULL) {
3271 ksp->ks_update = cpu_stat_ks_update;
3272 ksp->ks_private = cp;
3273 kstat_install(ksp);
3274 }
3275 }
3276
3277 static void
3278 cpu_stats_kstat_destroy(cpu_t *cp)
3279 {
3280 char ks_name[KSTAT_STRLEN];
3281
3282 (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3283 kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3284
3285 kstat_delete_byname("cpu", cp->cpu_id, "sys");
3286 kstat_delete_byname("cpu", cp->cpu_id, "vm");
3287 }
3288
3289 static int
3290 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3291 {
3292 cpu_t *cp = (cpu_t *)ksp->ks_private;
3293 struct cpu_sys_stats_ks_data *csskd;
3294 cpu_sys_stats_t *css;
3295 hrtime_t msnsecs[NCMSTATES];
3296 int i;
3297
3298 if (rw == KSTAT_WRITE)
3299 return (EACCES);
3300
3301 csskd = ksp->ks_data;
3302 css = &cp->cpu_stats.sys;
3303
3304 /*
3305 * Read CPU mstate, but compare with the last values we
3306 * received to make sure that the returned kstats never
3307 * decrease.
3308 */
3309
3310 get_cpu_mstate(cp, msnsecs);
3311 if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3312 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3313 if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3314 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3315 if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3316 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3317
3318 bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3319 sizeof (cpu_sys_stats_ks_data_template));
3320
3321 csskd->cpu_ticks_wait.value.ui64 = 0;
3322 csskd->wait_ticks_io.value.ui64 = 0;
3323
3324 csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3325 csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3326 csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3327 csskd->cpu_ticks_idle.value.ui64 =
3328 NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3329 csskd->cpu_ticks_user.value.ui64 =
3330 NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3331 csskd->cpu_ticks_kernel.value.ui64 =
3332 NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3333 csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3334 csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3335 csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3336 csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3337 csskd->bread.value.ui64 = css->bread;
3338 csskd->bwrite.value.ui64 = css->bwrite;
3339 csskd->lread.value.ui64 = css->lread;
3340 csskd->lwrite.value.ui64 = css->lwrite;
3341 csskd->phread.value.ui64 = css->phread;
3342 csskd->phwrite.value.ui64 = css->phwrite;
3343 csskd->pswitch.value.ui64 = css->pswitch;
3344 csskd->trap.value.ui64 = css->trap;
3345 csskd->intr.value.ui64 = 0;
3346 for (i = 0; i < PIL_MAX; i++)
3347 csskd->intr.value.ui64 += css->intr[i];
3348 csskd->syscall.value.ui64 = css->syscall;
3349 csskd->sysread.value.ui64 = css->sysread;
3350 csskd->syswrite.value.ui64 = css->syswrite;
3351 csskd->sysfork.value.ui64 = css->sysfork;
3352 csskd->sysvfork.value.ui64 = css->sysvfork;
3353 csskd->sysexec.value.ui64 = css->sysexec;
3354 csskd->readch.value.ui64 = css->readch;
3355 csskd->writech.value.ui64 = css->writech;
3356 csskd->rcvint.value.ui64 = css->rcvint;
3357 csskd->xmtint.value.ui64 = css->xmtint;
3358 csskd->mdmint.value.ui64 = css->mdmint;
3359 csskd->rawch.value.ui64 = css->rawch;
3360 csskd->canch.value.ui64 = css->canch;
3361 csskd->outch.value.ui64 = css->outch;
3362 csskd->msg.value.ui64 = css->msg;
3363 csskd->sema.value.ui64 = css->sema;
3364 csskd->namei.value.ui64 = css->namei;
3365 csskd->ufsiget.value.ui64 = css->ufsiget;
3366 csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3367 csskd->ufsipage.value.ui64 = css->ufsipage;
3368 csskd->ufsinopage.value.ui64 = css->ufsinopage;
3369 csskd->procovf.value.ui64 = css->procovf;
3370 csskd->intrthread.value.ui64 = 0;
3371 for (i = 0; i < LOCK_LEVEL - 1; i++)
3372 csskd->intrthread.value.ui64 += css->intr[i];
3373 csskd->intrblk.value.ui64 = css->intrblk;
3374 csskd->intrunpin.value.ui64 = css->intrunpin;
3375 csskd->idlethread.value.ui64 = css->idlethread;
3376 csskd->inv_swtch.value.ui64 = css->inv_swtch;
3377 csskd->nthreads.value.ui64 = css->nthreads;
3378 csskd->cpumigrate.value.ui64 = css->cpumigrate;
3379 csskd->xcalls.value.ui64 = css->xcalls;
3380 csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3381 csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3382 csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3383 csskd->modload.value.ui64 = css->modload;
3384 csskd->modunload.value.ui64 = css->modunload;
3385 csskd->bawrite.value.ui64 = css->bawrite;
3386 csskd->iowait.value.ui64 = css->iowait;
3387
3388 return (0);
3389 }
3390
3391 static int
3392 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3393 {
3394 cpu_t *cp = (cpu_t *)ksp->ks_private;
3395 struct cpu_vm_stats_ks_data *cvskd;
3396 cpu_vm_stats_t *cvs;
3397
3398 if (rw == KSTAT_WRITE)
3399 return (EACCES);
3400
3401 cvs = &cp->cpu_stats.vm;
3402 cvskd = ksp->ks_data;
3403
3404 bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3405 sizeof (cpu_vm_stats_ks_data_template));
3406 cvskd->pgrec.value.ui64 = cvs->pgrec;
3407 cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3408 cvskd->pgin.value.ui64 = cvs->pgin;
3409 cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3410 cvskd->pgout.value.ui64 = cvs->pgout;
3411 cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3412 cvskd->swapin.value.ui64 = cvs->swapin;
3413 cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3414 cvskd->swapout.value.ui64 = cvs->swapout;
3415 cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3416 cvskd->zfod.value.ui64 = cvs->zfod;
3417 cvskd->dfree.value.ui64 = cvs->dfree;
3418 cvskd->scan.value.ui64 = cvs->scan;
3419 cvskd->rev.value.ui64 = cvs->rev;
3420 cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3421 cvskd->as_fault.value.ui64 = cvs->as_fault;
3422 cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3423 cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3424 cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3425 cvskd->softlock.value.ui64 = cvs->softlock;
3426 cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3427 cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3428 cvskd->execpgin.value.ui64 = cvs->execpgin;
3429 cvskd->execpgout.value.ui64 = cvs->execpgout;
3430 cvskd->execfree.value.ui64 = cvs->execfree;
3431 cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3432 cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3433 cvskd->anonfree.value.ui64 = cvs->anonfree;
3434 cvskd->fspgin.value.ui64 = cvs->fspgin;
3435 cvskd->fspgout.value.ui64 = cvs->fspgout;
3436 cvskd->fsfree.value.ui64 = cvs->fsfree;
3437
3438 return (0);
3439 }
3440
3441 static int
3442 cpu_stat_ks_update(kstat_t *ksp, int rw)
3443 {
3444 cpu_stat_t *cso;
3445 cpu_t *cp;
3446 int i;
3447 hrtime_t msnsecs[NCMSTATES];
3448
3449 cso = (cpu_stat_t *)ksp->ks_data;
3450 cp = (cpu_t *)ksp->ks_private;
3451
3452 if (rw == KSTAT_WRITE)
3453 return (EACCES);
3454
3455 /*
3456 * Read CPU mstate, but compare with the last values we
3457 * received to make sure that the returned kstats never
3458 * decrease.
3459 */
3460
3461 get_cpu_mstate(cp, msnsecs);
3462 msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3463 msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3464 msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3465 if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3466 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3467 if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3468 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3469 if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3470 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3471 cso->cpu_sysinfo.cpu[CPU_WAIT] = 0;
3472 cso->cpu_sysinfo.wait[W_IO] = 0;
3473 cso->cpu_sysinfo.wait[W_SWAP] = 0;
3474 cso->cpu_sysinfo.wait[W_PIO] = 0;
3475 cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread);
3476 cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite);
3477 cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread);
3478 cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite);
3479 cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread);
3480 cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite);
3481 cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch);
3482 cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap);
3483 cso->cpu_sysinfo.intr = 0;
3484 for (i = 0; i < PIL_MAX; i++)
3485 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3486 cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall);
3487 cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread);
3488 cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite);
3489 cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork);
3490 cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork);
3491 cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec);
3492 cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch);
3493 cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech);
3494 cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint);
3495 cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint);
3496 cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint);
3497 cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch);
3498 cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch);
3499 cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch);
3500 cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg);
3501 cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema);
3502 cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei);
3503 cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget);
3504 cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk);
3505 cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage);
3506 cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage);
3507 cso->cpu_sysinfo.inodeovf = 0;
3508 cso->cpu_sysinfo.fileovf = 0;
3509 cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf);
3510 cso->cpu_sysinfo.intrthread = 0;
3511 for (i = 0; i < LOCK_LEVEL - 1; i++)
3512 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3513 cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk);
3514 cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread);
3515 cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch);
3516 cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads);
3517 cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate);
3518 cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls);
3519 cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters);
3520 cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails);
3521 cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails);
3522 cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload);
3523 cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload);
3524 cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite);
3525 cso->cpu_sysinfo.rw_enters = 0;
3526 cso->cpu_sysinfo.win_uo_cnt = 0;
3527 cso->cpu_sysinfo.win_uu_cnt = 0;
3528 cso->cpu_sysinfo.win_so_cnt = 0;
3529 cso->cpu_sysinfo.win_su_cnt = 0;
3530 cso->cpu_sysinfo.win_suo_cnt = 0;
3531
3532 cso->cpu_syswait.iowait = CPU_STATS(cp, sys.iowait);
3533 cso->cpu_syswait.swap = 0;
3534 cso->cpu_syswait.physio = 0;
3535
3536 cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec);
3537 cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec);
3538 cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin);
3539 cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin);
3540 cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout);
3541 cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout);
3542 cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin);
3543 cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin);
3544 cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout);
3545 cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout);
3546 cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod);
3547 cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree);
3548 cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan);
3549 cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev);
3550 cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault);
3551 cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault);
3552 cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault);
3553 cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault);
3554 cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault);
3555 cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock);
3556 cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt);
3557 cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun);
3558 cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin);
3559 cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout);
3560 cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree);
3561 cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin);
3562 cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout);
3563 cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree);
3564 cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin);
3565 cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout);
3566 cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree);
3567
3568 return (0);
3569 }