1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * Architecture-independent CPU control functions.
27 */
28
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/var.h>
32 #include <sys/thread.h>
33 #include <sys/cpuvar.h>
34 #include <sys/cpu_event.h>
35 #include <sys/kstat.h>
36 #include <sys/uadmin.h>
37 #include <sys/systm.h>
38 #include <sys/errno.h>
39 #include <sys/cmn_err.h>
40 #include <sys/procset.h>
41 #include <sys/processor.h>
42 #include <sys/debug.h>
43 #include <sys/cpupart.h>
44 #include <sys/lgrp.h>
45 #include <sys/pset.h>
46 #include <sys/pghw.h>
47 #include <sys/kmem.h>
48 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */
49 #include <sys/atomic.h>
50 #include <sys/callb.h>
51 #include <sys/vtrace.h>
52 #include <sys/cyclic.h>
53 #include <sys/bitmap.h>
54 #include <sys/nvpair.h>
55 #include <sys/pool_pset.h>
56 #include <sys/msacct.h>
57 #include <sys/time.h>
58 #include <sys/archsystm.h>
59 #include <sys/sdt.h>
60 #if defined(__x86) || defined(__amd64)
61 #include <sys/x86_archext.h>
62 #endif
63 #include <sys/callo.h>
64
65 extern int mp_cpu_start(cpu_t *);
66 extern int mp_cpu_stop(cpu_t *);
67 extern int mp_cpu_poweron(cpu_t *);
68 extern int mp_cpu_poweroff(cpu_t *);
69 extern int mp_cpu_configure(int);
70 extern int mp_cpu_unconfigure(int);
71 extern void mp_cpu_faulted_enter(cpu_t *);
72 extern void mp_cpu_faulted_exit(cpu_t *);
73
74 extern int cmp_cpu_to_chip(processorid_t cpuid);
75 #ifdef __sparcv9
76 extern char *cpu_fru_fmri(cpu_t *cp);
77 #endif
78
79 static void cpu_add_active_internal(cpu_t *cp);
80 static void cpu_remove_active(cpu_t *cp);
81 static void cpu_info_kstat_create(cpu_t *cp);
82 static void cpu_info_kstat_destroy(cpu_t *cp);
83 static void cpu_stats_kstat_create(cpu_t *cp);
84 static void cpu_stats_kstat_destroy(cpu_t *cp);
85
86 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
87 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
88 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
89 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
90
91 /*
92 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
93 * max_cpu_seqid_ever, and dispatch queue reallocations. The lock ordering with
94 * respect to related locks is:
95 *
96 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock()
97 *
98 * Warning: Certain sections of code do not use the cpu_lock when
99 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since
100 * all cpus are paused during modifications to this list, a solution
101 * to protect the list is too either disable kernel preemption while
102 * walking the list, *or* recheck the cpu_next pointer at each
103 * iteration in the loop. Note that in no cases can any cached
104 * copies of the cpu pointers be kept as they may become invalid.
105 */
106 kmutex_t cpu_lock;
107 cpu_t *cpu_list; /* list of all CPUs */
108 cpu_t *clock_cpu_list; /* used by clock to walk CPUs */
109 cpu_t *cpu_active; /* list of active CPUs */
110 static cpuset_t cpu_available; /* set of available CPUs */
111 cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */
112
113 cpu_t **cpu_seq; /* ptrs to CPUs, indexed by seq_id */
114
115 /*
116 * max_ncpus keeps the max cpus the system can have. Initially
117 * it's NCPU, but since most archs scan the devtree for cpus
118 * fairly early on during boot, the real max can be known before
119 * ncpus is set (useful for early NCPU based allocations).
120 */
121 int max_ncpus = NCPU;
122 /*
123 * platforms that set max_ncpus to maxiumum number of cpus that can be
124 * dynamically added will set boot_max_ncpus to the number of cpus found
125 * at device tree scan time during boot.
126 */
127 int boot_max_ncpus = -1;
128 int boot_ncpus = -1;
129 /*
130 * Maximum possible CPU id. This can never be >= NCPU since NCPU is
131 * used to size arrays that are indexed by CPU id.
132 */
133 processorid_t max_cpuid = NCPU - 1;
134
135 /*
136 * Maximum cpu_seqid was given. This number can only grow and never shrink. It
137 * can be used to optimize NCPU loops to avoid going through CPUs which were
138 * never on-line.
139 */
140 processorid_t max_cpu_seqid_ever = 0;
141
142 int ncpus = 1;
143 int ncpus_online = 1;
144
145 /*
146 * CPU that we're trying to offline. Protected by cpu_lock.
147 */
148 cpu_t *cpu_inmotion;
149
150 /*
151 * Can be raised to suppress further weakbinding, which are instead
152 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock,
153 * while individual thread weakbinding synchronization is done under thread
154 * lock.
155 */
156 int weakbindingbarrier;
157
158 /*
159 * Variables used in pause_cpus().
160 */
161 static volatile char safe_list[NCPU];
162
163 static struct _cpu_pause_info {
164 int cp_spl; /* spl saved in pause_cpus() */
165 volatile int cp_go; /* Go signal sent after all ready */
166 int cp_count; /* # of CPUs to pause */
167 ksema_t cp_sem; /* synch pause_cpus & cpu_pause */
168 kthread_id_t cp_paused;
169 } cpu_pause_info;
170
171 static kmutex_t pause_free_mutex;
172 static kcondvar_t pause_free_cv;
173
174 void *(*cpu_pause_func)(void *) = NULL;
175
176
177 static struct cpu_sys_stats_ks_data {
178 kstat_named_t cpu_ticks_idle;
179 kstat_named_t cpu_ticks_user;
180 kstat_named_t cpu_ticks_kernel;
181 kstat_named_t cpu_ticks_wait;
182 kstat_named_t cpu_nsec_idle;
183 kstat_named_t cpu_nsec_user;
184 kstat_named_t cpu_nsec_kernel;
185 kstat_named_t cpu_nsec_intr;
186 kstat_named_t cpu_load_intr;
187 kstat_named_t wait_ticks_io;
188 kstat_named_t bread;
189 kstat_named_t bwrite;
190 kstat_named_t lread;
191 kstat_named_t lwrite;
192 kstat_named_t phread;
193 kstat_named_t phwrite;
194 kstat_named_t pswitch;
195 kstat_named_t trap;
196 kstat_named_t intr;
197 kstat_named_t syscall;
198 kstat_named_t sysread;
199 kstat_named_t syswrite;
200 kstat_named_t sysfork;
201 kstat_named_t sysvfork;
202 kstat_named_t sysexec;
203 kstat_named_t readch;
204 kstat_named_t writech;
205 kstat_named_t rcvint;
206 kstat_named_t xmtint;
207 kstat_named_t mdmint;
208 kstat_named_t rawch;
209 kstat_named_t canch;
210 kstat_named_t outch;
211 kstat_named_t msg;
212 kstat_named_t sema;
213 kstat_named_t namei;
214 kstat_named_t ufsiget;
215 kstat_named_t ufsdirblk;
216 kstat_named_t ufsipage;
217 kstat_named_t ufsinopage;
218 kstat_named_t procovf;
219 kstat_named_t intrthread;
220 kstat_named_t intrblk;
221 kstat_named_t intrunpin;
222 kstat_named_t idlethread;
223 kstat_named_t inv_swtch;
224 kstat_named_t nthreads;
225 kstat_named_t cpumigrate;
226 kstat_named_t xcalls;
227 kstat_named_t mutex_adenters;
228 kstat_named_t rw_rdfails;
229 kstat_named_t rw_wrfails;
230 kstat_named_t modload;
231 kstat_named_t modunload;
232 kstat_named_t bawrite;
233 kstat_named_t iowait;
234 } cpu_sys_stats_ks_data_template = {
235 { "cpu_ticks_idle", KSTAT_DATA_UINT64 },
236 { "cpu_ticks_user", KSTAT_DATA_UINT64 },
237 { "cpu_ticks_kernel", KSTAT_DATA_UINT64 },
238 { "cpu_ticks_wait", KSTAT_DATA_UINT64 },
239 { "cpu_nsec_idle", KSTAT_DATA_UINT64 },
240 { "cpu_nsec_user", KSTAT_DATA_UINT64 },
241 { "cpu_nsec_kernel", KSTAT_DATA_UINT64 },
242 { "cpu_nsec_intr", KSTAT_DATA_UINT64 },
243 { "cpu_load_intr", KSTAT_DATA_UINT64 },
244 { "wait_ticks_io", KSTAT_DATA_UINT64 },
245 { "bread", KSTAT_DATA_UINT64 },
246 { "bwrite", KSTAT_DATA_UINT64 },
247 { "lread", KSTAT_DATA_UINT64 },
248 { "lwrite", KSTAT_DATA_UINT64 },
249 { "phread", KSTAT_DATA_UINT64 },
250 { "phwrite", KSTAT_DATA_UINT64 },
251 { "pswitch", KSTAT_DATA_UINT64 },
252 { "trap", KSTAT_DATA_UINT64 },
253 { "intr", KSTAT_DATA_UINT64 },
254 { "syscall", KSTAT_DATA_UINT64 },
255 { "sysread", KSTAT_DATA_UINT64 },
256 { "syswrite", KSTAT_DATA_UINT64 },
257 { "sysfork", KSTAT_DATA_UINT64 },
258 { "sysvfork", KSTAT_DATA_UINT64 },
259 { "sysexec", KSTAT_DATA_UINT64 },
260 { "readch", KSTAT_DATA_UINT64 },
261 { "writech", KSTAT_DATA_UINT64 },
262 { "rcvint", KSTAT_DATA_UINT64 },
263 { "xmtint", KSTAT_DATA_UINT64 },
264 { "mdmint", KSTAT_DATA_UINT64 },
265 { "rawch", KSTAT_DATA_UINT64 },
266 { "canch", KSTAT_DATA_UINT64 },
267 { "outch", KSTAT_DATA_UINT64 },
268 { "msg", KSTAT_DATA_UINT64 },
269 { "sema", KSTAT_DATA_UINT64 },
270 { "namei", KSTAT_DATA_UINT64 },
271 { "ufsiget", KSTAT_DATA_UINT64 },
272 { "ufsdirblk", KSTAT_DATA_UINT64 },
273 { "ufsipage", KSTAT_DATA_UINT64 },
274 { "ufsinopage", KSTAT_DATA_UINT64 },
275 { "procovf", KSTAT_DATA_UINT64 },
276 { "intrthread", KSTAT_DATA_UINT64 },
277 { "intrblk", KSTAT_DATA_UINT64 },
278 { "intrunpin", KSTAT_DATA_UINT64 },
279 { "idlethread", KSTAT_DATA_UINT64 },
280 { "inv_swtch", KSTAT_DATA_UINT64 },
281 { "nthreads", KSTAT_DATA_UINT64 },
282 { "cpumigrate", KSTAT_DATA_UINT64 },
283 { "xcalls", KSTAT_DATA_UINT64 },
284 { "mutex_adenters", KSTAT_DATA_UINT64 },
285 { "rw_rdfails", KSTAT_DATA_UINT64 },
286 { "rw_wrfails", KSTAT_DATA_UINT64 },
287 { "modload", KSTAT_DATA_UINT64 },
288 { "modunload", KSTAT_DATA_UINT64 },
289 { "bawrite", KSTAT_DATA_UINT64 },
290 { "iowait", KSTAT_DATA_UINT64 },
291 };
292
293 static struct cpu_vm_stats_ks_data {
294 kstat_named_t pgrec;
295 kstat_named_t pgfrec;
296 kstat_named_t pgin;
297 kstat_named_t pgpgin;
298 kstat_named_t pgout;
299 kstat_named_t pgpgout;
300 kstat_named_t swapin;
301 kstat_named_t pgswapin;
302 kstat_named_t swapout;
303 kstat_named_t pgswapout;
304 kstat_named_t zfod;
305 kstat_named_t dfree;
306 kstat_named_t scan;
307 kstat_named_t rev;
308 kstat_named_t hat_fault;
309 kstat_named_t as_fault;
310 kstat_named_t maj_fault;
311 kstat_named_t cow_fault;
312 kstat_named_t prot_fault;
313 kstat_named_t softlock;
314 kstat_named_t kernel_asflt;
315 kstat_named_t pgrrun;
316 kstat_named_t execpgin;
317 kstat_named_t execpgout;
318 kstat_named_t execfree;
319 kstat_named_t anonpgin;
320 kstat_named_t anonpgout;
321 kstat_named_t anonfree;
322 kstat_named_t fspgin;
323 kstat_named_t fspgout;
324 kstat_named_t fsfree;
325 } cpu_vm_stats_ks_data_template = {
326 { "pgrec", KSTAT_DATA_UINT64 },
327 { "pgfrec", KSTAT_DATA_UINT64 },
328 { "pgin", KSTAT_DATA_UINT64 },
329 { "pgpgin", KSTAT_DATA_UINT64 },
330 { "pgout", KSTAT_DATA_UINT64 },
331 { "pgpgout", KSTAT_DATA_UINT64 },
332 { "swapin", KSTAT_DATA_UINT64 },
333 { "pgswapin", KSTAT_DATA_UINT64 },
334 { "swapout", KSTAT_DATA_UINT64 },
335 { "pgswapout", KSTAT_DATA_UINT64 },
336 { "zfod", KSTAT_DATA_UINT64 },
337 { "dfree", KSTAT_DATA_UINT64 },
338 { "scan", KSTAT_DATA_UINT64 },
339 { "rev", KSTAT_DATA_UINT64 },
340 { "hat_fault", KSTAT_DATA_UINT64 },
341 { "as_fault", KSTAT_DATA_UINT64 },
342 { "maj_fault", KSTAT_DATA_UINT64 },
343 { "cow_fault", KSTAT_DATA_UINT64 },
344 { "prot_fault", KSTAT_DATA_UINT64 },
345 { "softlock", KSTAT_DATA_UINT64 },
346 { "kernel_asflt", KSTAT_DATA_UINT64 },
347 { "pgrrun", KSTAT_DATA_UINT64 },
348 { "execpgin", KSTAT_DATA_UINT64 },
349 { "execpgout", KSTAT_DATA_UINT64 },
350 { "execfree", KSTAT_DATA_UINT64 },
351 { "anonpgin", KSTAT_DATA_UINT64 },
352 { "anonpgout", KSTAT_DATA_UINT64 },
353 { "anonfree", KSTAT_DATA_UINT64 },
354 { "fspgin", KSTAT_DATA_UINT64 },
355 { "fspgout", KSTAT_DATA_UINT64 },
356 { "fsfree", KSTAT_DATA_UINT64 },
357 };
358
359 /*
360 * Force the specified thread to migrate to the appropriate processor.
361 * Called with thread lock held, returns with it dropped.
362 */
363 static void
364 force_thread_migrate(kthread_id_t tp)
365 {
366 ASSERT(THREAD_LOCK_HELD(tp));
367 if (tp == curthread) {
368 THREAD_TRANSITION(tp);
369 CL_SETRUN(tp);
370 thread_unlock_nopreempt(tp);
371 swtch();
372 } else {
373 if (tp->t_state == TS_ONPROC) {
374 cpu_surrender(tp);
375 } else if (tp->t_state == TS_RUN) {
376 (void) dispdeq(tp);
377 setbackdq(tp);
378 }
379 thread_unlock(tp);
380 }
381 }
382
383 /*
384 * Set affinity for a specified CPU.
385 * A reference count is incremented and the affinity is held until the
386 * reference count is decremented to zero by thread_affinity_clear().
387 * This is so regions of code requiring affinity can be nested.
388 * Caller needs to ensure that cpu_id remains valid, which can be
389 * done by holding cpu_lock across this call, unless the caller
390 * specifies CPU_CURRENT in which case the cpu_lock will be acquired
391 * by thread_affinity_set and CPU->cpu_id will be the target CPU.
392 */
393 void
394 thread_affinity_set(kthread_id_t t, int cpu_id)
395 {
396 cpu_t *cp;
397 int c;
398
399 ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
400
401 if ((c = cpu_id) == CPU_CURRENT) {
402 mutex_enter(&cpu_lock);
403 cpu_id = CPU->cpu_id;
404 }
405 /*
406 * We should be asserting that cpu_lock is held here, but
407 * the NCA code doesn't acquire it. The following assert
408 * should be uncommented when the NCA code is fixed.
409 *
410 * ASSERT(MUTEX_HELD(&cpu_lock));
411 */
412 ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
413 cp = cpu[cpu_id];
414 ASSERT(cp != NULL); /* user must provide a good cpu_id */
415 /*
416 * If there is already a hard affinity requested, and this affinity
417 * conflicts with that, panic.
418 */
419 thread_lock(t);
420 if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
421 panic("affinity_set: setting %p but already bound to %p",
422 (void *)cp, (void *)t->t_bound_cpu);
423 }
424 t->t_affinitycnt++;
425 t->t_bound_cpu = cp;
426
427 /*
428 * Make sure we're running on the right CPU.
429 */
430 if (cp != t->t_cpu || t != curthread) {
431 force_thread_migrate(t); /* drops thread lock */
432 } else {
433 thread_unlock(t);
434 }
435
436 if (c == CPU_CURRENT)
437 mutex_exit(&cpu_lock);
438 }
439
440 /*
441 * Wrapper for backward compatibility.
442 */
443 void
444 affinity_set(int cpu_id)
445 {
446 thread_affinity_set(curthread, cpu_id);
447 }
448
449 /*
450 * Decrement the affinity reservation count and if it becomes zero,
451 * clear the CPU affinity for the current thread, or set it to the user's
452 * software binding request.
453 */
454 void
455 thread_affinity_clear(kthread_id_t t)
456 {
457 register processorid_t binding;
458
459 thread_lock(t);
460 if (--t->t_affinitycnt == 0) {
461 if ((binding = t->t_bind_cpu) == PBIND_NONE) {
462 /*
463 * Adjust disp_max_unbound_pri if necessary.
464 */
465 disp_adjust_unbound_pri(t);
466 t->t_bound_cpu = NULL;
467 if (t->t_cpu->cpu_part != t->t_cpupart) {
468 force_thread_migrate(t);
469 return;
470 }
471 } else {
472 t->t_bound_cpu = cpu[binding];
473 /*
474 * Make sure the thread is running on the bound CPU.
475 */
476 if (t->t_cpu != t->t_bound_cpu) {
477 force_thread_migrate(t);
478 return; /* already dropped lock */
479 }
480 }
481 }
482 thread_unlock(t);
483 }
484
485 /*
486 * Wrapper for backward compatibility.
487 */
488 void
489 affinity_clear(void)
490 {
491 thread_affinity_clear(curthread);
492 }
493
494 /*
495 * Weak cpu affinity. Bind to the "current" cpu for short periods
496 * of time during which the thread must not block (but may be preempted).
497 * Use this instead of kpreempt_disable() when it is only "no migration"
498 * rather than "no preemption" semantics that are required - disabling
499 * preemption holds higher priority threads off of cpu and if the
500 * operation that is protected is more than momentary this is not good
501 * for realtime etc.
502 *
503 * Weakly bound threads will not prevent a cpu from being offlined -
504 * we'll only run them on the cpu to which they are weakly bound but
505 * (because they do not block) we'll always be able to move them on to
506 * another cpu at offline time if we give them just a short moment to
507 * run during which they will unbind. To give a cpu a chance of offlining,
508 * however, we require a barrier to weak bindings that may be raised for a
509 * given cpu (offline/move code may set this and then wait a short time for
510 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
511 *
512 * There are few restrictions on the calling context of thread_nomigrate.
513 * The caller must not hold the thread lock. Calls may be nested.
514 *
515 * After weakbinding a thread must not perform actions that may block.
516 * In particular it must not call thread_affinity_set; calling that when
517 * already weakbound is nonsensical anyway.
518 *
519 * If curthread is prevented from migrating for other reasons
520 * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
521 * then the weak binding will succeed even if this cpu is the target of an
522 * offline/move request.
523 */
524 void
525 thread_nomigrate(void)
526 {
527 cpu_t *cp;
528 kthread_id_t t = curthread;
529
530 again:
531 kpreempt_disable();
532 cp = CPU;
533
534 /*
535 * A highlevel interrupt must not modify t_nomigrate or
536 * t_weakbound_cpu of the thread it has interrupted. A lowlevel
537 * interrupt thread cannot migrate and we can avoid the
538 * thread_lock call below by short-circuiting here. In either
539 * case we can just return since no migration is possible and
540 * the condition will persist (ie, when we test for these again
541 * in thread_allowmigrate they can't have changed). Migration
542 * is also impossible if we're at or above DISP_LEVEL pil.
543 */
544 if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
545 getpil() >= DISP_LEVEL) {
546 kpreempt_enable();
547 return;
548 }
549
550 /*
551 * We must be consistent with existing weak bindings. Since we
552 * may be interrupted between the increment of t_nomigrate and
553 * the store to t_weakbound_cpu below we cannot assume that
554 * t_weakbound_cpu will be set if t_nomigrate is. Note that we
555 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
556 * always the case.
557 */
558 if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
559 if (!panicstr)
560 panic("thread_nomigrate: binding to %p but already "
561 "bound to %p", (void *)cp,
562 (void *)t->t_weakbound_cpu);
563 }
564
565 /*
566 * At this point we have preemption disabled and we don't yet hold
567 * the thread lock. So it's possible that somebody else could
568 * set t_bind_cpu here and not be able to force us across to the
569 * new cpu (since we have preemption disabled).
570 */
571 thread_lock(curthread);
572
573 /*
574 * If further weak bindings are being (temporarily) suppressed then
575 * we'll settle for disabling kernel preemption (which assures
576 * no migration provided the thread does not block which it is
577 * not allowed to if using thread_nomigrate). We must remember
578 * this disposition so we can take appropriate action in
579 * thread_allowmigrate. If this is a nested call and the
580 * thread is already weakbound then fall through as normal.
581 * We remember the decision to settle for kpreempt_disable through
582 * negative nesting counting in t_nomigrate. Once a thread has had one
583 * weakbinding request satisfied in this way any further (nested)
584 * requests will continue to be satisfied in the same way,
585 * even if weak bindings have recommenced.
586 */
587 if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
588 --t->t_nomigrate;
589 thread_unlock(curthread);
590 return; /* with kpreempt_disable still active */
591 }
592
593 /*
594 * We hold thread_lock so t_bind_cpu cannot change. We could,
595 * however, be running on a different cpu to which we are t_bound_cpu
596 * to (as explained above). If we grant the weak binding request
597 * in that case then the dispatcher must favour our weak binding
598 * over our strong (in which case, just as when preemption is
599 * disabled, we can continue to run on a cpu other than the one to
600 * which we are strongbound; the difference in this case is that
601 * this thread can be preempted and so can appear on the dispatch
602 * queues of a cpu other than the one it is strongbound to).
603 *
604 * If the cpu we are running on does not appear to be a current
605 * offline target (we check cpu_inmotion to determine this - since
606 * we don't hold cpu_lock we may not see a recent store to that,
607 * so it's possible that we at times can grant a weak binding to a
608 * cpu that is an offline target, but that one request will not
609 * prevent the offline from succeeding) then we will always grant
610 * the weak binding request. This includes the case above where
611 * we grant a weakbinding not commensurate with our strong binding.
612 *
613 * If our cpu does appear to be an offline target then we're inclined
614 * not to grant the weakbinding request just yet - we'd prefer to
615 * migrate to another cpu and grant the request there. The
616 * exceptions are those cases where going through preemption code
617 * will not result in us changing cpu:
618 *
619 * . interrupts have already bypassed this case (see above)
620 * . we are already weakbound to this cpu (dispatcher code will
621 * always return us to the weakbound cpu)
622 * . preemption was disabled even before we disabled it above
623 * . we are strongbound to this cpu (if we're strongbound to
624 * another and not yet running there the trip through the
625 * dispatcher will move us to the strongbound cpu and we
626 * will grant the weak binding there)
627 */
628 if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
629 t->t_bound_cpu == cp) {
630 /*
631 * Don't be tempted to store to t_weakbound_cpu only on
632 * the first nested bind request - if we're interrupted
633 * after the increment of t_nomigrate and before the
634 * store to t_weakbound_cpu and the interrupt calls
635 * thread_nomigrate then the assertion in thread_allowmigrate
636 * would fail.
637 */
638 t->t_nomigrate++;
639 t->t_weakbound_cpu = cp;
640 membar_producer();
641 thread_unlock(curthread);
642 /*
643 * Now that we have dropped the thread_lock another thread
644 * can set our t_weakbound_cpu, and will try to migrate us
645 * to the strongbound cpu (which will not be prevented by
646 * preemption being disabled since we're about to enable
647 * preemption). We have granted the weakbinding to the current
648 * cpu, so again we are in the position that is is is possible
649 * that our weak and strong bindings differ. Again this
650 * is catered for by dispatcher code which will favour our
651 * weak binding.
652 */
653 kpreempt_enable();
654 } else {
655 /*
656 * Move to another cpu before granting the request by
657 * forcing this thread through preemption code. When we
658 * get to set{front,back}dq called from CL_PREEMPT()
659 * cpu_choose() will be used to select a cpu to queue
660 * us on - that will see cpu_inmotion and take
661 * steps to avoid returning us to this cpu.
662 */
663 cp->cpu_kprunrun = 1;
664 thread_unlock(curthread);
665 kpreempt_enable(); /* will call preempt() */
666 goto again;
667 }
668 }
669
670 void
671 thread_allowmigrate(void)
672 {
673 kthread_id_t t = curthread;
674
675 ASSERT(t->t_weakbound_cpu == CPU ||
676 (t->t_nomigrate < 0 && t->t_preempt > 0) ||
677 CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
678 getpil() >= DISP_LEVEL);
679
680 if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
681 getpil() >= DISP_LEVEL)
682 return;
683
684 if (t->t_nomigrate < 0) {
685 /*
686 * This thread was granted "weak binding" in the
687 * stronger form of kernel preemption disabling.
688 * Undo a level of nesting for both t_nomigrate
689 * and t_preempt.
690 */
691 ++t->t_nomigrate;
692 kpreempt_enable();
693 } else if (--t->t_nomigrate == 0) {
694 /*
695 * Time to drop the weak binding. We need to cater
696 * for the case where we're weakbound to a different
697 * cpu than that to which we're strongbound (a very
698 * temporary arrangement that must only persist until
699 * weak binding drops). We don't acquire thread_lock
700 * here so even as this code executes t_bound_cpu
701 * may be changing. So we disable preemption and
702 * a) in the case that t_bound_cpu changes while we
703 * have preemption disabled kprunrun will be set
704 * asynchronously, and b) if before disabling
705 * preemption we were already on a different cpu to
706 * our t_bound_cpu then we set kprunrun ourselves
707 * to force a trip through the dispatcher when
708 * preemption is enabled.
709 */
710 kpreempt_disable();
711 if (t->t_bound_cpu &&
712 t->t_weakbound_cpu != t->t_bound_cpu)
713 CPU->cpu_kprunrun = 1;
714 t->t_weakbound_cpu = NULL;
715 membar_producer();
716 kpreempt_enable();
717 }
718 }
719
720 /*
721 * weakbinding_stop can be used to temporarily cause weakbindings made
722 * with thread_nomigrate to be satisfied through the stronger action of
723 * kpreempt_disable. weakbinding_start recommences normal weakbinding.
724 */
725
726 void
727 weakbinding_stop(void)
728 {
729 ASSERT(MUTEX_HELD(&cpu_lock));
730 weakbindingbarrier = 1;
731 membar_producer(); /* make visible before subsequent thread_lock */
732 }
733
734 void
735 weakbinding_start(void)
736 {
737 ASSERT(MUTEX_HELD(&cpu_lock));
738 weakbindingbarrier = 0;
739 }
740
741 void
742 null_xcall(void)
743 {
744 }
745
746 /*
747 * This routine is called to place the CPUs in a safe place so that
748 * one of them can be taken off line or placed on line. What we are
749 * trying to do here is prevent a thread from traversing the list
750 * of active CPUs while we are changing it or from getting placed on
751 * the run queue of a CPU that has just gone off line. We do this by
752 * creating a thread with the highest possible prio for each CPU and
753 * having it call this routine. The advantage of this method is that
754 * we can eliminate all checks for CPU_ACTIVE in the disp routines.
755 * This makes disp faster at the expense of making p_online() slower
756 * which is a good trade off.
757 */
758 static void
759 cpu_pause(int index)
760 {
761 int s;
762 struct _cpu_pause_info *cpi = &cpu_pause_info;
763 volatile char *safe = &safe_list[index];
764 long lindex = index;
765
766 ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
767
768 while (*safe != PAUSE_DIE) {
769 *safe = PAUSE_READY;
770 membar_enter(); /* make sure stores are flushed */
771 sema_v(&cpi->cp_sem); /* signal requesting thread */
772
773 /*
774 * Wait here until all pause threads are running. That
775 * indicates that it's safe to do the spl. Until
776 * cpu_pause_info.cp_go is set, we don't want to spl
777 * because that might block clock interrupts needed
778 * to preempt threads on other CPUs.
779 */
780 while (cpi->cp_go == 0)
781 ;
782 /*
783 * Even though we are at the highest disp prio, we need
784 * to block out all interrupts below LOCK_LEVEL so that
785 * an intr doesn't come in, wake up a thread, and call
786 * setbackdq/setfrontdq.
787 */
788 s = splhigh();
789 /*
790 * if cpu_pause_func() has been set then call it using
791 * index as the argument, currently only used by
792 * cpr_suspend_cpus(). This function is used as the
793 * code to execute on the "paused" cpu's when a machine
794 * comes out of a sleep state and CPU's were powered off.
795 * (could also be used for hotplugging CPU's).
796 */
797 if (cpu_pause_func != NULL)
798 (*cpu_pause_func)((void *)lindex);
799
800 mach_cpu_pause(safe);
801
802 splx(s);
803 /*
804 * Waiting is at an end. Switch out of cpu_pause
805 * loop and resume useful work.
806 */
807 swtch();
808 }
809
810 mutex_enter(&pause_free_mutex);
811 *safe = PAUSE_DEAD;
812 cv_broadcast(&pause_free_cv);
813 mutex_exit(&pause_free_mutex);
814 }
815
816 /*
817 * Allow the cpus to start running again.
818 */
819 void
820 start_cpus()
821 {
822 int i;
823
824 ASSERT(MUTEX_HELD(&cpu_lock));
825 ASSERT(cpu_pause_info.cp_paused);
826 cpu_pause_info.cp_paused = NULL;
827 for (i = 0; i < NCPU; i++)
828 safe_list[i] = PAUSE_IDLE;
829 membar_enter(); /* make sure stores are flushed */
830 affinity_clear();
831 splx(cpu_pause_info.cp_spl);
832 kpreempt_enable();
833 }
834
835 /*
836 * Allocate a pause thread for a CPU.
837 */
838 static void
839 cpu_pause_alloc(cpu_t *cp)
840 {
841 kthread_id_t t;
842 long cpun = cp->cpu_id;
843
844 /*
845 * Note, v.v_nglobpris will not change value as long as I hold
846 * cpu_lock.
847 */
848 t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
849 0, &p0, TS_STOPPED, v.v_nglobpris - 1);
850 thread_lock(t);
851 t->t_bound_cpu = cp;
852 t->t_disp_queue = cp->cpu_disp;
853 t->t_affinitycnt = 1;
854 t->t_preempt = 1;
855 thread_unlock(t);
856 cp->cpu_pause_thread = t;
857 /*
858 * Registering a thread in the callback table is usually done
859 * in the initialization code of the thread. In this
860 * case, we do it right after thread creation because the
861 * thread itself may never run, and we need to register the
862 * fact that it is safe for cpr suspend.
863 */
864 CALLB_CPR_INIT_SAFE(t, "cpu_pause");
865 }
866
867 /*
868 * Free a pause thread for a CPU.
869 */
870 static void
871 cpu_pause_free(cpu_t *cp)
872 {
873 kthread_id_t t;
874 int cpun = cp->cpu_id;
875
876 ASSERT(MUTEX_HELD(&cpu_lock));
877 /*
878 * We have to get the thread and tell him to die.
879 */
880 if ((t = cp->cpu_pause_thread) == NULL) {
881 ASSERT(safe_list[cpun] == PAUSE_IDLE);
882 return;
883 }
884 thread_lock(t);
885 t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */
886 t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */
887 t->t_pri = v.v_nglobpris - 1;
888 ASSERT(safe_list[cpun] == PAUSE_IDLE);
889 safe_list[cpun] = PAUSE_DIE;
890 THREAD_TRANSITION(t);
891 setbackdq(t);
892 thread_unlock_nopreempt(t);
893
894 /*
895 * If we don't wait for the thread to actually die, it may try to
896 * run on the wrong cpu as part of an actual call to pause_cpus().
897 */
898 mutex_enter(&pause_free_mutex);
899 while (safe_list[cpun] != PAUSE_DEAD) {
900 cv_wait(&pause_free_cv, &pause_free_mutex);
901 }
902 mutex_exit(&pause_free_mutex);
903 safe_list[cpun] = PAUSE_IDLE;
904
905 cp->cpu_pause_thread = NULL;
906 }
907
908 /*
909 * Initialize basic structures for pausing CPUs.
910 */
911 void
912 cpu_pause_init()
913 {
914 sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
915 /*
916 * Create initial CPU pause thread.
917 */
918 cpu_pause_alloc(CPU);
919 }
920
921 /*
922 * Start the threads used to pause another CPU.
923 */
924 static int
925 cpu_pause_start(processorid_t cpu_id)
926 {
927 int i;
928 int cpu_count = 0;
929
930 for (i = 0; i < NCPU; i++) {
931 cpu_t *cp;
932 kthread_id_t t;
933
934 cp = cpu[i];
935 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
936 safe_list[i] = PAUSE_WAIT;
937 continue;
938 }
939
940 /*
941 * Skip CPU if it is quiesced or not yet started.
942 */
943 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
944 safe_list[i] = PAUSE_WAIT;
945 continue;
946 }
947
948 /*
949 * Start this CPU's pause thread.
950 */
951 t = cp->cpu_pause_thread;
952 thread_lock(t);
953 /*
954 * Reset the priority, since nglobpris may have
955 * changed since the thread was created, if someone
956 * has loaded the RT (or some other) scheduling
957 * class.
958 */
959 t->t_pri = v.v_nglobpris - 1;
960 THREAD_TRANSITION(t);
961 setbackdq(t);
962 thread_unlock_nopreempt(t);
963 ++cpu_count;
964 }
965 return (cpu_count);
966 }
967
968
969 /*
970 * Pause all of the CPUs except the one we are on by creating a high
971 * priority thread bound to those CPUs.
972 *
973 * Note that one must be extremely careful regarding code
974 * executed while CPUs are paused. Since a CPU may be paused
975 * while a thread scheduling on that CPU is holding an adaptive
976 * lock, code executed with CPUs paused must not acquire adaptive
977 * (or low-level spin) locks. Also, such code must not block,
978 * since the thread that is supposed to initiate the wakeup may
979 * never run.
980 *
981 * With a few exceptions, the restrictions on code executed with CPUs
982 * paused match those for code executed at high-level interrupt
983 * context.
984 */
985 void
986 pause_cpus(cpu_t *off_cp)
987 {
988 processorid_t cpu_id;
989 int i;
990 struct _cpu_pause_info *cpi = &cpu_pause_info;
991
992 ASSERT(MUTEX_HELD(&cpu_lock));
993 ASSERT(cpi->cp_paused == NULL);
994 cpi->cp_count = 0;
995 cpi->cp_go = 0;
996 for (i = 0; i < NCPU; i++)
997 safe_list[i] = PAUSE_IDLE;
998 kpreempt_disable();
999
1000 /*
1001 * If running on the cpu that is going offline, get off it.
1002 * This is so that it won't be necessary to rechoose a CPU
1003 * when done.
1004 */
1005 if (CPU == off_cp)
1006 cpu_id = off_cp->cpu_next_part->cpu_id;
1007 else
1008 cpu_id = CPU->cpu_id;
1009 affinity_set(cpu_id);
1010
1011 /*
1012 * Start the pause threads and record how many were started
1013 */
1014 cpi->cp_count = cpu_pause_start(cpu_id);
1015
1016 /*
1017 * Now wait for all CPUs to be running the pause thread.
1018 */
1019 while (cpi->cp_count > 0) {
1020 /*
1021 * Spin reading the count without grabbing the disp
1022 * lock to make sure we don't prevent the pause
1023 * threads from getting the lock.
1024 */
1025 while (sema_held(&cpi->cp_sem))
1026 ;
1027 if (sema_tryp(&cpi->cp_sem))
1028 --cpi->cp_count;
1029 }
1030 cpi->cp_go = 1; /* all have reached cpu_pause */
1031
1032 /*
1033 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1034 * to PAUSE_WAIT.)
1035 */
1036 for (i = 0; i < NCPU; i++) {
1037 while (safe_list[i] != PAUSE_WAIT)
1038 ;
1039 }
1040 cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */
1041 cpi->cp_paused = curthread;
1042 }
1043
1044 /*
1045 * Check whether the current thread has CPUs paused
1046 */
1047 int
1048 cpus_paused(void)
1049 {
1050 if (cpu_pause_info.cp_paused != NULL) {
1051 ASSERT(cpu_pause_info.cp_paused == curthread);
1052 return (1);
1053 }
1054 return (0);
1055 }
1056
1057 static cpu_t *
1058 cpu_get_all(processorid_t cpun)
1059 {
1060 ASSERT(MUTEX_HELD(&cpu_lock));
1061
1062 if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1063 return (NULL);
1064 return (cpu[cpun]);
1065 }
1066
1067 /*
1068 * Check whether cpun is a valid processor id and whether it should be
1069 * visible from the current zone. If it is, return a pointer to the
1070 * associated CPU structure.
1071 */
1072 cpu_t *
1073 cpu_get(processorid_t cpun)
1074 {
1075 cpu_t *c;
1076
1077 ASSERT(MUTEX_HELD(&cpu_lock));
1078 c = cpu_get_all(cpun);
1079 if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1080 zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1081 return (NULL);
1082 return (c);
1083 }
1084
1085 /*
1086 * The following functions should be used to check CPU states in the kernel.
1087 * They should be invoked with cpu_lock held. Kernel subsystems interested
1088 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1089 * states. Those are for user-land (and system call) use only.
1090 */
1091
1092 /*
1093 * Determine whether the CPU is online and handling interrupts.
1094 */
1095 int
1096 cpu_is_online(cpu_t *cpu)
1097 {
1098 ASSERT(MUTEX_HELD(&cpu_lock));
1099 return (cpu_flagged_online(cpu->cpu_flags));
1100 }
1101
1102 /*
1103 * Determine whether the CPU is offline (this includes spare and faulted).
1104 */
1105 int
1106 cpu_is_offline(cpu_t *cpu)
1107 {
1108 ASSERT(MUTEX_HELD(&cpu_lock));
1109 return (cpu_flagged_offline(cpu->cpu_flags));
1110 }
1111
1112 /*
1113 * Determine whether the CPU is powered off.
1114 */
1115 int
1116 cpu_is_poweredoff(cpu_t *cpu)
1117 {
1118 ASSERT(MUTEX_HELD(&cpu_lock));
1119 return (cpu_flagged_poweredoff(cpu->cpu_flags));
1120 }
1121
1122 /*
1123 * Determine whether the CPU is handling interrupts.
1124 */
1125 int
1126 cpu_is_nointr(cpu_t *cpu)
1127 {
1128 ASSERT(MUTEX_HELD(&cpu_lock));
1129 return (cpu_flagged_nointr(cpu->cpu_flags));
1130 }
1131
1132 /*
1133 * Determine whether the CPU is active (scheduling threads).
1134 */
1135 int
1136 cpu_is_active(cpu_t *cpu)
1137 {
1138 ASSERT(MUTEX_HELD(&cpu_lock));
1139 return (cpu_flagged_active(cpu->cpu_flags));
1140 }
1141
1142 /*
1143 * Same as above, but these require cpu_flags instead of cpu_t pointers.
1144 */
1145 int
1146 cpu_flagged_online(cpu_flag_t cpu_flags)
1147 {
1148 return (cpu_flagged_active(cpu_flags) &&
1149 (cpu_flags & CPU_ENABLE));
1150 }
1151
1152 int
1153 cpu_flagged_offline(cpu_flag_t cpu_flags)
1154 {
1155 return (((cpu_flags & CPU_POWEROFF) == 0) &&
1156 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1157 }
1158
1159 int
1160 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1161 {
1162 return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1163 }
1164
1165 int
1166 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1167 {
1168 return (cpu_flagged_active(cpu_flags) &&
1169 (cpu_flags & CPU_ENABLE) == 0);
1170 }
1171
1172 int
1173 cpu_flagged_active(cpu_flag_t cpu_flags)
1174 {
1175 return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1176 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1177 }
1178
1179 /*
1180 * Bring the indicated CPU online.
1181 */
1182 int
1183 cpu_online(cpu_t *cp)
1184 {
1185 int error = 0;
1186
1187 /*
1188 * Handle on-line request.
1189 * This code must put the new CPU on the active list before
1190 * starting it because it will not be paused, and will start
1191 * using the active list immediately. The real start occurs
1192 * when the CPU_QUIESCED flag is turned off.
1193 */
1194
1195 ASSERT(MUTEX_HELD(&cpu_lock));
1196
1197 /*
1198 * Put all the cpus into a known safe place.
1199 * No mutexes can be entered while CPUs are paused.
1200 */
1201 error = mp_cpu_start(cp); /* arch-dep hook */
1202 if (error == 0) {
1203 pg_cpupart_in(cp, cp->cpu_part);
1204 pause_cpus(NULL);
1205 cpu_add_active_internal(cp);
1206 if (cp->cpu_flags & CPU_FAULTED) {
1207 cp->cpu_flags &= ~CPU_FAULTED;
1208 mp_cpu_faulted_exit(cp);
1209 }
1210 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1211 CPU_SPARE);
1212 CPU_NEW_GENERATION(cp);
1213 start_cpus();
1214 cpu_stats_kstat_create(cp);
1215 cpu_create_intrstat(cp);
1216 lgrp_kstat_create(cp);
1217 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1218 cpu_intr_enable(cp); /* arch-dep hook */
1219 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1220 cpu_set_state(cp);
1221 cyclic_online(cp);
1222 /*
1223 * This has to be called only after cyclic_online(). This
1224 * function uses cyclics.
1225 */
1226 callout_cpu_online(cp);
1227 poke_cpu(cp->cpu_id);
1228 }
1229
1230 return (error);
1231 }
1232
1233 /*
1234 * Take the indicated CPU offline.
1235 */
1236 int
1237 cpu_offline(cpu_t *cp, int flags)
1238 {
1239 cpupart_t *pp;
1240 int error = 0;
1241 cpu_t *ncp;
1242 int intr_enable;
1243 int cyclic_off = 0;
1244 int callout_off = 0;
1245 int loop_count;
1246 int no_quiesce = 0;
1247 int (*bound_func)(struct cpu *, int);
1248 kthread_t *t;
1249 lpl_t *cpu_lpl;
1250 proc_t *p;
1251 int lgrp_diff_lpl;
1252 boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1253
1254 ASSERT(MUTEX_HELD(&cpu_lock));
1255
1256 /*
1257 * If we're going from faulted or spare to offline, just
1258 * clear these flags and update CPU state.
1259 */
1260 if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1261 if (cp->cpu_flags & CPU_FAULTED) {
1262 cp->cpu_flags &= ~CPU_FAULTED;
1263 mp_cpu_faulted_exit(cp);
1264 }
1265 cp->cpu_flags &= ~CPU_SPARE;
1266 cpu_set_state(cp);
1267 return (0);
1268 }
1269
1270 /*
1271 * Handle off-line request.
1272 */
1273 pp = cp->cpu_part;
1274 /*
1275 * Don't offline last online CPU in partition
1276 */
1277 if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1278 return (EBUSY);
1279 /*
1280 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1281 * if we were asked to.
1282 */
1283 error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1284 if (error != 0)
1285 return (error);
1286 /*
1287 * We shouldn't be bound to this CPU ourselves.
1288 */
1289 if (curthread->t_bound_cpu == cp)
1290 return (EBUSY);
1291
1292 /*
1293 * Tell interested parties that this CPU is going offline.
1294 */
1295 CPU_NEW_GENERATION(cp);
1296 cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1297
1298 /*
1299 * Tell the PG subsystem that the CPU is leaving the partition
1300 */
1301 pg_cpupart_out(cp, pp);
1302
1303 /*
1304 * Take the CPU out of interrupt participation so we won't find
1305 * bound kernel threads. If the architecture cannot completely
1306 * shut off interrupts on the CPU, don't quiesce it, but don't
1307 * run anything but interrupt thread... this is indicated by
1308 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1309 * off.
1310 */
1311 intr_enable = cp->cpu_flags & CPU_ENABLE;
1312 if (intr_enable)
1313 no_quiesce = cpu_intr_disable(cp);
1314
1315 /*
1316 * Record that we are aiming to offline this cpu. This acts as
1317 * a barrier to further weak binding requests in thread_nomigrate
1318 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1319 * lean away from this cpu. Further strong bindings are already
1320 * avoided since we hold cpu_lock. Since threads that are set
1321 * runnable around now and others coming off the target cpu are
1322 * directed away from the target, existing strong and weak bindings
1323 * (especially the latter) to the target cpu stand maximum chance of
1324 * being able to unbind during the short delay loop below (if other
1325 * unbound threads compete they may not see cpu in time to unbind
1326 * even if they would do so immediately.
1327 */
1328 cpu_inmotion = cp;
1329 membar_enter();
1330
1331 /*
1332 * Check for kernel threads (strong or weak) bound to that CPU.
1333 * Strongly bound threads may not unbind, and we'll have to return
1334 * EBUSY. Weakly bound threads should always disappear - we've
1335 * stopped more weak binding with cpu_inmotion and existing
1336 * bindings will drain imminently (they may not block). Nonetheless
1337 * we will wait for a fixed period for all bound threads to disappear.
1338 * Inactive interrupt threads are OK (they'll be in TS_FREE
1339 * state). If test finds some bound threads, wait a few ticks
1340 * to give short-lived threads (such as interrupts) chance to
1341 * complete. Note that if no_quiesce is set, i.e. this cpu
1342 * is required to service interrupts, then we take the route
1343 * that permits interrupt threads to be active (or bypassed).
1344 */
1345 bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1346
1347 again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1348 if (loop_count >= 5) {
1349 error = EBUSY; /* some threads still bound */
1350 break;
1351 }
1352
1353 /*
1354 * If some threads were assigned, give them
1355 * a chance to complete or move.
1356 *
1357 * This assumes that the clock_thread is not bound
1358 * to any CPU, because the clock_thread is needed to
1359 * do the delay(hz/100).
1360 *
1361 * Note: we still hold the cpu_lock while waiting for
1362 * the next clock tick. This is OK since it isn't
1363 * needed for anything else except processor_bind(2),
1364 * and system initialization. If we drop the lock,
1365 * we would risk another p_online disabling the last
1366 * processor.
1367 */
1368 delay(hz/100);
1369 }
1370
1371 if (error == 0 && callout_off == 0) {
1372 callout_cpu_offline(cp);
1373 callout_off = 1;
1374 }
1375
1376 if (error == 0 && cyclic_off == 0) {
1377 if (!cyclic_offline(cp)) {
1378 /*
1379 * We must have bound cyclics...
1380 */
1381 error = EBUSY;
1382 goto out;
1383 }
1384 cyclic_off = 1;
1385 }
1386
1387 /*
1388 * Call mp_cpu_stop() to perform any special operations
1389 * needed for this machine architecture to offline a CPU.
1390 */
1391 if (error == 0)
1392 error = mp_cpu_stop(cp); /* arch-dep hook */
1393
1394 /*
1395 * If that all worked, take the CPU offline and decrement
1396 * ncpus_online.
1397 */
1398 if (error == 0) {
1399 /*
1400 * Put all the cpus into a known safe place.
1401 * No mutexes can be entered while CPUs are paused.
1402 */
1403 pause_cpus(cp);
1404 /*
1405 * Repeat the operation, if necessary, to make sure that
1406 * all outstanding low-level interrupts run to completion
1407 * before we set the CPU_QUIESCED flag. It's also possible
1408 * that a thread has weak bound to the cpu despite our raising
1409 * cpu_inmotion above since it may have loaded that
1410 * value before the barrier became visible (this would have
1411 * to be the thread that was on the target cpu at the time
1412 * we raised the barrier).
1413 */
1414 if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1415 (*bound_func)(cp, 1)) {
1416 start_cpus();
1417 (void) mp_cpu_start(cp);
1418 goto again;
1419 }
1420 ncp = cp->cpu_next_part;
1421 cpu_lpl = cp->cpu_lpl;
1422 ASSERT(cpu_lpl != NULL);
1423
1424 /*
1425 * Remove the CPU from the list of active CPUs.
1426 */
1427 cpu_remove_active(cp);
1428
1429 /*
1430 * Walk the active process list and look for threads
1431 * whose home lgroup needs to be updated, or
1432 * the last CPU they run on is the one being offlined now.
1433 */
1434
1435 ASSERT(curthread->t_cpu != cp);
1436 for (p = practive; p != NULL; p = p->p_next) {
1437
1438 t = p->p_tlist;
1439
1440 if (t == NULL)
1441 continue;
1442
1443 lgrp_diff_lpl = 0;
1444
1445 do {
1446 ASSERT(t->t_lpl != NULL);
1447 /*
1448 * Taking last CPU in lpl offline
1449 * Rehome thread if it is in this lpl
1450 * Otherwise, update the count of how many
1451 * threads are in this CPU's lgroup but have
1452 * a different lpl.
1453 */
1454
1455 if (cpu_lpl->lpl_ncpu == 0) {
1456 if (t->t_lpl == cpu_lpl)
1457 lgrp_move_thread(t,
1458 lgrp_choose(t,
1459 t->t_cpupart), 0);
1460 else if (t->t_lpl->lpl_lgrpid ==
1461 cpu_lpl->lpl_lgrpid)
1462 lgrp_diff_lpl++;
1463 }
1464 ASSERT(t->t_lpl->lpl_ncpu > 0);
1465
1466 /*
1467 * Update CPU last ran on if it was this CPU
1468 */
1469 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1470 t->t_cpu = disp_lowpri_cpu(ncp,
1471 t->t_lpl, t->t_pri, NULL);
1472 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1473 t->t_weakbound_cpu == cp);
1474
1475 t = t->t_forw;
1476 } while (t != p->p_tlist);
1477
1478 /*
1479 * Didn't find any threads in the same lgroup as this
1480 * CPU with a different lpl, so remove the lgroup from
1481 * the process lgroup bitmask.
1482 */
1483
1484 if (lgrp_diff_lpl == 0)
1485 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1486 }
1487
1488 /*
1489 * Walk thread list looking for threads that need to be
1490 * rehomed, since there are some threads that are not in
1491 * their process's p_tlist.
1492 */
1493
1494 t = curthread;
1495 do {
1496 ASSERT(t != NULL && t->t_lpl != NULL);
1497
1498 /*
1499 * Rehome threads with same lpl as this CPU when this
1500 * is the last CPU in the lpl.
1501 */
1502
1503 if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1504 lgrp_move_thread(t,
1505 lgrp_choose(t, t->t_cpupart), 1);
1506
1507 ASSERT(t->t_lpl->lpl_ncpu > 0);
1508
1509 /*
1510 * Update CPU last ran on if it was this CPU
1511 */
1512
1513 if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1514 t->t_cpu = disp_lowpri_cpu(ncp,
1515 t->t_lpl, t->t_pri, NULL);
1516 }
1517 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1518 t->t_weakbound_cpu == cp);
1519 t = t->t_next;
1520
1521 } while (t != curthread);
1522 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1523 cp->cpu_flags |= CPU_OFFLINE;
1524 disp_cpu_inactive(cp);
1525 if (!no_quiesce)
1526 cp->cpu_flags |= CPU_QUIESCED;
1527 ncpus_online--;
1528 cpu_set_state(cp);
1529 cpu_inmotion = NULL;
1530 start_cpus();
1531 cpu_stats_kstat_destroy(cp);
1532 cpu_delete_intrstat(cp);
1533 lgrp_kstat_destroy(cp);
1534 }
1535
1536 out:
1537 cpu_inmotion = NULL;
1538
1539 /*
1540 * If we failed, re-enable interrupts.
1541 * Do this even if cpu_intr_disable returned an error, because
1542 * it may have partially disabled interrupts.
1543 */
1544 if (error && intr_enable)
1545 cpu_intr_enable(cp);
1546
1547 /*
1548 * If we failed, but managed to offline the cyclic subsystem on this
1549 * CPU, bring it back online.
1550 */
1551 if (error && cyclic_off)
1552 cyclic_online(cp);
1553
1554 /*
1555 * If we failed, but managed to offline callouts on this CPU,
1556 * bring it back online.
1557 */
1558 if (error && callout_off)
1559 callout_cpu_online(cp);
1560
1561 /*
1562 * If we failed, tell the PG subsystem that the CPU is back
1563 */
1564 pg_cpupart_in(cp, pp);
1565
1566 /*
1567 * If we failed, we need to notify everyone that this CPU is back on.
1568 */
1569 if (error != 0) {
1570 CPU_NEW_GENERATION(cp);
1571 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1572 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1573 }
1574
1575 return (error);
1576 }
1577
1578 /*
1579 * Mark the indicated CPU as faulted, taking it offline.
1580 */
1581 int
1582 cpu_faulted(cpu_t *cp, int flags)
1583 {
1584 int error = 0;
1585
1586 ASSERT(MUTEX_HELD(&cpu_lock));
1587 ASSERT(!cpu_is_poweredoff(cp));
1588
1589 if (cpu_is_offline(cp)) {
1590 cp->cpu_flags &= ~CPU_SPARE;
1591 cp->cpu_flags |= CPU_FAULTED;
1592 mp_cpu_faulted_enter(cp);
1593 cpu_set_state(cp);
1594 return (0);
1595 }
1596
1597 if ((error = cpu_offline(cp, flags)) == 0) {
1598 cp->cpu_flags |= CPU_FAULTED;
1599 mp_cpu_faulted_enter(cp);
1600 cpu_set_state(cp);
1601 }
1602
1603 return (error);
1604 }
1605
1606 /*
1607 * Mark the indicated CPU as a spare, taking it offline.
1608 */
1609 int
1610 cpu_spare(cpu_t *cp, int flags)
1611 {
1612 int error = 0;
1613
1614 ASSERT(MUTEX_HELD(&cpu_lock));
1615 ASSERT(!cpu_is_poweredoff(cp));
1616
1617 if (cpu_is_offline(cp)) {
1618 if (cp->cpu_flags & CPU_FAULTED) {
1619 cp->cpu_flags &= ~CPU_FAULTED;
1620 mp_cpu_faulted_exit(cp);
1621 }
1622 cp->cpu_flags |= CPU_SPARE;
1623 cpu_set_state(cp);
1624 return (0);
1625 }
1626
1627 if ((error = cpu_offline(cp, flags)) == 0) {
1628 cp->cpu_flags |= CPU_SPARE;
1629 cpu_set_state(cp);
1630 }
1631
1632 return (error);
1633 }
1634
1635 /*
1636 * Take the indicated CPU from poweroff to offline.
1637 */
1638 int
1639 cpu_poweron(cpu_t *cp)
1640 {
1641 int error = ENOTSUP;
1642
1643 ASSERT(MUTEX_HELD(&cpu_lock));
1644 ASSERT(cpu_is_poweredoff(cp));
1645
1646 error = mp_cpu_poweron(cp); /* arch-dep hook */
1647 if (error == 0)
1648 cpu_set_state(cp);
1649
1650 return (error);
1651 }
1652
1653 /*
1654 * Take the indicated CPU from any inactive state to powered off.
1655 */
1656 int
1657 cpu_poweroff(cpu_t *cp)
1658 {
1659 int error = ENOTSUP;
1660
1661 ASSERT(MUTEX_HELD(&cpu_lock));
1662 ASSERT(cpu_is_offline(cp));
1663
1664 if (!(cp->cpu_flags & CPU_QUIESCED))
1665 return (EBUSY); /* not completely idle */
1666
1667 error = mp_cpu_poweroff(cp); /* arch-dep hook */
1668 if (error == 0)
1669 cpu_set_state(cp);
1670
1671 return (error);
1672 }
1673
1674 /*
1675 * Initialize the Sequential CPU id lookup table
1676 */
1677 void
1678 cpu_seq_tbl_init()
1679 {
1680 cpu_t **tbl;
1681
1682 tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1683 tbl[0] = CPU;
1684
1685 cpu_seq = tbl;
1686 }
1687
1688 /*
1689 * Initialize the CPU lists for the first CPU.
1690 */
1691 void
1692 cpu_list_init(cpu_t *cp)
1693 {
1694 cp->cpu_next = cp;
1695 cp->cpu_prev = cp;
1696 cpu_list = cp;
1697 clock_cpu_list = cp;
1698
1699 cp->cpu_next_onln = cp;
1700 cp->cpu_prev_onln = cp;
1701 cpu_active = cp;
1702
1703 cp->cpu_seqid = 0;
1704 CPUSET_ADD(cpu_seqid_inuse, 0);
1705
1706 /*
1707 * Bootstrap cpu_seq using cpu_list
1708 * The cpu_seq[] table will be dynamically allocated
1709 * when kmem later becomes available (but before going MP)
1710 */
1711 cpu_seq = &cpu_list;
1712
1713 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1714 cp_default.cp_cpulist = cp;
1715 cp_default.cp_ncpus = 1;
1716 cp->cpu_next_part = cp;
1717 cp->cpu_prev_part = cp;
1718 cp->cpu_part = &cp_default;
1719
1720 CPUSET_ADD(cpu_available, cp->cpu_id);
1721 }
1722
1723 /*
1724 * Insert a CPU into the list of available CPUs.
1725 */
1726 void
1727 cpu_add_unit(cpu_t *cp)
1728 {
1729 int seqid;
1730
1731 ASSERT(MUTEX_HELD(&cpu_lock));
1732 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1733
1734 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1735
1736 /*
1737 * Note: most users of the cpu_list will grab the
1738 * cpu_lock to insure that it isn't modified. However,
1739 * certain users can't or won't do that. To allow this
1740 * we pause the other cpus. Users who walk the list
1741 * without cpu_lock, must disable kernel preemption
1742 * to insure that the list isn't modified underneath
1743 * them. Also, any cached pointers to cpu structures
1744 * must be revalidated by checking to see if the
1745 * cpu_next pointer points to itself. This check must
1746 * be done with the cpu_lock held or kernel preemption
1747 * disabled. This check relies upon the fact that
1748 * old cpu structures are not free'ed or cleared after
1749 * then are removed from the cpu_list.
1750 *
1751 * Note that the clock code walks the cpu list dereferencing
1752 * the cpu_part pointer, so we need to initialize it before
1753 * adding the cpu to the list.
1754 */
1755 cp->cpu_part = &cp_default;
1756 (void) pause_cpus(NULL);
1757 cp->cpu_next = cpu_list;
1758 cp->cpu_prev = cpu_list->cpu_prev;
1759 cpu_list->cpu_prev->cpu_next = cp;
1760 cpu_list->cpu_prev = cp;
1761 start_cpus();
1762
1763 for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1764 continue;
1765 CPUSET_ADD(cpu_seqid_inuse, seqid);
1766 cp->cpu_seqid = seqid;
1767
1768 if (seqid > max_cpu_seqid_ever)
1769 max_cpu_seqid_ever = seqid;
1770
1771 ASSERT(ncpus < max_ncpus);
1772 ncpus++;
1773 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1774 cpu[cp->cpu_id] = cp;
1775 CPUSET_ADD(cpu_available, cp->cpu_id);
1776 cpu_seq[cp->cpu_seqid] = cp;
1777
1778 /*
1779 * allocate a pause thread for this CPU.
1780 */
1781 cpu_pause_alloc(cp);
1782
1783 /*
1784 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1785 * link them into a list of just that CPU.
1786 * This is so that disp_lowpri_cpu will work for thread_create in
1787 * pause_cpus() when called from the startup thread in a new CPU.
1788 */
1789 cp->cpu_next_onln = cp;
1790 cp->cpu_prev_onln = cp;
1791 cpu_info_kstat_create(cp);
1792 cp->cpu_next_part = cp;
1793 cp->cpu_prev_part = cp;
1794
1795 init_cpu_mstate(cp, CMS_SYSTEM);
1796
1797 pool_pset_mod = gethrtime();
1798 }
1799
1800 /*
1801 * Do the opposite of cpu_add_unit().
1802 */
1803 void
1804 cpu_del_unit(int cpuid)
1805 {
1806 struct cpu *cp, *cpnext;
1807
1808 ASSERT(MUTEX_HELD(&cpu_lock));
1809 cp = cpu[cpuid];
1810 ASSERT(cp != NULL);
1811
1812 ASSERT(cp->cpu_next_onln == cp);
1813 ASSERT(cp->cpu_prev_onln == cp);
1814 ASSERT(cp->cpu_next_part == cp);
1815 ASSERT(cp->cpu_prev_part == cp);
1816
1817 /*
1818 * Tear down the CPU's physical ID cache, and update any
1819 * processor groups
1820 */
1821 pg_cpu_fini(cp, NULL);
1822 pghw_physid_destroy(cp);
1823
1824 /*
1825 * Destroy kstat stuff.
1826 */
1827 cpu_info_kstat_destroy(cp);
1828 term_cpu_mstate(cp);
1829 /*
1830 * Free up pause thread.
1831 */
1832 cpu_pause_free(cp);
1833 CPUSET_DEL(cpu_available, cp->cpu_id);
1834 cpu[cp->cpu_id] = NULL;
1835 cpu_seq[cp->cpu_seqid] = NULL;
1836
1837 /*
1838 * The clock thread and mutex_vector_enter cannot hold the
1839 * cpu_lock while traversing the cpu list, therefore we pause
1840 * all other threads by pausing the other cpus. These, and any
1841 * other routines holding cpu pointers while possibly sleeping
1842 * must be sure to call kpreempt_disable before processing the
1843 * list and be sure to check that the cpu has not been deleted
1844 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1845 * to keep the deleted cpu structure around.
1846 *
1847 * Note that this MUST be done AFTER cpu_available
1848 * has been updated so that we don't waste time
1849 * trying to pause the cpu we're trying to delete.
1850 */
1851 (void) pause_cpus(NULL);
1852
1853 cpnext = cp->cpu_next;
1854 cp->cpu_prev->cpu_next = cp->cpu_next;
1855 cp->cpu_next->cpu_prev = cp->cpu_prev;
1856 if (cp == cpu_list)
1857 cpu_list = cpnext;
1858
1859 /*
1860 * Signals that the cpu has been deleted (see above).
1861 */
1862 cp->cpu_next = NULL;
1863 cp->cpu_prev = NULL;
1864
1865 start_cpus();
1866
1867 CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1868 ncpus--;
1869 lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1870
1871 pool_pset_mod = gethrtime();
1872 }
1873
1874 /*
1875 * Add a CPU to the list of active CPUs.
1876 * This routine must not get any locks, because other CPUs are paused.
1877 */
1878 static void
1879 cpu_add_active_internal(cpu_t *cp)
1880 {
1881 cpupart_t *pp = cp->cpu_part;
1882
1883 ASSERT(MUTEX_HELD(&cpu_lock));
1884 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1885
1886 ncpus_online++;
1887 cpu_set_state(cp);
1888 cp->cpu_next_onln = cpu_active;
1889 cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1890 cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1891 cpu_active->cpu_prev_onln = cp;
1892
1893 if (pp->cp_cpulist) {
1894 cp->cpu_next_part = pp->cp_cpulist;
1895 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1896 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1897 pp->cp_cpulist->cpu_prev_part = cp;
1898 } else {
1899 ASSERT(pp->cp_ncpus == 0);
1900 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1901 }
1902 pp->cp_ncpus++;
1903 if (pp->cp_ncpus == 1) {
1904 cp_numparts_nonempty++;
1905 ASSERT(cp_numparts_nonempty != 0);
1906 }
1907
1908 pg_cpu_active(cp);
1909 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1910
1911 bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1912 }
1913
1914 /*
1915 * Add a CPU to the list of active CPUs.
1916 * This is called from machine-dependent layers when a new CPU is started.
1917 */
1918 void
1919 cpu_add_active(cpu_t *cp)
1920 {
1921 pg_cpupart_in(cp, cp->cpu_part);
1922
1923 pause_cpus(NULL);
1924 cpu_add_active_internal(cp);
1925 start_cpus();
1926
1927 cpu_stats_kstat_create(cp);
1928 cpu_create_intrstat(cp);
1929 lgrp_kstat_create(cp);
1930 cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1931 }
1932
1933
1934 /*
1935 * Remove a CPU from the list of active CPUs.
1936 * This routine must not get any locks, because other CPUs are paused.
1937 */
1938 /* ARGSUSED */
1939 static void
1940 cpu_remove_active(cpu_t *cp)
1941 {
1942 cpupart_t *pp = cp->cpu_part;
1943
1944 ASSERT(MUTEX_HELD(&cpu_lock));
1945 ASSERT(cp->cpu_next_onln != cp); /* not the last one */
1946 ASSERT(cp->cpu_prev_onln != cp); /* not the last one */
1947
1948 pg_cpu_inactive(cp);
1949
1950 lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1951
1952 if (cp == clock_cpu_list)
1953 clock_cpu_list = cp->cpu_next_onln;
1954
1955 cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1956 cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1957 if (cpu_active == cp) {
1958 cpu_active = cp->cpu_next_onln;
1959 }
1960 cp->cpu_next_onln = cp;
1961 cp->cpu_prev_onln = cp;
1962
1963 cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1964 cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1965 if (pp->cp_cpulist == cp) {
1966 pp->cp_cpulist = cp->cpu_next_part;
1967 ASSERT(pp->cp_cpulist != cp);
1968 }
1969 cp->cpu_next_part = cp;
1970 cp->cpu_prev_part = cp;
1971 pp->cp_ncpus--;
1972 if (pp->cp_ncpus == 0) {
1973 cp_numparts_nonempty--;
1974 ASSERT(cp_numparts_nonempty != 0);
1975 }
1976 }
1977
1978 /*
1979 * Routine used to setup a newly inserted CPU in preparation for starting
1980 * it running code.
1981 */
1982 int
1983 cpu_configure(int cpuid)
1984 {
1985 int retval = 0;
1986
1987 ASSERT(MUTEX_HELD(&cpu_lock));
1988
1989 /*
1990 * Some structures are statically allocated based upon
1991 * the maximum number of cpus the system supports. Do not
1992 * try to add anything beyond this limit.
1993 */
1994 if (cpuid < 0 || cpuid >= NCPU) {
1995 return (EINVAL);
1996 }
1997
1998 if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
1999 return (EALREADY);
2000 }
2001
2002 if ((retval = mp_cpu_configure(cpuid)) != 0) {
2003 return (retval);
2004 }
2005
2006 cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2007 cpu_set_state(cpu[cpuid]);
2008 retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2009 if (retval != 0)
2010 (void) mp_cpu_unconfigure(cpuid);
2011
2012 return (retval);
2013 }
2014
2015 /*
2016 * Routine used to cleanup a CPU that has been powered off. This will
2017 * destroy all per-cpu information related to this cpu.
2018 */
2019 int
2020 cpu_unconfigure(int cpuid)
2021 {
2022 int error;
2023
2024 ASSERT(MUTEX_HELD(&cpu_lock));
2025
2026 if (cpu[cpuid] == NULL) {
2027 return (ENODEV);
2028 }
2029
2030 if (cpu[cpuid]->cpu_flags == 0) {
2031 return (EALREADY);
2032 }
2033
2034 if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2035 return (EBUSY);
2036 }
2037
2038 if (cpu[cpuid]->cpu_props != NULL) {
2039 (void) nvlist_free(cpu[cpuid]->cpu_props);
2040 cpu[cpuid]->cpu_props = NULL;
2041 }
2042
2043 error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2044
2045 if (error != 0)
2046 return (error);
2047
2048 return (mp_cpu_unconfigure(cpuid));
2049 }
2050
2051 /*
2052 * Routines for registering and de-registering cpu_setup callback functions.
2053 *
2054 * Caller's context
2055 * These routines must not be called from a driver's attach(9E) or
2056 * detach(9E) entry point.
2057 *
2058 * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2059 */
2060
2061 /*
2062 * Ideally, these would be dynamically allocated and put into a linked
2063 * list; however that is not feasible because the registration routine
2064 * has to be available before the kmem allocator is working (in fact,
2065 * it is called by the kmem allocator init code). In any case, there
2066 * are quite a few extra entries for future users.
2067 */
2068 #define NCPU_SETUPS 20
2069
2070 struct cpu_setup {
2071 cpu_setup_func_t *func;
2072 void *arg;
2073 } cpu_setups[NCPU_SETUPS];
2074
2075 void
2076 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2077 {
2078 int i;
2079
2080 ASSERT(MUTEX_HELD(&cpu_lock));
2081
2082 for (i = 0; i < NCPU_SETUPS; i++)
2083 if (cpu_setups[i].func == NULL)
2084 break;
2085 if (i >= NCPU_SETUPS)
2086 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2087
2088 cpu_setups[i].func = func;
2089 cpu_setups[i].arg = arg;
2090 }
2091
2092 void
2093 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2094 {
2095 int i;
2096
2097 ASSERT(MUTEX_HELD(&cpu_lock));
2098
2099 for (i = 0; i < NCPU_SETUPS; i++)
2100 if ((cpu_setups[i].func == func) &&
2101 (cpu_setups[i].arg == arg))
2102 break;
2103 if (i >= NCPU_SETUPS)
2104 cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2105 "deregister");
2106
2107 cpu_setups[i].func = NULL;
2108 cpu_setups[i].arg = 0;
2109 }
2110
2111 /*
2112 * Call any state change hooks for this CPU, ignore any errors.
2113 */
2114 void
2115 cpu_state_change_notify(int id, cpu_setup_t what)
2116 {
2117 int i;
2118
2119 ASSERT(MUTEX_HELD(&cpu_lock));
2120
2121 for (i = 0; i < NCPU_SETUPS; i++) {
2122 if (cpu_setups[i].func != NULL) {
2123 cpu_setups[i].func(what, id, cpu_setups[i].arg);
2124 }
2125 }
2126 }
2127
2128 /*
2129 * Call any state change hooks for this CPU, undo it if error found.
2130 */
2131 static int
2132 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2133 {
2134 int i;
2135 int retval = 0;
2136
2137 ASSERT(MUTEX_HELD(&cpu_lock));
2138
2139 for (i = 0; i < NCPU_SETUPS; i++) {
2140 if (cpu_setups[i].func != NULL) {
2141 retval = cpu_setups[i].func(what, id,
2142 cpu_setups[i].arg);
2143 if (retval) {
2144 for (i--; i >= 0; i--) {
2145 if (cpu_setups[i].func != NULL)
2146 cpu_setups[i].func(undo,
2147 id, cpu_setups[i].arg);
2148 }
2149 break;
2150 }
2151 }
2152 }
2153 return (retval);
2154 }
2155
2156 /*
2157 * Export information about this CPU via the kstat mechanism.
2158 */
2159 static struct {
2160 kstat_named_t ci_state;
2161 kstat_named_t ci_state_begin;
2162 kstat_named_t ci_cpu_type;
2163 kstat_named_t ci_fpu_type;
2164 kstat_named_t ci_clock_MHz;
2165 kstat_named_t ci_chip_id;
2166 kstat_named_t ci_implementation;
2167 kstat_named_t ci_brandstr;
2168 kstat_named_t ci_core_id;
2169 kstat_named_t ci_curr_clock_Hz;
2170 kstat_named_t ci_supp_freq_Hz;
2171 kstat_named_t ci_pg_id;
2172 #if defined(__sparcv9)
2173 kstat_named_t ci_device_ID;
2174 kstat_named_t ci_cpu_fru;
2175 #endif
2176 #if defined(__x86)
2177 kstat_named_t ci_vendorstr;
2178 kstat_named_t ci_family;
2179 kstat_named_t ci_model;
2180 kstat_named_t ci_step;
2181 kstat_named_t ci_clogid;
2182 kstat_named_t ci_pkg_core_id;
2183 kstat_named_t ci_ncpuperchip;
2184 kstat_named_t ci_ncoreperchip;
2185 kstat_named_t ci_max_cstates;
2186 kstat_named_t ci_curr_cstate;
2187 kstat_named_t ci_cacheid;
2188 kstat_named_t ci_sktstr;
2189 #endif
2190 } cpu_info_template = {
2191 { "state", KSTAT_DATA_CHAR },
2192 { "state_begin", KSTAT_DATA_LONG },
2193 { "cpu_type", KSTAT_DATA_CHAR },
2194 { "fpu_type", KSTAT_DATA_CHAR },
2195 { "clock_MHz", KSTAT_DATA_LONG },
2196 { "chip_id", KSTAT_DATA_LONG },
2197 { "implementation", KSTAT_DATA_STRING },
2198 { "brand", KSTAT_DATA_STRING },
2199 { "core_id", KSTAT_DATA_LONG },
2200 { "current_clock_Hz", KSTAT_DATA_UINT64 },
2201 { "supported_frequencies_Hz", KSTAT_DATA_STRING },
2202 { "pg_id", KSTAT_DATA_LONG },
2203 #if defined(__sparcv9)
2204 { "device_ID", KSTAT_DATA_UINT64 },
2205 { "cpu_fru", KSTAT_DATA_STRING },
2206 #endif
2207 #if defined(__x86)
2208 { "vendor_id", KSTAT_DATA_STRING },
2209 { "family", KSTAT_DATA_INT32 },
2210 { "model", KSTAT_DATA_INT32 },
2211 { "stepping", KSTAT_DATA_INT32 },
2212 { "clog_id", KSTAT_DATA_INT32 },
2213 { "pkg_core_id", KSTAT_DATA_LONG },
2214 { "ncpu_per_chip", KSTAT_DATA_INT32 },
2215 { "ncore_per_chip", KSTAT_DATA_INT32 },
2216 { "supported_max_cstates", KSTAT_DATA_INT32 },
2217 { "current_cstate", KSTAT_DATA_INT32 },
2218 { "cache_id", KSTAT_DATA_INT32 },
2219 { "socket_type", KSTAT_DATA_STRING },
2220 #endif
2221 };
2222
2223 static kmutex_t cpu_info_template_lock;
2224
2225 static int
2226 cpu_info_kstat_update(kstat_t *ksp, int rw)
2227 {
2228 cpu_t *cp = ksp->ks_private;
2229 const char *pi_state;
2230
2231 if (rw == KSTAT_WRITE)
2232 return (EACCES);
2233
2234 #if defined(__x86)
2235 /* Is the cpu still initialising itself? */
2236 if (cpuid_checkpass(cp, 1) == 0)
2237 return (ENXIO);
2238 #endif
2239 switch (cp->cpu_type_info.pi_state) {
2240 case P_ONLINE:
2241 pi_state = PS_ONLINE;
2242 break;
2243 case P_POWEROFF:
2244 pi_state = PS_POWEROFF;
2245 break;
2246 case P_NOINTR:
2247 pi_state = PS_NOINTR;
2248 break;
2249 case P_FAULTED:
2250 pi_state = PS_FAULTED;
2251 break;
2252 case P_SPARE:
2253 pi_state = PS_SPARE;
2254 break;
2255 case P_OFFLINE:
2256 pi_state = PS_OFFLINE;
2257 break;
2258 default:
2259 pi_state = "unknown";
2260 }
2261 (void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2262 cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2263 (void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2264 cp->cpu_type_info.pi_processor_type, 15);
2265 (void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2266 cp->cpu_type_info.pi_fputypes, 15);
2267 cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2268 cpu_info_template.ci_chip_id.value.l =
2269 pg_plat_hw_instance_id(cp, PGHW_CHIP);
2270 kstat_named_setstr(&cpu_info_template.ci_implementation,
2271 cp->cpu_idstr);
2272 kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2273 cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2274 cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2275 cp->cpu_curr_clock;
2276 cpu_info_template.ci_pg_id.value.l =
2277 cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2278 cp->cpu_pg->cmt_lineage->pg_id : -1;
2279 kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2280 cp->cpu_supp_freqs);
2281 #if defined(__sparcv9)
2282 cpu_info_template.ci_device_ID.value.ui64 =
2283 cpunodes[cp->cpu_id].device_id;
2284 kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2285 #endif
2286 #if defined(__x86)
2287 kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2288 cpuid_getvendorstr(cp));
2289 cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2290 cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2291 cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2292 cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2293 cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2294 cpu_info_template.ci_ncoreperchip.value.l =
2295 cpuid_get_ncore_per_chip(cp);
2296 cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2297 cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2298 cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2299 cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2300 kstat_named_setstr(&cpu_info_template.ci_sktstr,
2301 cpuid_getsocketstr(cp));
2302 #endif
2303
2304 return (0);
2305 }
2306
2307 static void
2308 cpu_info_kstat_create(cpu_t *cp)
2309 {
2310 zoneid_t zoneid;
2311
2312 ASSERT(MUTEX_HELD(&cpu_lock));
2313
2314 if (pool_pset_enabled())
2315 zoneid = GLOBAL_ZONEID;
2316 else
2317 zoneid = ALL_ZONES;
2318 if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2319 NULL, "misc", KSTAT_TYPE_NAMED,
2320 sizeof (cpu_info_template) / sizeof (kstat_named_t),
2321 KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2322 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2323 #if defined(__sparcv9)
2324 cp->cpu_info_kstat->ks_data_size +=
2325 strlen(cpu_fru_fmri(cp)) + 1;
2326 #endif
2327 #if defined(__x86)
2328 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2329 #endif
2330 if (cp->cpu_supp_freqs != NULL)
2331 cp->cpu_info_kstat->ks_data_size +=
2332 strlen(cp->cpu_supp_freqs) + 1;
2333 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2334 cp->cpu_info_kstat->ks_data = &cpu_info_template;
2335 cp->cpu_info_kstat->ks_private = cp;
2336 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2337 kstat_install(cp->cpu_info_kstat);
2338 }
2339 }
2340
2341 static void
2342 cpu_info_kstat_destroy(cpu_t *cp)
2343 {
2344 ASSERT(MUTEX_HELD(&cpu_lock));
2345
2346 kstat_delete(cp->cpu_info_kstat);
2347 cp->cpu_info_kstat = NULL;
2348 }
2349
2350 /*
2351 * Create and install kstats for the boot CPU.
2352 */
2353 void
2354 cpu_kstat_init(cpu_t *cp)
2355 {
2356 mutex_enter(&cpu_lock);
2357 cpu_info_kstat_create(cp);
2358 cpu_stats_kstat_create(cp);
2359 cpu_create_intrstat(cp);
2360 cpu_set_state(cp);
2361 mutex_exit(&cpu_lock);
2362 }
2363
2364 /*
2365 * Make visible to the zone that subset of the cpu information that would be
2366 * initialized when a cpu is configured (but still offline).
2367 */
2368 void
2369 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2370 {
2371 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2372
2373 ASSERT(MUTEX_HELD(&cpu_lock));
2374 ASSERT(pool_pset_enabled());
2375 ASSERT(cp != NULL);
2376
2377 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2378 zone->zone_ncpus++;
2379 ASSERT(zone->zone_ncpus <= ncpus);
2380 }
2381 if (cp->cpu_info_kstat != NULL)
2382 kstat_zone_add(cp->cpu_info_kstat, zoneid);
2383 }
2384
2385 /*
2386 * Make visible to the zone that subset of the cpu information that would be
2387 * initialized when a previously configured cpu is onlined.
2388 */
2389 void
2390 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2391 {
2392 kstat_t *ksp;
2393 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2394 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2395 processorid_t cpun;
2396
2397 ASSERT(MUTEX_HELD(&cpu_lock));
2398 ASSERT(pool_pset_enabled());
2399 ASSERT(cp != NULL);
2400 ASSERT(cpu_is_active(cp));
2401
2402 cpun = cp->cpu_id;
2403 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2404 zone->zone_ncpus_online++;
2405 ASSERT(zone->zone_ncpus_online <= ncpus_online);
2406 }
2407 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2408 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2409 != NULL) {
2410 kstat_zone_add(ksp, zoneid);
2411 kstat_rele(ksp);
2412 }
2413 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2414 kstat_zone_add(ksp, zoneid);
2415 kstat_rele(ksp);
2416 }
2417 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2418 kstat_zone_add(ksp, zoneid);
2419 kstat_rele(ksp);
2420 }
2421 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2422 NULL) {
2423 kstat_zone_add(ksp, zoneid);
2424 kstat_rele(ksp);
2425 }
2426 }
2427
2428 /*
2429 * Update relevant kstats such that cpu is now visible to processes
2430 * executing in specified zone.
2431 */
2432 void
2433 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2434 {
2435 cpu_visibility_configure(cp, zone);
2436 if (cpu_is_active(cp))
2437 cpu_visibility_online(cp, zone);
2438 }
2439
2440 /*
2441 * Make invisible to the zone that subset of the cpu information that would be
2442 * torn down when a previously offlined cpu is unconfigured.
2443 */
2444 void
2445 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2446 {
2447 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2448
2449 ASSERT(MUTEX_HELD(&cpu_lock));
2450 ASSERT(pool_pset_enabled());
2451 ASSERT(cp != NULL);
2452
2453 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2454 ASSERT(zone->zone_ncpus != 0);
2455 zone->zone_ncpus--;
2456 }
2457 if (cp->cpu_info_kstat)
2458 kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2459 }
2460
2461 /*
2462 * Make invisible to the zone that subset of the cpu information that would be
2463 * torn down when a cpu is offlined (but still configured).
2464 */
2465 void
2466 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2467 {
2468 kstat_t *ksp;
2469 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2470 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2471 processorid_t cpun;
2472
2473 ASSERT(MUTEX_HELD(&cpu_lock));
2474 ASSERT(pool_pset_enabled());
2475 ASSERT(cp != NULL);
2476 ASSERT(cpu_is_active(cp));
2477
2478 cpun = cp->cpu_id;
2479 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2480 ASSERT(zone->zone_ncpus_online != 0);
2481 zone->zone_ncpus_online--;
2482 }
2483
2484 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2485 NULL) {
2486 kstat_zone_remove(ksp, zoneid);
2487 kstat_rele(ksp);
2488 }
2489 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2490 kstat_zone_remove(ksp, zoneid);
2491 kstat_rele(ksp);
2492 }
2493 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2494 kstat_zone_remove(ksp, zoneid);
2495 kstat_rele(ksp);
2496 }
2497 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2498 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2499 != NULL) {
2500 kstat_zone_remove(ksp, zoneid);
2501 kstat_rele(ksp);
2502 }
2503 }
2504
2505 /*
2506 * Update relevant kstats such that cpu is no longer visible to processes
2507 * executing in specified zone.
2508 */
2509 void
2510 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2511 {
2512 if (cpu_is_active(cp))
2513 cpu_visibility_offline(cp, zone);
2514 cpu_visibility_unconfigure(cp, zone);
2515 }
2516
2517 /*
2518 * Bind a thread to a CPU as requested.
2519 */
2520 int
2521 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2522 int *error)
2523 {
2524 processorid_t binding;
2525 cpu_t *cp = NULL;
2526
2527 ASSERT(MUTEX_HELD(&cpu_lock));
2528 ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2529
2530 thread_lock(tp);
2531
2532 /*
2533 * Record old binding, but change the obind, which was initialized
2534 * to PBIND_NONE, only if this thread has a binding. This avoids
2535 * reporting PBIND_NONE for a process when some LWPs are bound.
2536 */
2537 binding = tp->t_bind_cpu;
2538 if (binding != PBIND_NONE)
2539 *obind = binding; /* record old binding */
2540
2541 switch (bind) {
2542 case PBIND_QUERY:
2543 /* Just return the old binding */
2544 thread_unlock(tp);
2545 return (0);
2546
2547 case PBIND_QUERY_TYPE:
2548 /* Return the binding type */
2549 *obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2550 thread_unlock(tp);
2551 return (0);
2552
2553 case PBIND_SOFT:
2554 /*
2555 * Set soft binding for this thread and return the actual
2556 * binding
2557 */
2558 TB_CPU_SOFT_SET(tp);
2559 thread_unlock(tp);
2560 return (0);
2561
2562 case PBIND_HARD:
2563 /*
2564 * Set hard binding for this thread and return the actual
2565 * binding
2566 */
2567 TB_CPU_HARD_SET(tp);
2568 thread_unlock(tp);
2569 return (0);
2570
2571 default:
2572 break;
2573 }
2574
2575 /*
2576 * If this thread/LWP cannot be bound because of permission
2577 * problems, just note that and return success so that the
2578 * other threads/LWPs will be bound. This is the way
2579 * processor_bind() is defined to work.
2580 *
2581 * Binding will get EPERM if the thread is of system class
2582 * or hasprocperm() fails.
2583 */
2584 if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2585 *error = EPERM;
2586 thread_unlock(tp);
2587 return (0);
2588 }
2589
2590 binding = bind;
2591 if (binding != PBIND_NONE) {
2592 cp = cpu_get((processorid_t)binding);
2593 /*
2594 * Make sure binding is valid and is in right partition.
2595 */
2596 if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2597 *error = EINVAL;
2598 thread_unlock(tp);
2599 return (0);
2600 }
2601 }
2602 tp->t_bind_cpu = binding; /* set new binding */
2603
2604 /*
2605 * If there is no system-set reason for affinity, set
2606 * the t_bound_cpu field to reflect the binding.
2607 */
2608 if (tp->t_affinitycnt == 0) {
2609 if (binding == PBIND_NONE) {
2610 /*
2611 * We may need to adjust disp_max_unbound_pri
2612 * since we're becoming unbound.
2613 */
2614 disp_adjust_unbound_pri(tp);
2615
2616 tp->t_bound_cpu = NULL; /* set new binding */
2617
2618 /*
2619 * Move thread to lgroup with strongest affinity
2620 * after unbinding
2621 */
2622 if (tp->t_lgrp_affinity)
2623 lgrp_move_thread(tp,
2624 lgrp_choose(tp, tp->t_cpupart), 1);
2625
2626 if (tp->t_state == TS_ONPROC &&
2627 tp->t_cpu->cpu_part != tp->t_cpupart)
2628 cpu_surrender(tp);
2629 } else {
2630 lpl_t *lpl;
2631
2632 tp->t_bound_cpu = cp;
2633 ASSERT(cp->cpu_lpl != NULL);
2634
2635 /*
2636 * Set home to lgroup with most affinity containing CPU
2637 * that thread is being bound or minimum bounding
2638 * lgroup if no affinities set
2639 */
2640 if (tp->t_lgrp_affinity)
2641 lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2642 LGRP_NONE, B_FALSE);
2643 else
2644 lpl = cp->cpu_lpl;
2645
2646 if (tp->t_lpl != lpl) {
2647 /* can't grab cpu_lock */
2648 lgrp_move_thread(tp, lpl, 1);
2649 }
2650
2651 /*
2652 * Make the thread switch to the bound CPU.
2653 * If the thread is runnable, we need to
2654 * requeue it even if t_cpu is already set
2655 * to the right CPU, since it may be on a
2656 * kpreempt queue and need to move to a local
2657 * queue. We could check t_disp_queue to
2658 * avoid unnecessary overhead if it's already
2659 * on the right queue, but since this isn't
2660 * a performance-critical operation it doesn't
2661 * seem worth the extra code and complexity.
2662 *
2663 * If the thread is weakbound to the cpu then it will
2664 * resist the new binding request until the weak
2665 * binding drops. The cpu_surrender or requeueing
2666 * below could be skipped in such cases (since it
2667 * will have no effect), but that would require
2668 * thread_allowmigrate to acquire thread_lock so
2669 * we'll take the very occasional hit here instead.
2670 */
2671 if (tp->t_state == TS_ONPROC) {
2672 cpu_surrender(tp);
2673 } else if (tp->t_state == TS_RUN) {
2674 cpu_t *ocp = tp->t_cpu;
2675
2676 (void) dispdeq(tp);
2677 setbackdq(tp);
2678 /*
2679 * Either on the bound CPU's disp queue now,
2680 * or swapped out or on the swap queue.
2681 */
2682 ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2683 tp->t_weakbound_cpu == ocp ||
2684 (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2685 != TS_LOAD);
2686 }
2687 }
2688 }
2689
2690 /*
2691 * Our binding has changed; set TP_CHANGEBIND.
2692 */
2693 tp->t_proc_flag |= TP_CHANGEBIND;
2694 aston(tp);
2695
2696 thread_unlock(tp);
2697
2698 return (0);
2699 }
2700
2701 #if CPUSET_WORDS > 1
2702
2703 /*
2704 * Functions for implementing cpuset operations when a cpuset is more
2705 * than one word. On platforms where a cpuset is a single word these
2706 * are implemented as macros in cpuvar.h.
2707 */
2708
2709 void
2710 cpuset_all(cpuset_t *s)
2711 {
2712 int i;
2713
2714 for (i = 0; i < CPUSET_WORDS; i++)
2715 s->cpub[i] = ~0UL;
2716 }
2717
2718 void
2719 cpuset_all_but(cpuset_t *s, uint_t cpu)
2720 {
2721 cpuset_all(s);
2722 CPUSET_DEL(*s, cpu);
2723 }
2724
2725 void
2726 cpuset_only(cpuset_t *s, uint_t cpu)
2727 {
2728 CPUSET_ZERO(*s);
2729 CPUSET_ADD(*s, cpu);
2730 }
2731
2732 int
2733 cpuset_isnull(cpuset_t *s)
2734 {
2735 int i;
2736
2737 for (i = 0; i < CPUSET_WORDS; i++)
2738 if (s->cpub[i] != 0)
2739 return (0);
2740 return (1);
2741 }
2742
2743 int
2744 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2745 {
2746 int i;
2747
2748 for (i = 0; i < CPUSET_WORDS; i++)
2749 if (s1->cpub[i] != s2->cpub[i])
2750 return (0);
2751 return (1);
2752 }
2753
2754 uint_t
2755 cpuset_find(cpuset_t *s)
2756 {
2757
2758 uint_t i;
2759 uint_t cpu = (uint_t)-1;
2760
2761 /*
2762 * Find a cpu in the cpuset
2763 */
2764 for (i = 0; i < CPUSET_WORDS; i++) {
2765 cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2766 if (cpu != (uint_t)-1) {
2767 cpu += i * BT_NBIPUL;
2768 break;
2769 }
2770 }
2771 return (cpu);
2772 }
2773
2774 void
2775 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2776 {
2777 int i, j;
2778 uint_t bit;
2779
2780 /*
2781 * First, find the smallest cpu id in the set.
2782 */
2783 for (i = 0; i < CPUSET_WORDS; i++) {
2784 if (s->cpub[i] != 0) {
2785 bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2786 ASSERT(bit != (uint_t)-1);
2787 *smallestid = bit + (i * BT_NBIPUL);
2788
2789 /*
2790 * Now find the largest cpu id in
2791 * the set and return immediately.
2792 * Done in an inner loop to avoid
2793 * having to break out of the first
2794 * loop.
2795 */
2796 for (j = CPUSET_WORDS - 1; j >= i; j--) {
2797 if (s->cpub[j] != 0) {
2798 bit = (uint_t)(highbit(s->cpub[j]) - 1);
2799 ASSERT(bit != (uint_t)-1);
2800 *largestid = bit + (j * BT_NBIPUL);
2801 ASSERT(*largestid >= *smallestid);
2802 return;
2803 }
2804 }
2805
2806 /*
2807 * If this code is reached, a
2808 * smallestid was found, but not a
2809 * largestid. The cpuset must have
2810 * been changed during the course
2811 * of this function call.
2812 */
2813 ASSERT(0);
2814 }
2815 }
2816 *smallestid = *largestid = CPUSET_NOTINSET;
2817 }
2818
2819 #endif /* CPUSET_WORDS */
2820
2821 /*
2822 * Unbind threads bound to specified CPU.
2823 *
2824 * If `unbind_all_threads' is true, unbind all user threads bound to a given
2825 * CPU. Otherwise unbind all soft-bound user threads.
2826 */
2827 int
2828 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2829 {
2830 processorid_t obind;
2831 kthread_t *tp;
2832 int ret = 0;
2833 proc_t *pp;
2834 int err, berr = 0;
2835
2836 ASSERT(MUTEX_HELD(&cpu_lock));
2837
2838 mutex_enter(&pidlock);
2839 for (pp = practive; pp != NULL; pp = pp->p_next) {
2840 mutex_enter(&pp->p_lock);
2841 tp = pp->p_tlist;
2842 /*
2843 * Skip zombies, kernel processes, and processes in
2844 * other zones, if called from a non-global zone.
2845 */
2846 if (tp == NULL || (pp->p_flag & SSYS) ||
2847 !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2848 mutex_exit(&pp->p_lock);
2849 continue;
2850 }
2851 do {
2852 if (tp->t_bind_cpu != cpu)
2853 continue;
2854 /*
2855 * Skip threads with hard binding when
2856 * `unbind_all_threads' is not specified.
2857 */
2858 if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2859 continue;
2860 err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2861 if (ret == 0)
2862 ret = err;
2863 } while ((tp = tp->t_forw) != pp->p_tlist);
2864 mutex_exit(&pp->p_lock);
2865 }
2866 mutex_exit(&pidlock);
2867 if (ret == 0)
2868 ret = berr;
2869 return (ret);
2870 }
2871
2872
2873 /*
2874 * Destroy all remaining bound threads on a cpu.
2875 */
2876 void
2877 cpu_destroy_bound_threads(cpu_t *cp)
2878 {
2879 extern id_t syscid;
2880 register kthread_id_t t, tlist, tnext;
2881
2882 /*
2883 * Destroy all remaining bound threads on the cpu. This
2884 * should include both the interrupt threads and the idle thread.
2885 * This requires some care, since we need to traverse the
2886 * thread list with the pidlock mutex locked, but thread_free
2887 * also locks the pidlock mutex. So, we collect the threads
2888 * we're going to reap in a list headed by "tlist", then we
2889 * unlock the pidlock mutex and traverse the tlist list,
2890 * doing thread_free's on the thread's. Simple, n'est pas?
2891 * Also, this depends on thread_free not mucking with the
2892 * t_next and t_prev links of the thread.
2893 */
2894
2895 if ((t = curthread) != NULL) {
2896
2897 tlist = NULL;
2898 mutex_enter(&pidlock);
2899 do {
2900 tnext = t->t_next;
2901 if (t->t_bound_cpu == cp) {
2902
2903 /*
2904 * We've found a bound thread, carefully unlink
2905 * it out of the thread list, and add it to
2906 * our "tlist". We "know" we don't have to
2907 * worry about unlinking curthread (the thread
2908 * that is executing this code).
2909 */
2910 t->t_next->t_prev = t->t_prev;
2911 t->t_prev->t_next = t->t_next;
2912 t->t_next = tlist;
2913 tlist = t;
2914 ASSERT(t->t_cid == syscid);
2915 /* wake up anyone blocked in thread_join */
2916 cv_broadcast(&t->t_joincv);
2917 /*
2918 * t_lwp set by interrupt threads and not
2919 * cleared.
2920 */
2921 t->t_lwp = NULL;
2922 /*
2923 * Pause and idle threads always have
2924 * t_state set to TS_ONPROC.
2925 */
2926 t->t_state = TS_FREE;
2927 t->t_prev = NULL; /* Just in case */
2928 }
2929
2930 } while ((t = tnext) != curthread);
2931
2932 mutex_exit(&pidlock);
2933
2934 mutex_sync();
2935 for (t = tlist; t != NULL; t = tnext) {
2936 tnext = t->t_next;
2937 thread_free(t);
2938 }
2939 }
2940 }
2941
2942 /*
2943 * Update the cpu_supp_freqs of this cpu. This information is returned
2944 * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2945 * maintain the kstat data size.
2946 */
2947 void
2948 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2949 {
2950 char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2951 const char *lfreqs = clkstr;
2952 boolean_t kstat_exists = B_FALSE;
2953 kstat_t *ksp;
2954 size_t len;
2955
2956 /*
2957 * A NULL pointer means we only support one speed.
2958 */
2959 if (freqs == NULL)
2960 (void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2961 cp->cpu_curr_clock);
2962 else
2963 lfreqs = freqs;
2964
2965 /*
2966 * Make sure the frequency doesn't change while a snapshot is
2967 * going on. Of course, we only need to worry about this if
2968 * the kstat exists.
2969 */
2970 if ((ksp = cp->cpu_info_kstat) != NULL) {
2971 mutex_enter(ksp->ks_lock);
2972 kstat_exists = B_TRUE;
2973 }
2974
2975 /*
2976 * Free any previously allocated string and if the kstat
2977 * already exists, then update its data size.
2978 */
2979 if (cp->cpu_supp_freqs != NULL) {
2980 len = strlen(cp->cpu_supp_freqs) + 1;
2981 kmem_free(cp->cpu_supp_freqs, len);
2982 if (kstat_exists)
2983 ksp->ks_data_size -= len;
2984 }
2985
2986 /*
2987 * Allocate the new string and set the pointer.
2988 */
2989 len = strlen(lfreqs) + 1;
2990 cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2991 (void) strcpy(cp->cpu_supp_freqs, lfreqs);
2992
2993 /*
2994 * If the kstat already exists then update the data size and
2995 * free the lock.
2996 */
2997 if (kstat_exists) {
2998 ksp->ks_data_size += len;
2999 mutex_exit(ksp->ks_lock);
3000 }
3001 }
3002
3003 /*
3004 * Indicate the current CPU's clock freqency (in Hz).
3005 * The calling context must be such that CPU references are safe.
3006 */
3007 void
3008 cpu_set_curr_clock(uint64_t new_clk)
3009 {
3010 uint64_t old_clk;
3011
3012 old_clk = CPU->cpu_curr_clock;
3013 CPU->cpu_curr_clock = new_clk;
3014
3015 /*
3016 * The cpu-change-speed DTrace probe exports the frequency in Hz
3017 */
3018 DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3019 uint64_t, old_clk, uint64_t, new_clk);
3020 }
3021
3022 /*
3023 * processor_info(2) and p_online(2) status support functions
3024 * The constants returned by the cpu_get_state() and cpu_get_state_str() are
3025 * for use in communicating processor state information to userland. Kernel
3026 * subsystems should only be using the cpu_flags value directly. Subsystems
3027 * modifying cpu_flags should record the state change via a call to the
3028 * cpu_set_state().
3029 */
3030
3031 /*
3032 * Update the pi_state of this CPU. This function provides the CPU status for
3033 * the information returned by processor_info(2).
3034 */
3035 void
3036 cpu_set_state(cpu_t *cpu)
3037 {
3038 ASSERT(MUTEX_HELD(&cpu_lock));
3039 cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3040 cpu->cpu_state_begin = gethrestime_sec();
3041 pool_cpu_mod = gethrtime();
3042 }
3043
3044 /*
3045 * Return offline/online/other status for the indicated CPU. Use only for
3046 * communication with user applications; cpu_flags provides the in-kernel
3047 * interface.
3048 */
3049 int
3050 cpu_get_state(cpu_t *cpu)
3051 {
3052 ASSERT(MUTEX_HELD(&cpu_lock));
3053 if (cpu->cpu_flags & CPU_POWEROFF)
3054 return (P_POWEROFF);
3055 else if (cpu->cpu_flags & CPU_FAULTED)
3056 return (P_FAULTED);
3057 else if (cpu->cpu_flags & CPU_SPARE)
3058 return (P_SPARE);
3059 else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3060 return (P_OFFLINE);
3061 else if (cpu->cpu_flags & CPU_ENABLE)
3062 return (P_ONLINE);
3063 else
3064 return (P_NOINTR);
3065 }
3066
3067 /*
3068 * Return processor_info(2) state as a string.
3069 */
3070 const char *
3071 cpu_get_state_str(cpu_t *cpu)
3072 {
3073 const char *string;
3074
3075 switch (cpu_get_state(cpu)) {
3076 case P_ONLINE:
3077 string = PS_ONLINE;
3078 break;
3079 case P_POWEROFF:
3080 string = PS_POWEROFF;
3081 break;
3082 case P_NOINTR:
3083 string = PS_NOINTR;
3084 break;
3085 case P_SPARE:
3086 string = PS_SPARE;
3087 break;
3088 case P_FAULTED:
3089 string = PS_FAULTED;
3090 break;
3091 case P_OFFLINE:
3092 string = PS_OFFLINE;
3093 break;
3094 default:
3095 string = "unknown";
3096 break;
3097 }
3098 return (string);
3099 }
3100
3101 /*
3102 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3103 * kstats, respectively. This is done when a CPU is initialized or placed
3104 * online via p_online(2).
3105 */
3106 static void
3107 cpu_stats_kstat_create(cpu_t *cp)
3108 {
3109 int instance = cp->cpu_id;
3110 char *module = "cpu";
3111 char *class = "misc";
3112 kstat_t *ksp;
3113 zoneid_t zoneid;
3114
3115 ASSERT(MUTEX_HELD(&cpu_lock));
3116
3117 if (pool_pset_enabled())
3118 zoneid = GLOBAL_ZONEID;
3119 else
3120 zoneid = ALL_ZONES;
3121 /*
3122 * Create named kstats
3123 */
3124 #define CPU_STATS_KS_CREATE(name, template, update_func) \
3125 ksp = kstat_create_zone(module, instance, (name), class, \
3126 KSTAT_TYPE_NAMED, sizeof (template) / sizeof (kstat_named_t),\
3127 0, zoneid); \
3128 if (ksp != NULL) { \
3129 bcopy(&template, ksp->ks_data, sizeof (template)); \
3130 ksp->ks_private = cp; \
3131 ksp->ks_update = (update_func); \
3132 kstat_install(ksp); \
3133 } else { \
3134 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3135 module, instance, (name)); \
3136 } \
3137
3138 CPU_STATS_KS_CREATE("sys", cpu_sys_stats_ks_data_template,
3139 cpu_sys_stats_ks_update);
3140 CPU_STATS_KS_CREATE("vm", cpu_vm_stats_ks_data_template,
3141 cpu_vm_stats_ks_update);
3142
3143 /*
3144 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3145 */
3146 ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3147 "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3148 if (ksp != NULL) {
3149 ksp->ks_update = cpu_stat_ks_update;
3150 ksp->ks_private = cp;
3151 kstat_install(ksp);
3152 }
3153 }
3154
3155 static void
3156 cpu_stats_kstat_destroy(cpu_t *cp)
3157 {
3158 char ks_name[KSTAT_STRLEN];
3159
3160 (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3161 kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3162
3163 kstat_delete_byname("cpu", cp->cpu_id, "sys");
3164 kstat_delete_byname("cpu", cp->cpu_id, "vm");
3165 }
3166
3167 static int
3168 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3169 {
3170 cpu_t *cp = (cpu_t *)ksp->ks_private;
3171 struct cpu_sys_stats_ks_data *csskd;
3172 cpu_sys_stats_t *css;
3173 hrtime_t msnsecs[NCMSTATES];
3174 int i;
3175
3176 if (rw == KSTAT_WRITE)
3177 return (EACCES);
3178
3179 csskd = ksp->ks_data;
3180 css = &cp->cpu_stats.sys;
3181
3182 /*
3183 * Read CPU mstate, but compare with the last values we
3184 * received to make sure that the returned kstats never
3185 * decrease.
3186 */
3187
3188 get_cpu_mstate(cp, msnsecs);
3189 if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3190 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3191 if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3192 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3193 if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3194 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3195
3196 bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3197 sizeof (cpu_sys_stats_ks_data_template));
3198
3199 csskd->cpu_ticks_wait.value.ui64 = 0;
3200 csskd->wait_ticks_io.value.ui64 = 0;
3201
3202 csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3203 csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3204 csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3205 csskd->cpu_ticks_idle.value.ui64 =
3206 NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3207 csskd->cpu_ticks_user.value.ui64 =
3208 NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3209 csskd->cpu_ticks_kernel.value.ui64 =
3210 NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3211 csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3212 csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3213 csskd->bread.value.ui64 = css->bread;
3214 csskd->bwrite.value.ui64 = css->bwrite;
3215 csskd->lread.value.ui64 = css->lread;
3216 csskd->lwrite.value.ui64 = css->lwrite;
3217 csskd->phread.value.ui64 = css->phread;
3218 csskd->phwrite.value.ui64 = css->phwrite;
3219 csskd->pswitch.value.ui64 = css->pswitch;
3220 csskd->trap.value.ui64 = css->trap;
3221 csskd->intr.value.ui64 = 0;
3222 for (i = 0; i < PIL_MAX; i++)
3223 csskd->intr.value.ui64 += css->intr[i];
3224 csskd->syscall.value.ui64 = css->syscall;
3225 csskd->sysread.value.ui64 = css->sysread;
3226 csskd->syswrite.value.ui64 = css->syswrite;
3227 csskd->sysfork.value.ui64 = css->sysfork;
3228 csskd->sysvfork.value.ui64 = css->sysvfork;
3229 csskd->sysexec.value.ui64 = css->sysexec;
3230 csskd->readch.value.ui64 = css->readch;
3231 csskd->writech.value.ui64 = css->writech;
3232 csskd->rcvint.value.ui64 = css->rcvint;
3233 csskd->xmtint.value.ui64 = css->xmtint;
3234 csskd->mdmint.value.ui64 = css->mdmint;
3235 csskd->rawch.value.ui64 = css->rawch;
3236 csskd->canch.value.ui64 = css->canch;
3237 csskd->outch.value.ui64 = css->outch;
3238 csskd->msg.value.ui64 = css->msg;
3239 csskd->sema.value.ui64 = css->sema;
3240 csskd->namei.value.ui64 = css->namei;
3241 csskd->ufsiget.value.ui64 = css->ufsiget;
3242 csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3243 csskd->ufsipage.value.ui64 = css->ufsipage;
3244 csskd->ufsinopage.value.ui64 = css->ufsinopage;
3245 csskd->procovf.value.ui64 = css->procovf;
3246 csskd->intrthread.value.ui64 = 0;
3247 for (i = 0; i < LOCK_LEVEL - 1; i++)
3248 csskd->intrthread.value.ui64 += css->intr[i];
3249 csskd->intrblk.value.ui64 = css->intrblk;
3250 csskd->intrunpin.value.ui64 = css->intrunpin;
3251 csskd->idlethread.value.ui64 = css->idlethread;
3252 csskd->inv_swtch.value.ui64 = css->inv_swtch;
3253 csskd->nthreads.value.ui64 = css->nthreads;
3254 csskd->cpumigrate.value.ui64 = css->cpumigrate;
3255 csskd->xcalls.value.ui64 = css->xcalls;
3256 csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3257 csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3258 csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3259 csskd->modload.value.ui64 = css->modload;
3260 csskd->modunload.value.ui64 = css->modunload;
3261 csskd->bawrite.value.ui64 = css->bawrite;
3262 csskd->iowait.value.ui64 = css->iowait;
3263
3264 return (0);
3265 }
3266
3267 static int
3268 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3269 {
3270 cpu_t *cp = (cpu_t *)ksp->ks_private;
3271 struct cpu_vm_stats_ks_data *cvskd;
3272 cpu_vm_stats_t *cvs;
3273
3274 if (rw == KSTAT_WRITE)
3275 return (EACCES);
3276
3277 cvs = &cp->cpu_stats.vm;
3278 cvskd = ksp->ks_data;
3279
3280 bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3281 sizeof (cpu_vm_stats_ks_data_template));
3282 cvskd->pgrec.value.ui64 = cvs->pgrec;
3283 cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3284 cvskd->pgin.value.ui64 = cvs->pgin;
3285 cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3286 cvskd->pgout.value.ui64 = cvs->pgout;
3287 cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3288 cvskd->swapin.value.ui64 = cvs->swapin;
3289 cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3290 cvskd->swapout.value.ui64 = cvs->swapout;
3291 cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3292 cvskd->zfod.value.ui64 = cvs->zfod;
3293 cvskd->dfree.value.ui64 = cvs->dfree;
3294 cvskd->scan.value.ui64 = cvs->scan;
3295 cvskd->rev.value.ui64 = cvs->rev;
3296 cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3297 cvskd->as_fault.value.ui64 = cvs->as_fault;
3298 cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3299 cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3300 cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3301 cvskd->softlock.value.ui64 = cvs->softlock;
3302 cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3303 cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3304 cvskd->execpgin.value.ui64 = cvs->execpgin;
3305 cvskd->execpgout.value.ui64 = cvs->execpgout;
3306 cvskd->execfree.value.ui64 = cvs->execfree;
3307 cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3308 cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3309 cvskd->anonfree.value.ui64 = cvs->anonfree;
3310 cvskd->fspgin.value.ui64 = cvs->fspgin;
3311 cvskd->fspgout.value.ui64 = cvs->fspgout;
3312 cvskd->fsfree.value.ui64 = cvs->fsfree;
3313
3314 return (0);
3315 }
3316
3317 static int
3318 cpu_stat_ks_update(kstat_t *ksp, int rw)
3319 {
3320 cpu_stat_t *cso;
3321 cpu_t *cp;
3322 int i;
3323 hrtime_t msnsecs[NCMSTATES];
3324
3325 cso = (cpu_stat_t *)ksp->ks_data;
3326 cp = (cpu_t *)ksp->ks_private;
3327
3328 if (rw == KSTAT_WRITE)
3329 return (EACCES);
3330
3331 /*
3332 * Read CPU mstate, but compare with the last values we
3333 * received to make sure that the returned kstats never
3334 * decrease.
3335 */
3336
3337 get_cpu_mstate(cp, msnsecs);
3338 msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3339 msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3340 msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3341 if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3342 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3343 if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3344 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3345 if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3346 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3347 cso->cpu_sysinfo.cpu[CPU_WAIT] = 0;
3348 cso->cpu_sysinfo.wait[W_IO] = 0;
3349 cso->cpu_sysinfo.wait[W_SWAP] = 0;
3350 cso->cpu_sysinfo.wait[W_PIO] = 0;
3351 cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread);
3352 cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite);
3353 cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread);
3354 cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite);
3355 cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread);
3356 cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite);
3357 cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch);
3358 cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap);
3359 cso->cpu_sysinfo.intr = 0;
3360 for (i = 0; i < PIL_MAX; i++)
3361 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3362 cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall);
3363 cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread);
3364 cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite);
3365 cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork);
3366 cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork);
3367 cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec);
3368 cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch);
3369 cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech);
3370 cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint);
3371 cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint);
3372 cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint);
3373 cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch);
3374 cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch);
3375 cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch);
3376 cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg);
3377 cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema);
3378 cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei);
3379 cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget);
3380 cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk);
3381 cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage);
3382 cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage);
3383 cso->cpu_sysinfo.inodeovf = 0;
3384 cso->cpu_sysinfo.fileovf = 0;
3385 cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf);
3386 cso->cpu_sysinfo.intrthread = 0;
3387 for (i = 0; i < LOCK_LEVEL - 1; i++)
3388 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3389 cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk);
3390 cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread);
3391 cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch);
3392 cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads);
3393 cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate);
3394 cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls);
3395 cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters);
3396 cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails);
3397 cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails);
3398 cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload);
3399 cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload);
3400 cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite);
3401 cso->cpu_sysinfo.rw_enters = 0;
3402 cso->cpu_sysinfo.win_uo_cnt = 0;
3403 cso->cpu_sysinfo.win_uu_cnt = 0;
3404 cso->cpu_sysinfo.win_so_cnt = 0;
3405 cso->cpu_sysinfo.win_su_cnt = 0;
3406 cso->cpu_sysinfo.win_suo_cnt = 0;
3407
3408 cso->cpu_syswait.iowait = CPU_STATS(cp, sys.iowait);
3409 cso->cpu_syswait.swap = 0;
3410 cso->cpu_syswait.physio = 0;
3411
3412 cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec);
3413 cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec);
3414 cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin);
3415 cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin);
3416 cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout);
3417 cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout);
3418 cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin);
3419 cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin);
3420 cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout);
3421 cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout);
3422 cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod);
3423 cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree);
3424 cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan);
3425 cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev);
3426 cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault);
3427 cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault);
3428 cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault);
3429 cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault);
3430 cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault);
3431 cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock);
3432 cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt);
3433 cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun);
3434 cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin);
3435 cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout);
3436 cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree);
3437 cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin);
3438 cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout);
3439 cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree);
3440 cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin);
3441 cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout);
3442 cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree);
3443
3444 return (0);
3445 }