1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
24 */
25
26 /*
27 * Architecture-independent CPU control functions.
28 */
29
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/var.h>
33 #include <sys/thread.h>
34 #include <sys/cpuvar.h>
35 #include <sys/cpu_event.h>
36 #include <sys/kstat.h>
37 #include <sys/uadmin.h>
38 #include <sys/systm.h>
39 #include <sys/errno.h>
40 #include <sys/cmn_err.h>
41 #include <sys/procset.h>
42 #include <sys/processor.h>
43 #include <sys/debug.h>
44 #include <sys/cpupart.h>
45 #include <sys/lgrp.h>
46 #include <sys/pset.h>
47 #include <sys/pghw.h>
48 #include <sys/kmem.h>
49 #include <sys/kmem_impl.h> /* to set per-cpu kmem_cache offset */
50 #include <sys/atomic.h>
51 #include <sys/callb.h>
52 #include <sys/vtrace.h>
53 #include <sys/cyclic.h>
54 #include <sys/bitmap.h>
55 #include <sys/nvpair.h>
56 #include <sys/pool_pset.h>
57 #include <sys/msacct.h>
58 #include <sys/time.h>
59 #include <sys/archsystm.h>
60 #include <sys/sdt.h>
61 #if defined(__x86) || defined(__amd64)
62 #include <sys/x86_archext.h>
63 #endif
64 #include <sys/callo.h>
65
66 extern int mp_cpu_start(cpu_t *);
67 extern int mp_cpu_stop(cpu_t *);
68 extern int mp_cpu_poweron(cpu_t *);
69 extern int mp_cpu_poweroff(cpu_t *);
70 extern int mp_cpu_configure(int);
71 extern int mp_cpu_unconfigure(int);
72 extern void mp_cpu_faulted_enter(cpu_t *);
73 extern void mp_cpu_faulted_exit(cpu_t *);
74
75 extern int cmp_cpu_to_chip(processorid_t cpuid);
76 #ifdef __sparcv9
77 extern char *cpu_fru_fmri(cpu_t *cp);
78 #endif
79
80 static void cpu_add_active_internal(cpu_t *cp);
81 static void cpu_remove_active(cpu_t *cp);
82 static void cpu_info_kstat_create(cpu_t *cp);
83 static void cpu_info_kstat_destroy(cpu_t *cp);
84 static void cpu_stats_kstat_create(cpu_t *cp);
85 static void cpu_stats_kstat_destroy(cpu_t *cp);
86
87 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
88 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
89 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
90 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
91
92 /*
93 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
94 * max_cpu_seqid_ever, and dispatch queue reallocations. The lock ordering with
95 * respect to related locks is:
96 *
97 * cpu_lock --> thread_free_lock ---> p_lock ---> thread_lock()
98 *
99 * Warning: Certain sections of code do not use the cpu_lock when
100 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()). Since
101 * all cpus are paused during modifications to this list, a solution
102 * to protect the list is too either disable kernel preemption while
103 * walking the list, *or* recheck the cpu_next pointer at each
104 * iteration in the loop. Note that in no cases can any cached
105 * copies of the cpu pointers be kept as they may become invalid.
106 */
107 kmutex_t cpu_lock;
108 cpu_t *cpu_list; /* list of all CPUs */
109 cpu_t *clock_cpu_list; /* used by clock to walk CPUs */
110 cpu_t *cpu_active; /* list of active CPUs */
111 static cpuset_t cpu_available; /* set of available CPUs */
112 cpuset_t cpu_seqid_inuse; /* which cpu_seqids are in use */
113
114 cpu_t **cpu_seq; /* ptrs to CPUs, indexed by seq_id */
115
116 /*
117 * max_ncpus keeps the max cpus the system can have. Initially
118 * it's NCPU, but since most archs scan the devtree for cpus
119 * fairly early on during boot, the real max can be known before
120 * ncpus is set (useful for early NCPU based allocations).
121 */
122 int max_ncpus = NCPU;
123 /*
124 * platforms that set max_ncpus to maxiumum number of cpus that can be
125 * dynamically added will set boot_max_ncpus to the number of cpus found
126 * at device tree scan time during boot.
127 */
128 int boot_max_ncpus = -1;
129 int boot_ncpus = -1;
130 /*
131 * Maximum possible CPU id. This can never be >= NCPU since NCPU is
132 * used to size arrays that are indexed by CPU id.
133 */
134 processorid_t max_cpuid = NCPU - 1;
135
136 /*
137 * Maximum cpu_seqid was given. This number can only grow and never shrink. It
138 * can be used to optimize NCPU loops to avoid going through CPUs which were
139 * never on-line.
140 */
141 processorid_t max_cpu_seqid_ever = 0;
142
143 int ncpus = 1;
144 int ncpus_online = 1;
145
146 /*
147 * CPU that we're trying to offline. Protected by cpu_lock.
148 */
149 cpu_t *cpu_inmotion;
150
151 /*
152 * Can be raised to suppress further weakbinding, which are instead
153 * satisfied by disabling preemption. Must be raised/lowered under cpu_lock,
154 * while individual thread weakbinding synchronization is done under thread
155 * lock.
156 */
157 int weakbindingbarrier;
158
159 /*
160 * Variables used in pause_cpus().
161 */
162 static volatile char safe_list[NCPU];
163
164 static struct _cpu_pause_info {
165 int cp_spl; /* spl saved in pause_cpus() */
166 volatile int cp_go; /* Go signal sent after all ready */
167 int cp_count; /* # of CPUs to pause */
168 ksema_t cp_sem; /* synch pause_cpus & cpu_pause */
169 kthread_id_t cp_paused;
170 void *(*cp_func)(void *);
171 } cpu_pause_info;
172
173 static kmutex_t pause_free_mutex;
174 static kcondvar_t pause_free_cv;
175
176
177 static struct cpu_sys_stats_ks_data {
178 kstat_named_t cpu_ticks_idle;
179 kstat_named_t cpu_ticks_user;
180 kstat_named_t cpu_ticks_kernel;
181 kstat_named_t cpu_ticks_wait;
182 kstat_named_t cpu_nsec_idle;
183 kstat_named_t cpu_nsec_user;
184 kstat_named_t cpu_nsec_kernel;
185 kstat_named_t cpu_nsec_dtrace;
186 kstat_named_t cpu_nsec_intr;
187 kstat_named_t cpu_load_intr;
188 kstat_named_t wait_ticks_io;
189 kstat_named_t dtrace_probes;
190 kstat_named_t bread;
191 kstat_named_t bwrite;
192 kstat_named_t lread;
193 kstat_named_t lwrite;
194 kstat_named_t phread;
195 kstat_named_t phwrite;
196 kstat_named_t pswitch;
197 kstat_named_t trap;
198 kstat_named_t intr;
199 kstat_named_t syscall;
200 kstat_named_t sysread;
201 kstat_named_t syswrite;
202 kstat_named_t sysfork;
203 kstat_named_t sysvfork;
204 kstat_named_t sysexec;
205 kstat_named_t readch;
206 kstat_named_t writech;
207 kstat_named_t rcvint;
208 kstat_named_t xmtint;
209 kstat_named_t mdmint;
210 kstat_named_t rawch;
211 kstat_named_t canch;
212 kstat_named_t outch;
213 kstat_named_t msg;
214 kstat_named_t sema;
215 kstat_named_t namei;
216 kstat_named_t ufsiget;
217 kstat_named_t ufsdirblk;
218 kstat_named_t ufsipage;
219 kstat_named_t ufsinopage;
220 kstat_named_t procovf;
221 kstat_named_t intrthread;
222 kstat_named_t intrblk;
223 kstat_named_t intrunpin;
224 kstat_named_t idlethread;
225 kstat_named_t inv_swtch;
226 kstat_named_t nthreads;
227 kstat_named_t cpumigrate;
228 kstat_named_t xcalls;
229 kstat_named_t mutex_adenters;
230 kstat_named_t rw_rdfails;
231 kstat_named_t rw_wrfails;
232 kstat_named_t modload;
233 kstat_named_t modunload;
234 kstat_named_t bawrite;
235 kstat_named_t iowait;
236 } cpu_sys_stats_ks_data_template = {
237 { "cpu_ticks_idle", KSTAT_DATA_UINT64 },
238 { "cpu_ticks_user", KSTAT_DATA_UINT64 },
239 { "cpu_ticks_kernel", KSTAT_DATA_UINT64 },
240 { "cpu_ticks_wait", KSTAT_DATA_UINT64 },
241 { "cpu_nsec_idle", KSTAT_DATA_UINT64 },
242 { "cpu_nsec_user", KSTAT_DATA_UINT64 },
243 { "cpu_nsec_kernel", KSTAT_DATA_UINT64 },
244 { "cpu_nsec_dtrace", KSTAT_DATA_UINT64 },
245 { "cpu_nsec_intr", KSTAT_DATA_UINT64 },
246 { "cpu_load_intr", KSTAT_DATA_UINT64 },
247 { "wait_ticks_io", KSTAT_DATA_UINT64 },
248 { "dtrace_probes", KSTAT_DATA_UINT64 },
249 { "bread", KSTAT_DATA_UINT64 },
250 { "bwrite", KSTAT_DATA_UINT64 },
251 { "lread", KSTAT_DATA_UINT64 },
252 { "lwrite", KSTAT_DATA_UINT64 },
253 { "phread", KSTAT_DATA_UINT64 },
254 { "phwrite", KSTAT_DATA_UINT64 },
255 { "pswitch", KSTAT_DATA_UINT64 },
256 { "trap", KSTAT_DATA_UINT64 },
257 { "intr", KSTAT_DATA_UINT64 },
258 { "syscall", KSTAT_DATA_UINT64 },
259 { "sysread", KSTAT_DATA_UINT64 },
260 { "syswrite", KSTAT_DATA_UINT64 },
261 { "sysfork", KSTAT_DATA_UINT64 },
262 { "sysvfork", KSTAT_DATA_UINT64 },
263 { "sysexec", KSTAT_DATA_UINT64 },
264 { "readch", KSTAT_DATA_UINT64 },
265 { "writech", KSTAT_DATA_UINT64 },
266 { "rcvint", KSTAT_DATA_UINT64 },
267 { "xmtint", KSTAT_DATA_UINT64 },
268 { "mdmint", KSTAT_DATA_UINT64 },
269 { "rawch", KSTAT_DATA_UINT64 },
270 { "canch", KSTAT_DATA_UINT64 },
271 { "outch", KSTAT_DATA_UINT64 },
272 { "msg", KSTAT_DATA_UINT64 },
273 { "sema", KSTAT_DATA_UINT64 },
274 { "namei", KSTAT_DATA_UINT64 },
275 { "ufsiget", KSTAT_DATA_UINT64 },
276 { "ufsdirblk", KSTAT_DATA_UINT64 },
277 { "ufsipage", KSTAT_DATA_UINT64 },
278 { "ufsinopage", KSTAT_DATA_UINT64 },
279 { "procovf", KSTAT_DATA_UINT64 },
280 { "intrthread", KSTAT_DATA_UINT64 },
281 { "intrblk", KSTAT_DATA_UINT64 },
282 { "intrunpin", KSTAT_DATA_UINT64 },
283 { "idlethread", KSTAT_DATA_UINT64 },
284 { "inv_swtch", KSTAT_DATA_UINT64 },
285 { "nthreads", KSTAT_DATA_UINT64 },
286 { "cpumigrate", KSTAT_DATA_UINT64 },
287 { "xcalls", KSTAT_DATA_UINT64 },
288 { "mutex_adenters", KSTAT_DATA_UINT64 },
289 { "rw_rdfails", KSTAT_DATA_UINT64 },
290 { "rw_wrfails", KSTAT_DATA_UINT64 },
291 { "modload", KSTAT_DATA_UINT64 },
292 { "modunload", KSTAT_DATA_UINT64 },
293 { "bawrite", KSTAT_DATA_UINT64 },
294 { "iowait", KSTAT_DATA_UINT64 },
295 };
296
297 static struct cpu_vm_stats_ks_data {
298 kstat_named_t pgrec;
299 kstat_named_t pgfrec;
300 kstat_named_t pgin;
301 kstat_named_t pgpgin;
302 kstat_named_t pgout;
303 kstat_named_t pgpgout;
304 kstat_named_t swapin;
305 kstat_named_t pgswapin;
306 kstat_named_t swapout;
307 kstat_named_t pgswapout;
308 kstat_named_t zfod;
309 kstat_named_t dfree;
310 kstat_named_t scan;
311 kstat_named_t rev;
312 kstat_named_t hat_fault;
313 kstat_named_t as_fault;
314 kstat_named_t maj_fault;
315 kstat_named_t cow_fault;
316 kstat_named_t prot_fault;
317 kstat_named_t softlock;
318 kstat_named_t kernel_asflt;
319 kstat_named_t pgrrun;
320 kstat_named_t execpgin;
321 kstat_named_t execpgout;
322 kstat_named_t execfree;
323 kstat_named_t anonpgin;
324 kstat_named_t anonpgout;
325 kstat_named_t anonfree;
326 kstat_named_t fspgin;
327 kstat_named_t fspgout;
328 kstat_named_t fsfree;
329 } cpu_vm_stats_ks_data_template = {
330 { "pgrec", KSTAT_DATA_UINT64 },
331 { "pgfrec", KSTAT_DATA_UINT64 },
332 { "pgin", KSTAT_DATA_UINT64 },
333 { "pgpgin", KSTAT_DATA_UINT64 },
334 { "pgout", KSTAT_DATA_UINT64 },
335 { "pgpgout", KSTAT_DATA_UINT64 },
336 { "swapin", KSTAT_DATA_UINT64 },
337 { "pgswapin", KSTAT_DATA_UINT64 },
338 { "swapout", KSTAT_DATA_UINT64 },
339 { "pgswapout", KSTAT_DATA_UINT64 },
340 { "zfod", KSTAT_DATA_UINT64 },
341 { "dfree", KSTAT_DATA_UINT64 },
342 { "scan", KSTAT_DATA_UINT64 },
343 { "rev", KSTAT_DATA_UINT64 },
344 { "hat_fault", KSTAT_DATA_UINT64 },
345 { "as_fault", KSTAT_DATA_UINT64 },
346 { "maj_fault", KSTAT_DATA_UINT64 },
347 { "cow_fault", KSTAT_DATA_UINT64 },
348 { "prot_fault", KSTAT_DATA_UINT64 },
349 { "softlock", KSTAT_DATA_UINT64 },
350 { "kernel_asflt", KSTAT_DATA_UINT64 },
351 { "pgrrun", KSTAT_DATA_UINT64 },
352 { "execpgin", KSTAT_DATA_UINT64 },
353 { "execpgout", KSTAT_DATA_UINT64 },
354 { "execfree", KSTAT_DATA_UINT64 },
355 { "anonpgin", KSTAT_DATA_UINT64 },
356 { "anonpgout", KSTAT_DATA_UINT64 },
357 { "anonfree", KSTAT_DATA_UINT64 },
358 { "fspgin", KSTAT_DATA_UINT64 },
359 { "fspgout", KSTAT_DATA_UINT64 },
360 { "fsfree", KSTAT_DATA_UINT64 },
361 };
362
363 /*
364 * Force the specified thread to migrate to the appropriate processor.
365 * Called with thread lock held, returns with it dropped.
366 */
367 static void
368 force_thread_migrate(kthread_id_t tp)
369 {
370 ASSERT(THREAD_LOCK_HELD(tp));
371 if (tp == curthread) {
372 THREAD_TRANSITION(tp);
373 CL_SETRUN(tp);
374 thread_unlock_nopreempt(tp);
375 swtch();
376 } else {
377 if (tp->t_state == TS_ONPROC) {
378 cpu_surrender(tp);
379 } else if (tp->t_state == TS_RUN) {
380 (void) dispdeq(tp);
381 setbackdq(tp);
382 }
383 thread_unlock(tp);
384 }
385 }
386
387 /*
388 * Set affinity for a specified CPU.
389 * A reference count is incremented and the affinity is held until the
390 * reference count is decremented to zero by thread_affinity_clear().
391 * This is so regions of code requiring affinity can be nested.
392 * Caller needs to ensure that cpu_id remains valid, which can be
393 * done by holding cpu_lock across this call, unless the caller
394 * specifies CPU_CURRENT in which case the cpu_lock will be acquired
395 * by thread_affinity_set and CPU->cpu_id will be the target CPU.
396 */
397 void
398 thread_affinity_set(kthread_id_t t, int cpu_id)
399 {
400 cpu_t *cp;
401 int c;
402
403 ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
404
405 if ((c = cpu_id) == CPU_CURRENT) {
406 mutex_enter(&cpu_lock);
407 cpu_id = CPU->cpu_id;
408 }
409 /*
410 * We should be asserting that cpu_lock is held here, but
411 * the NCA code doesn't acquire it. The following assert
412 * should be uncommented when the NCA code is fixed.
413 *
414 * ASSERT(MUTEX_HELD(&cpu_lock));
415 */
416 ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
417 cp = cpu[cpu_id];
418 ASSERT(cp != NULL); /* user must provide a good cpu_id */
419 /*
420 * If there is already a hard affinity requested, and this affinity
421 * conflicts with that, panic.
422 */
423 thread_lock(t);
424 if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
425 panic("affinity_set: setting %p but already bound to %p",
426 (void *)cp, (void *)t->t_bound_cpu);
427 }
428 t->t_affinitycnt++;
429 t->t_bound_cpu = cp;
430
431 /*
432 * Make sure we're running on the right CPU.
433 */
434 if (cp != t->t_cpu || t != curthread) {
435 force_thread_migrate(t); /* drops thread lock */
436 } else {
437 thread_unlock(t);
438 }
439
440 if (c == CPU_CURRENT)
441 mutex_exit(&cpu_lock);
442 }
443
444 /*
445 * Wrapper for backward compatibility.
446 */
447 void
448 affinity_set(int cpu_id)
449 {
450 thread_affinity_set(curthread, cpu_id);
451 }
452
453 /*
454 * Decrement the affinity reservation count and if it becomes zero,
455 * clear the CPU affinity for the current thread, or set it to the user's
456 * software binding request.
457 */
458 void
459 thread_affinity_clear(kthread_id_t t)
460 {
461 register processorid_t binding;
462
463 thread_lock(t);
464 if (--t->t_affinitycnt == 0) {
465 if ((binding = t->t_bind_cpu) == PBIND_NONE) {
466 /*
467 * Adjust disp_max_unbound_pri if necessary.
468 */
469 disp_adjust_unbound_pri(t);
470 t->t_bound_cpu = NULL;
471 if (t->t_cpu->cpu_part != t->t_cpupart) {
472 force_thread_migrate(t);
473 return;
474 }
475 } else {
476 t->t_bound_cpu = cpu[binding];
477 /*
478 * Make sure the thread is running on the bound CPU.
479 */
480 if (t->t_cpu != t->t_bound_cpu) {
481 force_thread_migrate(t);
482 return; /* already dropped lock */
483 }
484 }
485 }
486 thread_unlock(t);
487 }
488
489 /*
490 * Wrapper for backward compatibility.
491 */
492 void
493 affinity_clear(void)
494 {
495 thread_affinity_clear(curthread);
496 }
497
498 /*
499 * Weak cpu affinity. Bind to the "current" cpu for short periods
500 * of time during which the thread must not block (but may be preempted).
501 * Use this instead of kpreempt_disable() when it is only "no migration"
502 * rather than "no preemption" semantics that are required - disabling
503 * preemption holds higher priority threads off of cpu and if the
504 * operation that is protected is more than momentary this is not good
505 * for realtime etc.
506 *
507 * Weakly bound threads will not prevent a cpu from being offlined -
508 * we'll only run them on the cpu to which they are weakly bound but
509 * (because they do not block) we'll always be able to move them on to
510 * another cpu at offline time if we give them just a short moment to
511 * run during which they will unbind. To give a cpu a chance of offlining,
512 * however, we require a barrier to weak bindings that may be raised for a
513 * given cpu (offline/move code may set this and then wait a short time for
514 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
515 *
516 * There are few restrictions on the calling context of thread_nomigrate.
517 * The caller must not hold the thread lock. Calls may be nested.
518 *
519 * After weakbinding a thread must not perform actions that may block.
520 * In particular it must not call thread_affinity_set; calling that when
521 * already weakbound is nonsensical anyway.
522 *
523 * If curthread is prevented from migrating for other reasons
524 * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
525 * then the weak binding will succeed even if this cpu is the target of an
526 * offline/move request.
527 */
528 void
529 thread_nomigrate(void)
530 {
531 cpu_t *cp;
532 kthread_id_t t = curthread;
533
534 again:
535 kpreempt_disable();
536 cp = CPU;
537
538 /*
539 * A highlevel interrupt must not modify t_nomigrate or
540 * t_weakbound_cpu of the thread it has interrupted. A lowlevel
541 * interrupt thread cannot migrate and we can avoid the
542 * thread_lock call below by short-circuiting here. In either
543 * case we can just return since no migration is possible and
544 * the condition will persist (ie, when we test for these again
545 * in thread_allowmigrate they can't have changed). Migration
546 * is also impossible if we're at or above DISP_LEVEL pil.
547 */
548 if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
549 getpil() >= DISP_LEVEL) {
550 kpreempt_enable();
551 return;
552 }
553
554 /*
555 * We must be consistent with existing weak bindings. Since we
556 * may be interrupted between the increment of t_nomigrate and
557 * the store to t_weakbound_cpu below we cannot assume that
558 * t_weakbound_cpu will be set if t_nomigrate is. Note that we
559 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
560 * always the case.
561 */
562 if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
563 if (!panicstr)
564 panic("thread_nomigrate: binding to %p but already "
565 "bound to %p", (void *)cp,
566 (void *)t->t_weakbound_cpu);
567 }
568
569 /*
570 * At this point we have preemption disabled and we don't yet hold
571 * the thread lock. So it's possible that somebody else could
572 * set t_bind_cpu here and not be able to force us across to the
573 * new cpu (since we have preemption disabled).
574 */
575 thread_lock(curthread);
576
577 /*
578 * If further weak bindings are being (temporarily) suppressed then
579 * we'll settle for disabling kernel preemption (which assures
580 * no migration provided the thread does not block which it is
581 * not allowed to if using thread_nomigrate). We must remember
582 * this disposition so we can take appropriate action in
583 * thread_allowmigrate. If this is a nested call and the
584 * thread is already weakbound then fall through as normal.
585 * We remember the decision to settle for kpreempt_disable through
586 * negative nesting counting in t_nomigrate. Once a thread has had one
587 * weakbinding request satisfied in this way any further (nested)
588 * requests will continue to be satisfied in the same way,
589 * even if weak bindings have recommenced.
590 */
591 if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
592 --t->t_nomigrate;
593 thread_unlock(curthread);
594 return; /* with kpreempt_disable still active */
595 }
596
597 /*
598 * We hold thread_lock so t_bind_cpu cannot change. We could,
599 * however, be running on a different cpu to which we are t_bound_cpu
600 * to (as explained above). If we grant the weak binding request
601 * in that case then the dispatcher must favour our weak binding
602 * over our strong (in which case, just as when preemption is
603 * disabled, we can continue to run on a cpu other than the one to
604 * which we are strongbound; the difference in this case is that
605 * this thread can be preempted and so can appear on the dispatch
606 * queues of a cpu other than the one it is strongbound to).
607 *
608 * If the cpu we are running on does not appear to be a current
609 * offline target (we check cpu_inmotion to determine this - since
610 * we don't hold cpu_lock we may not see a recent store to that,
611 * so it's possible that we at times can grant a weak binding to a
612 * cpu that is an offline target, but that one request will not
613 * prevent the offline from succeeding) then we will always grant
614 * the weak binding request. This includes the case above where
615 * we grant a weakbinding not commensurate with our strong binding.
616 *
617 * If our cpu does appear to be an offline target then we're inclined
618 * not to grant the weakbinding request just yet - we'd prefer to
619 * migrate to another cpu and grant the request there. The
620 * exceptions are those cases where going through preemption code
621 * will not result in us changing cpu:
622 *
623 * . interrupts have already bypassed this case (see above)
624 * . we are already weakbound to this cpu (dispatcher code will
625 * always return us to the weakbound cpu)
626 * . preemption was disabled even before we disabled it above
627 * . we are strongbound to this cpu (if we're strongbound to
628 * another and not yet running there the trip through the
629 * dispatcher will move us to the strongbound cpu and we
630 * will grant the weak binding there)
631 */
632 if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
633 t->t_bound_cpu == cp) {
634 /*
635 * Don't be tempted to store to t_weakbound_cpu only on
636 * the first nested bind request - if we're interrupted
637 * after the increment of t_nomigrate and before the
638 * store to t_weakbound_cpu and the interrupt calls
639 * thread_nomigrate then the assertion in thread_allowmigrate
640 * would fail.
641 */
642 t->t_nomigrate++;
643 t->t_weakbound_cpu = cp;
644 membar_producer();
645 thread_unlock(curthread);
646 /*
647 * Now that we have dropped the thread_lock another thread
648 * can set our t_weakbound_cpu, and will try to migrate us
649 * to the strongbound cpu (which will not be prevented by
650 * preemption being disabled since we're about to enable
651 * preemption). We have granted the weakbinding to the current
652 * cpu, so again we are in the position that is is is possible
653 * that our weak and strong bindings differ. Again this
654 * is catered for by dispatcher code which will favour our
655 * weak binding.
656 */
657 kpreempt_enable();
658 } else {
659 /*
660 * Move to another cpu before granting the request by
661 * forcing this thread through preemption code. When we
662 * get to set{front,back}dq called from CL_PREEMPT()
663 * cpu_choose() will be used to select a cpu to queue
664 * us on - that will see cpu_inmotion and take
665 * steps to avoid returning us to this cpu.
666 */
667 cp->cpu_kprunrun = 1;
668 thread_unlock(curthread);
669 kpreempt_enable(); /* will call preempt() */
670 goto again;
671 }
672 }
673
674 void
675 thread_allowmigrate(void)
676 {
677 kthread_id_t t = curthread;
678
679 ASSERT(t->t_weakbound_cpu == CPU ||
680 (t->t_nomigrate < 0 && t->t_preempt > 0) ||
681 CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
682 getpil() >= DISP_LEVEL);
683
684 if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
685 getpil() >= DISP_LEVEL)
686 return;
687
688 if (t->t_nomigrate < 0) {
689 /*
690 * This thread was granted "weak binding" in the
691 * stronger form of kernel preemption disabling.
692 * Undo a level of nesting for both t_nomigrate
693 * and t_preempt.
694 */
695 ++t->t_nomigrate;
696 kpreempt_enable();
697 } else if (--t->t_nomigrate == 0) {
698 /*
699 * Time to drop the weak binding. We need to cater
700 * for the case where we're weakbound to a different
701 * cpu than that to which we're strongbound (a very
702 * temporary arrangement that must only persist until
703 * weak binding drops). We don't acquire thread_lock
704 * here so even as this code executes t_bound_cpu
705 * may be changing. So we disable preemption and
706 * a) in the case that t_bound_cpu changes while we
707 * have preemption disabled kprunrun will be set
708 * asynchronously, and b) if before disabling
709 * preemption we were already on a different cpu to
710 * our t_bound_cpu then we set kprunrun ourselves
711 * to force a trip through the dispatcher when
712 * preemption is enabled.
713 */
714 kpreempt_disable();
715 if (t->t_bound_cpu &&
716 t->t_weakbound_cpu != t->t_bound_cpu)
717 CPU->cpu_kprunrun = 1;
718 t->t_weakbound_cpu = NULL;
719 membar_producer();
720 kpreempt_enable();
721 }
722 }
723
724 /*
725 * weakbinding_stop can be used to temporarily cause weakbindings made
726 * with thread_nomigrate to be satisfied through the stronger action of
727 * kpreempt_disable. weakbinding_start recommences normal weakbinding.
728 */
729
730 void
731 weakbinding_stop(void)
732 {
733 ASSERT(MUTEX_HELD(&cpu_lock));
734 weakbindingbarrier = 1;
735 membar_producer(); /* make visible before subsequent thread_lock */
736 }
737
738 void
739 weakbinding_start(void)
740 {
741 ASSERT(MUTEX_HELD(&cpu_lock));
742 weakbindingbarrier = 0;
743 }
744
745 void
746 null_xcall(void)
747 {
748 }
749
750 /*
751 * This routine is called to place the CPUs in a safe place so that
752 * one of them can be taken off line or placed on line. What we are
753 * trying to do here is prevent a thread from traversing the list
754 * of active CPUs while we are changing it or from getting placed on
755 * the run queue of a CPU that has just gone off line. We do this by
756 * creating a thread with the highest possible prio for each CPU and
757 * having it call this routine. The advantage of this method is that
758 * we can eliminate all checks for CPU_ACTIVE in the disp routines.
759 * This makes disp faster at the expense of making p_online() slower
760 * which is a good trade off.
761 */
762 static void
763 cpu_pause(int index)
764 {
765 int s;
766 struct _cpu_pause_info *cpi = &cpu_pause_info;
767 volatile char *safe = &safe_list[index];
768 long lindex = index;
769
770 ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
771
772 while (*safe != PAUSE_DIE) {
773 *safe = PAUSE_READY;
774 membar_enter(); /* make sure stores are flushed */
775 sema_v(&cpi->cp_sem); /* signal requesting thread */
776
777 /*
778 * Wait here until all pause threads are running. That
779 * indicates that it's safe to do the spl. Until
780 * cpu_pause_info.cp_go is set, we don't want to spl
781 * because that might block clock interrupts needed
782 * to preempt threads on other CPUs.
783 */
784 while (cpi->cp_go == 0)
785 ;
786 /*
787 * Even though we are at the highest disp prio, we need
788 * to block out all interrupts below LOCK_LEVEL so that
789 * an intr doesn't come in, wake up a thread, and call
790 * setbackdq/setfrontdq.
791 */
792 s = splhigh();
793 /*
794 * if cp_func has been set then call it using index as the
795 * argument, currently only used by cpr_suspend_cpus().
796 * This function is used as the code to execute on the
797 * "paused" cpu's when a machine comes out of a sleep state
798 * and CPU's were powered off. (could also be used for
799 * hotplugging CPU's).
800 */
801 if (cpi->cp_func != NULL)
802 (*cpi->cp_func)((void *)lindex);
803
804 mach_cpu_pause(safe);
805
806 splx(s);
807 /*
808 * Waiting is at an end. Switch out of cpu_pause
809 * loop and resume useful work.
810 */
811 swtch();
812 }
813
814 mutex_enter(&pause_free_mutex);
815 *safe = PAUSE_DEAD;
816 cv_broadcast(&pause_free_cv);
817 mutex_exit(&pause_free_mutex);
818 }
819
820 /*
821 * Allow the cpus to start running again.
822 */
823 void
824 start_cpus()
825 {
826 int i;
827
828 ASSERT(MUTEX_HELD(&cpu_lock));
829 ASSERT(cpu_pause_info.cp_paused);
830 cpu_pause_info.cp_paused = NULL;
831 for (i = 0; i < NCPU; i++)
832 safe_list[i] = PAUSE_IDLE;
833 membar_enter(); /* make sure stores are flushed */
834 affinity_clear();
835 splx(cpu_pause_info.cp_spl);
836 kpreempt_enable();
837 }
838
839 /*
840 * Allocate a pause thread for a CPU.
841 */
842 static void
843 cpu_pause_alloc(cpu_t *cp)
844 {
845 kthread_id_t t;
846 long cpun = cp->cpu_id;
847
848 /*
849 * Note, v.v_nglobpris will not change value as long as I hold
850 * cpu_lock.
851 */
852 t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
853 0, &p0, TS_STOPPED, v.v_nglobpris - 1);
854 thread_lock(t);
855 t->t_bound_cpu = cp;
856 t->t_disp_queue = cp->cpu_disp;
857 t->t_affinitycnt = 1;
858 t->t_preempt = 1;
859 thread_unlock(t);
860 cp->cpu_pause_thread = t;
861 /*
862 * Registering a thread in the callback table is usually done
863 * in the initialization code of the thread. In this
864 * case, we do it right after thread creation because the
865 * thread itself may never run, and we need to register the
866 * fact that it is safe for cpr suspend.
867 */
868 CALLB_CPR_INIT_SAFE(t, "cpu_pause");
869 }
870
871 /*
872 * Free a pause thread for a CPU.
873 */
874 static void
875 cpu_pause_free(cpu_t *cp)
876 {
877 kthread_id_t t;
878 int cpun = cp->cpu_id;
879
880 ASSERT(MUTEX_HELD(&cpu_lock));
881 /*
882 * We have to get the thread and tell it to die.
883 */
884 if ((t = cp->cpu_pause_thread) == NULL) {
885 ASSERT(safe_list[cpun] == PAUSE_IDLE);
886 return;
887 }
888 thread_lock(t);
889 t->t_cpu = CPU; /* disp gets upset if last cpu is quiesced. */
890 t->t_bound_cpu = NULL; /* Must un-bind; cpu may not be running. */
891 t->t_pri = v.v_nglobpris - 1;
892 ASSERT(safe_list[cpun] == PAUSE_IDLE);
893 safe_list[cpun] = PAUSE_DIE;
894 THREAD_TRANSITION(t);
895 setbackdq(t);
896 thread_unlock_nopreempt(t);
897
898 /*
899 * If we don't wait for the thread to actually die, it may try to
900 * run on the wrong cpu as part of an actual call to pause_cpus().
901 */
902 mutex_enter(&pause_free_mutex);
903 while (safe_list[cpun] != PAUSE_DEAD) {
904 cv_wait(&pause_free_cv, &pause_free_mutex);
905 }
906 mutex_exit(&pause_free_mutex);
907 safe_list[cpun] = PAUSE_IDLE;
908
909 cp->cpu_pause_thread = NULL;
910 }
911
912 /*
913 * Initialize basic structures for pausing CPUs.
914 */
915 void
916 cpu_pause_init()
917 {
918 sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
919 /*
920 * Create initial CPU pause thread.
921 */
922 cpu_pause_alloc(CPU);
923 }
924
925 /*
926 * Start the threads used to pause another CPU.
927 */
928 static int
929 cpu_pause_start(processorid_t cpu_id)
930 {
931 int i;
932 int cpu_count = 0;
933
934 for (i = 0; i < NCPU; i++) {
935 cpu_t *cp;
936 kthread_id_t t;
937
938 cp = cpu[i];
939 if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
940 safe_list[i] = PAUSE_WAIT;
941 continue;
942 }
943
944 /*
945 * Skip CPU if it is quiesced or not yet started.
946 */
947 if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
948 safe_list[i] = PAUSE_WAIT;
949 continue;
950 }
951
952 /*
953 * Start this CPU's pause thread.
954 */
955 t = cp->cpu_pause_thread;
956 thread_lock(t);
957 /*
958 * Reset the priority, since nglobpris may have
959 * changed since the thread was created, if someone
960 * has loaded the RT (or some other) scheduling
961 * class.
962 */
963 t->t_pri = v.v_nglobpris - 1;
964 THREAD_TRANSITION(t);
965 setbackdq(t);
966 thread_unlock_nopreempt(t);
967 ++cpu_count;
968 }
969 return (cpu_count);
970 }
971
972
973 /*
974 * Pause all of the CPUs except the one we are on by creating a high
975 * priority thread bound to those CPUs.
976 *
977 * Note that one must be extremely careful regarding code
978 * executed while CPUs are paused. Since a CPU may be paused
979 * while a thread scheduling on that CPU is holding an adaptive
980 * lock, code executed with CPUs paused must not acquire adaptive
981 * (or low-level spin) locks. Also, such code must not block,
982 * since the thread that is supposed to initiate the wakeup may
983 * never run.
984 *
985 * With a few exceptions, the restrictions on code executed with CPUs
986 * paused match those for code executed at high-level interrupt
987 * context.
988 */
989 void
990 pause_cpus(cpu_t *off_cp, void *(*func)(void *))
991 {
992 processorid_t cpu_id;
993 int i;
994 struct _cpu_pause_info *cpi = &cpu_pause_info;
995
996 ASSERT(MUTEX_HELD(&cpu_lock));
997 ASSERT(cpi->cp_paused == NULL);
998 cpi->cp_count = 0;
999 cpi->cp_go = 0;
1000 for (i = 0; i < NCPU; i++)
1001 safe_list[i] = PAUSE_IDLE;
1002 kpreempt_disable();
1003
1004 cpi->cp_func = func;
1005
1006 /*
1007 * If running on the cpu that is going offline, get off it.
1008 * This is so that it won't be necessary to rechoose a CPU
1009 * when done.
1010 */
1011 if (CPU == off_cp)
1012 cpu_id = off_cp->cpu_next_part->cpu_id;
1013 else
1014 cpu_id = CPU->cpu_id;
1015 affinity_set(cpu_id);
1016
1017 /*
1018 * Start the pause threads and record how many were started
1019 */
1020 cpi->cp_count = cpu_pause_start(cpu_id);
1021
1022 /*
1023 * Now wait for all CPUs to be running the pause thread.
1024 */
1025 while (cpi->cp_count > 0) {
1026 /*
1027 * Spin reading the count without grabbing the disp
1028 * lock to make sure we don't prevent the pause
1029 * threads from getting the lock.
1030 */
1031 while (sema_held(&cpi->cp_sem))
1032 ;
1033 if (sema_tryp(&cpi->cp_sem))
1034 --cpi->cp_count;
1035 }
1036 cpi->cp_go = 1; /* all have reached cpu_pause */
1037
1038 /*
1039 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1040 * to PAUSE_WAIT.)
1041 */
1042 for (i = 0; i < NCPU; i++) {
1043 while (safe_list[i] != PAUSE_WAIT)
1044 ;
1045 }
1046 cpi->cp_spl = splhigh(); /* block dispatcher on this CPU */
1047 cpi->cp_paused = curthread;
1048 }
1049
1050 /*
1051 * Check whether the current thread has CPUs paused
1052 */
1053 int
1054 cpus_paused(void)
1055 {
1056 if (cpu_pause_info.cp_paused != NULL) {
1057 ASSERT(cpu_pause_info.cp_paused == curthread);
1058 return (1);
1059 }
1060 return (0);
1061 }
1062
1063 static cpu_t *
1064 cpu_get_all(processorid_t cpun)
1065 {
1066 ASSERT(MUTEX_HELD(&cpu_lock));
1067
1068 if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1069 return (NULL);
1070 return (cpu[cpun]);
1071 }
1072
1073 /*
1074 * Check whether cpun is a valid processor id and whether it should be
1075 * visible from the current zone. If it is, return a pointer to the
1076 * associated CPU structure.
1077 */
1078 cpu_t *
1079 cpu_get(processorid_t cpun)
1080 {
1081 cpu_t *c;
1082
1083 ASSERT(MUTEX_HELD(&cpu_lock));
1084 c = cpu_get_all(cpun);
1085 if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1086 zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1087 return (NULL);
1088 return (c);
1089 }
1090
1091 /*
1092 * The following functions should be used to check CPU states in the kernel.
1093 * They should be invoked with cpu_lock held. Kernel subsystems interested
1094 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1095 * states. Those are for user-land (and system call) use only.
1096 */
1097
1098 /*
1099 * Determine whether the CPU is online and handling interrupts.
1100 */
1101 int
1102 cpu_is_online(cpu_t *cpu)
1103 {
1104 ASSERT(MUTEX_HELD(&cpu_lock));
1105 return (cpu_flagged_online(cpu->cpu_flags));
1106 }
1107
1108 /*
1109 * Determine whether the CPU is offline (this includes spare and faulted).
1110 */
1111 int
1112 cpu_is_offline(cpu_t *cpu)
1113 {
1114 ASSERT(MUTEX_HELD(&cpu_lock));
1115 return (cpu_flagged_offline(cpu->cpu_flags));
1116 }
1117
1118 /*
1119 * Determine whether the CPU is powered off.
1120 */
1121 int
1122 cpu_is_poweredoff(cpu_t *cpu)
1123 {
1124 ASSERT(MUTEX_HELD(&cpu_lock));
1125 return (cpu_flagged_poweredoff(cpu->cpu_flags));
1126 }
1127
1128 /*
1129 * Determine whether the CPU is handling interrupts.
1130 */
1131 int
1132 cpu_is_nointr(cpu_t *cpu)
1133 {
1134 ASSERT(MUTEX_HELD(&cpu_lock));
1135 return (cpu_flagged_nointr(cpu->cpu_flags));
1136 }
1137
1138 /*
1139 * Determine whether the CPU is active (scheduling threads).
1140 */
1141 int
1142 cpu_is_active(cpu_t *cpu)
1143 {
1144 ASSERT(MUTEX_HELD(&cpu_lock));
1145 return (cpu_flagged_active(cpu->cpu_flags));
1146 }
1147
1148 /*
1149 * Same as above, but these require cpu_flags instead of cpu_t pointers.
1150 */
1151 int
1152 cpu_flagged_online(cpu_flag_t cpu_flags)
1153 {
1154 return (cpu_flagged_active(cpu_flags) &&
1155 (cpu_flags & CPU_ENABLE));
1156 }
1157
1158 int
1159 cpu_flagged_offline(cpu_flag_t cpu_flags)
1160 {
1161 return (((cpu_flags & CPU_POWEROFF) == 0) &&
1162 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1163 }
1164
1165 int
1166 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1167 {
1168 return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1169 }
1170
1171 int
1172 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1173 {
1174 return (cpu_flagged_active(cpu_flags) &&
1175 (cpu_flags & CPU_ENABLE) == 0);
1176 }
1177
1178 int
1179 cpu_flagged_active(cpu_flag_t cpu_flags)
1180 {
1181 return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1182 ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1183 }
1184
1185 /*
1186 * Bring the indicated CPU online.
1187 */
1188 int
1189 cpu_online(cpu_t *cp)
1190 {
1191 int error = 0;
1192
1193 /*
1194 * Handle on-line request.
1195 * This code must put the new CPU on the active list before
1196 * starting it because it will not be paused, and will start
1197 * using the active list immediately. The real start occurs
1198 * when the CPU_QUIESCED flag is turned off.
1199 */
1200
1201 ASSERT(MUTEX_HELD(&cpu_lock));
1202
1203 /*
1204 * Put all the cpus into a known safe place.
1205 * No mutexes can be entered while CPUs are paused.
1206 */
1207 error = mp_cpu_start(cp); /* arch-dep hook */
1208 if (error == 0) {
1209 pg_cpupart_in(cp, cp->cpu_part);
1210 pause_cpus(NULL, NULL);
1211 cpu_add_active_internal(cp);
1212 if (cp->cpu_flags & CPU_FAULTED) {
1213 cp->cpu_flags &= ~CPU_FAULTED;
1214 mp_cpu_faulted_exit(cp);
1215 }
1216 cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1217 CPU_SPARE);
1218 CPU_NEW_GENERATION(cp);
1219 start_cpus();
1220 cpu_stats_kstat_create(cp);
1221 cpu_create_intrstat(cp);
1222 lgrp_kstat_create(cp);
1223 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1224 cpu_intr_enable(cp); /* arch-dep hook */
1225 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1226 cpu_set_state(cp);
1227 cyclic_online(cp);
1228 /*
1229 * This has to be called only after cyclic_online(). This
1230 * function uses cyclics.
1231 */
1232 callout_cpu_online(cp);
1233 poke_cpu(cp->cpu_id);
1234 }
1235
1236 return (error);
1237 }
1238
1239 /*
1240 * Take the indicated CPU offline.
1241 */
1242 int
1243 cpu_offline(cpu_t *cp, int flags)
1244 {
1245 cpupart_t *pp;
1246 int error = 0;
1247 cpu_t *ncp;
1248 int intr_enable;
1249 int cyclic_off = 0;
1250 int callout_off = 0;
1251 int loop_count;
1252 int no_quiesce = 0;
1253 int (*bound_func)(struct cpu *, int);
1254 kthread_t *t;
1255 lpl_t *cpu_lpl;
1256 proc_t *p;
1257 int lgrp_diff_lpl;
1258 boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1259
1260 ASSERT(MUTEX_HELD(&cpu_lock));
1261
1262 /*
1263 * If we're going from faulted or spare to offline, just
1264 * clear these flags and update CPU state.
1265 */
1266 if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1267 if (cp->cpu_flags & CPU_FAULTED) {
1268 cp->cpu_flags &= ~CPU_FAULTED;
1269 mp_cpu_faulted_exit(cp);
1270 }
1271 cp->cpu_flags &= ~CPU_SPARE;
1272 cpu_set_state(cp);
1273 return (0);
1274 }
1275
1276 /*
1277 * Handle off-line request.
1278 */
1279 pp = cp->cpu_part;
1280 /*
1281 * Don't offline last online CPU in partition
1282 */
1283 if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1284 return (EBUSY);
1285 /*
1286 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1287 * if we were asked to.
1288 */
1289 error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1290 if (error != 0)
1291 return (error);
1292 /*
1293 * We shouldn't be bound to this CPU ourselves.
1294 */
1295 if (curthread->t_bound_cpu == cp)
1296 return (EBUSY);
1297
1298 /*
1299 * Tell interested parties that this CPU is going offline.
1300 */
1301 CPU_NEW_GENERATION(cp);
1302 cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1303
1304 /*
1305 * Tell the PG subsystem that the CPU is leaving the partition
1306 */
1307 pg_cpupart_out(cp, pp);
1308
1309 /*
1310 * Take the CPU out of interrupt participation so we won't find
1311 * bound kernel threads. If the architecture cannot completely
1312 * shut off interrupts on the CPU, don't quiesce it, but don't
1313 * run anything but interrupt thread... this is indicated by
1314 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1315 * off.
1316 */
1317 intr_enable = cp->cpu_flags & CPU_ENABLE;
1318 if (intr_enable)
1319 no_quiesce = cpu_intr_disable(cp);
1320
1321 /*
1322 * Record that we are aiming to offline this cpu. This acts as
1323 * a barrier to further weak binding requests in thread_nomigrate
1324 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1325 * lean away from this cpu. Further strong bindings are already
1326 * avoided since we hold cpu_lock. Since threads that are set
1327 * runnable around now and others coming off the target cpu are
1328 * directed away from the target, existing strong and weak bindings
1329 * (especially the latter) to the target cpu stand maximum chance of
1330 * being able to unbind during the short delay loop below (if other
1331 * unbound threads compete they may not see cpu in time to unbind
1332 * even if they would do so immediately.
1333 */
1334 cpu_inmotion = cp;
1335 membar_enter();
1336
1337 /*
1338 * Check for kernel threads (strong or weak) bound to that CPU.
1339 * Strongly bound threads may not unbind, and we'll have to return
1340 * EBUSY. Weakly bound threads should always disappear - we've
1341 * stopped more weak binding with cpu_inmotion and existing
1342 * bindings will drain imminently (they may not block). Nonetheless
1343 * we will wait for a fixed period for all bound threads to disappear.
1344 * Inactive interrupt threads are OK (they'll be in TS_FREE
1345 * state). If test finds some bound threads, wait a few ticks
1346 * to give short-lived threads (such as interrupts) chance to
1347 * complete. Note that if no_quiesce is set, i.e. this cpu
1348 * is required to service interrupts, then we take the route
1349 * that permits interrupt threads to be active (or bypassed).
1350 */
1351 bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1352
1353 again: for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1354 if (loop_count >= 5) {
1355 error = EBUSY; /* some threads still bound */
1356 break;
1357 }
1358
1359 /*
1360 * If some threads were assigned, give them
1361 * a chance to complete or move.
1362 *
1363 * This assumes that the clock_thread is not bound
1364 * to any CPU, because the clock_thread is needed to
1365 * do the delay(hz/100).
1366 *
1367 * Note: we still hold the cpu_lock while waiting for
1368 * the next clock tick. This is OK since it isn't
1369 * needed for anything else except processor_bind(2),
1370 * and system initialization. If we drop the lock,
1371 * we would risk another p_online disabling the last
1372 * processor.
1373 */
1374 delay(hz/100);
1375 }
1376
1377 if (error == 0 && callout_off == 0) {
1378 callout_cpu_offline(cp);
1379 callout_off = 1;
1380 }
1381
1382 if (error == 0 && cyclic_off == 0) {
1383 if (!cyclic_offline(cp)) {
1384 /*
1385 * We must have bound cyclics...
1386 */
1387 error = EBUSY;
1388 goto out;
1389 }
1390 cyclic_off = 1;
1391 }
1392
1393 /*
1394 * Call mp_cpu_stop() to perform any special operations
1395 * needed for this machine architecture to offline a CPU.
1396 */
1397 if (error == 0)
1398 error = mp_cpu_stop(cp); /* arch-dep hook */
1399
1400 /*
1401 * If that all worked, take the CPU offline and decrement
1402 * ncpus_online.
1403 */
1404 if (error == 0) {
1405 /*
1406 * Put all the cpus into a known safe place.
1407 * No mutexes can be entered while CPUs are paused.
1408 */
1409 pause_cpus(cp, NULL);
1410 /*
1411 * Repeat the operation, if necessary, to make sure that
1412 * all outstanding low-level interrupts run to completion
1413 * before we set the CPU_QUIESCED flag. It's also possible
1414 * that a thread has weak bound to the cpu despite our raising
1415 * cpu_inmotion above since it may have loaded that
1416 * value before the barrier became visible (this would have
1417 * to be the thread that was on the target cpu at the time
1418 * we raised the barrier).
1419 */
1420 if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1421 (*bound_func)(cp, 1)) {
1422 start_cpus();
1423 (void) mp_cpu_start(cp);
1424 goto again;
1425 }
1426 ncp = cp->cpu_next_part;
1427 cpu_lpl = cp->cpu_lpl;
1428 ASSERT(cpu_lpl != NULL);
1429
1430 /*
1431 * Remove the CPU from the list of active CPUs.
1432 */
1433 cpu_remove_active(cp);
1434
1435 /*
1436 * Walk the active process list and look for threads
1437 * whose home lgroup needs to be updated, or
1438 * the last CPU they run on is the one being offlined now.
1439 */
1440
1441 ASSERT(curthread->t_cpu != cp);
1442 for (p = practive; p != NULL; p = p->p_next) {
1443
1444 t = p->p_tlist;
1445
1446 if (t == NULL)
1447 continue;
1448
1449 lgrp_diff_lpl = 0;
1450
1451 do {
1452 ASSERT(t->t_lpl != NULL);
1453 /*
1454 * Taking last CPU in lpl offline
1455 * Rehome thread if it is in this lpl
1456 * Otherwise, update the count of how many
1457 * threads are in this CPU's lgroup but have
1458 * a different lpl.
1459 */
1460
1461 if (cpu_lpl->lpl_ncpu == 0) {
1462 if (t->t_lpl == cpu_lpl)
1463 lgrp_move_thread(t,
1464 lgrp_choose(t,
1465 t->t_cpupart), 0);
1466 else if (t->t_lpl->lpl_lgrpid ==
1467 cpu_lpl->lpl_lgrpid)
1468 lgrp_diff_lpl++;
1469 }
1470 ASSERT(t->t_lpl->lpl_ncpu > 0);
1471
1472 /*
1473 * Update CPU last ran on if it was this CPU
1474 */
1475 if (t->t_cpu == cp && t->t_bound_cpu != cp)
1476 t->t_cpu = disp_lowpri_cpu(ncp,
1477 t->t_lpl, t->t_pri, NULL);
1478 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1479 t->t_weakbound_cpu == cp);
1480
1481 t = t->t_forw;
1482 } while (t != p->p_tlist);
1483
1484 /*
1485 * Didn't find any threads in the same lgroup as this
1486 * CPU with a different lpl, so remove the lgroup from
1487 * the process lgroup bitmask.
1488 */
1489
1490 if (lgrp_diff_lpl == 0)
1491 klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1492 }
1493
1494 /*
1495 * Walk thread list looking for threads that need to be
1496 * rehomed, since there are some threads that are not in
1497 * their process's p_tlist.
1498 */
1499
1500 t = curthread;
1501 do {
1502 ASSERT(t != NULL && t->t_lpl != NULL);
1503
1504 /*
1505 * Rehome threads with same lpl as this CPU when this
1506 * is the last CPU in the lpl.
1507 */
1508
1509 if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1510 lgrp_move_thread(t,
1511 lgrp_choose(t, t->t_cpupart), 1);
1512
1513 ASSERT(t->t_lpl->lpl_ncpu > 0);
1514
1515 /*
1516 * Update CPU last ran on if it was this CPU
1517 */
1518
1519 if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1520 t->t_cpu = disp_lowpri_cpu(ncp,
1521 t->t_lpl, t->t_pri, NULL);
1522 }
1523 ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1524 t->t_weakbound_cpu == cp);
1525 t = t->t_next;
1526
1527 } while (t != curthread);
1528 ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1529 cp->cpu_flags |= CPU_OFFLINE;
1530 disp_cpu_inactive(cp);
1531 if (!no_quiesce)
1532 cp->cpu_flags |= CPU_QUIESCED;
1533 ncpus_online--;
1534 cpu_set_state(cp);
1535 cpu_inmotion = NULL;
1536 start_cpus();
1537 cpu_stats_kstat_destroy(cp);
1538 cpu_delete_intrstat(cp);
1539 lgrp_kstat_destroy(cp);
1540 }
1541
1542 out:
1543 cpu_inmotion = NULL;
1544
1545 /*
1546 * If we failed, re-enable interrupts.
1547 * Do this even if cpu_intr_disable returned an error, because
1548 * it may have partially disabled interrupts.
1549 */
1550 if (error && intr_enable)
1551 cpu_intr_enable(cp);
1552
1553 /*
1554 * If we failed, but managed to offline the cyclic subsystem on this
1555 * CPU, bring it back online.
1556 */
1557 if (error && cyclic_off)
1558 cyclic_online(cp);
1559
1560 /*
1561 * If we failed, but managed to offline callouts on this CPU,
1562 * bring it back online.
1563 */
1564 if (error && callout_off)
1565 callout_cpu_online(cp);
1566
1567 /*
1568 * If we failed, tell the PG subsystem that the CPU is back
1569 */
1570 pg_cpupart_in(cp, pp);
1571
1572 /*
1573 * If we failed, we need to notify everyone that this CPU is back on.
1574 */
1575 if (error != 0) {
1576 CPU_NEW_GENERATION(cp);
1577 cpu_state_change_notify(cp->cpu_id, CPU_ON);
1578 cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1579 }
1580
1581 return (error);
1582 }
1583
1584 /*
1585 * Mark the indicated CPU as faulted, taking it offline.
1586 */
1587 int
1588 cpu_faulted(cpu_t *cp, int flags)
1589 {
1590 int error = 0;
1591
1592 ASSERT(MUTEX_HELD(&cpu_lock));
1593 ASSERT(!cpu_is_poweredoff(cp));
1594
1595 if (cpu_is_offline(cp)) {
1596 cp->cpu_flags &= ~CPU_SPARE;
1597 cp->cpu_flags |= CPU_FAULTED;
1598 mp_cpu_faulted_enter(cp);
1599 cpu_set_state(cp);
1600 return (0);
1601 }
1602
1603 if ((error = cpu_offline(cp, flags)) == 0) {
1604 cp->cpu_flags |= CPU_FAULTED;
1605 mp_cpu_faulted_enter(cp);
1606 cpu_set_state(cp);
1607 }
1608
1609 return (error);
1610 }
1611
1612 /*
1613 * Mark the indicated CPU as a spare, taking it offline.
1614 */
1615 int
1616 cpu_spare(cpu_t *cp, int flags)
1617 {
1618 int error = 0;
1619
1620 ASSERT(MUTEX_HELD(&cpu_lock));
1621 ASSERT(!cpu_is_poweredoff(cp));
1622
1623 if (cpu_is_offline(cp)) {
1624 if (cp->cpu_flags & CPU_FAULTED) {
1625 cp->cpu_flags &= ~CPU_FAULTED;
1626 mp_cpu_faulted_exit(cp);
1627 }
1628 cp->cpu_flags |= CPU_SPARE;
1629 cpu_set_state(cp);
1630 return (0);
1631 }
1632
1633 if ((error = cpu_offline(cp, flags)) == 0) {
1634 cp->cpu_flags |= CPU_SPARE;
1635 cpu_set_state(cp);
1636 }
1637
1638 return (error);
1639 }
1640
1641 /*
1642 * Take the indicated CPU from poweroff to offline.
1643 */
1644 int
1645 cpu_poweron(cpu_t *cp)
1646 {
1647 int error = ENOTSUP;
1648
1649 ASSERT(MUTEX_HELD(&cpu_lock));
1650 ASSERT(cpu_is_poweredoff(cp));
1651
1652 error = mp_cpu_poweron(cp); /* arch-dep hook */
1653 if (error == 0)
1654 cpu_set_state(cp);
1655
1656 return (error);
1657 }
1658
1659 /*
1660 * Take the indicated CPU from any inactive state to powered off.
1661 */
1662 int
1663 cpu_poweroff(cpu_t *cp)
1664 {
1665 int error = ENOTSUP;
1666
1667 ASSERT(MUTEX_HELD(&cpu_lock));
1668 ASSERT(cpu_is_offline(cp));
1669
1670 if (!(cp->cpu_flags & CPU_QUIESCED))
1671 return (EBUSY); /* not completely idle */
1672
1673 error = mp_cpu_poweroff(cp); /* arch-dep hook */
1674 if (error == 0)
1675 cpu_set_state(cp);
1676
1677 return (error);
1678 }
1679
1680 /*
1681 * Initialize the Sequential CPU id lookup table
1682 */
1683 void
1684 cpu_seq_tbl_init()
1685 {
1686 cpu_t **tbl;
1687
1688 tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1689 tbl[0] = CPU;
1690
1691 cpu_seq = tbl;
1692 }
1693
1694 /*
1695 * Initialize the CPU lists for the first CPU.
1696 */
1697 void
1698 cpu_list_init(cpu_t *cp)
1699 {
1700 cp->cpu_next = cp;
1701 cp->cpu_prev = cp;
1702 cpu_list = cp;
1703 clock_cpu_list = cp;
1704
1705 cp->cpu_next_onln = cp;
1706 cp->cpu_prev_onln = cp;
1707 cpu_active = cp;
1708
1709 cp->cpu_seqid = 0;
1710 CPUSET_ADD(cpu_seqid_inuse, 0);
1711
1712 /*
1713 * Bootstrap cpu_seq using cpu_list
1714 * The cpu_seq[] table will be dynamically allocated
1715 * when kmem later becomes available (but before going MP)
1716 */
1717 cpu_seq = &cpu_list;
1718
1719 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1720 cp_default.cp_cpulist = cp;
1721 cp_default.cp_ncpus = 1;
1722 cp->cpu_next_part = cp;
1723 cp->cpu_prev_part = cp;
1724 cp->cpu_part = &cp_default;
1725
1726 CPUSET_ADD(cpu_available, cp->cpu_id);
1727 }
1728
1729 /*
1730 * Insert a CPU into the list of available CPUs.
1731 */
1732 void
1733 cpu_add_unit(cpu_t *cp)
1734 {
1735 int seqid;
1736
1737 ASSERT(MUTEX_HELD(&cpu_lock));
1738 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1739
1740 lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1741
1742 /*
1743 * Note: most users of the cpu_list will grab the
1744 * cpu_lock to insure that it isn't modified. However,
1745 * certain users can't or won't do that. To allow this
1746 * we pause the other cpus. Users who walk the list
1747 * without cpu_lock, must disable kernel preemption
1748 * to insure that the list isn't modified underneath
1749 * them. Also, any cached pointers to cpu structures
1750 * must be revalidated by checking to see if the
1751 * cpu_next pointer points to itself. This check must
1752 * be done with the cpu_lock held or kernel preemption
1753 * disabled. This check relies upon the fact that
1754 * old cpu structures are not free'ed or cleared after
1755 * then are removed from the cpu_list.
1756 *
1757 * Note that the clock code walks the cpu list dereferencing
1758 * the cpu_part pointer, so we need to initialize it before
1759 * adding the cpu to the list.
1760 */
1761 cp->cpu_part = &cp_default;
1762 pause_cpus(NULL, NULL);
1763 cp->cpu_next = cpu_list;
1764 cp->cpu_prev = cpu_list->cpu_prev;
1765 cpu_list->cpu_prev->cpu_next = cp;
1766 cpu_list->cpu_prev = cp;
1767 start_cpus();
1768
1769 for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1770 continue;
1771 CPUSET_ADD(cpu_seqid_inuse, seqid);
1772 cp->cpu_seqid = seqid;
1773
1774 if (seqid > max_cpu_seqid_ever)
1775 max_cpu_seqid_ever = seqid;
1776
1777 ASSERT(ncpus < max_ncpus);
1778 ncpus++;
1779 cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1780 cpu[cp->cpu_id] = cp;
1781 CPUSET_ADD(cpu_available, cp->cpu_id);
1782 cpu_seq[cp->cpu_seqid] = cp;
1783
1784 /*
1785 * allocate a pause thread for this CPU.
1786 */
1787 cpu_pause_alloc(cp);
1788
1789 /*
1790 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1791 * link them into a list of just that CPU.
1792 * This is so that disp_lowpri_cpu will work for thread_create in
1793 * pause_cpus() when called from the startup thread in a new CPU.
1794 */
1795 cp->cpu_next_onln = cp;
1796 cp->cpu_prev_onln = cp;
1797 cpu_info_kstat_create(cp);
1798 cp->cpu_next_part = cp;
1799 cp->cpu_prev_part = cp;
1800
1801 init_cpu_mstate(cp, CMS_SYSTEM);
1802
1803 pool_pset_mod = gethrtime();
1804 }
1805
1806 /*
1807 * Do the opposite of cpu_add_unit().
1808 */
1809 void
1810 cpu_del_unit(int cpuid)
1811 {
1812 struct cpu *cp, *cpnext;
1813
1814 ASSERT(MUTEX_HELD(&cpu_lock));
1815 cp = cpu[cpuid];
1816 ASSERT(cp != NULL);
1817
1818 ASSERT(cp->cpu_next_onln == cp);
1819 ASSERT(cp->cpu_prev_onln == cp);
1820 ASSERT(cp->cpu_next_part == cp);
1821 ASSERT(cp->cpu_prev_part == cp);
1822
1823 /*
1824 * Tear down the CPU's physical ID cache, and update any
1825 * processor groups
1826 */
1827 pg_cpu_fini(cp, NULL);
1828 pghw_physid_destroy(cp);
1829
1830 /*
1831 * Destroy kstat stuff.
1832 */
1833 cpu_info_kstat_destroy(cp);
1834 term_cpu_mstate(cp);
1835 /*
1836 * Free up pause thread.
1837 */
1838 cpu_pause_free(cp);
1839 CPUSET_DEL(cpu_available, cp->cpu_id);
1840 cpu[cp->cpu_id] = NULL;
1841 cpu_seq[cp->cpu_seqid] = NULL;
1842
1843 /*
1844 * The clock thread and mutex_vector_enter cannot hold the
1845 * cpu_lock while traversing the cpu list, therefore we pause
1846 * all other threads by pausing the other cpus. These, and any
1847 * other routines holding cpu pointers while possibly sleeping
1848 * must be sure to call kpreempt_disable before processing the
1849 * list and be sure to check that the cpu has not been deleted
1850 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1851 * to keep the deleted cpu structure around.
1852 *
1853 * Note that this MUST be done AFTER cpu_available
1854 * has been updated so that we don't waste time
1855 * trying to pause the cpu we're trying to delete.
1856 */
1857 pause_cpus(NULL, NULL);
1858
1859 cpnext = cp->cpu_next;
1860 cp->cpu_prev->cpu_next = cp->cpu_next;
1861 cp->cpu_next->cpu_prev = cp->cpu_prev;
1862 if (cp == cpu_list)
1863 cpu_list = cpnext;
1864
1865 /*
1866 * Signals that the cpu has been deleted (see above).
1867 */
1868 cp->cpu_next = NULL;
1869 cp->cpu_prev = NULL;
1870
1871 start_cpus();
1872
1873 CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1874 ncpus--;
1875 lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1876
1877 pool_pset_mod = gethrtime();
1878 }
1879
1880 /*
1881 * Add a CPU to the list of active CPUs.
1882 * This routine must not get any locks, because other CPUs are paused.
1883 */
1884 static void
1885 cpu_add_active_internal(cpu_t *cp)
1886 {
1887 cpupart_t *pp = cp->cpu_part;
1888
1889 ASSERT(MUTEX_HELD(&cpu_lock));
1890 ASSERT(cpu_list != NULL); /* list started in cpu_list_init */
1891
1892 ncpus_online++;
1893 cpu_set_state(cp);
1894 cp->cpu_next_onln = cpu_active;
1895 cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1896 cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1897 cpu_active->cpu_prev_onln = cp;
1898
1899 if (pp->cp_cpulist) {
1900 cp->cpu_next_part = pp->cp_cpulist;
1901 cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1902 pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1903 pp->cp_cpulist->cpu_prev_part = cp;
1904 } else {
1905 ASSERT(pp->cp_ncpus == 0);
1906 pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1907 }
1908 pp->cp_ncpus++;
1909 if (pp->cp_ncpus == 1) {
1910 cp_numparts_nonempty++;
1911 ASSERT(cp_numparts_nonempty != 0);
1912 }
1913
1914 pg_cpu_active(cp);
1915 lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1916
1917 bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1918 }
1919
1920 /*
1921 * Add a CPU to the list of active CPUs.
1922 * This is called from machine-dependent layers when a new CPU is started.
1923 */
1924 void
1925 cpu_add_active(cpu_t *cp)
1926 {
1927 pg_cpupart_in(cp, cp->cpu_part);
1928
1929 pause_cpus(NULL, NULL);
1930 cpu_add_active_internal(cp);
1931 start_cpus();
1932
1933 cpu_stats_kstat_create(cp);
1934 cpu_create_intrstat(cp);
1935 lgrp_kstat_create(cp);
1936 cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1937 }
1938
1939
1940 /*
1941 * Remove a CPU from the list of active CPUs.
1942 * This routine must not get any locks, because other CPUs are paused.
1943 */
1944 /* ARGSUSED */
1945 static void
1946 cpu_remove_active(cpu_t *cp)
1947 {
1948 cpupart_t *pp = cp->cpu_part;
1949
1950 ASSERT(MUTEX_HELD(&cpu_lock));
1951 ASSERT(cp->cpu_next_onln != cp); /* not the last one */
1952 ASSERT(cp->cpu_prev_onln != cp); /* not the last one */
1953
1954 pg_cpu_inactive(cp);
1955
1956 lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1957
1958 if (cp == clock_cpu_list)
1959 clock_cpu_list = cp->cpu_next_onln;
1960
1961 cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1962 cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1963 if (cpu_active == cp) {
1964 cpu_active = cp->cpu_next_onln;
1965 }
1966 cp->cpu_next_onln = cp;
1967 cp->cpu_prev_onln = cp;
1968
1969 cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1970 cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1971 if (pp->cp_cpulist == cp) {
1972 pp->cp_cpulist = cp->cpu_next_part;
1973 ASSERT(pp->cp_cpulist != cp);
1974 }
1975 cp->cpu_next_part = cp;
1976 cp->cpu_prev_part = cp;
1977 pp->cp_ncpus--;
1978 if (pp->cp_ncpus == 0) {
1979 cp_numparts_nonempty--;
1980 ASSERT(cp_numparts_nonempty != 0);
1981 }
1982 }
1983
1984 /*
1985 * Routine used to setup a newly inserted CPU in preparation for starting
1986 * it running code.
1987 */
1988 int
1989 cpu_configure(int cpuid)
1990 {
1991 int retval = 0;
1992
1993 ASSERT(MUTEX_HELD(&cpu_lock));
1994
1995 /*
1996 * Some structures are statically allocated based upon
1997 * the maximum number of cpus the system supports. Do not
1998 * try to add anything beyond this limit.
1999 */
2000 if (cpuid < 0 || cpuid >= NCPU) {
2001 return (EINVAL);
2002 }
2003
2004 if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2005 return (EALREADY);
2006 }
2007
2008 if ((retval = mp_cpu_configure(cpuid)) != 0) {
2009 return (retval);
2010 }
2011
2012 cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2013 cpu_set_state(cpu[cpuid]);
2014 retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2015 if (retval != 0)
2016 (void) mp_cpu_unconfigure(cpuid);
2017
2018 return (retval);
2019 }
2020
2021 /*
2022 * Routine used to cleanup a CPU that has been powered off. This will
2023 * destroy all per-cpu information related to this cpu.
2024 */
2025 int
2026 cpu_unconfigure(int cpuid)
2027 {
2028 int error;
2029
2030 ASSERT(MUTEX_HELD(&cpu_lock));
2031
2032 if (cpu[cpuid] == NULL) {
2033 return (ENODEV);
2034 }
2035
2036 if (cpu[cpuid]->cpu_flags == 0) {
2037 return (EALREADY);
2038 }
2039
2040 if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2041 return (EBUSY);
2042 }
2043
2044 if (cpu[cpuid]->cpu_props != NULL) {
2045 (void) nvlist_free(cpu[cpuid]->cpu_props);
2046 cpu[cpuid]->cpu_props = NULL;
2047 }
2048
2049 error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2050
2051 if (error != 0)
2052 return (error);
2053
2054 return (mp_cpu_unconfigure(cpuid));
2055 }
2056
2057 /*
2058 * Routines for registering and de-registering cpu_setup callback functions.
2059 *
2060 * Caller's context
2061 * These routines must not be called from a driver's attach(9E) or
2062 * detach(9E) entry point.
2063 *
2064 * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2065 */
2066
2067 /*
2068 * Ideally, these would be dynamically allocated and put into a linked
2069 * list; however that is not feasible because the registration routine
2070 * has to be available before the kmem allocator is working (in fact,
2071 * it is called by the kmem allocator init code). In any case, there
2072 * are quite a few extra entries for future users.
2073 */
2074 #define NCPU_SETUPS 20
2075
2076 struct cpu_setup {
2077 cpu_setup_func_t *func;
2078 void *arg;
2079 } cpu_setups[NCPU_SETUPS];
2080
2081 void
2082 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2083 {
2084 int i;
2085
2086 ASSERT(MUTEX_HELD(&cpu_lock));
2087
2088 for (i = 0; i < NCPU_SETUPS; i++)
2089 if (cpu_setups[i].func == NULL)
2090 break;
2091 if (i >= NCPU_SETUPS)
2092 cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2093
2094 cpu_setups[i].func = func;
2095 cpu_setups[i].arg = arg;
2096 }
2097
2098 void
2099 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2100 {
2101 int i;
2102
2103 ASSERT(MUTEX_HELD(&cpu_lock));
2104
2105 for (i = 0; i < NCPU_SETUPS; i++)
2106 if ((cpu_setups[i].func == func) &&
2107 (cpu_setups[i].arg == arg))
2108 break;
2109 if (i >= NCPU_SETUPS)
2110 cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2111 "deregister");
2112
2113 cpu_setups[i].func = NULL;
2114 cpu_setups[i].arg = 0;
2115 }
2116
2117 /*
2118 * Call any state change hooks for this CPU, ignore any errors.
2119 */
2120 void
2121 cpu_state_change_notify(int id, cpu_setup_t what)
2122 {
2123 int i;
2124
2125 ASSERT(MUTEX_HELD(&cpu_lock));
2126
2127 for (i = 0; i < NCPU_SETUPS; i++) {
2128 if (cpu_setups[i].func != NULL) {
2129 cpu_setups[i].func(what, id, cpu_setups[i].arg);
2130 }
2131 }
2132 }
2133
2134 /*
2135 * Call any state change hooks for this CPU, undo it if error found.
2136 */
2137 static int
2138 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2139 {
2140 int i;
2141 int retval = 0;
2142
2143 ASSERT(MUTEX_HELD(&cpu_lock));
2144
2145 for (i = 0; i < NCPU_SETUPS; i++) {
2146 if (cpu_setups[i].func != NULL) {
2147 retval = cpu_setups[i].func(what, id,
2148 cpu_setups[i].arg);
2149 if (retval) {
2150 for (i--; i >= 0; i--) {
2151 if (cpu_setups[i].func != NULL)
2152 cpu_setups[i].func(undo,
2153 id, cpu_setups[i].arg);
2154 }
2155 break;
2156 }
2157 }
2158 }
2159 return (retval);
2160 }
2161
2162 /*
2163 * Export information about this CPU via the kstat mechanism.
2164 */
2165 static struct {
2166 kstat_named_t ci_state;
2167 kstat_named_t ci_state_begin;
2168 kstat_named_t ci_cpu_type;
2169 kstat_named_t ci_fpu_type;
2170 kstat_named_t ci_clock_MHz;
2171 kstat_named_t ci_chip_id;
2172 kstat_named_t ci_implementation;
2173 kstat_named_t ci_brandstr;
2174 kstat_named_t ci_core_id;
2175 kstat_named_t ci_curr_clock_Hz;
2176 kstat_named_t ci_supp_freq_Hz;
2177 kstat_named_t ci_pg_id;
2178 #if defined(__sparcv9)
2179 kstat_named_t ci_device_ID;
2180 kstat_named_t ci_cpu_fru;
2181 #endif
2182 #if defined(__x86)
2183 kstat_named_t ci_vendorstr;
2184 kstat_named_t ci_family;
2185 kstat_named_t ci_model;
2186 kstat_named_t ci_step;
2187 kstat_named_t ci_clogid;
2188 kstat_named_t ci_pkg_core_id;
2189 kstat_named_t ci_ncpuperchip;
2190 kstat_named_t ci_ncoreperchip;
2191 kstat_named_t ci_max_cstates;
2192 kstat_named_t ci_curr_cstate;
2193 kstat_named_t ci_cacheid;
2194 kstat_named_t ci_sktstr;
2195 #endif
2196 } cpu_info_template = {
2197 { "state", KSTAT_DATA_CHAR },
2198 { "state_begin", KSTAT_DATA_LONG },
2199 { "cpu_type", KSTAT_DATA_CHAR },
2200 { "fpu_type", KSTAT_DATA_CHAR },
2201 { "clock_MHz", KSTAT_DATA_LONG },
2202 { "chip_id", KSTAT_DATA_LONG },
2203 { "implementation", KSTAT_DATA_STRING },
2204 { "brand", KSTAT_DATA_STRING },
2205 { "core_id", KSTAT_DATA_LONG },
2206 { "current_clock_Hz", KSTAT_DATA_UINT64 },
2207 { "supported_frequencies_Hz", KSTAT_DATA_STRING },
2208 { "pg_id", KSTAT_DATA_LONG },
2209 #if defined(__sparcv9)
2210 { "device_ID", KSTAT_DATA_UINT64 },
2211 { "cpu_fru", KSTAT_DATA_STRING },
2212 #endif
2213 #if defined(__x86)
2214 { "vendor_id", KSTAT_DATA_STRING },
2215 { "family", KSTAT_DATA_INT32 },
2216 { "model", KSTAT_DATA_INT32 },
2217 { "stepping", KSTAT_DATA_INT32 },
2218 { "clog_id", KSTAT_DATA_INT32 },
2219 { "pkg_core_id", KSTAT_DATA_LONG },
2220 { "ncpu_per_chip", KSTAT_DATA_INT32 },
2221 { "ncore_per_chip", KSTAT_DATA_INT32 },
2222 { "supported_max_cstates", KSTAT_DATA_INT32 },
2223 { "current_cstate", KSTAT_DATA_INT32 },
2224 { "cache_id", KSTAT_DATA_INT32 },
2225 { "socket_type", KSTAT_DATA_STRING },
2226 #endif
2227 };
2228
2229 static kmutex_t cpu_info_template_lock;
2230
2231 static int
2232 cpu_info_kstat_update(kstat_t *ksp, int rw)
2233 {
2234 cpu_t *cp = ksp->ks_private;
2235 const char *pi_state;
2236
2237 if (rw == KSTAT_WRITE)
2238 return (EACCES);
2239
2240 #if defined(__x86)
2241 /* Is the cpu still initialising itself? */
2242 if (cpuid_checkpass(cp, 1) == 0)
2243 return (ENXIO);
2244 #endif
2245 switch (cp->cpu_type_info.pi_state) {
2246 case P_ONLINE:
2247 pi_state = PS_ONLINE;
2248 break;
2249 case P_POWEROFF:
2250 pi_state = PS_POWEROFF;
2251 break;
2252 case P_NOINTR:
2253 pi_state = PS_NOINTR;
2254 break;
2255 case P_FAULTED:
2256 pi_state = PS_FAULTED;
2257 break;
2258 case P_SPARE:
2259 pi_state = PS_SPARE;
2260 break;
2261 case P_OFFLINE:
2262 pi_state = PS_OFFLINE;
2263 break;
2264 default:
2265 pi_state = "unknown";
2266 }
2267 (void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2268 cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2269 (void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2270 cp->cpu_type_info.pi_processor_type, 15);
2271 (void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2272 cp->cpu_type_info.pi_fputypes, 15);
2273 cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2274 cpu_info_template.ci_chip_id.value.l =
2275 pg_plat_hw_instance_id(cp, PGHW_CHIP);
2276 kstat_named_setstr(&cpu_info_template.ci_implementation,
2277 cp->cpu_idstr);
2278 kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2279 cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2280 cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2281 cp->cpu_curr_clock;
2282 cpu_info_template.ci_pg_id.value.l =
2283 cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2284 cp->cpu_pg->cmt_lineage->pg_id : -1;
2285 kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2286 cp->cpu_supp_freqs);
2287 #if defined(__sparcv9)
2288 cpu_info_template.ci_device_ID.value.ui64 =
2289 cpunodes[cp->cpu_id].device_id;
2290 kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2291 #endif
2292 #if defined(__x86)
2293 kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2294 cpuid_getvendorstr(cp));
2295 cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2296 cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2297 cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2298 cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2299 cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2300 cpu_info_template.ci_ncoreperchip.value.l =
2301 cpuid_get_ncore_per_chip(cp);
2302 cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2303 cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2304 cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2305 cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2306 kstat_named_setstr(&cpu_info_template.ci_sktstr,
2307 cpuid_getsocketstr(cp));
2308 #endif
2309
2310 return (0);
2311 }
2312
2313 static void
2314 cpu_info_kstat_create(cpu_t *cp)
2315 {
2316 zoneid_t zoneid;
2317
2318 ASSERT(MUTEX_HELD(&cpu_lock));
2319
2320 if (pool_pset_enabled())
2321 zoneid = GLOBAL_ZONEID;
2322 else
2323 zoneid = ALL_ZONES;
2324 if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2325 NULL, "misc", KSTAT_TYPE_NAMED,
2326 sizeof (cpu_info_template) / sizeof (kstat_named_t),
2327 KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2328 cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2329 #if defined(__sparcv9)
2330 cp->cpu_info_kstat->ks_data_size +=
2331 strlen(cpu_fru_fmri(cp)) + 1;
2332 #endif
2333 #if defined(__x86)
2334 cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2335 #endif
2336 if (cp->cpu_supp_freqs != NULL)
2337 cp->cpu_info_kstat->ks_data_size +=
2338 strlen(cp->cpu_supp_freqs) + 1;
2339 cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2340 cp->cpu_info_kstat->ks_data = &cpu_info_template;
2341 cp->cpu_info_kstat->ks_private = cp;
2342 cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2343 kstat_install(cp->cpu_info_kstat);
2344 }
2345 }
2346
2347 static void
2348 cpu_info_kstat_destroy(cpu_t *cp)
2349 {
2350 ASSERT(MUTEX_HELD(&cpu_lock));
2351
2352 kstat_delete(cp->cpu_info_kstat);
2353 cp->cpu_info_kstat = NULL;
2354 }
2355
2356 /*
2357 * Create and install kstats for the boot CPU.
2358 */
2359 void
2360 cpu_kstat_init(cpu_t *cp)
2361 {
2362 mutex_enter(&cpu_lock);
2363 cpu_info_kstat_create(cp);
2364 cpu_stats_kstat_create(cp);
2365 cpu_create_intrstat(cp);
2366 cpu_set_state(cp);
2367 mutex_exit(&cpu_lock);
2368 }
2369
2370 /*
2371 * Make visible to the zone that subset of the cpu information that would be
2372 * initialized when a cpu is configured (but still offline).
2373 */
2374 void
2375 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2376 {
2377 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2378
2379 ASSERT(MUTEX_HELD(&cpu_lock));
2380 ASSERT(pool_pset_enabled());
2381 ASSERT(cp != NULL);
2382
2383 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2384 zone->zone_ncpus++;
2385 ASSERT(zone->zone_ncpus <= ncpus);
2386 }
2387 if (cp->cpu_info_kstat != NULL)
2388 kstat_zone_add(cp->cpu_info_kstat, zoneid);
2389 }
2390
2391 /*
2392 * Make visible to the zone that subset of the cpu information that would be
2393 * initialized when a previously configured cpu is onlined.
2394 */
2395 void
2396 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2397 {
2398 kstat_t *ksp;
2399 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2400 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2401 processorid_t cpun;
2402
2403 ASSERT(MUTEX_HELD(&cpu_lock));
2404 ASSERT(pool_pset_enabled());
2405 ASSERT(cp != NULL);
2406 ASSERT(cpu_is_active(cp));
2407
2408 cpun = cp->cpu_id;
2409 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2410 zone->zone_ncpus_online++;
2411 ASSERT(zone->zone_ncpus_online <= ncpus_online);
2412 }
2413 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2414 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2415 != NULL) {
2416 kstat_zone_add(ksp, zoneid);
2417 kstat_rele(ksp);
2418 }
2419 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2420 kstat_zone_add(ksp, zoneid);
2421 kstat_rele(ksp);
2422 }
2423 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2424 kstat_zone_add(ksp, zoneid);
2425 kstat_rele(ksp);
2426 }
2427 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2428 NULL) {
2429 kstat_zone_add(ksp, zoneid);
2430 kstat_rele(ksp);
2431 }
2432 }
2433
2434 /*
2435 * Update relevant kstats such that cpu is now visible to processes
2436 * executing in specified zone.
2437 */
2438 void
2439 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2440 {
2441 cpu_visibility_configure(cp, zone);
2442 if (cpu_is_active(cp))
2443 cpu_visibility_online(cp, zone);
2444 }
2445
2446 /*
2447 * Make invisible to the zone that subset of the cpu information that would be
2448 * torn down when a previously offlined cpu is unconfigured.
2449 */
2450 void
2451 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2452 {
2453 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2454
2455 ASSERT(MUTEX_HELD(&cpu_lock));
2456 ASSERT(pool_pset_enabled());
2457 ASSERT(cp != NULL);
2458
2459 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2460 ASSERT(zone->zone_ncpus != 0);
2461 zone->zone_ncpus--;
2462 }
2463 if (cp->cpu_info_kstat)
2464 kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2465 }
2466
2467 /*
2468 * Make invisible to the zone that subset of the cpu information that would be
2469 * torn down when a cpu is offlined (but still configured).
2470 */
2471 void
2472 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2473 {
2474 kstat_t *ksp;
2475 char name[sizeof ("cpu_stat") + 10]; /* enough for 32-bit cpuids */
2476 zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2477 processorid_t cpun;
2478
2479 ASSERT(MUTEX_HELD(&cpu_lock));
2480 ASSERT(pool_pset_enabled());
2481 ASSERT(cp != NULL);
2482 ASSERT(cpu_is_active(cp));
2483
2484 cpun = cp->cpu_id;
2485 if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2486 ASSERT(zone->zone_ncpus_online != 0);
2487 zone->zone_ncpus_online--;
2488 }
2489
2490 if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2491 NULL) {
2492 kstat_zone_remove(ksp, zoneid);
2493 kstat_rele(ksp);
2494 }
2495 if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2496 kstat_zone_remove(ksp, zoneid);
2497 kstat_rele(ksp);
2498 }
2499 if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2500 kstat_zone_remove(ksp, zoneid);
2501 kstat_rele(ksp);
2502 }
2503 (void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2504 if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2505 != NULL) {
2506 kstat_zone_remove(ksp, zoneid);
2507 kstat_rele(ksp);
2508 }
2509 }
2510
2511 /*
2512 * Update relevant kstats such that cpu is no longer visible to processes
2513 * executing in specified zone.
2514 */
2515 void
2516 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2517 {
2518 if (cpu_is_active(cp))
2519 cpu_visibility_offline(cp, zone);
2520 cpu_visibility_unconfigure(cp, zone);
2521 }
2522
2523 /*
2524 * Bind a thread to a CPU as requested.
2525 */
2526 int
2527 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2528 int *error)
2529 {
2530 processorid_t binding;
2531 cpu_t *cp = NULL;
2532
2533 ASSERT(MUTEX_HELD(&cpu_lock));
2534 ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2535
2536 thread_lock(tp);
2537
2538 /*
2539 * Record old binding, but change the obind, which was initialized
2540 * to PBIND_NONE, only if this thread has a binding. This avoids
2541 * reporting PBIND_NONE for a process when some LWPs are bound.
2542 */
2543 binding = tp->t_bind_cpu;
2544 if (binding != PBIND_NONE)
2545 *obind = binding; /* record old binding */
2546
2547 switch (bind) {
2548 case PBIND_QUERY:
2549 /* Just return the old binding */
2550 thread_unlock(tp);
2551 return (0);
2552
2553 case PBIND_QUERY_TYPE:
2554 /* Return the binding type */
2555 *obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2556 thread_unlock(tp);
2557 return (0);
2558
2559 case PBIND_SOFT:
2560 /*
2561 * Set soft binding for this thread and return the actual
2562 * binding
2563 */
2564 TB_CPU_SOFT_SET(tp);
2565 thread_unlock(tp);
2566 return (0);
2567
2568 case PBIND_HARD:
2569 /*
2570 * Set hard binding for this thread and return the actual
2571 * binding
2572 */
2573 TB_CPU_HARD_SET(tp);
2574 thread_unlock(tp);
2575 return (0);
2576
2577 default:
2578 break;
2579 }
2580
2581 /*
2582 * If this thread/LWP cannot be bound because of permission
2583 * problems, just note that and return success so that the
2584 * other threads/LWPs will be bound. This is the way
2585 * processor_bind() is defined to work.
2586 *
2587 * Binding will get EPERM if the thread is of system class
2588 * or hasprocperm() fails.
2589 */
2590 if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2591 *error = EPERM;
2592 thread_unlock(tp);
2593 return (0);
2594 }
2595
2596 binding = bind;
2597 if (binding != PBIND_NONE) {
2598 cp = cpu_get((processorid_t)binding);
2599 /*
2600 * Make sure binding is valid and is in right partition.
2601 */
2602 if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2603 *error = EINVAL;
2604 thread_unlock(tp);
2605 return (0);
2606 }
2607 }
2608 tp->t_bind_cpu = binding; /* set new binding */
2609
2610 /*
2611 * If there is no system-set reason for affinity, set
2612 * the t_bound_cpu field to reflect the binding.
2613 */
2614 if (tp->t_affinitycnt == 0) {
2615 if (binding == PBIND_NONE) {
2616 /*
2617 * We may need to adjust disp_max_unbound_pri
2618 * since we're becoming unbound.
2619 */
2620 disp_adjust_unbound_pri(tp);
2621
2622 tp->t_bound_cpu = NULL; /* set new binding */
2623
2624 /*
2625 * Move thread to lgroup with strongest affinity
2626 * after unbinding
2627 */
2628 if (tp->t_lgrp_affinity)
2629 lgrp_move_thread(tp,
2630 lgrp_choose(tp, tp->t_cpupart), 1);
2631
2632 if (tp->t_state == TS_ONPROC &&
2633 tp->t_cpu->cpu_part != tp->t_cpupart)
2634 cpu_surrender(tp);
2635 } else {
2636 lpl_t *lpl;
2637
2638 tp->t_bound_cpu = cp;
2639 ASSERT(cp->cpu_lpl != NULL);
2640
2641 /*
2642 * Set home to lgroup with most affinity containing CPU
2643 * that thread is being bound or minimum bounding
2644 * lgroup if no affinities set
2645 */
2646 if (tp->t_lgrp_affinity)
2647 lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2648 LGRP_NONE, B_FALSE);
2649 else
2650 lpl = cp->cpu_lpl;
2651
2652 if (tp->t_lpl != lpl) {
2653 /* can't grab cpu_lock */
2654 lgrp_move_thread(tp, lpl, 1);
2655 }
2656
2657 /*
2658 * Make the thread switch to the bound CPU.
2659 * If the thread is runnable, we need to
2660 * requeue it even if t_cpu is already set
2661 * to the right CPU, since it may be on a
2662 * kpreempt queue and need to move to a local
2663 * queue. We could check t_disp_queue to
2664 * avoid unnecessary overhead if it's already
2665 * on the right queue, but since this isn't
2666 * a performance-critical operation it doesn't
2667 * seem worth the extra code and complexity.
2668 *
2669 * If the thread is weakbound to the cpu then it will
2670 * resist the new binding request until the weak
2671 * binding drops. The cpu_surrender or requeueing
2672 * below could be skipped in such cases (since it
2673 * will have no effect), but that would require
2674 * thread_allowmigrate to acquire thread_lock so
2675 * we'll take the very occasional hit here instead.
2676 */
2677 if (tp->t_state == TS_ONPROC) {
2678 cpu_surrender(tp);
2679 } else if (tp->t_state == TS_RUN) {
2680 cpu_t *ocp = tp->t_cpu;
2681
2682 (void) dispdeq(tp);
2683 setbackdq(tp);
2684 /*
2685 * Either on the bound CPU's disp queue now,
2686 * or swapped out or on the swap queue.
2687 */
2688 ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2689 tp->t_weakbound_cpu == ocp ||
2690 (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2691 != TS_LOAD);
2692 }
2693 }
2694 }
2695
2696 /*
2697 * Our binding has changed; set TP_CHANGEBIND.
2698 */
2699 tp->t_proc_flag |= TP_CHANGEBIND;
2700 aston(tp);
2701
2702 thread_unlock(tp);
2703
2704 return (0);
2705 }
2706
2707 #if CPUSET_WORDS > 1
2708
2709 /*
2710 * Functions for implementing cpuset operations when a cpuset is more
2711 * than one word. On platforms where a cpuset is a single word these
2712 * are implemented as macros in cpuvar.h.
2713 */
2714
2715 void
2716 cpuset_all(cpuset_t *s)
2717 {
2718 int i;
2719
2720 for (i = 0; i < CPUSET_WORDS; i++)
2721 s->cpub[i] = ~0UL;
2722 }
2723
2724 void
2725 cpuset_all_but(cpuset_t *s, uint_t cpu)
2726 {
2727 cpuset_all(s);
2728 CPUSET_DEL(*s, cpu);
2729 }
2730
2731 void
2732 cpuset_only(cpuset_t *s, uint_t cpu)
2733 {
2734 CPUSET_ZERO(*s);
2735 CPUSET_ADD(*s, cpu);
2736 }
2737
2738 int
2739 cpuset_isnull(cpuset_t *s)
2740 {
2741 int i;
2742
2743 for (i = 0; i < CPUSET_WORDS; i++)
2744 if (s->cpub[i] != 0)
2745 return (0);
2746 return (1);
2747 }
2748
2749 int
2750 cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2751 {
2752 int i;
2753
2754 for (i = 0; i < CPUSET_WORDS; i++)
2755 if (s1->cpub[i] != s2->cpub[i])
2756 return (0);
2757 return (1);
2758 }
2759
2760 uint_t
2761 cpuset_find(cpuset_t *s)
2762 {
2763
2764 uint_t i;
2765 uint_t cpu = (uint_t)-1;
2766
2767 /*
2768 * Find a cpu in the cpuset
2769 */
2770 for (i = 0; i < CPUSET_WORDS; i++) {
2771 cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2772 if (cpu != (uint_t)-1) {
2773 cpu += i * BT_NBIPUL;
2774 break;
2775 }
2776 }
2777 return (cpu);
2778 }
2779
2780 void
2781 cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2782 {
2783 int i, j;
2784 uint_t bit;
2785
2786 /*
2787 * First, find the smallest cpu id in the set.
2788 */
2789 for (i = 0; i < CPUSET_WORDS; i++) {
2790 if (s->cpub[i] != 0) {
2791 bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2792 ASSERT(bit != (uint_t)-1);
2793 *smallestid = bit + (i * BT_NBIPUL);
2794
2795 /*
2796 * Now find the largest cpu id in
2797 * the set and return immediately.
2798 * Done in an inner loop to avoid
2799 * having to break out of the first
2800 * loop.
2801 */
2802 for (j = CPUSET_WORDS - 1; j >= i; j--) {
2803 if (s->cpub[j] != 0) {
2804 bit = (uint_t)(highbit(s->cpub[j]) - 1);
2805 ASSERT(bit != (uint_t)-1);
2806 *largestid = bit + (j * BT_NBIPUL);
2807 ASSERT(*largestid >= *smallestid);
2808 return;
2809 }
2810 }
2811
2812 /*
2813 * If this code is reached, a
2814 * smallestid was found, but not a
2815 * largestid. The cpuset must have
2816 * been changed during the course
2817 * of this function call.
2818 */
2819 ASSERT(0);
2820 }
2821 }
2822 *smallestid = *largestid = CPUSET_NOTINSET;
2823 }
2824
2825 #endif /* CPUSET_WORDS */
2826
2827 /*
2828 * Unbind threads bound to specified CPU.
2829 *
2830 * If `unbind_all_threads' is true, unbind all user threads bound to a given
2831 * CPU. Otherwise unbind all soft-bound user threads.
2832 */
2833 int
2834 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2835 {
2836 processorid_t obind;
2837 kthread_t *tp;
2838 int ret = 0;
2839 proc_t *pp;
2840 int err, berr = 0;
2841
2842 ASSERT(MUTEX_HELD(&cpu_lock));
2843
2844 mutex_enter(&pidlock);
2845 for (pp = practive; pp != NULL; pp = pp->p_next) {
2846 mutex_enter(&pp->p_lock);
2847 tp = pp->p_tlist;
2848 /*
2849 * Skip zombies, kernel processes, and processes in
2850 * other zones, if called from a non-global zone.
2851 */
2852 if (tp == NULL || (pp->p_flag & SSYS) ||
2853 !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2854 mutex_exit(&pp->p_lock);
2855 continue;
2856 }
2857 do {
2858 if (tp->t_bind_cpu != cpu)
2859 continue;
2860 /*
2861 * Skip threads with hard binding when
2862 * `unbind_all_threads' is not specified.
2863 */
2864 if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2865 continue;
2866 err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2867 if (ret == 0)
2868 ret = err;
2869 } while ((tp = tp->t_forw) != pp->p_tlist);
2870 mutex_exit(&pp->p_lock);
2871 }
2872 mutex_exit(&pidlock);
2873 if (ret == 0)
2874 ret = berr;
2875 return (ret);
2876 }
2877
2878
2879 /*
2880 * Destroy all remaining bound threads on a cpu.
2881 */
2882 void
2883 cpu_destroy_bound_threads(cpu_t *cp)
2884 {
2885 extern id_t syscid;
2886 register kthread_id_t t, tlist, tnext;
2887
2888 /*
2889 * Destroy all remaining bound threads on the cpu. This
2890 * should include both the interrupt threads and the idle thread.
2891 * This requires some care, since we need to traverse the
2892 * thread list with the pidlock mutex locked, but thread_free
2893 * also locks the pidlock mutex. So, we collect the threads
2894 * we're going to reap in a list headed by "tlist", then we
2895 * unlock the pidlock mutex and traverse the tlist list,
2896 * doing thread_free's on the thread's. Simple, n'est pas?
2897 * Also, this depends on thread_free not mucking with the
2898 * t_next and t_prev links of the thread.
2899 */
2900
2901 if ((t = curthread) != NULL) {
2902
2903 tlist = NULL;
2904 mutex_enter(&pidlock);
2905 do {
2906 tnext = t->t_next;
2907 if (t->t_bound_cpu == cp) {
2908
2909 /*
2910 * We've found a bound thread, carefully unlink
2911 * it out of the thread list, and add it to
2912 * our "tlist". We "know" we don't have to
2913 * worry about unlinking curthread (the thread
2914 * that is executing this code).
2915 */
2916 t->t_next->t_prev = t->t_prev;
2917 t->t_prev->t_next = t->t_next;
2918 t->t_next = tlist;
2919 tlist = t;
2920 ASSERT(t->t_cid == syscid);
2921 /* wake up anyone blocked in thread_join */
2922 cv_broadcast(&t->t_joincv);
2923 /*
2924 * t_lwp set by interrupt threads and not
2925 * cleared.
2926 */
2927 t->t_lwp = NULL;
2928 /*
2929 * Pause and idle threads always have
2930 * t_state set to TS_ONPROC.
2931 */
2932 t->t_state = TS_FREE;
2933 t->t_prev = NULL; /* Just in case */
2934 }
2935
2936 } while ((t = tnext) != curthread);
2937
2938 mutex_exit(&pidlock);
2939
2940 mutex_sync();
2941 for (t = tlist; t != NULL; t = tnext) {
2942 tnext = t->t_next;
2943 thread_free(t);
2944 }
2945 }
2946 }
2947
2948 /*
2949 * Update the cpu_supp_freqs of this cpu. This information is returned
2950 * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2951 * maintain the kstat data size.
2952 */
2953 void
2954 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2955 {
2956 char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2957 const char *lfreqs = clkstr;
2958 boolean_t kstat_exists = B_FALSE;
2959 kstat_t *ksp;
2960 size_t len;
2961
2962 /*
2963 * A NULL pointer means we only support one speed.
2964 */
2965 if (freqs == NULL)
2966 (void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2967 cp->cpu_curr_clock);
2968 else
2969 lfreqs = freqs;
2970
2971 /*
2972 * Make sure the frequency doesn't change while a snapshot is
2973 * going on. Of course, we only need to worry about this if
2974 * the kstat exists.
2975 */
2976 if ((ksp = cp->cpu_info_kstat) != NULL) {
2977 mutex_enter(ksp->ks_lock);
2978 kstat_exists = B_TRUE;
2979 }
2980
2981 /*
2982 * Free any previously allocated string and if the kstat
2983 * already exists, then update its data size.
2984 */
2985 if (cp->cpu_supp_freqs != NULL) {
2986 len = strlen(cp->cpu_supp_freqs) + 1;
2987 kmem_free(cp->cpu_supp_freqs, len);
2988 if (kstat_exists)
2989 ksp->ks_data_size -= len;
2990 }
2991
2992 /*
2993 * Allocate the new string and set the pointer.
2994 */
2995 len = strlen(lfreqs) + 1;
2996 cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2997 (void) strcpy(cp->cpu_supp_freqs, lfreqs);
2998
2999 /*
3000 * If the kstat already exists then update the data size and
3001 * free the lock.
3002 */
3003 if (kstat_exists) {
3004 ksp->ks_data_size += len;
3005 mutex_exit(ksp->ks_lock);
3006 }
3007 }
3008
3009 /*
3010 * Indicate the current CPU's clock freqency (in Hz).
3011 * The calling context must be such that CPU references are safe.
3012 */
3013 void
3014 cpu_set_curr_clock(uint64_t new_clk)
3015 {
3016 uint64_t old_clk;
3017
3018 old_clk = CPU->cpu_curr_clock;
3019 CPU->cpu_curr_clock = new_clk;
3020
3021 /*
3022 * The cpu-change-speed DTrace probe exports the frequency in Hz
3023 */
3024 DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3025 uint64_t, old_clk, uint64_t, new_clk);
3026 }
3027
3028 /*
3029 * processor_info(2) and p_online(2) status support functions
3030 * The constants returned by the cpu_get_state() and cpu_get_state_str() are
3031 * for use in communicating processor state information to userland. Kernel
3032 * subsystems should only be using the cpu_flags value directly. Subsystems
3033 * modifying cpu_flags should record the state change via a call to the
3034 * cpu_set_state().
3035 */
3036
3037 /*
3038 * Update the pi_state of this CPU. This function provides the CPU status for
3039 * the information returned by processor_info(2).
3040 */
3041 void
3042 cpu_set_state(cpu_t *cpu)
3043 {
3044 ASSERT(MUTEX_HELD(&cpu_lock));
3045 cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3046 cpu->cpu_state_begin = gethrestime_sec();
3047 pool_cpu_mod = gethrtime();
3048 }
3049
3050 /*
3051 * Return offline/online/other status for the indicated CPU. Use only for
3052 * communication with user applications; cpu_flags provides the in-kernel
3053 * interface.
3054 */
3055 int
3056 cpu_get_state(cpu_t *cpu)
3057 {
3058 ASSERT(MUTEX_HELD(&cpu_lock));
3059 if (cpu->cpu_flags & CPU_POWEROFF)
3060 return (P_POWEROFF);
3061 else if (cpu->cpu_flags & CPU_FAULTED)
3062 return (P_FAULTED);
3063 else if (cpu->cpu_flags & CPU_SPARE)
3064 return (P_SPARE);
3065 else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3066 return (P_OFFLINE);
3067 else if (cpu->cpu_flags & CPU_ENABLE)
3068 return (P_ONLINE);
3069 else
3070 return (P_NOINTR);
3071 }
3072
3073 /*
3074 * Return processor_info(2) state as a string.
3075 */
3076 const char *
3077 cpu_get_state_str(cpu_t *cpu)
3078 {
3079 const char *string;
3080
3081 switch (cpu_get_state(cpu)) {
3082 case P_ONLINE:
3083 string = PS_ONLINE;
3084 break;
3085 case P_POWEROFF:
3086 string = PS_POWEROFF;
3087 break;
3088 case P_NOINTR:
3089 string = PS_NOINTR;
3090 break;
3091 case P_SPARE:
3092 string = PS_SPARE;
3093 break;
3094 case P_FAULTED:
3095 string = PS_FAULTED;
3096 break;
3097 case P_OFFLINE:
3098 string = PS_OFFLINE;
3099 break;
3100 default:
3101 string = "unknown";
3102 break;
3103 }
3104 return (string);
3105 }
3106
3107 /*
3108 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3109 * kstats, respectively. This is done when a CPU is initialized or placed
3110 * online via p_online(2).
3111 */
3112 static void
3113 cpu_stats_kstat_create(cpu_t *cp)
3114 {
3115 int instance = cp->cpu_id;
3116 char *module = "cpu";
3117 char *class = "misc";
3118 kstat_t *ksp;
3119 zoneid_t zoneid;
3120
3121 ASSERT(MUTEX_HELD(&cpu_lock));
3122
3123 if (pool_pset_enabled())
3124 zoneid = GLOBAL_ZONEID;
3125 else
3126 zoneid = ALL_ZONES;
3127 /*
3128 * Create named kstats
3129 */
3130 #define CPU_STATS_KS_CREATE(name, tsize, update_func) \
3131 ksp = kstat_create_zone(module, instance, (name), class, \
3132 KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0, \
3133 zoneid); \
3134 if (ksp != NULL) { \
3135 ksp->ks_private = cp; \
3136 ksp->ks_update = (update_func); \
3137 kstat_install(ksp); \
3138 } else \
3139 cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3140 module, instance, (name));
3141
3142 CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3143 cpu_sys_stats_ks_update);
3144 CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3145 cpu_vm_stats_ks_update);
3146
3147 /*
3148 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3149 */
3150 ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3151 "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3152 if (ksp != NULL) {
3153 ksp->ks_update = cpu_stat_ks_update;
3154 ksp->ks_private = cp;
3155 kstat_install(ksp);
3156 }
3157 }
3158
3159 static void
3160 cpu_stats_kstat_destroy(cpu_t *cp)
3161 {
3162 char ks_name[KSTAT_STRLEN];
3163
3164 (void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3165 kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3166
3167 kstat_delete_byname("cpu", cp->cpu_id, "sys");
3168 kstat_delete_byname("cpu", cp->cpu_id, "vm");
3169 }
3170
3171 static int
3172 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3173 {
3174 cpu_t *cp = (cpu_t *)ksp->ks_private;
3175 struct cpu_sys_stats_ks_data *csskd;
3176 cpu_sys_stats_t *css;
3177 hrtime_t msnsecs[NCMSTATES];
3178 int i;
3179
3180 if (rw == KSTAT_WRITE)
3181 return (EACCES);
3182
3183 csskd = ksp->ks_data;
3184 css = &cp->cpu_stats.sys;
3185
3186 /*
3187 * Read CPU mstate, but compare with the last values we
3188 * received to make sure that the returned kstats never
3189 * decrease.
3190 */
3191
3192 get_cpu_mstate(cp, msnsecs);
3193 if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3194 msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3195 if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3196 msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3197 if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3198 msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3199
3200 bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3201 sizeof (cpu_sys_stats_ks_data_template));
3202
3203 csskd->cpu_ticks_wait.value.ui64 = 0;
3204 csskd->wait_ticks_io.value.ui64 = 0;
3205
3206 csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3207 csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3208 csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3209 csskd->cpu_ticks_idle.value.ui64 =
3210 NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3211 csskd->cpu_ticks_user.value.ui64 =
3212 NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3213 csskd->cpu_ticks_kernel.value.ui64 =
3214 NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3215 csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3216 csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3217 csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3218 csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3219 csskd->bread.value.ui64 = css->bread;
3220 csskd->bwrite.value.ui64 = css->bwrite;
3221 csskd->lread.value.ui64 = css->lread;
3222 csskd->lwrite.value.ui64 = css->lwrite;
3223 csskd->phread.value.ui64 = css->phread;
3224 csskd->phwrite.value.ui64 = css->phwrite;
3225 csskd->pswitch.value.ui64 = css->pswitch;
3226 csskd->trap.value.ui64 = css->trap;
3227 csskd->intr.value.ui64 = 0;
3228 for (i = 0; i < PIL_MAX; i++)
3229 csskd->intr.value.ui64 += css->intr[i];
3230 csskd->syscall.value.ui64 = css->syscall;
3231 csskd->sysread.value.ui64 = css->sysread;
3232 csskd->syswrite.value.ui64 = css->syswrite;
3233 csskd->sysfork.value.ui64 = css->sysfork;
3234 csskd->sysvfork.value.ui64 = css->sysvfork;
3235 csskd->sysexec.value.ui64 = css->sysexec;
3236 csskd->readch.value.ui64 = css->readch;
3237 csskd->writech.value.ui64 = css->writech;
3238 csskd->rcvint.value.ui64 = css->rcvint;
3239 csskd->xmtint.value.ui64 = css->xmtint;
3240 csskd->mdmint.value.ui64 = css->mdmint;
3241 csskd->rawch.value.ui64 = css->rawch;
3242 csskd->canch.value.ui64 = css->canch;
3243 csskd->outch.value.ui64 = css->outch;
3244 csskd->msg.value.ui64 = css->msg;
3245 csskd->sema.value.ui64 = css->sema;
3246 csskd->namei.value.ui64 = css->namei;
3247 csskd->ufsiget.value.ui64 = css->ufsiget;
3248 csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3249 csskd->ufsipage.value.ui64 = css->ufsipage;
3250 csskd->ufsinopage.value.ui64 = css->ufsinopage;
3251 csskd->procovf.value.ui64 = css->procovf;
3252 csskd->intrthread.value.ui64 = 0;
3253 for (i = 0; i < LOCK_LEVEL - 1; i++)
3254 csskd->intrthread.value.ui64 += css->intr[i];
3255 csskd->intrblk.value.ui64 = css->intrblk;
3256 csskd->intrunpin.value.ui64 = css->intrunpin;
3257 csskd->idlethread.value.ui64 = css->idlethread;
3258 csskd->inv_swtch.value.ui64 = css->inv_swtch;
3259 csskd->nthreads.value.ui64 = css->nthreads;
3260 csskd->cpumigrate.value.ui64 = css->cpumigrate;
3261 csskd->xcalls.value.ui64 = css->xcalls;
3262 csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3263 csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3264 csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3265 csskd->modload.value.ui64 = css->modload;
3266 csskd->modunload.value.ui64 = css->modunload;
3267 csskd->bawrite.value.ui64 = css->bawrite;
3268 csskd->iowait.value.ui64 = css->iowait;
3269
3270 return (0);
3271 }
3272
3273 static int
3274 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3275 {
3276 cpu_t *cp = (cpu_t *)ksp->ks_private;
3277 struct cpu_vm_stats_ks_data *cvskd;
3278 cpu_vm_stats_t *cvs;
3279
3280 if (rw == KSTAT_WRITE)
3281 return (EACCES);
3282
3283 cvs = &cp->cpu_stats.vm;
3284 cvskd = ksp->ks_data;
3285
3286 bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3287 sizeof (cpu_vm_stats_ks_data_template));
3288 cvskd->pgrec.value.ui64 = cvs->pgrec;
3289 cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3290 cvskd->pgin.value.ui64 = cvs->pgin;
3291 cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3292 cvskd->pgout.value.ui64 = cvs->pgout;
3293 cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3294 cvskd->swapin.value.ui64 = cvs->swapin;
3295 cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3296 cvskd->swapout.value.ui64 = cvs->swapout;
3297 cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3298 cvskd->zfod.value.ui64 = cvs->zfod;
3299 cvskd->dfree.value.ui64 = cvs->dfree;
3300 cvskd->scan.value.ui64 = cvs->scan;
3301 cvskd->rev.value.ui64 = cvs->rev;
3302 cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3303 cvskd->as_fault.value.ui64 = cvs->as_fault;
3304 cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3305 cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3306 cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3307 cvskd->softlock.value.ui64 = cvs->softlock;
3308 cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3309 cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3310 cvskd->execpgin.value.ui64 = cvs->execpgin;
3311 cvskd->execpgout.value.ui64 = cvs->execpgout;
3312 cvskd->execfree.value.ui64 = cvs->execfree;
3313 cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3314 cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3315 cvskd->anonfree.value.ui64 = cvs->anonfree;
3316 cvskd->fspgin.value.ui64 = cvs->fspgin;
3317 cvskd->fspgout.value.ui64 = cvs->fspgout;
3318 cvskd->fsfree.value.ui64 = cvs->fsfree;
3319
3320 return (0);
3321 }
3322
3323 static int
3324 cpu_stat_ks_update(kstat_t *ksp, int rw)
3325 {
3326 cpu_stat_t *cso;
3327 cpu_t *cp;
3328 int i;
3329 hrtime_t msnsecs[NCMSTATES];
3330
3331 cso = (cpu_stat_t *)ksp->ks_data;
3332 cp = (cpu_t *)ksp->ks_private;
3333
3334 if (rw == KSTAT_WRITE)
3335 return (EACCES);
3336
3337 /*
3338 * Read CPU mstate, but compare with the last values we
3339 * received to make sure that the returned kstats never
3340 * decrease.
3341 */
3342
3343 get_cpu_mstate(cp, msnsecs);
3344 msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3345 msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3346 msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3347 if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3348 cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3349 if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3350 cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3351 if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3352 cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3353 cso->cpu_sysinfo.cpu[CPU_WAIT] = 0;
3354 cso->cpu_sysinfo.wait[W_IO] = 0;
3355 cso->cpu_sysinfo.wait[W_SWAP] = 0;
3356 cso->cpu_sysinfo.wait[W_PIO] = 0;
3357 cso->cpu_sysinfo.bread = CPU_STATS(cp, sys.bread);
3358 cso->cpu_sysinfo.bwrite = CPU_STATS(cp, sys.bwrite);
3359 cso->cpu_sysinfo.lread = CPU_STATS(cp, sys.lread);
3360 cso->cpu_sysinfo.lwrite = CPU_STATS(cp, sys.lwrite);
3361 cso->cpu_sysinfo.phread = CPU_STATS(cp, sys.phread);
3362 cso->cpu_sysinfo.phwrite = CPU_STATS(cp, sys.phwrite);
3363 cso->cpu_sysinfo.pswitch = CPU_STATS(cp, sys.pswitch);
3364 cso->cpu_sysinfo.trap = CPU_STATS(cp, sys.trap);
3365 cso->cpu_sysinfo.intr = 0;
3366 for (i = 0; i < PIL_MAX; i++)
3367 cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3368 cso->cpu_sysinfo.syscall = CPU_STATS(cp, sys.syscall);
3369 cso->cpu_sysinfo.sysread = CPU_STATS(cp, sys.sysread);
3370 cso->cpu_sysinfo.syswrite = CPU_STATS(cp, sys.syswrite);
3371 cso->cpu_sysinfo.sysfork = CPU_STATS(cp, sys.sysfork);
3372 cso->cpu_sysinfo.sysvfork = CPU_STATS(cp, sys.sysvfork);
3373 cso->cpu_sysinfo.sysexec = CPU_STATS(cp, sys.sysexec);
3374 cso->cpu_sysinfo.readch = CPU_STATS(cp, sys.readch);
3375 cso->cpu_sysinfo.writech = CPU_STATS(cp, sys.writech);
3376 cso->cpu_sysinfo.rcvint = CPU_STATS(cp, sys.rcvint);
3377 cso->cpu_sysinfo.xmtint = CPU_STATS(cp, sys.xmtint);
3378 cso->cpu_sysinfo.mdmint = CPU_STATS(cp, sys.mdmint);
3379 cso->cpu_sysinfo.rawch = CPU_STATS(cp, sys.rawch);
3380 cso->cpu_sysinfo.canch = CPU_STATS(cp, sys.canch);
3381 cso->cpu_sysinfo.outch = CPU_STATS(cp, sys.outch);
3382 cso->cpu_sysinfo.msg = CPU_STATS(cp, sys.msg);
3383 cso->cpu_sysinfo.sema = CPU_STATS(cp, sys.sema);
3384 cso->cpu_sysinfo.namei = CPU_STATS(cp, sys.namei);
3385 cso->cpu_sysinfo.ufsiget = CPU_STATS(cp, sys.ufsiget);
3386 cso->cpu_sysinfo.ufsdirblk = CPU_STATS(cp, sys.ufsdirblk);
3387 cso->cpu_sysinfo.ufsipage = CPU_STATS(cp, sys.ufsipage);
3388 cso->cpu_sysinfo.ufsinopage = CPU_STATS(cp, sys.ufsinopage);
3389 cso->cpu_sysinfo.inodeovf = 0;
3390 cso->cpu_sysinfo.fileovf = 0;
3391 cso->cpu_sysinfo.procovf = CPU_STATS(cp, sys.procovf);
3392 cso->cpu_sysinfo.intrthread = 0;
3393 for (i = 0; i < LOCK_LEVEL - 1; i++)
3394 cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3395 cso->cpu_sysinfo.intrblk = CPU_STATS(cp, sys.intrblk);
3396 cso->cpu_sysinfo.idlethread = CPU_STATS(cp, sys.idlethread);
3397 cso->cpu_sysinfo.inv_swtch = CPU_STATS(cp, sys.inv_swtch);
3398 cso->cpu_sysinfo.nthreads = CPU_STATS(cp, sys.nthreads);
3399 cso->cpu_sysinfo.cpumigrate = CPU_STATS(cp, sys.cpumigrate);
3400 cso->cpu_sysinfo.xcalls = CPU_STATS(cp, sys.xcalls);
3401 cso->cpu_sysinfo.mutex_adenters = CPU_STATS(cp, sys.mutex_adenters);
3402 cso->cpu_sysinfo.rw_rdfails = CPU_STATS(cp, sys.rw_rdfails);
3403 cso->cpu_sysinfo.rw_wrfails = CPU_STATS(cp, sys.rw_wrfails);
3404 cso->cpu_sysinfo.modload = CPU_STATS(cp, sys.modload);
3405 cso->cpu_sysinfo.modunload = CPU_STATS(cp, sys.modunload);
3406 cso->cpu_sysinfo.bawrite = CPU_STATS(cp, sys.bawrite);
3407 cso->cpu_sysinfo.rw_enters = 0;
3408 cso->cpu_sysinfo.win_uo_cnt = 0;
3409 cso->cpu_sysinfo.win_uu_cnt = 0;
3410 cso->cpu_sysinfo.win_so_cnt = 0;
3411 cso->cpu_sysinfo.win_su_cnt = 0;
3412 cso->cpu_sysinfo.win_suo_cnt = 0;
3413
3414 cso->cpu_syswait.iowait = CPU_STATS(cp, sys.iowait);
3415 cso->cpu_syswait.swap = 0;
3416 cso->cpu_syswait.physio = 0;
3417
3418 cso->cpu_vminfo.pgrec = CPU_STATS(cp, vm.pgrec);
3419 cso->cpu_vminfo.pgfrec = CPU_STATS(cp, vm.pgfrec);
3420 cso->cpu_vminfo.pgin = CPU_STATS(cp, vm.pgin);
3421 cso->cpu_vminfo.pgpgin = CPU_STATS(cp, vm.pgpgin);
3422 cso->cpu_vminfo.pgout = CPU_STATS(cp, vm.pgout);
3423 cso->cpu_vminfo.pgpgout = CPU_STATS(cp, vm.pgpgout);
3424 cso->cpu_vminfo.swapin = CPU_STATS(cp, vm.swapin);
3425 cso->cpu_vminfo.pgswapin = CPU_STATS(cp, vm.pgswapin);
3426 cso->cpu_vminfo.swapout = CPU_STATS(cp, vm.swapout);
3427 cso->cpu_vminfo.pgswapout = CPU_STATS(cp, vm.pgswapout);
3428 cso->cpu_vminfo.zfod = CPU_STATS(cp, vm.zfod);
3429 cso->cpu_vminfo.dfree = CPU_STATS(cp, vm.dfree);
3430 cso->cpu_vminfo.scan = CPU_STATS(cp, vm.scan);
3431 cso->cpu_vminfo.rev = CPU_STATS(cp, vm.rev);
3432 cso->cpu_vminfo.hat_fault = CPU_STATS(cp, vm.hat_fault);
3433 cso->cpu_vminfo.as_fault = CPU_STATS(cp, vm.as_fault);
3434 cso->cpu_vminfo.maj_fault = CPU_STATS(cp, vm.maj_fault);
3435 cso->cpu_vminfo.cow_fault = CPU_STATS(cp, vm.cow_fault);
3436 cso->cpu_vminfo.prot_fault = CPU_STATS(cp, vm.prot_fault);
3437 cso->cpu_vminfo.softlock = CPU_STATS(cp, vm.softlock);
3438 cso->cpu_vminfo.kernel_asflt = CPU_STATS(cp, vm.kernel_asflt);
3439 cso->cpu_vminfo.pgrrun = CPU_STATS(cp, vm.pgrrun);
3440 cso->cpu_vminfo.execpgin = CPU_STATS(cp, vm.execpgin);
3441 cso->cpu_vminfo.execpgout = CPU_STATS(cp, vm.execpgout);
3442 cso->cpu_vminfo.execfree = CPU_STATS(cp, vm.execfree);
3443 cso->cpu_vminfo.anonpgin = CPU_STATS(cp, vm.anonpgin);
3444 cso->cpu_vminfo.anonpgout = CPU_STATS(cp, vm.anonpgout);
3445 cso->cpu_vminfo.anonfree = CPU_STATS(cp, vm.anonfree);
3446 cso->cpu_vminfo.fspgin = CPU_STATS(cp, vm.fspgin);
3447 cso->cpu_vminfo.fspgout = CPU_STATS(cp, vm.fspgout);
3448 cso->cpu_vminfo.fsfree = CPU_STATS(cp, vm.fsfree);
3449
3450 return (0);
3451 }