1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2018 Joyent, Inc.
25 */
26
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/signal.h>
31 #include <sys/stack.h>
32 #include <sys/pcb.h>
33 #include <sys/user.h>
34 #include <sys/systm.h>
35 #include <sys/sysinfo.h>
36 #include <sys/errno.h>
37 #include <sys/cmn_err.h>
38 #include <sys/cred.h>
39 #include <sys/resource.h>
40 #include <sys/task.h>
41 #include <sys/project.h>
42 #include <sys/proc.h>
43 #include <sys/debug.h>
44 #include <sys/disp.h>
45 #include <sys/class.h>
46 #include <vm/seg_kmem.h>
47 #include <vm/seg_kp.h>
48 #include <sys/machlock.h>
49 #include <sys/kmem.h>
50 #include <sys/varargs.h>
51 #include <sys/turnstile.h>
52 #include <sys/poll.h>
53 #include <sys/vtrace.h>
54 #include <sys/callb.h>
55 #include <c2/audit.h>
56 #include <sys/tnf.h>
57 #include <sys/sobject.h>
58 #include <sys/cpupart.h>
59 #include <sys/pset.h>
60 #include <sys/door.h>
61 #include <sys/spl.h>
62 #include <sys/copyops.h>
63 #include <sys/rctl.h>
64 #include <sys/brand.h>
65 #include <sys/pool.h>
66 #include <sys/zone.h>
67 #include <sys/tsol/label.h>
68 #include <sys/tsol/tndb.h>
69 #include <sys/cpc_impl.h>
70 #include <sys/sdt.h>
71 #include <sys/reboot.h>
72 #include <sys/kdi.h>
73 #include <sys/schedctl.h>
74 #include <sys/waitq.h>
75 #include <sys/cpucaps.h>
76 #include <sys/kiconv.h>
77 #include <sys/ctype.h>
78 #include <sys/ht.h>
79
80 #ifndef STACK_GROWTH_DOWN
81 #error Stacks do not grow downward; 3b2 zombie attack detected!
82 #endif
83
84 struct kmem_cache *thread_cache; /* cache of free threads */
85 struct kmem_cache *lwp_cache; /* cache of free lwps */
86 struct kmem_cache *turnstile_cache; /* cache of free turnstiles */
87
88 /*
89 * allthreads is only for use by kmem_readers. All kernel loops can use
90 * the current thread as a start/end point.
91 */
92 kthread_t *allthreads = &t0; /* circular list of all threads */
93
94 static kcondvar_t reaper_cv; /* synchronization var */
95 kthread_t *thread_deathrow; /* circular list of reapable threads */
96 kthread_t *lwp_deathrow; /* circular list of reapable threads */
97 kmutex_t reaplock; /* protects lwp and thread deathrows */
98 int thread_reapcnt = 0; /* number of threads on deathrow */
99 int lwp_reapcnt = 0; /* number of lwps on deathrow */
100 int reaplimit = 16; /* delay reaping until reaplimit */
101
102 thread_free_lock_t *thread_free_lock;
103 /* protects tick thread from reaper */
104
105 extern int nthread;
106
107 /* System Scheduling classes. */
108 id_t syscid; /* system scheduling class ID */
109 id_t sysdccid = CLASS_UNUSED; /* reset when SDC loads */
110
111 void *segkp_thread; /* cookie for segkp pool */
112
113 int lwp_cache_sz = 32;
114 int t_cache_sz = 8;
115 static kt_did_t next_t_id = 1;
116
117 /* Default mode for thread binding to CPUs and processor sets */
118 int default_binding_mode = TB_ALLHARD;
119
120 /*
121 * Min/Max stack sizes for stack size parameters
122 */
123 #define MAX_STKSIZE (32 * DEFAULTSTKSZ)
124 #define MIN_STKSIZE DEFAULTSTKSZ
125
126 /*
127 * default_stksize overrides lwp_default_stksize if it is set.
128 */
129 int default_stksize;
130 int lwp_default_stksize;
131
132 static zone_key_t zone_thread_key;
133
134 unsigned int kmem_stackinfo; /* stackinfo feature on-off */
135 kmem_stkinfo_t *kmem_stkinfo_log; /* stackinfo circular log */
136 static kmutex_t kmem_stkinfo_lock; /* protects kmem_stkinfo_log */
137
138 /*
139 * forward declarations for internal thread specific data (tsd)
140 */
141 static void *tsd_realloc(void *, size_t, size_t);
142
143 void thread_reaper(void);
144
145 /* forward declarations for stackinfo feature */
146 static void stkinfo_begin(kthread_t *);
147 static void stkinfo_end(kthread_t *);
148 static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
149
150 /*ARGSUSED*/
151 static int
152 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
153 {
154 bzero(buf, sizeof (turnstile_t));
155 return (0);
156 }
157
158 /*ARGSUSED*/
159 static void
160 turnstile_destructor(void *buf, void *cdrarg)
161 {
162 turnstile_t *ts = buf;
163
164 ASSERT(ts->ts_free == NULL);
165 ASSERT(ts->ts_waiters == 0);
166 ASSERT(ts->ts_inheritor == NULL);
167 ASSERT(ts->ts_sleepq[0].sq_first == NULL);
168 ASSERT(ts->ts_sleepq[1].sq_first == NULL);
169 }
170
171 void
172 thread_init(void)
173 {
174 kthread_t *tp;
175 extern char sys_name[];
176 extern void idle();
177 struct cpu *cpu = CPU;
178 int i;
179 kmutex_t *lp;
180
181 mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
182 thread_free_lock =
183 kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
184 for (i = 0; i < THREAD_FREE_NUM; i++) {
185 lp = &thread_free_lock[i].tf_lock;
186 mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
187 }
188
189 #if defined(__i386) || defined(__amd64)
190 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
191 PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
192
193 /*
194 * "struct _klwp" includes a "struct pcb", which includes a
195 * "struct fpu", which needs to be 64-byte aligned on amd64
196 * (and even on i386) for xsave/xrstor.
197 */
198 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
199 64, NULL, NULL, NULL, NULL, NULL, 0);
200 #else
201 /*
202 * Allocate thread structures from static_arena. This prevents
203 * issues where a thread tries to relocate its own thread
204 * structure and touches it after the mapping has been suspended.
205 */
206 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
207 PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
208
209 lwp_stk_cache_init();
210
211 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
212 0, NULL, NULL, NULL, NULL, NULL, 0);
213 #endif
214
215 turnstile_cache = kmem_cache_create("turnstile_cache",
216 sizeof (turnstile_t), 0,
217 turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
218
219 label_init();
220 cred_init();
221
222 /*
223 * Initialize various resource management facilities.
224 */
225 rctl_init();
226 cpucaps_init();
227 /*
228 * Zone_init() should be called before project_init() so that project ID
229 * for the first project is initialized correctly.
230 */
231 zone_init();
232 project_init();
233 brand_init();
234 kiconv_init();
235 task_init();
236 tcache_init();
237 pool_init();
238
239 curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
240
241 /*
242 * Originally, we had two parameters to set default stack
243 * size: one for lwp's (lwp_default_stksize), and one for
244 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
245 * Now we have a third parameter that overrides both if it is
246 * set to a legal stack size, called default_stksize.
247 */
248
249 if (default_stksize == 0) {
250 default_stksize = DEFAULTSTKSZ;
251 } else if (default_stksize % PAGESIZE != 0 ||
252 default_stksize > MAX_STKSIZE ||
253 default_stksize < MIN_STKSIZE) {
254 cmn_err(CE_WARN, "Illegal stack size. Using %d",
255 (int)DEFAULTSTKSZ);
256 default_stksize = DEFAULTSTKSZ;
257 } else {
258 lwp_default_stksize = default_stksize;
259 }
260
261 if (lwp_default_stksize == 0) {
262 lwp_default_stksize = default_stksize;
263 } else if (lwp_default_stksize % PAGESIZE != 0 ||
264 lwp_default_stksize > MAX_STKSIZE ||
265 lwp_default_stksize < MIN_STKSIZE) {
266 cmn_err(CE_WARN, "Illegal stack size. Using %d",
267 default_stksize);
268 lwp_default_stksize = default_stksize;
269 }
270
271 segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
272 lwp_default_stksize,
273 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
274
275 segkp_thread = segkp_cache_init(segkp, t_cache_sz,
276 default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
277
278 (void) getcid(sys_name, &syscid);
279 curthread->t_cid = syscid; /* current thread is t0 */
280
281 /*
282 * Set up the first CPU's idle thread.
283 * It runs whenever the CPU has nothing worthwhile to do.
284 */
285 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
286 cpu->cpu_idle_thread = tp;
287 tp->t_preempt = 1;
288 tp->t_disp_queue = cpu->cpu_disp;
289 ASSERT(tp->t_disp_queue != NULL);
290 tp->t_bound_cpu = cpu;
291 tp->t_affinitycnt = 1;
292
293 /*
294 * Registering a thread in the callback table is usually
295 * done in the initialization code of the thread. In this
296 * case, we do it right after thread creation to avoid
297 * blocking idle thread while registering itself. It also
298 * avoids the possibility of reregistration in case a CPU
299 * restarts its idle thread.
300 */
301 CALLB_CPR_INIT_SAFE(tp, "idle");
302
303 /*
304 * Create the thread_reaper daemon. From this point on, exited
305 * threads will get reaped.
306 */
307 (void) thread_create(NULL, 0, (void (*)())thread_reaper,
308 NULL, 0, &p0, TS_RUN, minclsyspri);
309
310 /*
311 * Finish initializing the kernel memory allocator now that
312 * thread_create() is available.
313 */
314 kmem_thread_init();
315
316 if (boothowto & RB_DEBUG)
317 kdi_dvec_thravail();
318 }
319
320 /*
321 * Create a thread.
322 *
323 * thread_create() blocks for memory if necessary. It never fails.
324 *
325 * If stk is NULL, the thread is created at the base of the stack
326 * and cannot be swapped.
327 */
328 kthread_t *
329 thread_create(
330 caddr_t stk,
331 size_t stksize,
332 void (*proc)(),
333 void *arg,
334 size_t len,
335 proc_t *pp,
336 int state,
337 pri_t pri)
338 {
339 kthread_t *t;
340 extern struct classfuncs sys_classfuncs;
341 turnstile_t *ts;
342
343 /*
344 * Every thread keeps a turnstile around in case it needs to block.
345 * The only reason the turnstile is not simply part of the thread
346 * structure is that we may have to break the association whenever
347 * more than one thread blocks on a given synchronization object.
348 * From a memory-management standpoint, turnstiles are like the
349 * "attached mblks" that hang off dblks in the streams allocator.
350 */
351 ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
352
353 if (stk == NULL) {
354 /*
355 * alloc both thread and stack in segkp chunk
356 */
357
358 if (stksize < default_stksize)
359 stksize = default_stksize;
360
361 if (stksize == default_stksize) {
362 stk = (caddr_t)segkp_cache_get(segkp_thread);
363 } else {
364 stksize = roundup(stksize, PAGESIZE);
365 stk = (caddr_t)segkp_get(segkp, stksize,
366 (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
367 }
368
369 ASSERT(stk != NULL);
370
371 /*
372 * The machine-dependent mutex code may require that
373 * thread pointers (since they may be used for mutex owner
374 * fields) have certain alignment requirements.
375 * PTR24_ALIGN is the size of the alignment quanta.
376 * XXX - assumes stack grows toward low addresses.
377 */
378 if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
379 cmn_err(CE_PANIC, "thread_create: proposed stack size"
380 " too small to hold thread.");
381 #ifdef STACK_GROWTH_DOWN
382 stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
383 stksize &= -PTR24_ALIGN; /* make thread aligned */
384 t = (kthread_t *)(stk + stksize);
385 bzero(t, sizeof (kthread_t));
386 if (audit_active)
387 audit_thread_create(t);
388 t->t_stk = stk + stksize;
389 t->t_stkbase = stk;
390 #else /* stack grows to larger addresses */
391 stksize -= SA(sizeof (kthread_t));
392 t = (kthread_t *)(stk);
393 bzero(t, sizeof (kthread_t));
394 t->t_stk = stk + sizeof (kthread_t);
395 t->t_stkbase = stk + stksize + sizeof (kthread_t);
396 #endif /* STACK_GROWTH_DOWN */
397 t->t_flag |= T_TALLOCSTK;
398 t->t_swap = stk;
399 } else {
400 t = kmem_cache_alloc(thread_cache, KM_SLEEP);
401 bzero(t, sizeof (kthread_t));
402 ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
403 if (audit_active)
404 audit_thread_create(t);
405 /*
406 * Initialize t_stk to the kernel stack pointer to use
407 * upon entry to the kernel
408 */
409 #ifdef STACK_GROWTH_DOWN
410 t->t_stk = stk + stksize;
411 t->t_stkbase = stk;
412 #else
413 t->t_stk = stk; /* 3b2-like */
414 t->t_stkbase = stk + stksize;
415 #endif /* STACK_GROWTH_DOWN */
416 }
417
418 if (kmem_stackinfo != 0) {
419 stkinfo_begin(t);
420 }
421
422 t->t_ts = ts;
423
424 /*
425 * p_cred could be NULL if it thread_create is called before cred_init
426 * is called in main.
427 */
428 mutex_enter(&pp->p_crlock);
429 if (pp->p_cred)
430 crhold(t->t_cred = pp->p_cred);
431 mutex_exit(&pp->p_crlock);
432 t->t_start = gethrestime_sec();
433 t->t_startpc = proc;
434 t->t_procp = pp;
435 t->t_clfuncs = &sys_classfuncs.thread;
436 t->t_cid = syscid;
437 t->t_pri = pri;
438 t->t_stime = ddi_get_lbolt();
439 t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
440 t->t_bind_cpu = PBIND_NONE;
441 t->t_bindflag = (uchar_t)default_binding_mode;
442 t->t_bind_pset = PS_NONE;
443 t->t_plockp = &pp->p_lock;
444 t->t_copyops = NULL;
445 t->t_taskq = NULL;
446 t->t_anttime = 0;
447 t->t_hatdepth = 0;
448
449 t->t_dtrace_vtime = 1; /* assure vtimestamp is always non-zero */
450
451 CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
452 #ifndef NPROBE
453 /* Kernel probe */
454 tnf_thread_create(t);
455 #endif /* NPROBE */
456 LOCK_INIT_CLEAR(&t->t_lock);
457
458 /*
459 * Callers who give us a NULL proc must do their own
460 * stack initialization. e.g. lwp_create()
461 */
462 if (proc != NULL) {
463 t->t_stk = thread_stk_init(t->t_stk);
464 thread_load(t, proc, arg, len);
465 }
466
467 /*
468 * Put a hold on project0. If this thread is actually in a
469 * different project, then t_proj will be changed later in
470 * lwp_create(). All kernel-only threads must be in project 0.
471 */
472 t->t_proj = project_hold(proj0p);
473
474 lgrp_affinity_init(&t->t_lgrp_affinity);
475
476 mutex_enter(&pidlock);
477 nthread++;
478 t->t_did = next_t_id++;
479 t->t_prev = curthread->t_prev;
480 t->t_next = curthread;
481
482 /*
483 * Add the thread to the list of all threads, and initialize
484 * its t_cpu pointer. We need to block preemption since
485 * cpu_offline walks the thread list looking for threads
486 * with t_cpu pointing to the CPU being offlined. We want
487 * to make sure that the list is consistent and that if t_cpu
488 * is set, the thread is on the list.
489 */
490 kpreempt_disable();
491 curthread->t_prev->t_next = t;
492 curthread->t_prev = t;
493
494 /*
495 * Threads should never have a NULL t_cpu pointer so assign it
496 * here. If the thread is being created with state TS_RUN a
497 * better CPU may be chosen when it is placed on the run queue.
498 *
499 * We need to keep kernel preemption disabled when setting all
500 * three fields to keep them in sync. Also, always create in
501 * the default partition since that's where kernel threads go
502 * (if this isn't a kernel thread, t_cpupart will be changed
503 * in lwp_create before setting the thread runnable).
504 */
505 t->t_cpupart = &cp_default;
506
507 /*
508 * For now, affiliate this thread with the root lgroup.
509 * Since the kernel does not (presently) allocate its memory
510 * in a locality aware fashion, the root is an appropriate home.
511 * If this thread is later associated with an lwp, it will have
512 * it's lgroup re-assigned at that time.
513 */
514 lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
515
516 /*
517 * Inherit the current cpu. If this cpu isn't part of the chosen
518 * lgroup, a new cpu will be chosen by cpu_choose when the thread
519 * is ready to run.
520 */
521 if (CPU->cpu_part == &cp_default)
522 t->t_cpu = CPU;
523 else
524 t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t,
525 t->t_pri);
526
527 t->t_disp_queue = t->t_cpu->cpu_disp;
528 kpreempt_enable();
529
530 /*
531 * Initialize thread state and the dispatcher lock pointer.
532 * Need to hold onto pidlock to block allthreads walkers until
533 * the state is set.
534 */
535 switch (state) {
536 case TS_RUN:
537 curthread->t_oldspl = splhigh(); /* get dispatcher spl */
538 THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
539 CL_SETRUN(t);
540 thread_unlock(t);
541 break;
542
543 case TS_ONPROC:
544 THREAD_ONPROC(t, t->t_cpu);
545 break;
546
547 case TS_FREE:
548 /*
549 * Free state will be used for intr threads.
550 * The interrupt routine must set the thread dispatcher
551 * lock pointer (t_lockp) if starting on a CPU
552 * other than the current one.
553 */
554 THREAD_FREEINTR(t, CPU);
555 break;
556
557 case TS_STOPPED:
558 THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
559 break;
560
561 default: /* TS_SLEEP, TS_ZOMB or TS_TRANS */
562 cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
563 }
564 mutex_exit(&pidlock);
565 return (t);
566 }
567
568 /*
569 * Move thread to project0 and take care of project reference counters.
570 */
571 void
572 thread_rele(kthread_t *t)
573 {
574 kproject_t *kpj;
575
576 thread_lock(t);
577
578 ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
579 kpj = ttoproj(t);
580 t->t_proj = proj0p;
581
582 thread_unlock(t);
583
584 if (kpj != proj0p) {
585 project_rele(kpj);
586 (void) project_hold(proj0p);
587 }
588 }
589
590 void
591 thread_exit(void)
592 {
593 kthread_t *t = curthread;
594
595 if ((t->t_proc_flag & TP_ZTHREAD) != 0)
596 cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
597
598 tsd_exit(); /* Clean up this thread's TSD */
599
600 kcpc_passivate(); /* clean up performance counter state */
601
602 /*
603 * No kernel thread should have called poll() without arranging
604 * calling pollcleanup() here.
605 */
606 ASSERT(t->t_pollstate == NULL);
607 ASSERT(t->t_schedctl == NULL);
608 if (t->t_door)
609 door_slam(); /* in case thread did an upcall */
610
611 #ifndef NPROBE
612 /* Kernel probe */
613 if (t->t_tnf_tpdp)
614 tnf_thread_exit();
615 #endif /* NPROBE */
616
617 thread_rele(t);
618 t->t_preempt++;
619
620 /*
621 * remove thread from the all threads list so that
622 * death-row can use the same pointers.
623 */
624 mutex_enter(&pidlock);
625 t->t_next->t_prev = t->t_prev;
626 t->t_prev->t_next = t->t_next;
627 ASSERT(allthreads != t); /* t0 never exits */
628 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */
629 mutex_exit(&pidlock);
630
631 if (t->t_ctx != NULL)
632 exitctx(t);
633 if (t->t_procp->p_pctx != NULL)
634 exitpctx(t->t_procp);
635
636 if (kmem_stackinfo != 0) {
637 stkinfo_end(t);
638 }
639
640 t->t_state = TS_ZOMB; /* set zombie thread */
641
642 swtch_from_zombie(); /* give up the CPU */
643 /* NOTREACHED */
644 }
645
646 /*
647 * Check to see if the specified thread is active (defined as being on
648 * the thread list). This is certainly a slow way to do this; if there's
649 * ever a reason to speed it up, we could maintain a hash table of active
650 * threads indexed by their t_did.
651 */
652 static kthread_t *
653 did_to_thread(kt_did_t tid)
654 {
655 kthread_t *t;
656
657 ASSERT(MUTEX_HELD(&pidlock));
658 for (t = curthread->t_next; t != curthread; t = t->t_next) {
659 if (t->t_did == tid)
660 break;
661 }
662 if (t->t_did == tid)
663 return (t);
664 else
665 return (NULL);
666 }
667
668 /*
669 * Wait for specified thread to exit. Returns immediately if the thread
670 * could not be found, meaning that it has either already exited or never
671 * existed.
672 */
673 void
674 thread_join(kt_did_t tid)
675 {
676 kthread_t *t;
677
678 ASSERT(tid != curthread->t_did);
679 ASSERT(tid != t0.t_did);
680
681 mutex_enter(&pidlock);
682 /*
683 * Make sure we check that the thread is on the thread list
684 * before blocking on it; otherwise we could end up blocking on
685 * a cv that's already been freed. In other words, don't cache
686 * the thread pointer across calls to cv_wait.
687 *
688 * The choice of loop invariant means that whenever a thread
689 * is taken off the allthreads list, a cv_broadcast must be
690 * performed on that thread's t_joincv to wake up any waiters.
691 * The broadcast doesn't have to happen right away, but it
692 * shouldn't be postponed indefinitely (e.g., by doing it in
693 * thread_free which may only be executed when the deathrow
694 * queue is processed.
695 */
696 while (t = did_to_thread(tid))
697 cv_wait(&t->t_joincv, &pidlock);
698 mutex_exit(&pidlock);
699 }
700
701 void
702 thread_free_prevent(kthread_t *t)
703 {
704 kmutex_t *lp;
705
706 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
707 mutex_enter(lp);
708 }
709
710 void
711 thread_free_allow(kthread_t *t)
712 {
713 kmutex_t *lp;
714
715 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
716 mutex_exit(lp);
717 }
718
719 static void
720 thread_free_barrier(kthread_t *t)
721 {
722 kmutex_t *lp;
723
724 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
725 mutex_enter(lp);
726 mutex_exit(lp);
727 }
728
729 void
730 thread_free(kthread_t *t)
731 {
732 boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
733 klwp_t *lwp = t->t_lwp;
734 caddr_t swap = t->t_swap;
735
736 ASSERT(t != &t0 && t->t_state == TS_FREE);
737 ASSERT(t->t_door == NULL);
738 ASSERT(t->t_schedctl == NULL);
739 ASSERT(t->t_pollstate == NULL);
740
741 t->t_pri = 0;
742 t->t_pc = 0;
743 t->t_sp = 0;
744 t->t_wchan0 = NULL;
745 t->t_wchan = NULL;
746 if (t->t_cred != NULL) {
747 crfree(t->t_cred);
748 t->t_cred = 0;
749 }
750 if (t->t_pdmsg) {
751 kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
752 t->t_pdmsg = NULL;
753 }
754 if (audit_active)
755 audit_thread_free(t);
756 #ifndef NPROBE
757 if (t->t_tnf_tpdp)
758 tnf_thread_free(t);
759 #endif /* NPROBE */
760 if (t->t_cldata) {
761 CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
762 }
763 if (t->t_rprof != NULL) {
764 kmem_free(t->t_rprof, sizeof (*t->t_rprof));
765 t->t_rprof = NULL;
766 }
767 t->t_lockp = NULL; /* nothing should try to lock this thread now */
768 if (lwp)
769 lwp_freeregs(lwp, 0);
770 if (t->t_ctx)
771 freectx(t, 0);
772 t->t_stk = NULL;
773 if (lwp)
774 lwp_stk_fini(lwp);
775 lock_clear(&t->t_lock);
776
777 if (t->t_ts->ts_waiters > 0)
778 panic("thread_free: turnstile still active");
779
780 kmem_cache_free(turnstile_cache, t->t_ts);
781
782 free_afd(&t->t_activefd);
783
784 /*
785 * Barrier for the tick accounting code. The tick accounting code
786 * holds this lock to keep the thread from going away while it's
787 * looking at it.
788 */
789 thread_free_barrier(t);
790
791 ASSERT(ttoproj(t) == proj0p);
792 project_rele(ttoproj(t));
793
794 lgrp_affinity_free(&t->t_lgrp_affinity);
795
796 mutex_enter(&pidlock);
797 nthread--;
798 mutex_exit(&pidlock);
799
800 if (t->t_name != NULL) {
801 kmem_free(t->t_name, THREAD_NAME_MAX);
802 t->t_name = NULL;
803 }
804
805 /*
806 * Free thread, lwp and stack. This needs to be done carefully, since
807 * if T_TALLOCSTK is set, the thread is part of the stack.
808 */
809 t->t_lwp = NULL;
810 t->t_swap = NULL;
811
812 if (swap) {
813 segkp_release(segkp, swap);
814 }
815 if (lwp) {
816 kmem_cache_free(lwp_cache, lwp);
817 }
818 if (!allocstk) {
819 kmem_cache_free(thread_cache, t);
820 }
821 }
822
823 /*
824 * Removes threads associated with the given zone from a deathrow queue.
825 * tp is a pointer to the head of the deathrow queue, and countp is a
826 * pointer to the current deathrow count. Returns a linked list of
827 * threads removed from the list.
828 */
829 static kthread_t *
830 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
831 {
832 kthread_t *tmp, *list = NULL;
833 cred_t *cr;
834
835 ASSERT(MUTEX_HELD(&reaplock));
836 while (*tp != NULL) {
837 if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
838 tmp = *tp;
839 *tp = tmp->t_forw;
840 tmp->t_forw = list;
841 list = tmp;
842 (*countp)--;
843 } else {
844 tp = &(*tp)->t_forw;
845 }
846 }
847 return (list);
848 }
849
850 static void
851 thread_reap_list(kthread_t *t)
852 {
853 kthread_t *next;
854
855 while (t != NULL) {
856 next = t->t_forw;
857 thread_free(t);
858 t = next;
859 }
860 }
861
862 /* ARGSUSED */
863 static void
864 thread_zone_destroy(zoneid_t zoneid, void *unused)
865 {
866 kthread_t *t, *l;
867
868 mutex_enter(&reaplock);
869 /*
870 * Pull threads and lwps associated with zone off deathrow lists.
871 */
872 t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
873 l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
874 mutex_exit(&reaplock);
875
876 /*
877 * Guard against race condition in mutex_owner_running:
878 * thread=owner(mutex)
879 * <interrupt>
880 * thread exits mutex
881 * thread exits
882 * thread reaped
883 * thread struct freed
884 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
885 * A cross call to all cpus will cause the interrupt handler
886 * to reset the PC if it is in mutex_owner_running, refreshing
887 * stale thread pointers.
888 */
889 mutex_sync(); /* sync with mutex code */
890
891 /*
892 * Reap threads
893 */
894 thread_reap_list(t);
895
896 /*
897 * Reap lwps
898 */
899 thread_reap_list(l);
900 }
901
902 /*
903 * cleanup zombie threads that are on deathrow.
904 */
905 void
906 thread_reaper()
907 {
908 kthread_t *t, *l;
909 callb_cpr_t cprinfo;
910
911 /*
912 * Register callback to clean up threads when zone is destroyed.
913 */
914 zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
915
916 CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
917 for (;;) {
918 mutex_enter(&reaplock);
919 while (thread_deathrow == NULL && lwp_deathrow == NULL) {
920 CALLB_CPR_SAFE_BEGIN(&cprinfo);
921 cv_wait(&reaper_cv, &reaplock);
922 CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
923 }
924 /*
925 * mutex_sync() needs to be called when reaping, but
926 * not too often. We limit reaping rate to once
927 * per second. Reaplimit is max rate at which threads can
928 * be freed. Does not impact thread destruction/creation.
929 */
930 t = thread_deathrow;
931 l = lwp_deathrow;
932 thread_deathrow = NULL;
933 lwp_deathrow = NULL;
934 thread_reapcnt = 0;
935 lwp_reapcnt = 0;
936 mutex_exit(&reaplock);
937
938 /*
939 * Guard against race condition in mutex_owner_running:
940 * thread=owner(mutex)
941 * <interrupt>
942 * thread exits mutex
943 * thread exits
944 * thread reaped
945 * thread struct freed
946 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
947 * A cross call to all cpus will cause the interrupt handler
948 * to reset the PC if it is in mutex_owner_running, refreshing
949 * stale thread pointers.
950 */
951 mutex_sync(); /* sync with mutex code */
952 /*
953 * Reap threads
954 */
955 thread_reap_list(t);
956
957 /*
958 * Reap lwps
959 */
960 thread_reap_list(l);
961 delay(hz);
962 }
963 }
964
965 /*
966 * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
967 * thread_deathrow. The thread's state is changed already TS_FREE to indicate
968 * that is reapable. The thread already holds the reaplock, and was already
969 * freed.
970 */
971 void
972 reapq_move_lq_to_tq(kthread_t *t)
973 {
974 ASSERT(t->t_state == TS_FREE);
975 ASSERT(MUTEX_HELD(&reaplock));
976 t->t_forw = thread_deathrow;
977 thread_deathrow = t;
978 thread_reapcnt++;
979 if (lwp_reapcnt + thread_reapcnt > reaplimit)
980 cv_signal(&reaper_cv); /* wake the reaper */
981 }
982
983 /*
984 * This is called by resume() to put a zombie thread onto deathrow.
985 * The thread's state is changed to TS_FREE to indicate that is reapable.
986 * This is called from the idle thread so it must not block - just spin.
987 */
988 void
989 reapq_add(kthread_t *t)
990 {
991 mutex_enter(&reaplock);
992
993 /*
994 * lwp_deathrow contains threads with lwp linkage and
995 * swappable thread stacks which have the default stacksize.
996 * These threads' lwps and stacks may be reused by lwp_create().
997 *
998 * Anything else goes on thread_deathrow(), where it will eventually
999 * be thread_free()d.
1000 */
1001 if (t->t_flag & T_LWPREUSE) {
1002 ASSERT(ttolwp(t) != NULL);
1003 t->t_forw = lwp_deathrow;
1004 lwp_deathrow = t;
1005 lwp_reapcnt++;
1006 } else {
1007 t->t_forw = thread_deathrow;
1008 thread_deathrow = t;
1009 thread_reapcnt++;
1010 }
1011 if (lwp_reapcnt + thread_reapcnt > reaplimit)
1012 cv_signal(&reaper_cv); /* wake the reaper */
1013 t->t_state = TS_FREE;
1014 lock_clear(&t->t_lock);
1015
1016 /*
1017 * Before we return, we need to grab and drop the thread lock for
1018 * the dead thread. At this point, the current thread is the idle
1019 * thread, and the dead thread's CPU lock points to the current
1020 * CPU -- and we must grab and drop the lock to synchronize with
1021 * a racing thread walking a blocking chain that the zombie thread
1022 * was recently in. By this point, that blocking chain is (by
1023 * definition) stale: the dead thread is not holding any locks, and
1024 * is therefore not in any blocking chains -- but if we do not regrab
1025 * our lock before freeing the dead thread's data structures, the
1026 * thread walking the (stale) blocking chain will die on memory
1027 * corruption when it attempts to drop the dead thread's lock. We
1028 * only need do this once because there is no way for the dead thread
1029 * to ever again be on a blocking chain: once we have grabbed and
1030 * dropped the thread lock, we are guaranteed that anyone that could
1031 * have seen this thread in a blocking chain can no longer see it.
1032 */
1033 thread_lock(t);
1034 thread_unlock(t);
1035
1036 mutex_exit(&reaplock);
1037 }
1038
1039 /*
1040 * Install thread context ops for the current thread.
1041 */
1042 void
1043 installctx(
1044 kthread_t *t,
1045 void *arg,
1046 void (*save)(void *),
1047 void (*restore)(void *),
1048 void (*fork)(void *, void *),
1049 void (*lwp_create)(void *, void *),
1050 void (*exit)(void *),
1051 void (*free)(void *, int))
1052 {
1053 struct ctxop *ctx;
1054
1055 ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1056 ctx->save_op = save;
1057 ctx->restore_op = restore;
1058 ctx->fork_op = fork;
1059 ctx->lwp_create_op = lwp_create;
1060 ctx->exit_op = exit;
1061 ctx->free_op = free;
1062 ctx->arg = arg;
1063 ctx->next = t->t_ctx;
1064 t->t_ctx = ctx;
1065 }
1066
1067 /*
1068 * Remove the thread context ops from a thread.
1069 */
1070 int
1071 removectx(
1072 kthread_t *t,
1073 void *arg,
1074 void (*save)(void *),
1075 void (*restore)(void *),
1076 void (*fork)(void *, void *),
1077 void (*lwp_create)(void *, void *),
1078 void (*exit)(void *),
1079 void (*free)(void *, int))
1080 {
1081 struct ctxop *ctx, *prev_ctx;
1082
1083 /*
1084 * The incoming kthread_t (which is the thread for which the
1085 * context ops will be removed) should be one of the following:
1086 *
1087 * a) the current thread,
1088 *
1089 * b) a thread of a process that's being forked (SIDL),
1090 *
1091 * c) a thread that belongs to the same process as the current
1092 * thread and for which the current thread is the agent thread,
1093 *
1094 * d) a thread that is TS_STOPPED which is indicative of it
1095 * being (if curthread is not an agent) a thread being created
1096 * as part of an lwp creation.
1097 */
1098 ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1099 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1100
1101 /*
1102 * Serialize modifications to t->t_ctx to prevent the agent thread
1103 * and the target thread from racing with each other during lwp exit.
1104 */
1105 mutex_enter(&t->t_ctx_lock);
1106 prev_ctx = NULL;
1107 kpreempt_disable();
1108 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
1109 if (ctx->save_op == save && ctx->restore_op == restore &&
1110 ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
1111 ctx->exit_op == exit && ctx->free_op == free &&
1112 ctx->arg == arg) {
1113 if (prev_ctx)
1114 prev_ctx->next = ctx->next;
1115 else
1116 t->t_ctx = ctx->next;
1117 mutex_exit(&t->t_ctx_lock);
1118 if (ctx->free_op != NULL)
1119 (ctx->free_op)(ctx->arg, 0);
1120 kmem_free(ctx, sizeof (struct ctxop));
1121 kpreempt_enable();
1122 return (1);
1123 }
1124 prev_ctx = ctx;
1125 }
1126 mutex_exit(&t->t_ctx_lock);
1127 kpreempt_enable();
1128
1129 return (0);
1130 }
1131
1132 void
1133 savectx(kthread_t *t)
1134 {
1135 struct ctxop *ctx;
1136
1137 ASSERT(t == curthread);
1138 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1139 if (ctx->save_op != NULL)
1140 (ctx->save_op)(ctx->arg);
1141 }
1142
1143 void
1144 restorectx(kthread_t *t)
1145 {
1146 struct ctxop *ctx;
1147
1148 ASSERT(t == curthread);
1149 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1150 if (ctx->restore_op != NULL)
1151 (ctx->restore_op)(ctx->arg);
1152 }
1153
1154 void
1155 forkctx(kthread_t *t, kthread_t *ct)
1156 {
1157 struct ctxop *ctx;
1158
1159 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1160 if (ctx->fork_op != NULL)
1161 (ctx->fork_op)(t, ct);
1162 }
1163
1164 /*
1165 * Note that this operator is only invoked via the _lwp_create
1166 * system call. The system may have other reasons to create lwps
1167 * e.g. the agent lwp or the doors unreferenced lwp.
1168 */
1169 void
1170 lwp_createctx(kthread_t *t, kthread_t *ct)
1171 {
1172 struct ctxop *ctx;
1173
1174 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1175 if (ctx->lwp_create_op != NULL)
1176 (ctx->lwp_create_op)(t, ct);
1177 }
1178
1179 /*
1180 * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1181 * needed when the thread/LWP leaves the processor for the last time. This
1182 * routine is not intended to deal with freeing memory; freectx() is used for
1183 * that purpose during thread_free(). This routine is provided to allow for
1184 * clean-up that can't wait until thread_free().
1185 */
1186 void
1187 exitctx(kthread_t *t)
1188 {
1189 struct ctxop *ctx;
1190
1191 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1192 if (ctx->exit_op != NULL)
1193 (ctx->exit_op)(t);
1194 }
1195
1196 /*
1197 * freectx is called from thread_free() and exec() to get
1198 * rid of old thread context ops.
1199 */
1200 void
1201 freectx(kthread_t *t, int isexec)
1202 {
1203 struct ctxop *ctx;
1204
1205 kpreempt_disable();
1206 while ((ctx = t->t_ctx) != NULL) {
1207 t->t_ctx = ctx->next;
1208 if (ctx->free_op != NULL)
1209 (ctx->free_op)(ctx->arg, isexec);
1210 kmem_free(ctx, sizeof (struct ctxop));
1211 }
1212 kpreempt_enable();
1213 }
1214
1215 /*
1216 * freectx_ctx is called from lwp_create() when lwp is reused from
1217 * lwp_deathrow and its thread structure is added to thread_deathrow.
1218 * The thread structure to which this ctx was attached may be already
1219 * freed by the thread reaper so free_op implementations shouldn't rely
1220 * on thread structure to which this ctx was attached still being around.
1221 */
1222 void
1223 freectx_ctx(struct ctxop *ctx)
1224 {
1225 struct ctxop *nctx;
1226
1227 ASSERT(ctx != NULL);
1228
1229 kpreempt_disable();
1230 do {
1231 nctx = ctx->next;
1232 if (ctx->free_op != NULL)
1233 (ctx->free_op)(ctx->arg, 0);
1234 kmem_free(ctx, sizeof (struct ctxop));
1235 } while ((ctx = nctx) != NULL);
1236 kpreempt_enable();
1237 }
1238
1239 /*
1240 * Set the thread running; arrange for it to be swapped in if necessary.
1241 */
1242 void
1243 setrun_locked(kthread_t *t)
1244 {
1245 ASSERT(THREAD_LOCK_HELD(t));
1246 if (t->t_state == TS_SLEEP) {
1247 /*
1248 * Take off sleep queue.
1249 */
1250 SOBJ_UNSLEEP(t->t_sobj_ops, t);
1251 } else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1252 /*
1253 * Already on dispatcher queue.
1254 */
1255 return;
1256 } else if (t->t_state == TS_WAIT) {
1257 waitq_setrun(t);
1258 } else if (t->t_state == TS_STOPPED) {
1259 /*
1260 * All of the sending of SIGCONT (TC_XSTART) and /proc
1261 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1262 * requested that the thread be run.
1263 * Just calling setrun() is not sufficient to set a stopped
1264 * thread running. TP_TXSTART is always set if the thread
1265 * is not stopped by a jobcontrol stop signal.
1266 * TP_TPSTART is always set if /proc is not controlling it.
1267 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1268 * The thread won't be stopped unless one of these
1269 * three mechanisms did it.
1270 *
1271 * These flags must be set before calling setrun_locked(t).
1272 * They can't be passed as arguments because the streams
1273 * code calls setrun() indirectly and the mechanism for
1274 * doing so admits only one argument. Note that the
1275 * thread must be locked in order to change t_schedflags.
1276 */
1277 if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1278 return;
1279 /*
1280 * Process is no longer stopped (a thread is running).
1281 */
1282 t->t_whystop = 0;
1283 t->t_whatstop = 0;
1284 /*
1285 * Strictly speaking, we do not have to clear these
1286 * flags here; they are cleared on entry to stop().
1287 * However, they are confusing when doing kernel
1288 * debugging or when they are revealed by ps(1).
1289 */
1290 t->t_schedflag &= ~TS_ALLSTART;
1291 THREAD_TRANSITION(t); /* drop stopped-thread lock */
1292 ASSERT(t->t_lockp == &transition_lock);
1293 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1294 /*
1295 * Let the class put the process on the dispatcher queue.
1296 */
1297 CL_SETRUN(t);
1298 }
1299 }
1300
1301 void
1302 setrun(kthread_t *t)
1303 {
1304 thread_lock(t);
1305 setrun_locked(t);
1306 thread_unlock(t);
1307 }
1308
1309 /*
1310 * Unpin an interrupted thread.
1311 * When an interrupt occurs, the interrupt is handled on the stack
1312 * of an interrupt thread, taken from a pool linked to the CPU structure.
1313 *
1314 * When swtch() is switching away from an interrupt thread because it
1315 * blocked or was preempted, this routine is called to complete the
1316 * saving of the interrupted thread state, and returns the interrupted
1317 * thread pointer so it may be resumed.
1318 *
1319 * Called by swtch() only at high spl.
1320 */
1321 kthread_t *
1322 thread_unpin()
1323 {
1324 kthread_t *t = curthread; /* current thread */
1325 kthread_t *itp; /* interrupted thread */
1326 int i; /* interrupt level */
1327 extern int intr_passivate();
1328
1329 ASSERT(t->t_intr != NULL);
1330
1331 itp = t->t_intr; /* interrupted thread */
1332 t->t_intr = NULL; /* clear interrupt ptr */
1333
1334 ht_end_intr();
1335
1336 /*
1337 * Get state from interrupt thread for the one
1338 * it interrupted.
1339 */
1340
1341 i = intr_passivate(t, itp);
1342
1343 TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1344 "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1345 i, t, t, itp, itp);
1346
1347 /*
1348 * Dissociate the current thread from the interrupted thread's LWP.
1349 */
1350 t->t_lwp = NULL;
1351
1352 /*
1353 * Interrupt handlers above the level that spinlocks block must
1354 * not block.
1355 */
1356 #if DEBUG
1357 if (i < 0 || i > LOCK_LEVEL)
1358 cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1359 #endif
1360
1361 /*
1362 * Compute the CPU's base interrupt level based on the active
1363 * interrupts.
1364 */
1365 ASSERT(CPU->cpu_intr_actv & (1 << i));
1366 set_base_spl();
1367
1368 return (itp);
1369 }
1370
1371 /*
1372 * Create and initialize an interrupt thread.
1373 * Returns non-zero on error.
1374 * Called at spl7() or better.
1375 */
1376 void
1377 thread_create_intr(struct cpu *cp)
1378 {
1379 kthread_t *tp;
1380
1381 tp = thread_create(NULL, 0,
1382 (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1383
1384 /*
1385 * Set the thread in the TS_FREE state. The state will change
1386 * to TS_ONPROC only while the interrupt is active. Think of these
1387 * as being on a private free list for the CPU. Being TS_FREE keeps
1388 * inactive interrupt threads out of debugger thread lists.
1389 *
1390 * We cannot call thread_create with TS_FREE because of the current
1391 * checks there for ONPROC. Fix this when thread_create takes flags.
1392 */
1393 THREAD_FREEINTR(tp, cp);
1394
1395 /*
1396 * Nobody should ever reference the credentials of an interrupt
1397 * thread so make it NULL to catch any such references.
1398 */
1399 tp->t_cred = NULL;
1400 tp->t_flag |= T_INTR_THREAD;
1401 tp->t_cpu = cp;
1402 tp->t_bound_cpu = cp;
1403 tp->t_disp_queue = cp->cpu_disp;
1404 tp->t_affinitycnt = 1;
1405 tp->t_preempt = 1;
1406
1407 /*
1408 * Don't make a user-requested binding on this thread so that
1409 * the processor can be offlined.
1410 */
1411 tp->t_bind_cpu = PBIND_NONE; /* no USER-requested binding */
1412 tp->t_bind_pset = PS_NONE;
1413
1414 #if defined(__i386) || defined(__amd64)
1415 tp->t_stk -= STACK_ALIGN;
1416 *(tp->t_stk) = 0; /* terminate intr thread stack */
1417 #endif
1418
1419 /*
1420 * Link onto CPU's interrupt pool.
1421 */
1422 tp->t_link = cp->cpu_intr_thread;
1423 cp->cpu_intr_thread = tp;
1424 }
1425
1426 /*
1427 * TSD -- THREAD SPECIFIC DATA
1428 */
1429 static kmutex_t tsd_mutex; /* linked list spin lock */
1430 static uint_t tsd_nkeys; /* size of destructor array */
1431 /* per-key destructor funcs */
1432 static void (**tsd_destructor)(void *);
1433 /* list of tsd_thread's */
1434 static struct tsd_thread *tsd_list;
1435
1436 /*
1437 * Default destructor
1438 * Needed because NULL destructor means that the key is unused
1439 */
1440 /* ARGSUSED */
1441 void
1442 tsd_defaultdestructor(void *value)
1443 {}
1444
1445 /*
1446 * Create a key (index into per thread array)
1447 * Locks out tsd_create, tsd_destroy, and tsd_exit
1448 * May allocate memory with lock held
1449 */
1450 void
1451 tsd_create(uint_t *keyp, void (*destructor)(void *))
1452 {
1453 int i;
1454 uint_t nkeys;
1455
1456 /*
1457 * if key is allocated, do nothing
1458 */
1459 mutex_enter(&tsd_mutex);
1460 if (*keyp) {
1461 mutex_exit(&tsd_mutex);
1462 return;
1463 }
1464 /*
1465 * find an unused key
1466 */
1467 if (destructor == NULL)
1468 destructor = tsd_defaultdestructor;
1469
1470 for (i = 0; i < tsd_nkeys; ++i)
1471 if (tsd_destructor[i] == NULL)
1472 break;
1473
1474 /*
1475 * if no unused keys, increase the size of the destructor array
1476 */
1477 if (i == tsd_nkeys) {
1478 if ((nkeys = (tsd_nkeys << 1)) == 0)
1479 nkeys = 1;
1480 tsd_destructor =
1481 (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1482 (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1483 (size_t)(nkeys * sizeof (void (*)(void *))));
1484 tsd_nkeys = nkeys;
1485 }
1486
1487 /*
1488 * allocate the next available unused key
1489 */
1490 tsd_destructor[i] = destructor;
1491 *keyp = i + 1;
1492 mutex_exit(&tsd_mutex);
1493 }
1494
1495 /*
1496 * Destroy a key -- this is for unloadable modules
1497 *
1498 * Assumes that the caller is preventing tsd_set and tsd_get
1499 * Locks out tsd_create, tsd_destroy, and tsd_exit
1500 * May free memory with lock held
1501 */
1502 void
1503 tsd_destroy(uint_t *keyp)
1504 {
1505 uint_t key;
1506 struct tsd_thread *tsd;
1507
1508 /*
1509 * protect the key namespace and our destructor lists
1510 */
1511 mutex_enter(&tsd_mutex);
1512 key = *keyp;
1513 *keyp = 0;
1514
1515 ASSERT(key <= tsd_nkeys);
1516
1517 /*
1518 * if the key is valid
1519 */
1520 if (key != 0) {
1521 uint_t k = key - 1;
1522 /*
1523 * for every thread with TSD, call key's destructor
1524 */
1525 for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1526 /*
1527 * no TSD for key in this thread
1528 */
1529 if (key > tsd->ts_nkeys)
1530 continue;
1531 /*
1532 * call destructor for key
1533 */
1534 if (tsd->ts_value[k] && tsd_destructor[k])
1535 (*tsd_destructor[k])(tsd->ts_value[k]);
1536 /*
1537 * reset value for key
1538 */
1539 tsd->ts_value[k] = NULL;
1540 }
1541 /*
1542 * actually free the key (NULL destructor == unused)
1543 */
1544 tsd_destructor[k] = NULL;
1545 }
1546
1547 mutex_exit(&tsd_mutex);
1548 }
1549
1550 /*
1551 * Quickly return the per thread value that was stored with the specified key
1552 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1553 */
1554 void *
1555 tsd_get(uint_t key)
1556 {
1557 return (tsd_agent_get(curthread, key));
1558 }
1559
1560 /*
1561 * Set a per thread value indexed with the specified key
1562 */
1563 int
1564 tsd_set(uint_t key, void *value)
1565 {
1566 return (tsd_agent_set(curthread, key, value));
1567 }
1568
1569 /*
1570 * Like tsd_get(), except that the agent lwp can get the tsd of
1571 * another thread in the same process (the agent thread only runs when the
1572 * process is completely stopped by /proc), or syslwp is creating a new lwp.
1573 */
1574 void *
1575 tsd_agent_get(kthread_t *t, uint_t key)
1576 {
1577 struct tsd_thread *tsd = t->t_tsd;
1578
1579 ASSERT(t == curthread ||
1580 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1581
1582 if (key && tsd != NULL && key <= tsd->ts_nkeys)
1583 return (tsd->ts_value[key - 1]);
1584 return (NULL);
1585 }
1586
1587 /*
1588 * Like tsd_set(), except that the agent lwp can set the tsd of
1589 * another thread in the same process, or syslwp can set the tsd
1590 * of a thread it's in the middle of creating.
1591 *
1592 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1593 * May lock out tsd_destroy (and tsd_create), may allocate memory with
1594 * lock held
1595 */
1596 int
1597 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1598 {
1599 struct tsd_thread *tsd = t->t_tsd;
1600
1601 ASSERT(t == curthread ||
1602 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1603
1604 if (key == 0)
1605 return (EINVAL);
1606 if (tsd == NULL)
1607 tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1608 if (key <= tsd->ts_nkeys) {
1609 tsd->ts_value[key - 1] = value;
1610 return (0);
1611 }
1612
1613 ASSERT(key <= tsd_nkeys);
1614
1615 /*
1616 * lock out tsd_destroy()
1617 */
1618 mutex_enter(&tsd_mutex);
1619 if (tsd->ts_nkeys == 0) {
1620 /*
1621 * Link onto list of threads with TSD
1622 */
1623 if ((tsd->ts_next = tsd_list) != NULL)
1624 tsd_list->ts_prev = tsd;
1625 tsd_list = tsd;
1626 }
1627
1628 /*
1629 * Allocate thread local storage and set the value for key
1630 */
1631 tsd->ts_value = tsd_realloc(tsd->ts_value,
1632 tsd->ts_nkeys * sizeof (void *),
1633 key * sizeof (void *));
1634 tsd->ts_nkeys = key;
1635 tsd->ts_value[key - 1] = value;
1636 mutex_exit(&tsd_mutex);
1637
1638 return (0);
1639 }
1640
1641
1642 /*
1643 * Return the per thread value that was stored with the specified key
1644 * If necessary, create the key and the value
1645 * Assumes the caller is protecting *keyp from tsd_destroy
1646 */
1647 void *
1648 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1649 {
1650 void *value;
1651 uint_t key = *keyp;
1652 struct tsd_thread *tsd = curthread->t_tsd;
1653
1654 if (tsd == NULL)
1655 tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1656 if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1657 return (value);
1658 if (key == 0)
1659 tsd_create(keyp, destroy);
1660 (void) tsd_set(*keyp, value = (*allocate)());
1661
1662 return (value);
1663 }
1664
1665 /*
1666 * Called from thread_exit() to run the destructor function for each tsd
1667 * Locks out tsd_create and tsd_destroy
1668 * Assumes that the destructor *DOES NOT* use tsd
1669 */
1670 void
1671 tsd_exit(void)
1672 {
1673 int i;
1674 struct tsd_thread *tsd = curthread->t_tsd;
1675
1676 if (tsd == NULL)
1677 return;
1678
1679 if (tsd->ts_nkeys == 0) {
1680 kmem_free(tsd, sizeof (*tsd));
1681 curthread->t_tsd = NULL;
1682 return;
1683 }
1684
1685 /*
1686 * lock out tsd_create and tsd_destroy, call
1687 * the destructor, and mark the value as destroyed.
1688 */
1689 mutex_enter(&tsd_mutex);
1690
1691 for (i = 0; i < tsd->ts_nkeys; i++) {
1692 if (tsd->ts_value[i] && tsd_destructor[i])
1693 (*tsd_destructor[i])(tsd->ts_value[i]);
1694 tsd->ts_value[i] = NULL;
1695 }
1696
1697 /*
1698 * remove from linked list of threads with TSD
1699 */
1700 if (tsd->ts_next)
1701 tsd->ts_next->ts_prev = tsd->ts_prev;
1702 if (tsd->ts_prev)
1703 tsd->ts_prev->ts_next = tsd->ts_next;
1704 if (tsd_list == tsd)
1705 tsd_list = tsd->ts_next;
1706
1707 mutex_exit(&tsd_mutex);
1708
1709 /*
1710 * free up the TSD
1711 */
1712 kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1713 kmem_free(tsd, sizeof (struct tsd_thread));
1714 curthread->t_tsd = NULL;
1715 }
1716
1717 /*
1718 * realloc
1719 */
1720 static void *
1721 tsd_realloc(void *old, size_t osize, size_t nsize)
1722 {
1723 void *new;
1724
1725 new = kmem_zalloc(nsize, KM_SLEEP);
1726 if (old) {
1727 bcopy(old, new, osize);
1728 kmem_free(old, osize);
1729 }
1730 return (new);
1731 }
1732
1733 /*
1734 * Return non-zero if an interrupt is being serviced.
1735 */
1736 int
1737 servicing_interrupt()
1738 {
1739 int onintr = 0;
1740
1741 /* Are we an interrupt thread */
1742 if (curthread->t_flag & T_INTR_THREAD)
1743 return (1);
1744 /* Are we servicing a high level interrupt? */
1745 if (CPU_ON_INTR(CPU)) {
1746 kpreempt_disable();
1747 onintr = CPU_ON_INTR(CPU);
1748 kpreempt_enable();
1749 }
1750 return (onintr);
1751 }
1752
1753
1754 /*
1755 * Change the dispatch priority of a thread in the system.
1756 * Used when raising or lowering a thread's priority.
1757 * (E.g., priority inheritance)
1758 *
1759 * Since threads are queued according to their priority, we
1760 * we must check the thread's state to determine whether it
1761 * is on a queue somewhere. If it is, we've got to:
1762 *
1763 * o Dequeue the thread.
1764 * o Change its effective priority.
1765 * o Enqueue the thread.
1766 *
1767 * Assumptions: The thread whose priority we wish to change
1768 * must be locked before we call thread_change_(e)pri().
1769 * The thread_change(e)pri() function doesn't drop the thread
1770 * lock--that must be done by its caller.
1771 */
1772 void
1773 thread_change_epri(kthread_t *t, pri_t disp_pri)
1774 {
1775 uint_t state;
1776
1777 ASSERT(THREAD_LOCK_HELD(t));
1778
1779 /*
1780 * If the inherited priority hasn't actually changed,
1781 * just return.
1782 */
1783 if (t->t_epri == disp_pri)
1784 return;
1785
1786 state = t->t_state;
1787
1788 /*
1789 * If it's not on a queue, change the priority with impunity.
1790 */
1791 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1792 t->t_epri = disp_pri;
1793 if (state == TS_ONPROC) {
1794 cpu_t *cp = t->t_disp_queue->disp_cpu;
1795
1796 if (t == cp->cpu_dispthread)
1797 cp->cpu_dispatch_pri = DISP_PRIO(t);
1798 }
1799 } else if (state == TS_SLEEP) {
1800 /*
1801 * Take the thread out of its sleep queue.
1802 * Change the inherited priority.
1803 * Re-enqueue the thread.
1804 * Each synchronization object exports a function
1805 * to do this in an appropriate manner.
1806 */
1807 SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1808 } else if (state == TS_WAIT) {
1809 /*
1810 * Re-enqueue a thread on the wait queue if its
1811 * effective priority needs to change.
1812 */
1813 if (disp_pri != t->t_epri)
1814 waitq_change_pri(t, disp_pri);
1815 } else {
1816 /*
1817 * The thread is on a run queue.
1818 * Note: setbackdq() may not put the thread
1819 * back on the same run queue where it originally
1820 * resided.
1821 */
1822 (void) dispdeq(t);
1823 t->t_epri = disp_pri;
1824 setbackdq(t);
1825 }
1826 schedctl_set_cidpri(t);
1827 }
1828
1829 /*
1830 * Function: Change the t_pri field of a thread.
1831 * Side Effects: Adjust the thread ordering on a run queue
1832 * or sleep queue, if necessary.
1833 * Returns: 1 if the thread was on a run queue, else 0.
1834 */
1835 int
1836 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1837 {
1838 uint_t state;
1839 int on_rq = 0;
1840
1841 ASSERT(THREAD_LOCK_HELD(t));
1842
1843 state = t->t_state;
1844 THREAD_WILLCHANGE_PRI(t, disp_pri);
1845
1846 /*
1847 * If it's not on a queue, change the priority with impunity.
1848 */
1849 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1850 t->t_pri = disp_pri;
1851
1852 if (state == TS_ONPROC) {
1853 cpu_t *cp = t->t_disp_queue->disp_cpu;
1854
1855 if (t == cp->cpu_dispthread)
1856 cp->cpu_dispatch_pri = DISP_PRIO(t);
1857 }
1858 } else if (state == TS_SLEEP) {
1859 /*
1860 * If the priority has changed, take the thread out of
1861 * its sleep queue and change the priority.
1862 * Re-enqueue the thread.
1863 * Each synchronization object exports a function
1864 * to do this in an appropriate manner.
1865 */
1866 if (disp_pri != t->t_pri)
1867 SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1868 } else if (state == TS_WAIT) {
1869 /*
1870 * Re-enqueue a thread on the wait queue if its
1871 * priority needs to change.
1872 */
1873 if (disp_pri != t->t_pri)
1874 waitq_change_pri(t, disp_pri);
1875 } else {
1876 /*
1877 * The thread is on a run queue.
1878 * Note: setbackdq() may not put the thread
1879 * back on the same run queue where it originally
1880 * resided.
1881 *
1882 * We still requeue the thread even if the priority
1883 * is unchanged to preserve round-robin (and other)
1884 * effects between threads of the same priority.
1885 */
1886 on_rq = dispdeq(t);
1887 ASSERT(on_rq);
1888 t->t_pri = disp_pri;
1889 if (front) {
1890 setfrontdq(t);
1891 } else {
1892 setbackdq(t);
1893 }
1894 }
1895 schedctl_set_cidpri(t);
1896 return (on_rq);
1897 }
1898
1899 /*
1900 * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
1901 * specific pattern.
1902 */
1903 static void
1904 stkinfo_begin(kthread_t *t)
1905 {
1906 caddr_t start; /* stack start */
1907 caddr_t end; /* stack end */
1908 uint64_t *ptr; /* pattern pointer */
1909
1910 /*
1911 * Stack grows up or down, see thread_create(),
1912 * compute stack memory area start and end (start < end).
1913 */
1914 if (t->t_stk > t->t_stkbase) {
1915 /* stack grows down */
1916 start = t->t_stkbase;
1917 end = t->t_stk;
1918 } else {
1919 /* stack grows up */
1920 start = t->t_stk;
1921 end = t->t_stkbase;
1922 }
1923
1924 /*
1925 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1926 * alignement for start and end in stack area boundaries
1927 * (protection against corrupt t_stkbase/t_stk data).
1928 */
1929 if ((((uintptr_t)start) & 0x7) != 0) {
1930 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1931 }
1932 end = (caddr_t)(((uintptr_t)end) & (~0x7));
1933
1934 if ((end <= start) || (end - start) > (1024 * 1024)) {
1935 /* negative or stack size > 1 meg, assume bogus */
1936 return;
1937 }
1938
1939 /* fill stack area with a pattern (instead of zeros) */
1940 ptr = (uint64_t *)((void *)start);
1941 while (ptr < (uint64_t *)((void *)end)) {
1942 *ptr++ = KMEM_STKINFO_PATTERN;
1943 }
1944 }
1945
1946
1947 /*
1948 * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
1949 * compute the percentage of kernel stack really used, and set in the log
1950 * if it's the latest highest percentage.
1951 */
1952 static void
1953 stkinfo_end(kthread_t *t)
1954 {
1955 caddr_t start; /* stack start */
1956 caddr_t end; /* stack end */
1957 uint64_t *ptr; /* pattern pointer */
1958 size_t stksz; /* stack size */
1959 size_t smallest = 0;
1960 size_t percent = 0;
1961 uint_t index = 0;
1962 uint_t i;
1963 static size_t smallest_percent = (size_t)-1;
1964 static uint_t full = 0;
1965
1966 /* create the stackinfo log, if doesn't already exist */
1967 mutex_enter(&kmem_stkinfo_lock);
1968 if (kmem_stkinfo_log == NULL) {
1969 kmem_stkinfo_log = (kmem_stkinfo_t *)
1970 kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
1971 (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
1972 if (kmem_stkinfo_log == NULL) {
1973 mutex_exit(&kmem_stkinfo_lock);
1974 return;
1975 }
1976 }
1977 mutex_exit(&kmem_stkinfo_lock);
1978
1979 /*
1980 * Stack grows up or down, see thread_create(),
1981 * compute stack memory area start and end (start < end).
1982 */
1983 if (t->t_stk > t->t_stkbase) {
1984 /* stack grows down */
1985 start = t->t_stkbase;
1986 end = t->t_stk;
1987 } else {
1988 /* stack grows up */
1989 start = t->t_stk;
1990 end = t->t_stkbase;
1991 }
1992
1993 /* stack size as found in kthread_t */
1994 stksz = end - start;
1995
1996 /*
1997 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1998 * alignement for start and end in stack area boundaries
1999 * (protection against corrupt t_stkbase/t_stk data).
2000 */
2001 if ((((uintptr_t)start) & 0x7) != 0) {
2002 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2003 }
2004 end = (caddr_t)(((uintptr_t)end) & (~0x7));
2005
2006 if ((end <= start) || (end - start) > (1024 * 1024)) {
2007 /* negative or stack size > 1 meg, assume bogus */
2008 return;
2009 }
2010
2011 /* search until no pattern in the stack */
2012 if (t->t_stk > t->t_stkbase) {
2013 /* stack grows down */
2014 #if defined(__i386) || defined(__amd64)
2015 /*
2016 * 6 longs are pushed on stack, see thread_load(). Skip
2017 * them, so if kthread has never run, percent is zero.
2018 * 8 bytes alignement is preserved for a 32 bit kernel,
2019 * 6 x 4 = 24, 24 is a multiple of 8.
2020 *
2021 */
2022 end -= (6 * sizeof (long));
2023 #endif
2024 ptr = (uint64_t *)((void *)start);
2025 while (ptr < (uint64_t *)((void *)end)) {
2026 if (*ptr != KMEM_STKINFO_PATTERN) {
2027 percent = stkinfo_percent(end,
2028 start, (caddr_t)ptr);
2029 break;
2030 }
2031 ptr++;
2032 }
2033 } else {
2034 /* stack grows up */
2035 ptr = (uint64_t *)((void *)end);
2036 ptr--;
2037 while (ptr >= (uint64_t *)((void *)start)) {
2038 if (*ptr != KMEM_STKINFO_PATTERN) {
2039 percent = stkinfo_percent(start,
2040 end, (caddr_t)ptr);
2041 break;
2042 }
2043 ptr--;
2044 }
2045 }
2046
2047 DTRACE_PROBE3(stack__usage, kthread_t *, t,
2048 size_t, stksz, size_t, percent);
2049
2050 if (percent == 0) {
2051 return;
2052 }
2053
2054 mutex_enter(&kmem_stkinfo_lock);
2055 if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2056 /*
2057 * The log is full and already contains the highest values
2058 */
2059 mutex_exit(&kmem_stkinfo_lock);
2060 return;
2061 }
2062
2063 /* keep a log of the highest used stack */
2064 for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2065 if (kmem_stkinfo_log[i].percent == 0) {
2066 index = i;
2067 full++;
2068 break;
2069 }
2070 if (smallest == 0) {
2071 smallest = kmem_stkinfo_log[i].percent;
2072 index = i;
2073 continue;
2074 }
2075 if (kmem_stkinfo_log[i].percent < smallest) {
2076 smallest = kmem_stkinfo_log[i].percent;
2077 index = i;
2078 }
2079 }
2080
2081 if (percent >= kmem_stkinfo_log[index].percent) {
2082 kmem_stkinfo_log[index].kthread = (caddr_t)t;
2083 kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2084 kmem_stkinfo_log[index].start = start;
2085 kmem_stkinfo_log[index].stksz = stksz;
2086 kmem_stkinfo_log[index].percent = percent;
2087 kmem_stkinfo_log[index].t_tid = t->t_tid;
2088 kmem_stkinfo_log[index].cmd[0] = '\0';
2089 if (t->t_tid != 0) {
2090 stksz = strlen((t->t_procp)->p_user.u_comm);
2091 if (stksz >= KMEM_STKINFO_STR_SIZE) {
2092 stksz = KMEM_STKINFO_STR_SIZE - 1;
2093 kmem_stkinfo_log[index].cmd[stksz] = '\0';
2094 } else {
2095 stksz += 1;
2096 }
2097 (void) memcpy(kmem_stkinfo_log[index].cmd,
2098 (t->t_procp)->p_user.u_comm, stksz);
2099 }
2100 if (percent < smallest_percent) {
2101 smallest_percent = percent;
2102 }
2103 }
2104 mutex_exit(&kmem_stkinfo_lock);
2105 }
2106
2107 /*
2108 * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2109 */
2110 static size_t
2111 stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2112 {
2113 size_t percent;
2114 size_t s;
2115
2116 if (t_stk > t_stkbase) {
2117 /* stack grows down */
2118 if (sp > t_stk) {
2119 return (0);
2120 }
2121 if (sp < t_stkbase) {
2122 return (100);
2123 }
2124 percent = t_stk - sp + 1;
2125 s = t_stk - t_stkbase + 1;
2126 } else {
2127 /* stack grows up */
2128 if (sp < t_stk) {
2129 return (0);
2130 }
2131 if (sp > t_stkbase) {
2132 return (100);
2133 }
2134 percent = sp - t_stk + 1;
2135 s = t_stkbase - t_stk + 1;
2136 }
2137 percent = ((100 * percent) / s) + 1;
2138 if (percent > 100) {
2139 percent = 100;
2140 }
2141 return (percent);
2142 }
2143
2144 /*
2145 * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2146 * long. It is expected that callers (acting on behalf of userland clients)
2147 * will perform any required checks to return the correct error semantics.
2148 * It is also expected callers on behalf of userland clients have done
2149 * any necessary permission checks.
2150 */
2151 int
2152 thread_setname(kthread_t *t, const char *name)
2153 {
2154 char *buf = NULL;
2155
2156 /*
2157 * We optimistically assume that a thread's name will only be set
2158 * once and so allocate memory in preparation of setting t_name.
2159 * If it turns out a name has already been set, we just discard (free)
2160 * the buffer we just allocated and reuse the current buffer
2161 * (as all should be THREAD_NAME_MAX large).
2162 *
2163 * Such an arrangement means over the lifetime of a kthread_t, t_name
2164 * is either NULL or has one value (the address of the buffer holding
2165 * the current thread name). The assumption is that most kthread_t
2166 * instances will not have a name assigned, so dynamically allocating
2167 * the memory should minimize the footprint of this feature, but by
2168 * having the buffer persist for the life of the thread, it simplifies
2169 * usage in highly constrained situations (e.g. dtrace).
2170 */
2171 if (name != NULL && name[0] != '\0') {
2172 for (size_t i = 0; name[i] != '\0'; i++) {
2173 if (!isprint(name[i]))
2174 return (EINVAL);
2175 }
2176
2177 buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2178 (void) strlcpy(buf, name, THREAD_NAME_MAX);
2179 }
2180
2181 mutex_enter(&ttoproc(t)->p_lock);
2182 if (t->t_name == NULL) {
2183 t->t_name = buf;
2184 } else {
2185 if (buf != NULL) {
2186 (void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2187 kmem_free(buf, THREAD_NAME_MAX);
2188 } else {
2189 bzero(t->t_name, THREAD_NAME_MAX);
2190 }
2191 }
2192 mutex_exit(&ttoproc(t)->p_lock);
2193 return (0);
2194 }
2195
2196 int
2197 thread_vsetname(kthread_t *t, const char *fmt, ...)
2198 {
2199 char name[THREAD_NAME_MAX];
2200 va_list va;
2201 int rc;
2202
2203 va_start(va, fmt);
2204 rc = vsnprintf(name, sizeof (name), fmt, va);
2205 va_end(va);
2206
2207 if (rc < 0)
2208 return (EINVAL);
2209
2210 if (rc >= sizeof (name))
2211 return (ENAMETOOLONG);
2212
2213 return (thread_setname(t, name));
2214 }