Print this page
OS-7125 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/disp/thread.c
+++ new/usr/src/uts/common/disp/thread.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2018 Joyent, Inc.
25 25 */
26 26
27 27 #include <sys/types.h>
28 28 #include <sys/param.h>
29 29 #include <sys/sysmacros.h>
30 30 #include <sys/signal.h>
31 31 #include <sys/stack.h>
32 32 #include <sys/pcb.h>
33 33 #include <sys/user.h>
34 34 #include <sys/systm.h>
35 35 #include <sys/sysinfo.h>
36 36 #include <sys/errno.h>
37 37 #include <sys/cmn_err.h>
38 38 #include <sys/cred.h>
39 39 #include <sys/resource.h>
40 40 #include <sys/task.h>
41 41 #include <sys/project.h>
42 42 #include <sys/proc.h>
43 43 #include <sys/debug.h>
44 44 #include <sys/disp.h>
45 45 #include <sys/class.h>
46 46 #include <vm/seg_kmem.h>
47 47 #include <vm/seg_kp.h>
48 48 #include <sys/machlock.h>
49 49 #include <sys/kmem.h>
50 50 #include <sys/varargs.h>
51 51 #include <sys/turnstile.h>
52 52 #include <sys/poll.h>
53 53 #include <sys/vtrace.h>
54 54 #include <sys/callb.h>
55 55 #include <c2/audit.h>
56 56 #include <sys/tnf.h>
57 57 #include <sys/sobject.h>
58 58 #include <sys/cpupart.h>
59 59 #include <sys/pset.h>
60 60 #include <sys/door.h>
61 61 #include <sys/spl.h>
62 62 #include <sys/copyops.h>
63 63 #include <sys/rctl.h>
64 64 #include <sys/brand.h>
65 65 #include <sys/pool.h>
66 66 #include <sys/zone.h>
67 67 #include <sys/tsol/label.h>
|
↓ open down ↓ |
67 lines elided |
↑ open up ↑ |
68 68 #include <sys/tsol/tndb.h>
69 69 #include <sys/cpc_impl.h>
70 70 #include <sys/sdt.h>
71 71 #include <sys/reboot.h>
72 72 #include <sys/kdi.h>
73 73 #include <sys/schedctl.h>
74 74 #include <sys/waitq.h>
75 75 #include <sys/cpucaps.h>
76 76 #include <sys/kiconv.h>
77 77 #include <sys/ctype.h>
78 +#include <sys/ht.h>
78 79
80 +#ifndef STACK_GROWTH_DOWN
81 +#error Stacks do not grow downward; 3b2 zombie attack detected!
82 +#endif
83 +
79 84 struct kmem_cache *thread_cache; /* cache of free threads */
80 85 struct kmem_cache *lwp_cache; /* cache of free lwps */
81 86 struct kmem_cache *turnstile_cache; /* cache of free turnstiles */
82 87
83 88 /*
84 89 * allthreads is only for use by kmem_readers. All kernel loops can use
85 90 * the current thread as a start/end point.
86 91 */
87 92 kthread_t *allthreads = &t0; /* circular list of all threads */
88 93
89 94 static kcondvar_t reaper_cv; /* synchronization var */
90 95 kthread_t *thread_deathrow; /* circular list of reapable threads */
91 96 kthread_t *lwp_deathrow; /* circular list of reapable threads */
92 97 kmutex_t reaplock; /* protects lwp and thread deathrows */
93 98 int thread_reapcnt = 0; /* number of threads on deathrow */
94 99 int lwp_reapcnt = 0; /* number of lwps on deathrow */
95 100 int reaplimit = 16; /* delay reaping until reaplimit */
96 101
97 102 thread_free_lock_t *thread_free_lock;
98 103 /* protects tick thread from reaper */
99 104
100 105 extern int nthread;
101 106
102 107 /* System Scheduling classes. */
103 108 id_t syscid; /* system scheduling class ID */
104 109 id_t sysdccid = CLASS_UNUSED; /* reset when SDC loads */
105 110
106 111 void *segkp_thread; /* cookie for segkp pool */
107 112
108 113 int lwp_cache_sz = 32;
109 114 int t_cache_sz = 8;
110 115 static kt_did_t next_t_id = 1;
111 116
112 117 /* Default mode for thread binding to CPUs and processor sets */
113 118 int default_binding_mode = TB_ALLHARD;
114 119
115 120 /*
116 121 * Min/Max stack sizes for stack size parameters
117 122 */
118 123 #define MAX_STKSIZE (32 * DEFAULTSTKSZ)
119 124 #define MIN_STKSIZE DEFAULTSTKSZ
120 125
121 126 /*
122 127 * default_stksize overrides lwp_default_stksize if it is set.
123 128 */
124 129 int default_stksize;
125 130 int lwp_default_stksize;
126 131
127 132 static zone_key_t zone_thread_key;
128 133
129 134 unsigned int kmem_stackinfo; /* stackinfo feature on-off */
130 135 kmem_stkinfo_t *kmem_stkinfo_log; /* stackinfo circular log */
131 136 static kmutex_t kmem_stkinfo_lock; /* protects kmem_stkinfo_log */
132 137
133 138 /*
134 139 * forward declarations for internal thread specific data (tsd)
135 140 */
136 141 static void *tsd_realloc(void *, size_t, size_t);
137 142
138 143 void thread_reaper(void);
139 144
140 145 /* forward declarations for stackinfo feature */
141 146 static void stkinfo_begin(kthread_t *);
142 147 static void stkinfo_end(kthread_t *);
143 148 static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
144 149
145 150 /*ARGSUSED*/
146 151 static int
147 152 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
148 153 {
149 154 bzero(buf, sizeof (turnstile_t));
150 155 return (0);
151 156 }
152 157
153 158 /*ARGSUSED*/
154 159 static void
155 160 turnstile_destructor(void *buf, void *cdrarg)
156 161 {
157 162 turnstile_t *ts = buf;
158 163
159 164 ASSERT(ts->ts_free == NULL);
160 165 ASSERT(ts->ts_waiters == 0);
161 166 ASSERT(ts->ts_inheritor == NULL);
162 167 ASSERT(ts->ts_sleepq[0].sq_first == NULL);
163 168 ASSERT(ts->ts_sleepq[1].sq_first == NULL);
164 169 }
165 170
166 171 void
167 172 thread_init(void)
168 173 {
169 174 kthread_t *tp;
170 175 extern char sys_name[];
171 176 extern void idle();
172 177 struct cpu *cpu = CPU;
173 178 int i;
174 179 kmutex_t *lp;
175 180
176 181 mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
177 182 thread_free_lock =
178 183 kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
179 184 for (i = 0; i < THREAD_FREE_NUM; i++) {
180 185 lp = &thread_free_lock[i].tf_lock;
181 186 mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
182 187 }
183 188
184 189 #if defined(__i386) || defined(__amd64)
185 190 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
186 191 PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
187 192
188 193 /*
189 194 * "struct _klwp" includes a "struct pcb", which includes a
190 195 * "struct fpu", which needs to be 64-byte aligned on amd64
191 196 * (and even on i386) for xsave/xrstor.
192 197 */
193 198 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
194 199 64, NULL, NULL, NULL, NULL, NULL, 0);
195 200 #else
196 201 /*
197 202 * Allocate thread structures from static_arena. This prevents
198 203 * issues where a thread tries to relocate its own thread
199 204 * structure and touches it after the mapping has been suspended.
200 205 */
201 206 thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
202 207 PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
203 208
204 209 lwp_stk_cache_init();
205 210
206 211 lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
207 212 0, NULL, NULL, NULL, NULL, NULL, 0);
208 213 #endif
209 214
210 215 turnstile_cache = kmem_cache_create("turnstile_cache",
211 216 sizeof (turnstile_t), 0,
212 217 turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
213 218
214 219 label_init();
215 220 cred_init();
216 221
217 222 /*
218 223 * Initialize various resource management facilities.
219 224 */
220 225 rctl_init();
221 226 cpucaps_init();
222 227 /*
223 228 * Zone_init() should be called before project_init() so that project ID
224 229 * for the first project is initialized correctly.
225 230 */
226 231 zone_init();
227 232 project_init();
228 233 brand_init();
229 234 kiconv_init();
230 235 task_init();
231 236 tcache_init();
232 237 pool_init();
233 238
234 239 curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
235 240
236 241 /*
237 242 * Originally, we had two parameters to set default stack
238 243 * size: one for lwp's (lwp_default_stksize), and one for
239 244 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
240 245 * Now we have a third parameter that overrides both if it is
241 246 * set to a legal stack size, called default_stksize.
242 247 */
243 248
244 249 if (default_stksize == 0) {
245 250 default_stksize = DEFAULTSTKSZ;
246 251 } else if (default_stksize % PAGESIZE != 0 ||
247 252 default_stksize > MAX_STKSIZE ||
248 253 default_stksize < MIN_STKSIZE) {
249 254 cmn_err(CE_WARN, "Illegal stack size. Using %d",
250 255 (int)DEFAULTSTKSZ);
251 256 default_stksize = DEFAULTSTKSZ;
252 257 } else {
253 258 lwp_default_stksize = default_stksize;
254 259 }
255 260
256 261 if (lwp_default_stksize == 0) {
257 262 lwp_default_stksize = default_stksize;
258 263 } else if (lwp_default_stksize % PAGESIZE != 0 ||
259 264 lwp_default_stksize > MAX_STKSIZE ||
260 265 lwp_default_stksize < MIN_STKSIZE) {
261 266 cmn_err(CE_WARN, "Illegal stack size. Using %d",
262 267 default_stksize);
263 268 lwp_default_stksize = default_stksize;
264 269 }
265 270
266 271 segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
267 272 lwp_default_stksize,
268 273 (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
269 274
270 275 segkp_thread = segkp_cache_init(segkp, t_cache_sz,
271 276 default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
272 277
273 278 (void) getcid(sys_name, &syscid);
274 279 curthread->t_cid = syscid; /* current thread is t0 */
275 280
276 281 /*
277 282 * Set up the first CPU's idle thread.
278 283 * It runs whenever the CPU has nothing worthwhile to do.
279 284 */
280 285 tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
281 286 cpu->cpu_idle_thread = tp;
282 287 tp->t_preempt = 1;
283 288 tp->t_disp_queue = cpu->cpu_disp;
284 289 ASSERT(tp->t_disp_queue != NULL);
285 290 tp->t_bound_cpu = cpu;
286 291 tp->t_affinitycnt = 1;
287 292
288 293 /*
289 294 * Registering a thread in the callback table is usually
290 295 * done in the initialization code of the thread. In this
291 296 * case, we do it right after thread creation to avoid
292 297 * blocking idle thread while registering itself. It also
293 298 * avoids the possibility of reregistration in case a CPU
294 299 * restarts its idle thread.
295 300 */
296 301 CALLB_CPR_INIT_SAFE(tp, "idle");
297 302
298 303 /*
299 304 * Create the thread_reaper daemon. From this point on, exited
300 305 * threads will get reaped.
301 306 */
302 307 (void) thread_create(NULL, 0, (void (*)())thread_reaper,
303 308 NULL, 0, &p0, TS_RUN, minclsyspri);
304 309
305 310 /*
306 311 * Finish initializing the kernel memory allocator now that
307 312 * thread_create() is available.
308 313 */
309 314 kmem_thread_init();
310 315
311 316 if (boothowto & RB_DEBUG)
312 317 kdi_dvec_thravail();
313 318 }
314 319
315 320 /*
316 321 * Create a thread.
317 322 *
318 323 * thread_create() blocks for memory if necessary. It never fails.
319 324 *
320 325 * If stk is NULL, the thread is created at the base of the stack
321 326 * and cannot be swapped.
322 327 */
323 328 kthread_t *
324 329 thread_create(
325 330 caddr_t stk,
326 331 size_t stksize,
327 332 void (*proc)(),
328 333 void *arg,
329 334 size_t len,
330 335 proc_t *pp,
331 336 int state,
332 337 pri_t pri)
333 338 {
334 339 kthread_t *t;
335 340 extern struct classfuncs sys_classfuncs;
336 341 turnstile_t *ts;
337 342
338 343 /*
339 344 * Every thread keeps a turnstile around in case it needs to block.
340 345 * The only reason the turnstile is not simply part of the thread
341 346 * structure is that we may have to break the association whenever
342 347 * more than one thread blocks on a given synchronization object.
343 348 * From a memory-management standpoint, turnstiles are like the
344 349 * "attached mblks" that hang off dblks in the streams allocator.
345 350 */
346 351 ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
347 352
348 353 if (stk == NULL) {
349 354 /*
350 355 * alloc both thread and stack in segkp chunk
351 356 */
352 357
353 358 if (stksize < default_stksize)
354 359 stksize = default_stksize;
355 360
356 361 if (stksize == default_stksize) {
357 362 stk = (caddr_t)segkp_cache_get(segkp_thread);
358 363 } else {
359 364 stksize = roundup(stksize, PAGESIZE);
360 365 stk = (caddr_t)segkp_get(segkp, stksize,
361 366 (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
362 367 }
363 368
364 369 ASSERT(stk != NULL);
365 370
366 371 /*
367 372 * The machine-dependent mutex code may require that
368 373 * thread pointers (since they may be used for mutex owner
369 374 * fields) have certain alignment requirements.
370 375 * PTR24_ALIGN is the size of the alignment quanta.
371 376 * XXX - assumes stack grows toward low addresses.
372 377 */
373 378 if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
374 379 cmn_err(CE_PANIC, "thread_create: proposed stack size"
375 380 " too small to hold thread.");
376 381 #ifdef STACK_GROWTH_DOWN
377 382 stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
378 383 stksize &= -PTR24_ALIGN; /* make thread aligned */
379 384 t = (kthread_t *)(stk + stksize);
380 385 bzero(t, sizeof (kthread_t));
381 386 if (audit_active)
382 387 audit_thread_create(t);
383 388 t->t_stk = stk + stksize;
384 389 t->t_stkbase = stk;
385 390 #else /* stack grows to larger addresses */
386 391 stksize -= SA(sizeof (kthread_t));
387 392 t = (kthread_t *)(stk);
388 393 bzero(t, sizeof (kthread_t));
389 394 t->t_stk = stk + sizeof (kthread_t);
390 395 t->t_stkbase = stk + stksize + sizeof (kthread_t);
391 396 #endif /* STACK_GROWTH_DOWN */
392 397 t->t_flag |= T_TALLOCSTK;
393 398 t->t_swap = stk;
394 399 } else {
395 400 t = kmem_cache_alloc(thread_cache, KM_SLEEP);
396 401 bzero(t, sizeof (kthread_t));
397 402 ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
398 403 if (audit_active)
399 404 audit_thread_create(t);
400 405 /*
401 406 * Initialize t_stk to the kernel stack pointer to use
402 407 * upon entry to the kernel
403 408 */
404 409 #ifdef STACK_GROWTH_DOWN
405 410 t->t_stk = stk + stksize;
406 411 t->t_stkbase = stk;
407 412 #else
408 413 t->t_stk = stk; /* 3b2-like */
409 414 t->t_stkbase = stk + stksize;
410 415 #endif /* STACK_GROWTH_DOWN */
411 416 }
412 417
413 418 if (kmem_stackinfo != 0) {
414 419 stkinfo_begin(t);
415 420 }
416 421
417 422 t->t_ts = ts;
418 423
419 424 /*
420 425 * p_cred could be NULL if it thread_create is called before cred_init
421 426 * is called in main.
422 427 */
423 428 mutex_enter(&pp->p_crlock);
424 429 if (pp->p_cred)
425 430 crhold(t->t_cred = pp->p_cred);
426 431 mutex_exit(&pp->p_crlock);
427 432 t->t_start = gethrestime_sec();
428 433 t->t_startpc = proc;
429 434 t->t_procp = pp;
430 435 t->t_clfuncs = &sys_classfuncs.thread;
431 436 t->t_cid = syscid;
432 437 t->t_pri = pri;
433 438 t->t_stime = ddi_get_lbolt();
434 439 t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
435 440 t->t_bind_cpu = PBIND_NONE;
436 441 t->t_bindflag = (uchar_t)default_binding_mode;
437 442 t->t_bind_pset = PS_NONE;
438 443 t->t_plockp = &pp->p_lock;
439 444 t->t_copyops = NULL;
440 445 t->t_taskq = NULL;
441 446 t->t_anttime = 0;
442 447 t->t_hatdepth = 0;
443 448
444 449 t->t_dtrace_vtime = 1; /* assure vtimestamp is always non-zero */
445 450
446 451 CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
447 452 #ifndef NPROBE
448 453 /* Kernel probe */
449 454 tnf_thread_create(t);
450 455 #endif /* NPROBE */
451 456 LOCK_INIT_CLEAR(&t->t_lock);
452 457
453 458 /*
454 459 * Callers who give us a NULL proc must do their own
455 460 * stack initialization. e.g. lwp_create()
456 461 */
457 462 if (proc != NULL) {
458 463 t->t_stk = thread_stk_init(t->t_stk);
459 464 thread_load(t, proc, arg, len);
460 465 }
461 466
462 467 /*
463 468 * Put a hold on project0. If this thread is actually in a
464 469 * different project, then t_proj will be changed later in
465 470 * lwp_create(). All kernel-only threads must be in project 0.
466 471 */
467 472 t->t_proj = project_hold(proj0p);
468 473
469 474 lgrp_affinity_init(&t->t_lgrp_affinity);
470 475
471 476 mutex_enter(&pidlock);
472 477 nthread++;
473 478 t->t_did = next_t_id++;
474 479 t->t_prev = curthread->t_prev;
475 480 t->t_next = curthread;
476 481
477 482 /*
478 483 * Add the thread to the list of all threads, and initialize
479 484 * its t_cpu pointer. We need to block preemption since
480 485 * cpu_offline walks the thread list looking for threads
481 486 * with t_cpu pointing to the CPU being offlined. We want
482 487 * to make sure that the list is consistent and that if t_cpu
483 488 * is set, the thread is on the list.
484 489 */
485 490 kpreempt_disable();
486 491 curthread->t_prev->t_next = t;
487 492 curthread->t_prev = t;
488 493
489 494 /*
490 495 * Threads should never have a NULL t_cpu pointer so assign it
491 496 * here. If the thread is being created with state TS_RUN a
492 497 * better CPU may be chosen when it is placed on the run queue.
493 498 *
494 499 * We need to keep kernel preemption disabled when setting all
495 500 * three fields to keep them in sync. Also, always create in
496 501 * the default partition since that's where kernel threads go
497 502 * (if this isn't a kernel thread, t_cpupart will be changed
498 503 * in lwp_create before setting the thread runnable).
499 504 */
500 505 t->t_cpupart = &cp_default;
501 506
502 507 /*
503 508 * For now, affiliate this thread with the root lgroup.
504 509 * Since the kernel does not (presently) allocate its memory
505 510 * in a locality aware fashion, the root is an appropriate home.
506 511 * If this thread is later associated with an lwp, it will have
507 512 * it's lgroup re-assigned at that time.
508 513 */
|
↓ open down ↓ |
420 lines elided |
↑ open up ↑ |
509 514 lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
510 515
511 516 /*
512 517 * Inherit the current cpu. If this cpu isn't part of the chosen
513 518 * lgroup, a new cpu will be chosen by cpu_choose when the thread
514 519 * is ready to run.
515 520 */
516 521 if (CPU->cpu_part == &cp_default)
517 522 t->t_cpu = CPU;
518 523 else
519 - t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl,
520 - t->t_pri, NULL);
524 + t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t,
525 + t->t_pri);
521 526
522 527 t->t_disp_queue = t->t_cpu->cpu_disp;
523 528 kpreempt_enable();
524 529
525 530 /*
526 531 * Initialize thread state and the dispatcher lock pointer.
527 532 * Need to hold onto pidlock to block allthreads walkers until
528 533 * the state is set.
529 534 */
530 535 switch (state) {
531 536 case TS_RUN:
532 537 curthread->t_oldspl = splhigh(); /* get dispatcher spl */
533 538 THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
534 539 CL_SETRUN(t);
535 540 thread_unlock(t);
536 541 break;
537 542
538 543 case TS_ONPROC:
539 544 THREAD_ONPROC(t, t->t_cpu);
540 545 break;
541 546
542 547 case TS_FREE:
543 548 /*
544 549 * Free state will be used for intr threads.
545 550 * The interrupt routine must set the thread dispatcher
546 551 * lock pointer (t_lockp) if starting on a CPU
547 552 * other than the current one.
548 553 */
549 554 THREAD_FREEINTR(t, CPU);
550 555 break;
551 556
552 557 case TS_STOPPED:
553 558 THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
554 559 break;
555 560
556 561 default: /* TS_SLEEP, TS_ZOMB or TS_TRANS */
557 562 cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
558 563 }
559 564 mutex_exit(&pidlock);
560 565 return (t);
561 566 }
562 567
563 568 /*
564 569 * Move thread to project0 and take care of project reference counters.
565 570 */
566 571 void
567 572 thread_rele(kthread_t *t)
568 573 {
569 574 kproject_t *kpj;
570 575
571 576 thread_lock(t);
572 577
573 578 ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
574 579 kpj = ttoproj(t);
575 580 t->t_proj = proj0p;
576 581
577 582 thread_unlock(t);
578 583
579 584 if (kpj != proj0p) {
580 585 project_rele(kpj);
581 586 (void) project_hold(proj0p);
582 587 }
583 588 }
584 589
585 590 void
586 591 thread_exit(void)
587 592 {
588 593 kthread_t *t = curthread;
589 594
590 595 if ((t->t_proc_flag & TP_ZTHREAD) != 0)
591 596 cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
592 597
593 598 tsd_exit(); /* Clean up this thread's TSD */
594 599
595 600 kcpc_passivate(); /* clean up performance counter state */
596 601
597 602 /*
598 603 * No kernel thread should have called poll() without arranging
599 604 * calling pollcleanup() here.
600 605 */
601 606 ASSERT(t->t_pollstate == NULL);
602 607 ASSERT(t->t_schedctl == NULL);
603 608 if (t->t_door)
604 609 door_slam(); /* in case thread did an upcall */
605 610
606 611 #ifndef NPROBE
607 612 /* Kernel probe */
608 613 if (t->t_tnf_tpdp)
609 614 tnf_thread_exit();
610 615 #endif /* NPROBE */
611 616
612 617 thread_rele(t);
613 618 t->t_preempt++;
614 619
615 620 /*
616 621 * remove thread from the all threads list so that
617 622 * death-row can use the same pointers.
618 623 */
619 624 mutex_enter(&pidlock);
620 625 t->t_next->t_prev = t->t_prev;
621 626 t->t_prev->t_next = t->t_next;
622 627 ASSERT(allthreads != t); /* t0 never exits */
623 628 cv_broadcast(&t->t_joincv); /* wake up anyone in thread_join */
624 629 mutex_exit(&pidlock);
625 630
626 631 if (t->t_ctx != NULL)
627 632 exitctx(t);
628 633 if (t->t_procp->p_pctx != NULL)
629 634 exitpctx(t->t_procp);
630 635
631 636 if (kmem_stackinfo != 0) {
632 637 stkinfo_end(t);
633 638 }
634 639
635 640 t->t_state = TS_ZOMB; /* set zombie thread */
636 641
637 642 swtch_from_zombie(); /* give up the CPU */
638 643 /* NOTREACHED */
639 644 }
640 645
641 646 /*
642 647 * Check to see if the specified thread is active (defined as being on
643 648 * the thread list). This is certainly a slow way to do this; if there's
644 649 * ever a reason to speed it up, we could maintain a hash table of active
645 650 * threads indexed by their t_did.
646 651 */
647 652 static kthread_t *
648 653 did_to_thread(kt_did_t tid)
649 654 {
650 655 kthread_t *t;
651 656
652 657 ASSERT(MUTEX_HELD(&pidlock));
653 658 for (t = curthread->t_next; t != curthread; t = t->t_next) {
654 659 if (t->t_did == tid)
655 660 break;
656 661 }
657 662 if (t->t_did == tid)
658 663 return (t);
659 664 else
660 665 return (NULL);
661 666 }
662 667
663 668 /*
664 669 * Wait for specified thread to exit. Returns immediately if the thread
665 670 * could not be found, meaning that it has either already exited or never
666 671 * existed.
667 672 */
668 673 void
669 674 thread_join(kt_did_t tid)
670 675 {
671 676 kthread_t *t;
672 677
673 678 ASSERT(tid != curthread->t_did);
674 679 ASSERT(tid != t0.t_did);
675 680
676 681 mutex_enter(&pidlock);
677 682 /*
678 683 * Make sure we check that the thread is on the thread list
679 684 * before blocking on it; otherwise we could end up blocking on
680 685 * a cv that's already been freed. In other words, don't cache
681 686 * the thread pointer across calls to cv_wait.
682 687 *
683 688 * The choice of loop invariant means that whenever a thread
684 689 * is taken off the allthreads list, a cv_broadcast must be
685 690 * performed on that thread's t_joincv to wake up any waiters.
686 691 * The broadcast doesn't have to happen right away, but it
687 692 * shouldn't be postponed indefinitely (e.g., by doing it in
688 693 * thread_free which may only be executed when the deathrow
689 694 * queue is processed.
690 695 */
691 696 while (t = did_to_thread(tid))
692 697 cv_wait(&t->t_joincv, &pidlock);
693 698 mutex_exit(&pidlock);
694 699 }
695 700
696 701 void
697 702 thread_free_prevent(kthread_t *t)
698 703 {
699 704 kmutex_t *lp;
700 705
701 706 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
702 707 mutex_enter(lp);
703 708 }
704 709
705 710 void
706 711 thread_free_allow(kthread_t *t)
707 712 {
708 713 kmutex_t *lp;
709 714
710 715 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
711 716 mutex_exit(lp);
712 717 }
713 718
714 719 static void
715 720 thread_free_barrier(kthread_t *t)
716 721 {
717 722 kmutex_t *lp;
718 723
719 724 lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
720 725 mutex_enter(lp);
721 726 mutex_exit(lp);
722 727 }
723 728
724 729 void
725 730 thread_free(kthread_t *t)
726 731 {
727 732 boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
728 733 klwp_t *lwp = t->t_lwp;
729 734 caddr_t swap = t->t_swap;
730 735
731 736 ASSERT(t != &t0 && t->t_state == TS_FREE);
732 737 ASSERT(t->t_door == NULL);
733 738 ASSERT(t->t_schedctl == NULL);
734 739 ASSERT(t->t_pollstate == NULL);
735 740
736 741 t->t_pri = 0;
737 742 t->t_pc = 0;
738 743 t->t_sp = 0;
739 744 t->t_wchan0 = NULL;
740 745 t->t_wchan = NULL;
741 746 if (t->t_cred != NULL) {
742 747 crfree(t->t_cred);
743 748 t->t_cred = 0;
744 749 }
745 750 if (t->t_pdmsg) {
746 751 kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
747 752 t->t_pdmsg = NULL;
748 753 }
749 754 if (audit_active)
750 755 audit_thread_free(t);
751 756 #ifndef NPROBE
752 757 if (t->t_tnf_tpdp)
753 758 tnf_thread_free(t);
754 759 #endif /* NPROBE */
755 760 if (t->t_cldata) {
756 761 CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
757 762 }
758 763 if (t->t_rprof != NULL) {
759 764 kmem_free(t->t_rprof, sizeof (*t->t_rprof));
760 765 t->t_rprof = NULL;
761 766 }
762 767 t->t_lockp = NULL; /* nothing should try to lock this thread now */
763 768 if (lwp)
764 769 lwp_freeregs(lwp, 0);
765 770 if (t->t_ctx)
766 771 freectx(t, 0);
767 772 t->t_stk = NULL;
768 773 if (lwp)
769 774 lwp_stk_fini(lwp);
770 775 lock_clear(&t->t_lock);
771 776
772 777 if (t->t_ts->ts_waiters > 0)
773 778 panic("thread_free: turnstile still active");
774 779
775 780 kmem_cache_free(turnstile_cache, t->t_ts);
776 781
777 782 free_afd(&t->t_activefd);
778 783
779 784 /*
780 785 * Barrier for the tick accounting code. The tick accounting code
781 786 * holds this lock to keep the thread from going away while it's
782 787 * looking at it.
783 788 */
784 789 thread_free_barrier(t);
785 790
786 791 ASSERT(ttoproj(t) == proj0p);
787 792 project_rele(ttoproj(t));
788 793
789 794 lgrp_affinity_free(&t->t_lgrp_affinity);
790 795
791 796 mutex_enter(&pidlock);
792 797 nthread--;
793 798 mutex_exit(&pidlock);
794 799
795 800 if (t->t_name != NULL) {
796 801 kmem_free(t->t_name, THREAD_NAME_MAX);
797 802 t->t_name = NULL;
798 803 }
799 804
800 805 /*
801 806 * Free thread, lwp and stack. This needs to be done carefully, since
802 807 * if T_TALLOCSTK is set, the thread is part of the stack.
803 808 */
804 809 t->t_lwp = NULL;
805 810 t->t_swap = NULL;
806 811
807 812 if (swap) {
808 813 segkp_release(segkp, swap);
809 814 }
810 815 if (lwp) {
811 816 kmem_cache_free(lwp_cache, lwp);
812 817 }
813 818 if (!allocstk) {
814 819 kmem_cache_free(thread_cache, t);
815 820 }
816 821 }
817 822
818 823 /*
819 824 * Removes threads associated with the given zone from a deathrow queue.
820 825 * tp is a pointer to the head of the deathrow queue, and countp is a
821 826 * pointer to the current deathrow count. Returns a linked list of
822 827 * threads removed from the list.
823 828 */
824 829 static kthread_t *
825 830 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
826 831 {
827 832 kthread_t *tmp, *list = NULL;
828 833 cred_t *cr;
829 834
830 835 ASSERT(MUTEX_HELD(&reaplock));
831 836 while (*tp != NULL) {
832 837 if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
833 838 tmp = *tp;
834 839 *tp = tmp->t_forw;
835 840 tmp->t_forw = list;
836 841 list = tmp;
837 842 (*countp)--;
838 843 } else {
839 844 tp = &(*tp)->t_forw;
840 845 }
841 846 }
842 847 return (list);
843 848 }
844 849
845 850 static void
846 851 thread_reap_list(kthread_t *t)
847 852 {
848 853 kthread_t *next;
849 854
850 855 while (t != NULL) {
851 856 next = t->t_forw;
852 857 thread_free(t);
853 858 t = next;
854 859 }
855 860 }
856 861
857 862 /* ARGSUSED */
858 863 static void
859 864 thread_zone_destroy(zoneid_t zoneid, void *unused)
860 865 {
861 866 kthread_t *t, *l;
862 867
863 868 mutex_enter(&reaplock);
864 869 /*
865 870 * Pull threads and lwps associated with zone off deathrow lists.
866 871 */
867 872 t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
868 873 l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
869 874 mutex_exit(&reaplock);
870 875
871 876 /*
872 877 * Guard against race condition in mutex_owner_running:
873 878 * thread=owner(mutex)
874 879 * <interrupt>
875 880 * thread exits mutex
876 881 * thread exits
877 882 * thread reaped
878 883 * thread struct freed
879 884 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
880 885 * A cross call to all cpus will cause the interrupt handler
881 886 * to reset the PC if it is in mutex_owner_running, refreshing
882 887 * stale thread pointers.
883 888 */
884 889 mutex_sync(); /* sync with mutex code */
885 890
886 891 /*
887 892 * Reap threads
888 893 */
889 894 thread_reap_list(t);
890 895
891 896 /*
892 897 * Reap lwps
893 898 */
894 899 thread_reap_list(l);
895 900 }
896 901
897 902 /*
898 903 * cleanup zombie threads that are on deathrow.
899 904 */
900 905 void
901 906 thread_reaper()
902 907 {
903 908 kthread_t *t, *l;
904 909 callb_cpr_t cprinfo;
905 910
906 911 /*
907 912 * Register callback to clean up threads when zone is destroyed.
908 913 */
909 914 zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
910 915
911 916 CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
912 917 for (;;) {
913 918 mutex_enter(&reaplock);
914 919 while (thread_deathrow == NULL && lwp_deathrow == NULL) {
915 920 CALLB_CPR_SAFE_BEGIN(&cprinfo);
916 921 cv_wait(&reaper_cv, &reaplock);
917 922 CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
918 923 }
919 924 /*
920 925 * mutex_sync() needs to be called when reaping, but
921 926 * not too often. We limit reaping rate to once
922 927 * per second. Reaplimit is max rate at which threads can
923 928 * be freed. Does not impact thread destruction/creation.
924 929 */
925 930 t = thread_deathrow;
926 931 l = lwp_deathrow;
927 932 thread_deathrow = NULL;
928 933 lwp_deathrow = NULL;
929 934 thread_reapcnt = 0;
930 935 lwp_reapcnt = 0;
931 936 mutex_exit(&reaplock);
932 937
933 938 /*
934 939 * Guard against race condition in mutex_owner_running:
935 940 * thread=owner(mutex)
936 941 * <interrupt>
937 942 * thread exits mutex
938 943 * thread exits
939 944 * thread reaped
940 945 * thread struct freed
941 946 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
942 947 * A cross call to all cpus will cause the interrupt handler
943 948 * to reset the PC if it is in mutex_owner_running, refreshing
944 949 * stale thread pointers.
945 950 */
946 951 mutex_sync(); /* sync with mutex code */
947 952 /*
948 953 * Reap threads
949 954 */
950 955 thread_reap_list(t);
951 956
952 957 /*
953 958 * Reap lwps
954 959 */
955 960 thread_reap_list(l);
956 961 delay(hz);
957 962 }
958 963 }
959 964
960 965 /*
961 966 * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
962 967 * thread_deathrow. The thread's state is changed already TS_FREE to indicate
963 968 * that is reapable. The thread already holds the reaplock, and was already
964 969 * freed.
965 970 */
966 971 void
967 972 reapq_move_lq_to_tq(kthread_t *t)
968 973 {
969 974 ASSERT(t->t_state == TS_FREE);
970 975 ASSERT(MUTEX_HELD(&reaplock));
971 976 t->t_forw = thread_deathrow;
972 977 thread_deathrow = t;
973 978 thread_reapcnt++;
974 979 if (lwp_reapcnt + thread_reapcnt > reaplimit)
975 980 cv_signal(&reaper_cv); /* wake the reaper */
976 981 }
977 982
978 983 /*
979 984 * This is called by resume() to put a zombie thread onto deathrow.
980 985 * The thread's state is changed to TS_FREE to indicate that is reapable.
981 986 * This is called from the idle thread so it must not block - just spin.
982 987 */
983 988 void
984 989 reapq_add(kthread_t *t)
985 990 {
986 991 mutex_enter(&reaplock);
987 992
988 993 /*
989 994 * lwp_deathrow contains threads with lwp linkage and
990 995 * swappable thread stacks which have the default stacksize.
991 996 * These threads' lwps and stacks may be reused by lwp_create().
992 997 *
993 998 * Anything else goes on thread_deathrow(), where it will eventually
994 999 * be thread_free()d.
995 1000 */
996 1001 if (t->t_flag & T_LWPREUSE) {
997 1002 ASSERT(ttolwp(t) != NULL);
998 1003 t->t_forw = lwp_deathrow;
999 1004 lwp_deathrow = t;
1000 1005 lwp_reapcnt++;
1001 1006 } else {
1002 1007 t->t_forw = thread_deathrow;
1003 1008 thread_deathrow = t;
1004 1009 thread_reapcnt++;
1005 1010 }
1006 1011 if (lwp_reapcnt + thread_reapcnt > reaplimit)
1007 1012 cv_signal(&reaper_cv); /* wake the reaper */
1008 1013 t->t_state = TS_FREE;
1009 1014 lock_clear(&t->t_lock);
1010 1015
1011 1016 /*
1012 1017 * Before we return, we need to grab and drop the thread lock for
1013 1018 * the dead thread. At this point, the current thread is the idle
1014 1019 * thread, and the dead thread's CPU lock points to the current
1015 1020 * CPU -- and we must grab and drop the lock to synchronize with
1016 1021 * a racing thread walking a blocking chain that the zombie thread
1017 1022 * was recently in. By this point, that blocking chain is (by
1018 1023 * definition) stale: the dead thread is not holding any locks, and
1019 1024 * is therefore not in any blocking chains -- but if we do not regrab
1020 1025 * our lock before freeing the dead thread's data structures, the
1021 1026 * thread walking the (stale) blocking chain will die on memory
1022 1027 * corruption when it attempts to drop the dead thread's lock. We
1023 1028 * only need do this once because there is no way for the dead thread
1024 1029 * to ever again be on a blocking chain: once we have grabbed and
1025 1030 * dropped the thread lock, we are guaranteed that anyone that could
1026 1031 * have seen this thread in a blocking chain can no longer see it.
1027 1032 */
1028 1033 thread_lock(t);
1029 1034 thread_unlock(t);
1030 1035
1031 1036 mutex_exit(&reaplock);
1032 1037 }
1033 1038
1034 1039 /*
1035 1040 * Install thread context ops for the current thread.
1036 1041 */
1037 1042 void
1038 1043 installctx(
1039 1044 kthread_t *t,
1040 1045 void *arg,
1041 1046 void (*save)(void *),
1042 1047 void (*restore)(void *),
1043 1048 void (*fork)(void *, void *),
1044 1049 void (*lwp_create)(void *, void *),
1045 1050 void (*exit)(void *),
1046 1051 void (*free)(void *, int))
1047 1052 {
1048 1053 struct ctxop *ctx;
1049 1054
1050 1055 ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1051 1056 ctx->save_op = save;
1052 1057 ctx->restore_op = restore;
1053 1058 ctx->fork_op = fork;
1054 1059 ctx->lwp_create_op = lwp_create;
1055 1060 ctx->exit_op = exit;
1056 1061 ctx->free_op = free;
1057 1062 ctx->arg = arg;
1058 1063 ctx->next = t->t_ctx;
1059 1064 t->t_ctx = ctx;
1060 1065 }
1061 1066
1062 1067 /*
1063 1068 * Remove the thread context ops from a thread.
1064 1069 */
1065 1070 int
1066 1071 removectx(
1067 1072 kthread_t *t,
1068 1073 void *arg,
1069 1074 void (*save)(void *),
1070 1075 void (*restore)(void *),
1071 1076 void (*fork)(void *, void *),
1072 1077 void (*lwp_create)(void *, void *),
1073 1078 void (*exit)(void *),
1074 1079 void (*free)(void *, int))
1075 1080 {
1076 1081 struct ctxop *ctx, *prev_ctx;
1077 1082
1078 1083 /*
1079 1084 * The incoming kthread_t (which is the thread for which the
1080 1085 * context ops will be removed) should be one of the following:
1081 1086 *
1082 1087 * a) the current thread,
1083 1088 *
1084 1089 * b) a thread of a process that's being forked (SIDL),
1085 1090 *
1086 1091 * c) a thread that belongs to the same process as the current
1087 1092 * thread and for which the current thread is the agent thread,
1088 1093 *
1089 1094 * d) a thread that is TS_STOPPED which is indicative of it
1090 1095 * being (if curthread is not an agent) a thread being created
1091 1096 * as part of an lwp creation.
1092 1097 */
1093 1098 ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1094 1099 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1095 1100
1096 1101 /*
1097 1102 * Serialize modifications to t->t_ctx to prevent the agent thread
1098 1103 * and the target thread from racing with each other during lwp exit.
1099 1104 */
1100 1105 mutex_enter(&t->t_ctx_lock);
1101 1106 prev_ctx = NULL;
1102 1107 kpreempt_disable();
1103 1108 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
1104 1109 if (ctx->save_op == save && ctx->restore_op == restore &&
1105 1110 ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
1106 1111 ctx->exit_op == exit && ctx->free_op == free &&
1107 1112 ctx->arg == arg) {
1108 1113 if (prev_ctx)
1109 1114 prev_ctx->next = ctx->next;
1110 1115 else
1111 1116 t->t_ctx = ctx->next;
1112 1117 mutex_exit(&t->t_ctx_lock);
1113 1118 if (ctx->free_op != NULL)
1114 1119 (ctx->free_op)(ctx->arg, 0);
1115 1120 kmem_free(ctx, sizeof (struct ctxop));
1116 1121 kpreempt_enable();
1117 1122 return (1);
1118 1123 }
1119 1124 prev_ctx = ctx;
1120 1125 }
1121 1126 mutex_exit(&t->t_ctx_lock);
1122 1127 kpreempt_enable();
1123 1128
1124 1129 return (0);
1125 1130 }
1126 1131
1127 1132 void
1128 1133 savectx(kthread_t *t)
1129 1134 {
1130 1135 struct ctxop *ctx;
1131 1136
1132 1137 ASSERT(t == curthread);
1133 1138 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1134 1139 if (ctx->save_op != NULL)
1135 1140 (ctx->save_op)(ctx->arg);
1136 1141 }
1137 1142
1138 1143 void
1139 1144 restorectx(kthread_t *t)
1140 1145 {
1141 1146 struct ctxop *ctx;
1142 1147
1143 1148 ASSERT(t == curthread);
1144 1149 for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1145 1150 if (ctx->restore_op != NULL)
1146 1151 (ctx->restore_op)(ctx->arg);
1147 1152 }
1148 1153
1149 1154 void
1150 1155 forkctx(kthread_t *t, kthread_t *ct)
1151 1156 {
1152 1157 struct ctxop *ctx;
1153 1158
1154 1159 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1155 1160 if (ctx->fork_op != NULL)
1156 1161 (ctx->fork_op)(t, ct);
1157 1162 }
1158 1163
1159 1164 /*
1160 1165 * Note that this operator is only invoked via the _lwp_create
1161 1166 * system call. The system may have other reasons to create lwps
1162 1167 * e.g. the agent lwp or the doors unreferenced lwp.
1163 1168 */
1164 1169 void
1165 1170 lwp_createctx(kthread_t *t, kthread_t *ct)
1166 1171 {
1167 1172 struct ctxop *ctx;
1168 1173
1169 1174 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1170 1175 if (ctx->lwp_create_op != NULL)
1171 1176 (ctx->lwp_create_op)(t, ct);
1172 1177 }
1173 1178
1174 1179 /*
1175 1180 * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1176 1181 * needed when the thread/LWP leaves the processor for the last time. This
1177 1182 * routine is not intended to deal with freeing memory; freectx() is used for
1178 1183 * that purpose during thread_free(). This routine is provided to allow for
1179 1184 * clean-up that can't wait until thread_free().
1180 1185 */
1181 1186 void
1182 1187 exitctx(kthread_t *t)
1183 1188 {
1184 1189 struct ctxop *ctx;
1185 1190
1186 1191 for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1187 1192 if (ctx->exit_op != NULL)
1188 1193 (ctx->exit_op)(t);
1189 1194 }
1190 1195
1191 1196 /*
1192 1197 * freectx is called from thread_free() and exec() to get
1193 1198 * rid of old thread context ops.
1194 1199 */
1195 1200 void
1196 1201 freectx(kthread_t *t, int isexec)
1197 1202 {
1198 1203 struct ctxop *ctx;
1199 1204
1200 1205 kpreempt_disable();
1201 1206 while ((ctx = t->t_ctx) != NULL) {
1202 1207 t->t_ctx = ctx->next;
1203 1208 if (ctx->free_op != NULL)
1204 1209 (ctx->free_op)(ctx->arg, isexec);
1205 1210 kmem_free(ctx, sizeof (struct ctxop));
1206 1211 }
1207 1212 kpreempt_enable();
1208 1213 }
1209 1214
1210 1215 /*
1211 1216 * freectx_ctx is called from lwp_create() when lwp is reused from
1212 1217 * lwp_deathrow and its thread structure is added to thread_deathrow.
1213 1218 * The thread structure to which this ctx was attached may be already
1214 1219 * freed by the thread reaper so free_op implementations shouldn't rely
1215 1220 * on thread structure to which this ctx was attached still being around.
1216 1221 */
1217 1222 void
1218 1223 freectx_ctx(struct ctxop *ctx)
1219 1224 {
1220 1225 struct ctxop *nctx;
1221 1226
1222 1227 ASSERT(ctx != NULL);
1223 1228
1224 1229 kpreempt_disable();
1225 1230 do {
1226 1231 nctx = ctx->next;
1227 1232 if (ctx->free_op != NULL)
1228 1233 (ctx->free_op)(ctx->arg, 0);
1229 1234 kmem_free(ctx, sizeof (struct ctxop));
1230 1235 } while ((ctx = nctx) != NULL);
1231 1236 kpreempt_enable();
1232 1237 }
1233 1238
1234 1239 /*
1235 1240 * Set the thread running; arrange for it to be swapped in if necessary.
1236 1241 */
1237 1242 void
1238 1243 setrun_locked(kthread_t *t)
1239 1244 {
1240 1245 ASSERT(THREAD_LOCK_HELD(t));
1241 1246 if (t->t_state == TS_SLEEP) {
1242 1247 /*
1243 1248 * Take off sleep queue.
1244 1249 */
1245 1250 SOBJ_UNSLEEP(t->t_sobj_ops, t);
1246 1251 } else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1247 1252 /*
1248 1253 * Already on dispatcher queue.
1249 1254 */
1250 1255 return;
1251 1256 } else if (t->t_state == TS_WAIT) {
1252 1257 waitq_setrun(t);
1253 1258 } else if (t->t_state == TS_STOPPED) {
1254 1259 /*
1255 1260 * All of the sending of SIGCONT (TC_XSTART) and /proc
1256 1261 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1257 1262 * requested that the thread be run.
1258 1263 * Just calling setrun() is not sufficient to set a stopped
1259 1264 * thread running. TP_TXSTART is always set if the thread
1260 1265 * is not stopped by a jobcontrol stop signal.
1261 1266 * TP_TPSTART is always set if /proc is not controlling it.
1262 1267 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1263 1268 * The thread won't be stopped unless one of these
1264 1269 * three mechanisms did it.
1265 1270 *
1266 1271 * These flags must be set before calling setrun_locked(t).
1267 1272 * They can't be passed as arguments because the streams
1268 1273 * code calls setrun() indirectly and the mechanism for
1269 1274 * doing so admits only one argument. Note that the
1270 1275 * thread must be locked in order to change t_schedflags.
1271 1276 */
1272 1277 if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1273 1278 return;
1274 1279 /*
1275 1280 * Process is no longer stopped (a thread is running).
1276 1281 */
1277 1282 t->t_whystop = 0;
1278 1283 t->t_whatstop = 0;
1279 1284 /*
1280 1285 * Strictly speaking, we do not have to clear these
1281 1286 * flags here; they are cleared on entry to stop().
1282 1287 * However, they are confusing when doing kernel
1283 1288 * debugging or when they are revealed by ps(1).
1284 1289 */
1285 1290 t->t_schedflag &= ~TS_ALLSTART;
1286 1291 THREAD_TRANSITION(t); /* drop stopped-thread lock */
1287 1292 ASSERT(t->t_lockp == &transition_lock);
1288 1293 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1289 1294 /*
1290 1295 * Let the class put the process on the dispatcher queue.
1291 1296 */
1292 1297 CL_SETRUN(t);
1293 1298 }
1294 1299 }
1295 1300
1296 1301 void
1297 1302 setrun(kthread_t *t)
1298 1303 {
1299 1304 thread_lock(t);
1300 1305 setrun_locked(t);
1301 1306 thread_unlock(t);
1302 1307 }
1303 1308
1304 1309 /*
1305 1310 * Unpin an interrupted thread.
1306 1311 * When an interrupt occurs, the interrupt is handled on the stack
1307 1312 * of an interrupt thread, taken from a pool linked to the CPU structure.
1308 1313 *
1309 1314 * When swtch() is switching away from an interrupt thread because it
1310 1315 * blocked or was preempted, this routine is called to complete the
1311 1316 * saving of the interrupted thread state, and returns the interrupted
1312 1317 * thread pointer so it may be resumed.
1313 1318 *
1314 1319 * Called by swtch() only at high spl.
1315 1320 */
1316 1321 kthread_t *
1317 1322 thread_unpin()
1318 1323 {
|
↓ open down ↓ |
788 lines elided |
↑ open up ↑ |
1319 1324 kthread_t *t = curthread; /* current thread */
1320 1325 kthread_t *itp; /* interrupted thread */
1321 1326 int i; /* interrupt level */
1322 1327 extern int intr_passivate();
1323 1328
1324 1329 ASSERT(t->t_intr != NULL);
1325 1330
1326 1331 itp = t->t_intr; /* interrupted thread */
1327 1332 t->t_intr = NULL; /* clear interrupt ptr */
1328 1333
1334 + ht_end_intr();
1335 +
1329 1336 /*
1330 1337 * Get state from interrupt thread for the one
1331 1338 * it interrupted.
1332 1339 */
1333 1340
1334 1341 i = intr_passivate(t, itp);
1335 1342
1336 1343 TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1337 1344 "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1338 1345 i, t, t, itp, itp);
1339 1346
1340 1347 /*
1341 1348 * Dissociate the current thread from the interrupted thread's LWP.
1342 1349 */
1343 1350 t->t_lwp = NULL;
1344 1351
1345 1352 /*
1346 1353 * Interrupt handlers above the level that spinlocks block must
1347 1354 * not block.
1348 1355 */
1349 1356 #if DEBUG
1350 1357 if (i < 0 || i > LOCK_LEVEL)
1351 1358 cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1352 1359 #endif
1353 1360
1354 1361 /*
1355 1362 * Compute the CPU's base interrupt level based on the active
1356 1363 * interrupts.
1357 1364 */
1358 1365 ASSERT(CPU->cpu_intr_actv & (1 << i));
1359 1366 set_base_spl();
1360 1367
1361 1368 return (itp);
1362 1369 }
1363 1370
1364 1371 /*
1365 1372 * Create and initialize an interrupt thread.
1366 1373 * Returns non-zero on error.
1367 1374 * Called at spl7() or better.
1368 1375 */
1369 1376 void
1370 1377 thread_create_intr(struct cpu *cp)
1371 1378 {
1372 1379 kthread_t *tp;
1373 1380
1374 1381 tp = thread_create(NULL, 0,
1375 1382 (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1376 1383
1377 1384 /*
1378 1385 * Set the thread in the TS_FREE state. The state will change
1379 1386 * to TS_ONPROC only while the interrupt is active. Think of these
1380 1387 * as being on a private free list for the CPU. Being TS_FREE keeps
1381 1388 * inactive interrupt threads out of debugger thread lists.
1382 1389 *
1383 1390 * We cannot call thread_create with TS_FREE because of the current
1384 1391 * checks there for ONPROC. Fix this when thread_create takes flags.
1385 1392 */
1386 1393 THREAD_FREEINTR(tp, cp);
1387 1394
1388 1395 /*
1389 1396 * Nobody should ever reference the credentials of an interrupt
1390 1397 * thread so make it NULL to catch any such references.
1391 1398 */
1392 1399 tp->t_cred = NULL;
1393 1400 tp->t_flag |= T_INTR_THREAD;
1394 1401 tp->t_cpu = cp;
1395 1402 tp->t_bound_cpu = cp;
1396 1403 tp->t_disp_queue = cp->cpu_disp;
1397 1404 tp->t_affinitycnt = 1;
1398 1405 tp->t_preempt = 1;
1399 1406
1400 1407 /*
1401 1408 * Don't make a user-requested binding on this thread so that
1402 1409 * the processor can be offlined.
1403 1410 */
1404 1411 tp->t_bind_cpu = PBIND_NONE; /* no USER-requested binding */
1405 1412 tp->t_bind_pset = PS_NONE;
1406 1413
1407 1414 #if defined(__i386) || defined(__amd64)
1408 1415 tp->t_stk -= STACK_ALIGN;
1409 1416 *(tp->t_stk) = 0; /* terminate intr thread stack */
1410 1417 #endif
1411 1418
1412 1419 /*
1413 1420 * Link onto CPU's interrupt pool.
1414 1421 */
1415 1422 tp->t_link = cp->cpu_intr_thread;
1416 1423 cp->cpu_intr_thread = tp;
1417 1424 }
1418 1425
1419 1426 /*
1420 1427 * TSD -- THREAD SPECIFIC DATA
1421 1428 */
1422 1429 static kmutex_t tsd_mutex; /* linked list spin lock */
1423 1430 static uint_t tsd_nkeys; /* size of destructor array */
1424 1431 /* per-key destructor funcs */
1425 1432 static void (**tsd_destructor)(void *);
1426 1433 /* list of tsd_thread's */
1427 1434 static struct tsd_thread *tsd_list;
1428 1435
1429 1436 /*
1430 1437 * Default destructor
1431 1438 * Needed because NULL destructor means that the key is unused
1432 1439 */
1433 1440 /* ARGSUSED */
1434 1441 void
1435 1442 tsd_defaultdestructor(void *value)
1436 1443 {}
1437 1444
1438 1445 /*
1439 1446 * Create a key (index into per thread array)
1440 1447 * Locks out tsd_create, tsd_destroy, and tsd_exit
1441 1448 * May allocate memory with lock held
1442 1449 */
1443 1450 void
1444 1451 tsd_create(uint_t *keyp, void (*destructor)(void *))
1445 1452 {
1446 1453 int i;
1447 1454 uint_t nkeys;
1448 1455
1449 1456 /*
1450 1457 * if key is allocated, do nothing
1451 1458 */
1452 1459 mutex_enter(&tsd_mutex);
1453 1460 if (*keyp) {
1454 1461 mutex_exit(&tsd_mutex);
1455 1462 return;
1456 1463 }
1457 1464 /*
1458 1465 * find an unused key
1459 1466 */
1460 1467 if (destructor == NULL)
1461 1468 destructor = tsd_defaultdestructor;
1462 1469
1463 1470 for (i = 0; i < tsd_nkeys; ++i)
1464 1471 if (tsd_destructor[i] == NULL)
1465 1472 break;
1466 1473
1467 1474 /*
1468 1475 * if no unused keys, increase the size of the destructor array
1469 1476 */
1470 1477 if (i == tsd_nkeys) {
1471 1478 if ((nkeys = (tsd_nkeys << 1)) == 0)
1472 1479 nkeys = 1;
1473 1480 tsd_destructor =
1474 1481 (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1475 1482 (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1476 1483 (size_t)(nkeys * sizeof (void (*)(void *))));
1477 1484 tsd_nkeys = nkeys;
1478 1485 }
1479 1486
1480 1487 /*
1481 1488 * allocate the next available unused key
1482 1489 */
1483 1490 tsd_destructor[i] = destructor;
1484 1491 *keyp = i + 1;
1485 1492 mutex_exit(&tsd_mutex);
1486 1493 }
1487 1494
1488 1495 /*
1489 1496 * Destroy a key -- this is for unloadable modules
1490 1497 *
1491 1498 * Assumes that the caller is preventing tsd_set and tsd_get
1492 1499 * Locks out tsd_create, tsd_destroy, and tsd_exit
1493 1500 * May free memory with lock held
1494 1501 */
1495 1502 void
1496 1503 tsd_destroy(uint_t *keyp)
1497 1504 {
1498 1505 uint_t key;
1499 1506 struct tsd_thread *tsd;
1500 1507
1501 1508 /*
1502 1509 * protect the key namespace and our destructor lists
1503 1510 */
1504 1511 mutex_enter(&tsd_mutex);
1505 1512 key = *keyp;
1506 1513 *keyp = 0;
1507 1514
1508 1515 ASSERT(key <= tsd_nkeys);
1509 1516
1510 1517 /*
1511 1518 * if the key is valid
1512 1519 */
1513 1520 if (key != 0) {
1514 1521 uint_t k = key - 1;
1515 1522 /*
1516 1523 * for every thread with TSD, call key's destructor
1517 1524 */
1518 1525 for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1519 1526 /*
1520 1527 * no TSD for key in this thread
1521 1528 */
1522 1529 if (key > tsd->ts_nkeys)
1523 1530 continue;
1524 1531 /*
1525 1532 * call destructor for key
1526 1533 */
1527 1534 if (tsd->ts_value[k] && tsd_destructor[k])
1528 1535 (*tsd_destructor[k])(tsd->ts_value[k]);
1529 1536 /*
1530 1537 * reset value for key
1531 1538 */
1532 1539 tsd->ts_value[k] = NULL;
1533 1540 }
1534 1541 /*
1535 1542 * actually free the key (NULL destructor == unused)
1536 1543 */
1537 1544 tsd_destructor[k] = NULL;
1538 1545 }
1539 1546
1540 1547 mutex_exit(&tsd_mutex);
1541 1548 }
1542 1549
1543 1550 /*
1544 1551 * Quickly return the per thread value that was stored with the specified key
1545 1552 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1546 1553 */
1547 1554 void *
1548 1555 tsd_get(uint_t key)
1549 1556 {
1550 1557 return (tsd_agent_get(curthread, key));
1551 1558 }
1552 1559
1553 1560 /*
1554 1561 * Set a per thread value indexed with the specified key
1555 1562 */
1556 1563 int
1557 1564 tsd_set(uint_t key, void *value)
1558 1565 {
1559 1566 return (tsd_agent_set(curthread, key, value));
1560 1567 }
1561 1568
1562 1569 /*
1563 1570 * Like tsd_get(), except that the agent lwp can get the tsd of
1564 1571 * another thread in the same process (the agent thread only runs when the
1565 1572 * process is completely stopped by /proc), or syslwp is creating a new lwp.
1566 1573 */
1567 1574 void *
1568 1575 tsd_agent_get(kthread_t *t, uint_t key)
1569 1576 {
1570 1577 struct tsd_thread *tsd = t->t_tsd;
1571 1578
1572 1579 ASSERT(t == curthread ||
1573 1580 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1574 1581
1575 1582 if (key && tsd != NULL && key <= tsd->ts_nkeys)
1576 1583 return (tsd->ts_value[key - 1]);
1577 1584 return (NULL);
1578 1585 }
1579 1586
1580 1587 /*
1581 1588 * Like tsd_set(), except that the agent lwp can set the tsd of
1582 1589 * another thread in the same process, or syslwp can set the tsd
1583 1590 * of a thread it's in the middle of creating.
1584 1591 *
1585 1592 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1586 1593 * May lock out tsd_destroy (and tsd_create), may allocate memory with
1587 1594 * lock held
1588 1595 */
1589 1596 int
1590 1597 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1591 1598 {
1592 1599 struct tsd_thread *tsd = t->t_tsd;
1593 1600
1594 1601 ASSERT(t == curthread ||
1595 1602 ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1596 1603
1597 1604 if (key == 0)
1598 1605 return (EINVAL);
1599 1606 if (tsd == NULL)
1600 1607 tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1601 1608 if (key <= tsd->ts_nkeys) {
1602 1609 tsd->ts_value[key - 1] = value;
1603 1610 return (0);
1604 1611 }
1605 1612
1606 1613 ASSERT(key <= tsd_nkeys);
1607 1614
1608 1615 /*
1609 1616 * lock out tsd_destroy()
1610 1617 */
1611 1618 mutex_enter(&tsd_mutex);
1612 1619 if (tsd->ts_nkeys == 0) {
1613 1620 /*
1614 1621 * Link onto list of threads with TSD
1615 1622 */
1616 1623 if ((tsd->ts_next = tsd_list) != NULL)
1617 1624 tsd_list->ts_prev = tsd;
1618 1625 tsd_list = tsd;
1619 1626 }
1620 1627
1621 1628 /*
1622 1629 * Allocate thread local storage and set the value for key
1623 1630 */
1624 1631 tsd->ts_value = tsd_realloc(tsd->ts_value,
1625 1632 tsd->ts_nkeys * sizeof (void *),
1626 1633 key * sizeof (void *));
1627 1634 tsd->ts_nkeys = key;
1628 1635 tsd->ts_value[key - 1] = value;
1629 1636 mutex_exit(&tsd_mutex);
1630 1637
1631 1638 return (0);
1632 1639 }
1633 1640
1634 1641
1635 1642 /*
1636 1643 * Return the per thread value that was stored with the specified key
1637 1644 * If necessary, create the key and the value
1638 1645 * Assumes the caller is protecting *keyp from tsd_destroy
1639 1646 */
1640 1647 void *
1641 1648 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1642 1649 {
1643 1650 void *value;
1644 1651 uint_t key = *keyp;
1645 1652 struct tsd_thread *tsd = curthread->t_tsd;
1646 1653
1647 1654 if (tsd == NULL)
1648 1655 tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1649 1656 if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1650 1657 return (value);
1651 1658 if (key == 0)
1652 1659 tsd_create(keyp, destroy);
1653 1660 (void) tsd_set(*keyp, value = (*allocate)());
1654 1661
1655 1662 return (value);
1656 1663 }
1657 1664
1658 1665 /*
1659 1666 * Called from thread_exit() to run the destructor function for each tsd
1660 1667 * Locks out tsd_create and tsd_destroy
1661 1668 * Assumes that the destructor *DOES NOT* use tsd
1662 1669 */
1663 1670 void
1664 1671 tsd_exit(void)
1665 1672 {
1666 1673 int i;
1667 1674 struct tsd_thread *tsd = curthread->t_tsd;
1668 1675
1669 1676 if (tsd == NULL)
1670 1677 return;
1671 1678
1672 1679 if (tsd->ts_nkeys == 0) {
1673 1680 kmem_free(tsd, sizeof (*tsd));
1674 1681 curthread->t_tsd = NULL;
1675 1682 return;
1676 1683 }
1677 1684
1678 1685 /*
1679 1686 * lock out tsd_create and tsd_destroy, call
1680 1687 * the destructor, and mark the value as destroyed.
1681 1688 */
1682 1689 mutex_enter(&tsd_mutex);
1683 1690
1684 1691 for (i = 0; i < tsd->ts_nkeys; i++) {
1685 1692 if (tsd->ts_value[i] && tsd_destructor[i])
1686 1693 (*tsd_destructor[i])(tsd->ts_value[i]);
1687 1694 tsd->ts_value[i] = NULL;
1688 1695 }
1689 1696
1690 1697 /*
1691 1698 * remove from linked list of threads with TSD
1692 1699 */
1693 1700 if (tsd->ts_next)
1694 1701 tsd->ts_next->ts_prev = tsd->ts_prev;
1695 1702 if (tsd->ts_prev)
1696 1703 tsd->ts_prev->ts_next = tsd->ts_next;
1697 1704 if (tsd_list == tsd)
1698 1705 tsd_list = tsd->ts_next;
1699 1706
1700 1707 mutex_exit(&tsd_mutex);
1701 1708
1702 1709 /*
1703 1710 * free up the TSD
1704 1711 */
1705 1712 kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1706 1713 kmem_free(tsd, sizeof (struct tsd_thread));
1707 1714 curthread->t_tsd = NULL;
1708 1715 }
1709 1716
1710 1717 /*
1711 1718 * realloc
1712 1719 */
1713 1720 static void *
1714 1721 tsd_realloc(void *old, size_t osize, size_t nsize)
1715 1722 {
1716 1723 void *new;
1717 1724
1718 1725 new = kmem_zalloc(nsize, KM_SLEEP);
1719 1726 if (old) {
1720 1727 bcopy(old, new, osize);
1721 1728 kmem_free(old, osize);
1722 1729 }
1723 1730 return (new);
1724 1731 }
1725 1732
1726 1733 /*
1727 1734 * Return non-zero if an interrupt is being serviced.
1728 1735 */
1729 1736 int
1730 1737 servicing_interrupt()
1731 1738 {
1732 1739 int onintr = 0;
1733 1740
1734 1741 /* Are we an interrupt thread */
1735 1742 if (curthread->t_flag & T_INTR_THREAD)
1736 1743 return (1);
1737 1744 /* Are we servicing a high level interrupt? */
1738 1745 if (CPU_ON_INTR(CPU)) {
1739 1746 kpreempt_disable();
1740 1747 onintr = CPU_ON_INTR(CPU);
1741 1748 kpreempt_enable();
1742 1749 }
1743 1750 return (onintr);
1744 1751 }
1745 1752
1746 1753
1747 1754 /*
1748 1755 * Change the dispatch priority of a thread in the system.
1749 1756 * Used when raising or lowering a thread's priority.
1750 1757 * (E.g., priority inheritance)
1751 1758 *
1752 1759 * Since threads are queued according to their priority, we
1753 1760 * we must check the thread's state to determine whether it
1754 1761 * is on a queue somewhere. If it is, we've got to:
1755 1762 *
1756 1763 * o Dequeue the thread.
1757 1764 * o Change its effective priority.
1758 1765 * o Enqueue the thread.
1759 1766 *
1760 1767 * Assumptions: The thread whose priority we wish to change
1761 1768 * must be locked before we call thread_change_(e)pri().
1762 1769 * The thread_change(e)pri() function doesn't drop the thread
1763 1770 * lock--that must be done by its caller.
1764 1771 */
1765 1772 void
1766 1773 thread_change_epri(kthread_t *t, pri_t disp_pri)
1767 1774 {
1768 1775 uint_t state;
1769 1776
1770 1777 ASSERT(THREAD_LOCK_HELD(t));
1771 1778
1772 1779 /*
1773 1780 * If the inherited priority hasn't actually changed,
1774 1781 * just return.
1775 1782 */
1776 1783 if (t->t_epri == disp_pri)
1777 1784 return;
1778 1785
1779 1786 state = t->t_state;
1780 1787
1781 1788 /*
1782 1789 * If it's not on a queue, change the priority with impunity.
1783 1790 */
1784 1791 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1785 1792 t->t_epri = disp_pri;
1786 1793 if (state == TS_ONPROC) {
1787 1794 cpu_t *cp = t->t_disp_queue->disp_cpu;
1788 1795
1789 1796 if (t == cp->cpu_dispthread)
1790 1797 cp->cpu_dispatch_pri = DISP_PRIO(t);
1791 1798 }
1792 1799 } else if (state == TS_SLEEP) {
1793 1800 /*
1794 1801 * Take the thread out of its sleep queue.
1795 1802 * Change the inherited priority.
1796 1803 * Re-enqueue the thread.
1797 1804 * Each synchronization object exports a function
1798 1805 * to do this in an appropriate manner.
1799 1806 */
1800 1807 SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1801 1808 } else if (state == TS_WAIT) {
1802 1809 /*
1803 1810 * Re-enqueue a thread on the wait queue if its
1804 1811 * effective priority needs to change.
1805 1812 */
1806 1813 if (disp_pri != t->t_epri)
1807 1814 waitq_change_pri(t, disp_pri);
1808 1815 } else {
1809 1816 /*
1810 1817 * The thread is on a run queue.
1811 1818 * Note: setbackdq() may not put the thread
1812 1819 * back on the same run queue where it originally
1813 1820 * resided.
1814 1821 */
1815 1822 (void) dispdeq(t);
1816 1823 t->t_epri = disp_pri;
1817 1824 setbackdq(t);
1818 1825 }
1819 1826 schedctl_set_cidpri(t);
1820 1827 }
1821 1828
1822 1829 /*
1823 1830 * Function: Change the t_pri field of a thread.
1824 1831 * Side Effects: Adjust the thread ordering on a run queue
1825 1832 * or sleep queue, if necessary.
1826 1833 * Returns: 1 if the thread was on a run queue, else 0.
1827 1834 */
1828 1835 int
1829 1836 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1830 1837 {
1831 1838 uint_t state;
1832 1839 int on_rq = 0;
1833 1840
1834 1841 ASSERT(THREAD_LOCK_HELD(t));
1835 1842
1836 1843 state = t->t_state;
1837 1844 THREAD_WILLCHANGE_PRI(t, disp_pri);
1838 1845
1839 1846 /*
1840 1847 * If it's not on a queue, change the priority with impunity.
1841 1848 */
1842 1849 if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1843 1850 t->t_pri = disp_pri;
1844 1851
1845 1852 if (state == TS_ONPROC) {
1846 1853 cpu_t *cp = t->t_disp_queue->disp_cpu;
1847 1854
1848 1855 if (t == cp->cpu_dispthread)
1849 1856 cp->cpu_dispatch_pri = DISP_PRIO(t);
1850 1857 }
1851 1858 } else if (state == TS_SLEEP) {
1852 1859 /*
1853 1860 * If the priority has changed, take the thread out of
1854 1861 * its sleep queue and change the priority.
1855 1862 * Re-enqueue the thread.
1856 1863 * Each synchronization object exports a function
1857 1864 * to do this in an appropriate manner.
1858 1865 */
1859 1866 if (disp_pri != t->t_pri)
1860 1867 SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1861 1868 } else if (state == TS_WAIT) {
1862 1869 /*
1863 1870 * Re-enqueue a thread on the wait queue if its
1864 1871 * priority needs to change.
1865 1872 */
1866 1873 if (disp_pri != t->t_pri)
1867 1874 waitq_change_pri(t, disp_pri);
1868 1875 } else {
1869 1876 /*
1870 1877 * The thread is on a run queue.
1871 1878 * Note: setbackdq() may not put the thread
1872 1879 * back on the same run queue where it originally
1873 1880 * resided.
1874 1881 *
1875 1882 * We still requeue the thread even if the priority
1876 1883 * is unchanged to preserve round-robin (and other)
1877 1884 * effects between threads of the same priority.
1878 1885 */
1879 1886 on_rq = dispdeq(t);
1880 1887 ASSERT(on_rq);
1881 1888 t->t_pri = disp_pri;
1882 1889 if (front) {
1883 1890 setfrontdq(t);
1884 1891 } else {
1885 1892 setbackdq(t);
1886 1893 }
1887 1894 }
1888 1895 schedctl_set_cidpri(t);
1889 1896 return (on_rq);
1890 1897 }
1891 1898
1892 1899 /*
1893 1900 * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
1894 1901 * specific pattern.
1895 1902 */
1896 1903 static void
1897 1904 stkinfo_begin(kthread_t *t)
1898 1905 {
1899 1906 caddr_t start; /* stack start */
1900 1907 caddr_t end; /* stack end */
1901 1908 uint64_t *ptr; /* pattern pointer */
1902 1909
1903 1910 /*
1904 1911 * Stack grows up or down, see thread_create(),
1905 1912 * compute stack memory area start and end (start < end).
1906 1913 */
1907 1914 if (t->t_stk > t->t_stkbase) {
1908 1915 /* stack grows down */
1909 1916 start = t->t_stkbase;
1910 1917 end = t->t_stk;
1911 1918 } else {
1912 1919 /* stack grows up */
1913 1920 start = t->t_stk;
1914 1921 end = t->t_stkbase;
1915 1922 }
1916 1923
1917 1924 /*
1918 1925 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1919 1926 * alignement for start and end in stack area boundaries
1920 1927 * (protection against corrupt t_stkbase/t_stk data).
1921 1928 */
1922 1929 if ((((uintptr_t)start) & 0x7) != 0) {
1923 1930 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1924 1931 }
1925 1932 end = (caddr_t)(((uintptr_t)end) & (~0x7));
1926 1933
1927 1934 if ((end <= start) || (end - start) > (1024 * 1024)) {
1928 1935 /* negative or stack size > 1 meg, assume bogus */
1929 1936 return;
1930 1937 }
1931 1938
1932 1939 /* fill stack area with a pattern (instead of zeros) */
1933 1940 ptr = (uint64_t *)((void *)start);
1934 1941 while (ptr < (uint64_t *)((void *)end)) {
1935 1942 *ptr++ = KMEM_STKINFO_PATTERN;
1936 1943 }
1937 1944 }
1938 1945
1939 1946
1940 1947 /*
1941 1948 * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
1942 1949 * compute the percentage of kernel stack really used, and set in the log
1943 1950 * if it's the latest highest percentage.
1944 1951 */
1945 1952 static void
1946 1953 stkinfo_end(kthread_t *t)
1947 1954 {
1948 1955 caddr_t start; /* stack start */
1949 1956 caddr_t end; /* stack end */
1950 1957 uint64_t *ptr; /* pattern pointer */
1951 1958 size_t stksz; /* stack size */
1952 1959 size_t smallest = 0;
1953 1960 size_t percent = 0;
1954 1961 uint_t index = 0;
1955 1962 uint_t i;
1956 1963 static size_t smallest_percent = (size_t)-1;
1957 1964 static uint_t full = 0;
1958 1965
1959 1966 /* create the stackinfo log, if doesn't already exist */
1960 1967 mutex_enter(&kmem_stkinfo_lock);
1961 1968 if (kmem_stkinfo_log == NULL) {
1962 1969 kmem_stkinfo_log = (kmem_stkinfo_t *)
1963 1970 kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
1964 1971 (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
1965 1972 if (kmem_stkinfo_log == NULL) {
1966 1973 mutex_exit(&kmem_stkinfo_lock);
1967 1974 return;
1968 1975 }
1969 1976 }
1970 1977 mutex_exit(&kmem_stkinfo_lock);
1971 1978
1972 1979 /*
1973 1980 * Stack grows up or down, see thread_create(),
1974 1981 * compute stack memory area start and end (start < end).
1975 1982 */
1976 1983 if (t->t_stk > t->t_stkbase) {
1977 1984 /* stack grows down */
1978 1985 start = t->t_stkbase;
1979 1986 end = t->t_stk;
1980 1987 } else {
1981 1988 /* stack grows up */
1982 1989 start = t->t_stk;
1983 1990 end = t->t_stkbase;
1984 1991 }
1985 1992
1986 1993 /* stack size as found in kthread_t */
1987 1994 stksz = end - start;
1988 1995
1989 1996 /*
1990 1997 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1991 1998 * alignement for start and end in stack area boundaries
1992 1999 * (protection against corrupt t_stkbase/t_stk data).
1993 2000 */
1994 2001 if ((((uintptr_t)start) & 0x7) != 0) {
1995 2002 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1996 2003 }
1997 2004 end = (caddr_t)(((uintptr_t)end) & (~0x7));
1998 2005
1999 2006 if ((end <= start) || (end - start) > (1024 * 1024)) {
2000 2007 /* negative or stack size > 1 meg, assume bogus */
2001 2008 return;
2002 2009 }
2003 2010
2004 2011 /* search until no pattern in the stack */
2005 2012 if (t->t_stk > t->t_stkbase) {
2006 2013 /* stack grows down */
2007 2014 #if defined(__i386) || defined(__amd64)
2008 2015 /*
2009 2016 * 6 longs are pushed on stack, see thread_load(). Skip
2010 2017 * them, so if kthread has never run, percent is zero.
2011 2018 * 8 bytes alignement is preserved for a 32 bit kernel,
2012 2019 * 6 x 4 = 24, 24 is a multiple of 8.
2013 2020 *
2014 2021 */
2015 2022 end -= (6 * sizeof (long));
2016 2023 #endif
2017 2024 ptr = (uint64_t *)((void *)start);
2018 2025 while (ptr < (uint64_t *)((void *)end)) {
2019 2026 if (*ptr != KMEM_STKINFO_PATTERN) {
2020 2027 percent = stkinfo_percent(end,
2021 2028 start, (caddr_t)ptr);
2022 2029 break;
2023 2030 }
2024 2031 ptr++;
2025 2032 }
2026 2033 } else {
2027 2034 /* stack grows up */
2028 2035 ptr = (uint64_t *)((void *)end);
2029 2036 ptr--;
2030 2037 while (ptr >= (uint64_t *)((void *)start)) {
2031 2038 if (*ptr != KMEM_STKINFO_PATTERN) {
2032 2039 percent = stkinfo_percent(start,
2033 2040 end, (caddr_t)ptr);
2034 2041 break;
2035 2042 }
2036 2043 ptr--;
2037 2044 }
2038 2045 }
2039 2046
2040 2047 DTRACE_PROBE3(stack__usage, kthread_t *, t,
2041 2048 size_t, stksz, size_t, percent);
2042 2049
2043 2050 if (percent == 0) {
2044 2051 return;
2045 2052 }
2046 2053
2047 2054 mutex_enter(&kmem_stkinfo_lock);
2048 2055 if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2049 2056 /*
2050 2057 * The log is full and already contains the highest values
2051 2058 */
2052 2059 mutex_exit(&kmem_stkinfo_lock);
2053 2060 return;
2054 2061 }
2055 2062
2056 2063 /* keep a log of the highest used stack */
2057 2064 for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2058 2065 if (kmem_stkinfo_log[i].percent == 0) {
2059 2066 index = i;
2060 2067 full++;
2061 2068 break;
2062 2069 }
2063 2070 if (smallest == 0) {
2064 2071 smallest = kmem_stkinfo_log[i].percent;
2065 2072 index = i;
2066 2073 continue;
2067 2074 }
2068 2075 if (kmem_stkinfo_log[i].percent < smallest) {
2069 2076 smallest = kmem_stkinfo_log[i].percent;
2070 2077 index = i;
2071 2078 }
2072 2079 }
2073 2080
2074 2081 if (percent >= kmem_stkinfo_log[index].percent) {
2075 2082 kmem_stkinfo_log[index].kthread = (caddr_t)t;
2076 2083 kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2077 2084 kmem_stkinfo_log[index].start = start;
2078 2085 kmem_stkinfo_log[index].stksz = stksz;
2079 2086 kmem_stkinfo_log[index].percent = percent;
2080 2087 kmem_stkinfo_log[index].t_tid = t->t_tid;
2081 2088 kmem_stkinfo_log[index].cmd[0] = '\0';
2082 2089 if (t->t_tid != 0) {
2083 2090 stksz = strlen((t->t_procp)->p_user.u_comm);
2084 2091 if (stksz >= KMEM_STKINFO_STR_SIZE) {
2085 2092 stksz = KMEM_STKINFO_STR_SIZE - 1;
2086 2093 kmem_stkinfo_log[index].cmd[stksz] = '\0';
2087 2094 } else {
2088 2095 stksz += 1;
2089 2096 }
2090 2097 (void) memcpy(kmem_stkinfo_log[index].cmd,
2091 2098 (t->t_procp)->p_user.u_comm, stksz);
2092 2099 }
2093 2100 if (percent < smallest_percent) {
2094 2101 smallest_percent = percent;
2095 2102 }
2096 2103 }
2097 2104 mutex_exit(&kmem_stkinfo_lock);
2098 2105 }
2099 2106
2100 2107 /*
2101 2108 * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2102 2109 */
2103 2110 static size_t
2104 2111 stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2105 2112 {
2106 2113 size_t percent;
2107 2114 size_t s;
2108 2115
2109 2116 if (t_stk > t_stkbase) {
2110 2117 /* stack grows down */
2111 2118 if (sp > t_stk) {
2112 2119 return (0);
2113 2120 }
2114 2121 if (sp < t_stkbase) {
2115 2122 return (100);
2116 2123 }
2117 2124 percent = t_stk - sp + 1;
2118 2125 s = t_stk - t_stkbase + 1;
2119 2126 } else {
2120 2127 /* stack grows up */
2121 2128 if (sp < t_stk) {
2122 2129 return (0);
2123 2130 }
2124 2131 if (sp > t_stkbase) {
2125 2132 return (100);
2126 2133 }
2127 2134 percent = sp - t_stk + 1;
2128 2135 s = t_stkbase - t_stk + 1;
2129 2136 }
2130 2137 percent = ((100 * percent) / s) + 1;
2131 2138 if (percent > 100) {
2132 2139 percent = 100;
2133 2140 }
2134 2141 return (percent);
2135 2142 }
2136 2143
2137 2144 /*
2138 2145 * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2139 2146 * long. It is expected that callers (acting on behalf of userland clients)
2140 2147 * will perform any required checks to return the correct error semantics.
2141 2148 * It is also expected callers on behalf of userland clients have done
2142 2149 * any necessary permission checks.
2143 2150 */
2144 2151 int
2145 2152 thread_setname(kthread_t *t, const char *name)
2146 2153 {
2147 2154 char *buf = NULL;
2148 2155
2149 2156 /*
2150 2157 * We optimistically assume that a thread's name will only be set
2151 2158 * once and so allocate memory in preparation of setting t_name.
2152 2159 * If it turns out a name has already been set, we just discard (free)
2153 2160 * the buffer we just allocated and reuse the current buffer
2154 2161 * (as all should be THREAD_NAME_MAX large).
2155 2162 *
2156 2163 * Such an arrangement means over the lifetime of a kthread_t, t_name
2157 2164 * is either NULL or has one value (the address of the buffer holding
2158 2165 * the current thread name). The assumption is that most kthread_t
2159 2166 * instances will not have a name assigned, so dynamically allocating
2160 2167 * the memory should minimize the footprint of this feature, but by
2161 2168 * having the buffer persist for the life of the thread, it simplifies
2162 2169 * usage in highly constrained situations (e.g. dtrace).
2163 2170 */
2164 2171 if (name != NULL && name[0] != '\0') {
2165 2172 for (size_t i = 0; name[i] != '\0'; i++) {
2166 2173 if (!isprint(name[i]))
2167 2174 return (EINVAL);
2168 2175 }
2169 2176
2170 2177 buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2171 2178 (void) strlcpy(buf, name, THREAD_NAME_MAX);
2172 2179 }
2173 2180
2174 2181 mutex_enter(&ttoproc(t)->p_lock);
2175 2182 if (t->t_name == NULL) {
2176 2183 t->t_name = buf;
2177 2184 } else {
2178 2185 if (buf != NULL) {
2179 2186 (void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2180 2187 kmem_free(buf, THREAD_NAME_MAX);
2181 2188 } else {
2182 2189 bzero(t->t_name, THREAD_NAME_MAX);
2183 2190 }
2184 2191 }
2185 2192 mutex_exit(&ttoproc(t)->p_lock);
2186 2193 return (0);
2187 2194 }
2188 2195
2189 2196 int
2190 2197 thread_vsetname(kthread_t *t, const char *fmt, ...)
2191 2198 {
2192 2199 char name[THREAD_NAME_MAX];
2193 2200 va_list va;
2194 2201 int rc;
2195 2202
2196 2203 va_start(va, fmt);
2197 2204 rc = vsnprintf(name, sizeof (name), fmt, va);
2198 2205 va_end(va);
2199 2206
2200 2207 if (rc < 0)
2201 2208 return (EINVAL);
2202 2209
2203 2210 if (rc >= sizeof (name))
2204 2211 return (ENAMETOOLONG);
2205 2212
2206 2213 return (thread_setname(t, name));
2207 2214 }
|
↓ open down ↓ |
869 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX