OS-7125 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2018 Joyent, Inc.
  25  */
  26 
  27 #include <sys/types.h>
  28 #include <sys/param.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/signal.h>
  31 #include <sys/stack.h>
  32 #include <sys/pcb.h>
  33 #include <sys/user.h>
  34 #include <sys/systm.h>
  35 #include <sys/sysinfo.h>
  36 #include <sys/errno.h>
  37 #include <sys/cmn_err.h>
  38 #include <sys/cred.h>
  39 #include <sys/resource.h>
  40 #include <sys/task.h>
  41 #include <sys/project.h>
  42 #include <sys/proc.h>
  43 #include <sys/debug.h>
  44 #include <sys/disp.h>
  45 #include <sys/class.h>
  46 #include <vm/seg_kmem.h>
  47 #include <vm/seg_kp.h>
  48 #include <sys/machlock.h>
  49 #include <sys/kmem.h>
  50 #include <sys/varargs.h>
  51 #include <sys/turnstile.h>
  52 #include <sys/poll.h>
  53 #include <sys/vtrace.h>
  54 #include <sys/callb.h>
  55 #include <c2/audit.h>
  56 #include <sys/tnf.h>
  57 #include <sys/sobject.h>
  58 #include <sys/cpupart.h>
  59 #include <sys/pset.h>
  60 #include <sys/door.h>
  61 #include <sys/spl.h>
  62 #include <sys/copyops.h>
  63 #include <sys/rctl.h>
  64 #include <sys/brand.h>
  65 #include <sys/pool.h>
  66 #include <sys/zone.h>
  67 #include <sys/tsol/label.h>
  68 #include <sys/tsol/tndb.h>
  69 #include <sys/cpc_impl.h>
  70 #include <sys/sdt.h>
  71 #include <sys/reboot.h>
  72 #include <sys/kdi.h>
  73 #include <sys/schedctl.h>
  74 #include <sys/waitq.h>
  75 #include <sys/cpucaps.h>
  76 #include <sys/kiconv.h>
  77 #include <sys/ctype.h>
  78 #include <sys/ht.h>
  79 
  80 #ifndef STACK_GROWTH_DOWN
  81 #error Stacks do not grow downward; 3b2 zombie attack detected!
  82 #endif
  83 
  84 struct kmem_cache *thread_cache;        /* cache of free threads */
  85 struct kmem_cache *lwp_cache;           /* cache of free lwps */
  86 struct kmem_cache *turnstile_cache;     /* cache of free turnstiles */
  87 
  88 /*
  89  * allthreads is only for use by kmem_readers.  All kernel loops can use
  90  * the current thread as a start/end point.
  91  */
  92 kthread_t *allthreads = &t0;        /* circular list of all threads */
  93 
  94 static kcondvar_t reaper_cv;            /* synchronization var */
  95 kthread_t       *thread_deathrow;       /* circular list of reapable threads */
  96 kthread_t       *lwp_deathrow;          /* circular list of reapable threads */
  97 kmutex_t        reaplock;               /* protects lwp and thread deathrows */
  98 int     thread_reapcnt = 0;             /* number of threads on deathrow */
  99 int     lwp_reapcnt = 0;                /* number of lwps on deathrow */
 100 int     reaplimit = 16;                 /* delay reaping until reaplimit */
 101 
 102 thread_free_lock_t      *thread_free_lock;
 103                                         /* protects tick thread from reaper */
 104 
 105 extern int nthread;
 106 
 107 /* System Scheduling classes. */
 108 id_t    syscid;                         /* system scheduling class ID */
 109 id_t    sysdccid = CLASS_UNUSED;        /* reset when SDC loads */
 110 
 111 void    *segkp_thread;                  /* cookie for segkp pool */
 112 
 113 int lwp_cache_sz = 32;
 114 int t_cache_sz = 8;
 115 static kt_did_t next_t_id = 1;
 116 
 117 /* Default mode for thread binding to CPUs and processor sets */
 118 int default_binding_mode = TB_ALLHARD;
 119 
 120 /*
 121  * Min/Max stack sizes for stack size parameters
 122  */
 123 #define MAX_STKSIZE     (32 * DEFAULTSTKSZ)
 124 #define MIN_STKSIZE     DEFAULTSTKSZ
 125 
 126 /*
 127  * default_stksize overrides lwp_default_stksize if it is set.
 128  */
 129 int     default_stksize;
 130 int     lwp_default_stksize;
 131 
 132 static zone_key_t zone_thread_key;
 133 
 134 unsigned int kmem_stackinfo;            /* stackinfo feature on-off */
 135 kmem_stkinfo_t *kmem_stkinfo_log;       /* stackinfo circular log */
 136 static kmutex_t kmem_stkinfo_lock;      /* protects kmem_stkinfo_log */
 137 
 138 /*
 139  * forward declarations for internal thread specific data (tsd)
 140  */
 141 static void *tsd_realloc(void *, size_t, size_t);
 142 
 143 void thread_reaper(void);
 144 
 145 /* forward declarations for stackinfo feature */
 146 static void stkinfo_begin(kthread_t *);
 147 static void stkinfo_end(kthread_t *);
 148 static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
 149 
 150 /*ARGSUSED*/
 151 static int
 152 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
 153 {
 154         bzero(buf, sizeof (turnstile_t));
 155         return (0);
 156 }
 157 
 158 /*ARGSUSED*/
 159 static void
 160 turnstile_destructor(void *buf, void *cdrarg)
 161 {
 162         turnstile_t *ts = buf;
 163 
 164         ASSERT(ts->ts_free == NULL);
 165         ASSERT(ts->ts_waiters == 0);
 166         ASSERT(ts->ts_inheritor == NULL);
 167         ASSERT(ts->ts_sleepq[0].sq_first == NULL);
 168         ASSERT(ts->ts_sleepq[1].sq_first == NULL);
 169 }
 170 
 171 void
 172 thread_init(void)
 173 {
 174         kthread_t *tp;
 175         extern char sys_name[];
 176         extern void idle();
 177         struct cpu *cpu = CPU;
 178         int i;
 179         kmutex_t *lp;
 180 
 181         mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
 182         thread_free_lock =
 183             kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
 184         for (i = 0; i < THREAD_FREE_NUM; i++) {
 185                 lp = &thread_free_lock[i].tf_lock;
 186                 mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
 187         }
 188 
 189 #if defined(__i386) || defined(__amd64)
 190         thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
 191             PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
 192 
 193         /*
 194          * "struct _klwp" includes a "struct pcb", which includes a
 195          * "struct fpu", which needs to be 64-byte aligned on amd64
 196          * (and even on i386) for xsave/xrstor.
 197          */
 198         lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
 199             64, NULL, NULL, NULL, NULL, NULL, 0);
 200 #else
 201         /*
 202          * Allocate thread structures from static_arena.  This prevents
 203          * issues where a thread tries to relocate its own thread
 204          * structure and touches it after the mapping has been suspended.
 205          */
 206         thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
 207             PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
 208 
 209         lwp_stk_cache_init();
 210 
 211         lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
 212             0, NULL, NULL, NULL, NULL, NULL, 0);
 213 #endif
 214 
 215         turnstile_cache = kmem_cache_create("turnstile_cache",
 216             sizeof (turnstile_t), 0,
 217             turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
 218 
 219         label_init();
 220         cred_init();
 221 
 222         /*
 223          * Initialize various resource management facilities.
 224          */
 225         rctl_init();
 226         cpucaps_init();
 227         /*
 228          * Zone_init() should be called before project_init() so that project ID
 229          * for the first project is initialized correctly.
 230          */
 231         zone_init();
 232         project_init();
 233         brand_init();
 234         kiconv_init();
 235         task_init();
 236         tcache_init();
 237         pool_init();
 238 
 239         curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
 240 
 241         /*
 242          * Originally, we had two parameters to set default stack
 243          * size: one for lwp's (lwp_default_stksize), and one for
 244          * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
 245          * Now we have a third parameter that overrides both if it is
 246          * set to a legal stack size, called default_stksize.
 247          */
 248 
 249         if (default_stksize == 0) {
 250                 default_stksize = DEFAULTSTKSZ;
 251         } else if (default_stksize % PAGESIZE != 0 ||
 252             default_stksize > MAX_STKSIZE ||
 253             default_stksize < MIN_STKSIZE) {
 254                 cmn_err(CE_WARN, "Illegal stack size. Using %d",
 255                     (int)DEFAULTSTKSZ);
 256                 default_stksize = DEFAULTSTKSZ;
 257         } else {
 258                 lwp_default_stksize = default_stksize;
 259         }
 260 
 261         if (lwp_default_stksize == 0) {
 262                 lwp_default_stksize = default_stksize;
 263         } else if (lwp_default_stksize % PAGESIZE != 0 ||
 264             lwp_default_stksize > MAX_STKSIZE ||
 265             lwp_default_stksize < MIN_STKSIZE) {
 266                 cmn_err(CE_WARN, "Illegal stack size. Using %d",
 267                     default_stksize);
 268                 lwp_default_stksize = default_stksize;
 269         }
 270 
 271         segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
 272             lwp_default_stksize,
 273             (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
 274 
 275         segkp_thread = segkp_cache_init(segkp, t_cache_sz,
 276             default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
 277 
 278         (void) getcid(sys_name, &syscid);
 279         curthread->t_cid = syscid;   /* current thread is t0 */
 280 
 281         /*
 282          * Set up the first CPU's idle thread.
 283          * It runs whenever the CPU has nothing worthwhile to do.
 284          */
 285         tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
 286         cpu->cpu_idle_thread = tp;
 287         tp->t_preempt = 1;
 288         tp->t_disp_queue = cpu->cpu_disp;
 289         ASSERT(tp->t_disp_queue != NULL);
 290         tp->t_bound_cpu = cpu;
 291         tp->t_affinitycnt = 1;
 292 
 293         /*
 294          * Registering a thread in the callback table is usually
 295          * done in the initialization code of the thread. In this
 296          * case, we do it right after thread creation to avoid
 297          * blocking idle thread while registering itself. It also
 298          * avoids the possibility of reregistration in case a CPU
 299          * restarts its idle thread.
 300          */
 301         CALLB_CPR_INIT_SAFE(tp, "idle");
 302 
 303         /*
 304          * Create the thread_reaper daemon. From this point on, exited
 305          * threads will get reaped.
 306          */
 307         (void) thread_create(NULL, 0, (void (*)())thread_reaper,
 308             NULL, 0, &p0, TS_RUN, minclsyspri);
 309 
 310         /*
 311          * Finish initializing the kernel memory allocator now that
 312          * thread_create() is available.
 313          */
 314         kmem_thread_init();
 315 
 316         if (boothowto & RB_DEBUG)
 317                 kdi_dvec_thravail();
 318 }
 319 
 320 /*
 321  * Create a thread.
 322  *
 323  * thread_create() blocks for memory if necessary.  It never fails.
 324  *
 325  * If stk is NULL, the thread is created at the base of the stack
 326  * and cannot be swapped.
 327  */
 328 kthread_t *
 329 thread_create(
 330         caddr_t stk,
 331         size_t  stksize,
 332         void    (*proc)(),
 333         void    *arg,
 334         size_t  len,
 335         proc_t   *pp,
 336         int     state,
 337         pri_t   pri)
 338 {
 339         kthread_t *t;
 340         extern struct classfuncs sys_classfuncs;
 341         turnstile_t *ts;
 342 
 343         /*
 344          * Every thread keeps a turnstile around in case it needs to block.
 345          * The only reason the turnstile is not simply part of the thread
 346          * structure is that we may have to break the association whenever
 347          * more than one thread blocks on a given synchronization object.
 348          * From a memory-management standpoint, turnstiles are like the
 349          * "attached mblks" that hang off dblks in the streams allocator.
 350          */
 351         ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
 352 
 353         if (stk == NULL) {
 354                 /*
 355                  * alloc both thread and stack in segkp chunk
 356                  */
 357 
 358                 if (stksize < default_stksize)
 359                         stksize = default_stksize;
 360 
 361                 if (stksize == default_stksize) {
 362                         stk = (caddr_t)segkp_cache_get(segkp_thread);
 363                 } else {
 364                         stksize = roundup(stksize, PAGESIZE);
 365                         stk = (caddr_t)segkp_get(segkp, stksize,
 366                             (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
 367                 }
 368 
 369                 ASSERT(stk != NULL);
 370 
 371                 /*
 372                  * The machine-dependent mutex code may require that
 373                  * thread pointers (since they may be used for mutex owner
 374                  * fields) have certain alignment requirements.
 375                  * PTR24_ALIGN is the size of the alignment quanta.
 376                  * XXX - assumes stack grows toward low addresses.
 377                  */
 378                 if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
 379                         cmn_err(CE_PANIC, "thread_create: proposed stack size"
 380                             " too small to hold thread.");
 381 #ifdef STACK_GROWTH_DOWN
 382                 stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
 383                 stksize &= -PTR24_ALIGN;    /* make thread aligned */
 384                 t = (kthread_t *)(stk + stksize);
 385                 bzero(t, sizeof (kthread_t));
 386                 if (audit_active)
 387                         audit_thread_create(t);
 388                 t->t_stk = stk + stksize;
 389                 t->t_stkbase = stk;
 390 #else   /* stack grows to larger addresses */
 391                 stksize -= SA(sizeof (kthread_t));
 392                 t = (kthread_t *)(stk);
 393                 bzero(t, sizeof (kthread_t));
 394                 t->t_stk = stk + sizeof (kthread_t);
 395                 t->t_stkbase = stk + stksize + sizeof (kthread_t);
 396 #endif  /* STACK_GROWTH_DOWN */
 397                 t->t_flag |= T_TALLOCSTK;
 398                 t->t_swap = stk;
 399         } else {
 400                 t = kmem_cache_alloc(thread_cache, KM_SLEEP);
 401                 bzero(t, sizeof (kthread_t));
 402                 ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
 403                 if (audit_active)
 404                         audit_thread_create(t);
 405                 /*
 406                  * Initialize t_stk to the kernel stack pointer to use
 407                  * upon entry to the kernel
 408                  */
 409 #ifdef STACK_GROWTH_DOWN
 410                 t->t_stk = stk + stksize;
 411                 t->t_stkbase = stk;
 412 #else
 413                 t->t_stk = stk;                      /* 3b2-like */
 414                 t->t_stkbase = stk + stksize;
 415 #endif /* STACK_GROWTH_DOWN */
 416         }
 417 
 418         if (kmem_stackinfo != 0) {
 419                 stkinfo_begin(t);
 420         }
 421 
 422         t->t_ts = ts;
 423 
 424         /*
 425          * p_cred could be NULL if it thread_create is called before cred_init
 426          * is called in main.
 427          */
 428         mutex_enter(&pp->p_crlock);
 429         if (pp->p_cred)
 430                 crhold(t->t_cred = pp->p_cred);
 431         mutex_exit(&pp->p_crlock);
 432         t->t_start = gethrestime_sec();
 433         t->t_startpc = proc;
 434         t->t_procp = pp;
 435         t->t_clfuncs = &sys_classfuncs.thread;
 436         t->t_cid = syscid;
 437         t->t_pri = pri;
 438         t->t_stime = ddi_get_lbolt();
 439         t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
 440         t->t_bind_cpu = PBIND_NONE;
 441         t->t_bindflag = (uchar_t)default_binding_mode;
 442         t->t_bind_pset = PS_NONE;
 443         t->t_plockp = &pp->p_lock;
 444         t->t_copyops = NULL;
 445         t->t_taskq = NULL;
 446         t->t_anttime = 0;
 447         t->t_hatdepth = 0;
 448 
 449         t->t_dtrace_vtime = 1;       /* assure vtimestamp is always non-zero */
 450 
 451         CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
 452 #ifndef NPROBE
 453         /* Kernel probe */
 454         tnf_thread_create(t);
 455 #endif /* NPROBE */
 456         LOCK_INIT_CLEAR(&t->t_lock);
 457 
 458         /*
 459          * Callers who give us a NULL proc must do their own
 460          * stack initialization.  e.g. lwp_create()
 461          */
 462         if (proc != NULL) {
 463                 t->t_stk = thread_stk_init(t->t_stk);
 464                 thread_load(t, proc, arg, len);
 465         }
 466 
 467         /*
 468          * Put a hold on project0. If this thread is actually in a
 469          * different project, then t_proj will be changed later in
 470          * lwp_create().  All kernel-only threads must be in project 0.
 471          */
 472         t->t_proj = project_hold(proj0p);
 473 
 474         lgrp_affinity_init(&t->t_lgrp_affinity);
 475 
 476         mutex_enter(&pidlock);
 477         nthread++;
 478         t->t_did = next_t_id++;
 479         t->t_prev = curthread->t_prev;
 480         t->t_next = curthread;
 481 
 482         /*
 483          * Add the thread to the list of all threads, and initialize
 484          * its t_cpu pointer.  We need to block preemption since
 485          * cpu_offline walks the thread list looking for threads
 486          * with t_cpu pointing to the CPU being offlined.  We want
 487          * to make sure that the list is consistent and that if t_cpu
 488          * is set, the thread is on the list.
 489          */
 490         kpreempt_disable();
 491         curthread->t_prev->t_next = t;
 492         curthread->t_prev = t;
 493 
 494         /*
 495          * Threads should never have a NULL t_cpu pointer so assign it
 496          * here.  If the thread is being created with state TS_RUN a
 497          * better CPU may be chosen when it is placed on the run queue.
 498          *
 499          * We need to keep kernel preemption disabled when setting all
 500          * three fields to keep them in sync.  Also, always create in
 501          * the default partition since that's where kernel threads go
 502          * (if this isn't a kernel thread, t_cpupart will be changed
 503          * in lwp_create before setting the thread runnable).
 504          */
 505         t->t_cpupart = &cp_default;
 506 
 507         /*
 508          * For now, affiliate this thread with the root lgroup.
 509          * Since the kernel does not (presently) allocate its memory
 510          * in a locality aware fashion, the root is an appropriate home.
 511          * If this thread is later associated with an lwp, it will have
 512          * it's lgroup re-assigned at that time.
 513          */
 514         lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
 515 
 516         /*
 517          * Inherit the current cpu.  If this cpu isn't part of the chosen
 518          * lgroup, a new cpu will be chosen by cpu_choose when the thread
 519          * is ready to run.
 520          */
 521         if (CPU->cpu_part == &cp_default)
 522                 t->t_cpu = CPU;
 523         else
 524                 t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t,
 525                     t->t_pri);
 526 
 527         t->t_disp_queue = t->t_cpu->cpu_disp;
 528         kpreempt_enable();
 529 
 530         /*
 531          * Initialize thread state and the dispatcher lock pointer.
 532          * Need to hold onto pidlock to block allthreads walkers until
 533          * the state is set.
 534          */
 535         switch (state) {
 536         case TS_RUN:
 537                 curthread->t_oldspl = splhigh();     /* get dispatcher spl */
 538                 THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
 539                 CL_SETRUN(t);
 540                 thread_unlock(t);
 541                 break;
 542 
 543         case TS_ONPROC:
 544                 THREAD_ONPROC(t, t->t_cpu);
 545                 break;
 546 
 547         case TS_FREE:
 548                 /*
 549                  * Free state will be used for intr threads.
 550                  * The interrupt routine must set the thread dispatcher
 551                  * lock pointer (t_lockp) if starting on a CPU
 552                  * other than the current one.
 553                  */
 554                 THREAD_FREEINTR(t, CPU);
 555                 break;
 556 
 557         case TS_STOPPED:
 558                 THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
 559                 break;
 560 
 561         default:                        /* TS_SLEEP, TS_ZOMB or TS_TRANS */
 562                 cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
 563         }
 564         mutex_exit(&pidlock);
 565         return (t);
 566 }
 567 
 568 /*
 569  * Move thread to project0 and take care of project reference counters.
 570  */
 571 void
 572 thread_rele(kthread_t *t)
 573 {
 574         kproject_t *kpj;
 575 
 576         thread_lock(t);
 577 
 578         ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
 579         kpj = ttoproj(t);
 580         t->t_proj = proj0p;
 581 
 582         thread_unlock(t);
 583 
 584         if (kpj != proj0p) {
 585                 project_rele(kpj);
 586                 (void) project_hold(proj0p);
 587         }
 588 }
 589 
 590 void
 591 thread_exit(void)
 592 {
 593         kthread_t *t = curthread;
 594 
 595         if ((t->t_proc_flag & TP_ZTHREAD) != 0)
 596                 cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
 597 
 598         tsd_exit();             /* Clean up this thread's TSD */
 599 
 600         kcpc_passivate();       /* clean up performance counter state */
 601 
 602         /*
 603          * No kernel thread should have called poll() without arranging
 604          * calling pollcleanup() here.
 605          */
 606         ASSERT(t->t_pollstate == NULL);
 607         ASSERT(t->t_schedctl == NULL);
 608         if (t->t_door)
 609                 door_slam();    /* in case thread did an upcall */
 610 
 611 #ifndef NPROBE
 612         /* Kernel probe */
 613         if (t->t_tnf_tpdp)
 614                 tnf_thread_exit();
 615 #endif /* NPROBE */
 616 
 617         thread_rele(t);
 618         t->t_preempt++;
 619 
 620         /*
 621          * remove thread from the all threads list so that
 622          * death-row can use the same pointers.
 623          */
 624         mutex_enter(&pidlock);
 625         t->t_next->t_prev = t->t_prev;
 626         t->t_prev->t_next = t->t_next;
 627         ASSERT(allthreads != t);        /* t0 never exits */
 628         cv_broadcast(&t->t_joincv);      /* wake up anyone in thread_join */
 629         mutex_exit(&pidlock);
 630 
 631         if (t->t_ctx != NULL)
 632                 exitctx(t);
 633         if (t->t_procp->p_pctx != NULL)
 634                 exitpctx(t->t_procp);
 635 
 636         if (kmem_stackinfo != 0) {
 637                 stkinfo_end(t);
 638         }
 639 
 640         t->t_state = TS_ZOMB;        /* set zombie thread */
 641 
 642         swtch_from_zombie();    /* give up the CPU */
 643         /* NOTREACHED */
 644 }
 645 
 646 /*
 647  * Check to see if the specified thread is active (defined as being on
 648  * the thread list).  This is certainly a slow way to do this; if there's
 649  * ever a reason to speed it up, we could maintain a hash table of active
 650  * threads indexed by their t_did.
 651  */
 652 static kthread_t *
 653 did_to_thread(kt_did_t tid)
 654 {
 655         kthread_t *t;
 656 
 657         ASSERT(MUTEX_HELD(&pidlock));
 658         for (t = curthread->t_next; t != curthread; t = t->t_next) {
 659                 if (t->t_did == tid)
 660                         break;
 661         }
 662         if (t->t_did == tid)
 663                 return (t);
 664         else
 665                 return (NULL);
 666 }
 667 
 668 /*
 669  * Wait for specified thread to exit.  Returns immediately if the thread
 670  * could not be found, meaning that it has either already exited or never
 671  * existed.
 672  */
 673 void
 674 thread_join(kt_did_t tid)
 675 {
 676         kthread_t *t;
 677 
 678         ASSERT(tid != curthread->t_did);
 679         ASSERT(tid != t0.t_did);
 680 
 681         mutex_enter(&pidlock);
 682         /*
 683          * Make sure we check that the thread is on the thread list
 684          * before blocking on it; otherwise we could end up blocking on
 685          * a cv that's already been freed.  In other words, don't cache
 686          * the thread pointer across calls to cv_wait.
 687          *
 688          * The choice of loop invariant means that whenever a thread
 689          * is taken off the allthreads list, a cv_broadcast must be
 690          * performed on that thread's t_joincv to wake up any waiters.
 691          * The broadcast doesn't have to happen right away, but it
 692          * shouldn't be postponed indefinitely (e.g., by doing it in
 693          * thread_free which may only be executed when the deathrow
 694          * queue is processed.
 695          */
 696         while (t = did_to_thread(tid))
 697                 cv_wait(&t->t_joincv, &pidlock);
 698         mutex_exit(&pidlock);
 699 }
 700 
 701 void
 702 thread_free_prevent(kthread_t *t)
 703 {
 704         kmutex_t *lp;
 705 
 706         lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
 707         mutex_enter(lp);
 708 }
 709 
 710 void
 711 thread_free_allow(kthread_t *t)
 712 {
 713         kmutex_t *lp;
 714 
 715         lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
 716         mutex_exit(lp);
 717 }
 718 
 719 static void
 720 thread_free_barrier(kthread_t *t)
 721 {
 722         kmutex_t *lp;
 723 
 724         lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
 725         mutex_enter(lp);
 726         mutex_exit(lp);
 727 }
 728 
 729 void
 730 thread_free(kthread_t *t)
 731 {
 732         boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
 733         klwp_t *lwp = t->t_lwp;
 734         caddr_t swap = t->t_swap;
 735 
 736         ASSERT(t != &t0 && t->t_state == TS_FREE);
 737         ASSERT(t->t_door == NULL);
 738         ASSERT(t->t_schedctl == NULL);
 739         ASSERT(t->t_pollstate == NULL);
 740 
 741         t->t_pri = 0;
 742         t->t_pc = 0;
 743         t->t_sp = 0;
 744         t->t_wchan0 = NULL;
 745         t->t_wchan = NULL;
 746         if (t->t_cred != NULL) {
 747                 crfree(t->t_cred);
 748                 t->t_cred = 0;
 749         }
 750         if (t->t_pdmsg) {
 751                 kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
 752                 t->t_pdmsg = NULL;
 753         }
 754         if (audit_active)
 755                 audit_thread_free(t);
 756 #ifndef NPROBE
 757         if (t->t_tnf_tpdp)
 758                 tnf_thread_free(t);
 759 #endif /* NPROBE */
 760         if (t->t_cldata) {
 761                 CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
 762         }
 763         if (t->t_rprof != NULL) {
 764                 kmem_free(t->t_rprof, sizeof (*t->t_rprof));
 765                 t->t_rprof = NULL;
 766         }
 767         t->t_lockp = NULL;   /* nothing should try to lock this thread now */
 768         if (lwp)
 769                 lwp_freeregs(lwp, 0);
 770         if (t->t_ctx)
 771                 freectx(t, 0);
 772         t->t_stk = NULL;
 773         if (lwp)
 774                 lwp_stk_fini(lwp);
 775         lock_clear(&t->t_lock);
 776 
 777         if (t->t_ts->ts_waiters > 0)
 778                 panic("thread_free: turnstile still active");
 779 
 780         kmem_cache_free(turnstile_cache, t->t_ts);
 781 
 782         free_afd(&t->t_activefd);
 783 
 784         /*
 785          * Barrier for the tick accounting code.  The tick accounting code
 786          * holds this lock to keep the thread from going away while it's
 787          * looking at it.
 788          */
 789         thread_free_barrier(t);
 790 
 791         ASSERT(ttoproj(t) == proj0p);
 792         project_rele(ttoproj(t));
 793 
 794         lgrp_affinity_free(&t->t_lgrp_affinity);
 795 
 796         mutex_enter(&pidlock);
 797         nthread--;
 798         mutex_exit(&pidlock);
 799 
 800         if (t->t_name != NULL) {
 801                 kmem_free(t->t_name, THREAD_NAME_MAX);
 802                 t->t_name = NULL;
 803         }
 804 
 805         /*
 806          * Free thread, lwp and stack.  This needs to be done carefully, since
 807          * if T_TALLOCSTK is set, the thread is part of the stack.
 808          */
 809         t->t_lwp = NULL;
 810         t->t_swap = NULL;
 811 
 812         if (swap) {
 813                 segkp_release(segkp, swap);
 814         }
 815         if (lwp) {
 816                 kmem_cache_free(lwp_cache, lwp);
 817         }
 818         if (!allocstk) {
 819                 kmem_cache_free(thread_cache, t);
 820         }
 821 }
 822 
 823 /*
 824  * Removes threads associated with the given zone from a deathrow queue.
 825  * tp is a pointer to the head of the deathrow queue, and countp is a
 826  * pointer to the current deathrow count.  Returns a linked list of
 827  * threads removed from the list.
 828  */
 829 static kthread_t *
 830 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
 831 {
 832         kthread_t *tmp, *list = NULL;
 833         cred_t *cr;
 834 
 835         ASSERT(MUTEX_HELD(&reaplock));
 836         while (*tp != NULL) {
 837                 if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
 838                         tmp = *tp;
 839                         *tp = tmp->t_forw;
 840                         tmp->t_forw = list;
 841                         list = tmp;
 842                         (*countp)--;
 843                 } else {
 844                         tp = &(*tp)->t_forw;
 845                 }
 846         }
 847         return (list);
 848 }
 849 
 850 static void
 851 thread_reap_list(kthread_t *t)
 852 {
 853         kthread_t *next;
 854 
 855         while (t != NULL) {
 856                 next = t->t_forw;
 857                 thread_free(t);
 858                 t = next;
 859         }
 860 }
 861 
 862 /* ARGSUSED */
 863 static void
 864 thread_zone_destroy(zoneid_t zoneid, void *unused)
 865 {
 866         kthread_t *t, *l;
 867 
 868         mutex_enter(&reaplock);
 869         /*
 870          * Pull threads and lwps associated with zone off deathrow lists.
 871          */
 872         t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
 873         l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
 874         mutex_exit(&reaplock);
 875 
 876         /*
 877          * Guard against race condition in mutex_owner_running:
 878          *      thread=owner(mutex)
 879          *      <interrupt>
 880          *                              thread exits mutex
 881          *                              thread exits
 882          *                              thread reaped
 883          *                              thread struct freed
 884          * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
 885          * A cross call to all cpus will cause the interrupt handler
 886          * to reset the PC if it is in mutex_owner_running, refreshing
 887          * stale thread pointers.
 888          */
 889         mutex_sync();   /* sync with mutex code */
 890 
 891         /*
 892          * Reap threads
 893          */
 894         thread_reap_list(t);
 895 
 896         /*
 897          * Reap lwps
 898          */
 899         thread_reap_list(l);
 900 }
 901 
 902 /*
 903  * cleanup zombie threads that are on deathrow.
 904  */
 905 void
 906 thread_reaper()
 907 {
 908         kthread_t *t, *l;
 909         callb_cpr_t cprinfo;
 910 
 911         /*
 912          * Register callback to clean up threads when zone is destroyed.
 913          */
 914         zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
 915 
 916         CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
 917         for (;;) {
 918                 mutex_enter(&reaplock);
 919                 while (thread_deathrow == NULL && lwp_deathrow == NULL) {
 920                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
 921                         cv_wait(&reaper_cv, &reaplock);
 922                         CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
 923                 }
 924                 /*
 925                  * mutex_sync() needs to be called when reaping, but
 926                  * not too often.  We limit reaping rate to once
 927                  * per second.  Reaplimit is max rate at which threads can
 928                  * be freed. Does not impact thread destruction/creation.
 929                  */
 930                 t = thread_deathrow;
 931                 l = lwp_deathrow;
 932                 thread_deathrow = NULL;
 933                 lwp_deathrow = NULL;
 934                 thread_reapcnt = 0;
 935                 lwp_reapcnt = 0;
 936                 mutex_exit(&reaplock);
 937 
 938                 /*
 939                  * Guard against race condition in mutex_owner_running:
 940                  *      thread=owner(mutex)
 941                  *      <interrupt>
 942                  *                              thread exits mutex
 943                  *                              thread exits
 944                  *                              thread reaped
 945                  *                              thread struct freed
 946                  * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
 947                  * A cross call to all cpus will cause the interrupt handler
 948                  * to reset the PC if it is in mutex_owner_running, refreshing
 949                  * stale thread pointers.
 950                  */
 951                 mutex_sync();   /* sync with mutex code */
 952                 /*
 953                  * Reap threads
 954                  */
 955                 thread_reap_list(t);
 956 
 957                 /*
 958                  * Reap lwps
 959                  */
 960                 thread_reap_list(l);
 961                 delay(hz);
 962         }
 963 }
 964 
 965 /*
 966  * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
 967  * thread_deathrow. The thread's state is changed already TS_FREE to indicate
 968  * that is reapable. The thread already holds the reaplock, and was already
 969  * freed.
 970  */
 971 void
 972 reapq_move_lq_to_tq(kthread_t *t)
 973 {
 974         ASSERT(t->t_state == TS_FREE);
 975         ASSERT(MUTEX_HELD(&reaplock));
 976         t->t_forw = thread_deathrow;
 977         thread_deathrow = t;
 978         thread_reapcnt++;
 979         if (lwp_reapcnt + thread_reapcnt > reaplimit)
 980                 cv_signal(&reaper_cv);  /* wake the reaper */
 981 }
 982 
 983 /*
 984  * This is called by resume() to put a zombie thread onto deathrow.
 985  * The thread's state is changed to TS_FREE to indicate that is reapable.
 986  * This is called from the idle thread so it must not block - just spin.
 987  */
 988 void
 989 reapq_add(kthread_t *t)
 990 {
 991         mutex_enter(&reaplock);
 992 
 993         /*
 994          * lwp_deathrow contains threads with lwp linkage and
 995          * swappable thread stacks which have the default stacksize.
 996          * These threads' lwps and stacks may be reused by lwp_create().
 997          *
 998          * Anything else goes on thread_deathrow(), where it will eventually
 999          * be thread_free()d.
1000          */
1001         if (t->t_flag & T_LWPREUSE) {
1002                 ASSERT(ttolwp(t) != NULL);
1003                 t->t_forw = lwp_deathrow;
1004                 lwp_deathrow = t;
1005                 lwp_reapcnt++;
1006         } else {
1007                 t->t_forw = thread_deathrow;
1008                 thread_deathrow = t;
1009                 thread_reapcnt++;
1010         }
1011         if (lwp_reapcnt + thread_reapcnt > reaplimit)
1012                 cv_signal(&reaper_cv);      /* wake the reaper */
1013         t->t_state = TS_FREE;
1014         lock_clear(&t->t_lock);
1015 
1016         /*
1017          * Before we return, we need to grab and drop the thread lock for
1018          * the dead thread.  At this point, the current thread is the idle
1019          * thread, and the dead thread's CPU lock points to the current
1020          * CPU -- and we must grab and drop the lock to synchronize with
1021          * a racing thread walking a blocking chain that the zombie thread
1022          * was recently in.  By this point, that blocking chain is (by
1023          * definition) stale:  the dead thread is not holding any locks, and
1024          * is therefore not in any blocking chains -- but if we do not regrab
1025          * our lock before freeing the dead thread's data structures, the
1026          * thread walking the (stale) blocking chain will die on memory
1027          * corruption when it attempts to drop the dead thread's lock.  We
1028          * only need do this once because there is no way for the dead thread
1029          * to ever again be on a blocking chain:  once we have grabbed and
1030          * dropped the thread lock, we are guaranteed that anyone that could
1031          * have seen this thread in a blocking chain can no longer see it.
1032          */
1033         thread_lock(t);
1034         thread_unlock(t);
1035 
1036         mutex_exit(&reaplock);
1037 }
1038 
1039 /*
1040  * Install thread context ops for the current thread.
1041  */
1042 void
1043 installctx(
1044         kthread_t *t,
1045         void    *arg,
1046         void    (*save)(void *),
1047         void    (*restore)(void *),
1048         void    (*fork)(void *, void *),
1049         void    (*lwp_create)(void *, void *),
1050         void    (*exit)(void *),
1051         void    (*free)(void *, int))
1052 {
1053         struct ctxop *ctx;
1054 
1055         ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1056         ctx->save_op = save;
1057         ctx->restore_op = restore;
1058         ctx->fork_op = fork;
1059         ctx->lwp_create_op = lwp_create;
1060         ctx->exit_op = exit;
1061         ctx->free_op = free;
1062         ctx->arg = arg;
1063         ctx->next = t->t_ctx;
1064         t->t_ctx = ctx;
1065 }
1066 
1067 /*
1068  * Remove the thread context ops from a thread.
1069  */
1070 int
1071 removectx(
1072         kthread_t *t,
1073         void    *arg,
1074         void    (*save)(void *),
1075         void    (*restore)(void *),
1076         void    (*fork)(void *, void *),
1077         void    (*lwp_create)(void *, void *),
1078         void    (*exit)(void *),
1079         void    (*free)(void *, int))
1080 {
1081         struct ctxop *ctx, *prev_ctx;
1082 
1083         /*
1084          * The incoming kthread_t (which is the thread for which the
1085          * context ops will be removed) should be one of the following:
1086          *
1087          * a) the current thread,
1088          *
1089          * b) a thread of a process that's being forked (SIDL),
1090          *
1091          * c) a thread that belongs to the same process as the current
1092          *    thread and for which the current thread is the agent thread,
1093          *
1094          * d) a thread that is TS_STOPPED which is indicative of it
1095          *    being (if curthread is not an agent) a thread being created
1096          *    as part of an lwp creation.
1097          */
1098         ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1099             ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1100 
1101         /*
1102          * Serialize modifications to t->t_ctx to prevent the agent thread
1103          * and the target thread from racing with each other during lwp exit.
1104          */
1105         mutex_enter(&t->t_ctx_lock);
1106         prev_ctx = NULL;
1107         kpreempt_disable();
1108         for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
1109                 if (ctx->save_op == save && ctx->restore_op == restore &&
1110                     ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
1111                     ctx->exit_op == exit && ctx->free_op == free &&
1112                     ctx->arg == arg) {
1113                         if (prev_ctx)
1114                                 prev_ctx->next = ctx->next;
1115                         else
1116                                 t->t_ctx = ctx->next;
1117                         mutex_exit(&t->t_ctx_lock);
1118                         if (ctx->free_op != NULL)
1119                                 (ctx->free_op)(ctx->arg, 0);
1120                         kmem_free(ctx, sizeof (struct ctxop));
1121                         kpreempt_enable();
1122                         return (1);
1123                 }
1124                 prev_ctx = ctx;
1125         }
1126         mutex_exit(&t->t_ctx_lock);
1127         kpreempt_enable();
1128 
1129         return (0);
1130 }
1131 
1132 void
1133 savectx(kthread_t *t)
1134 {
1135         struct ctxop *ctx;
1136 
1137         ASSERT(t == curthread);
1138         for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1139                 if (ctx->save_op != NULL)
1140                         (ctx->save_op)(ctx->arg);
1141 }
1142 
1143 void
1144 restorectx(kthread_t *t)
1145 {
1146         struct ctxop *ctx;
1147 
1148         ASSERT(t == curthread);
1149         for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1150                 if (ctx->restore_op != NULL)
1151                         (ctx->restore_op)(ctx->arg);
1152 }
1153 
1154 void
1155 forkctx(kthread_t *t, kthread_t *ct)
1156 {
1157         struct ctxop *ctx;
1158 
1159         for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1160                 if (ctx->fork_op != NULL)
1161                         (ctx->fork_op)(t, ct);
1162 }
1163 
1164 /*
1165  * Note that this operator is only invoked via the _lwp_create
1166  * system call.  The system may have other reasons to create lwps
1167  * e.g. the agent lwp or the doors unreferenced lwp.
1168  */
1169 void
1170 lwp_createctx(kthread_t *t, kthread_t *ct)
1171 {
1172         struct ctxop *ctx;
1173 
1174         for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1175                 if (ctx->lwp_create_op != NULL)
1176                         (ctx->lwp_create_op)(t, ct);
1177 }
1178 
1179 /*
1180  * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1181  * needed when the thread/LWP leaves the processor for the last time. This
1182  * routine is not intended to deal with freeing memory; freectx() is used for
1183  * that purpose during thread_free(). This routine is provided to allow for
1184  * clean-up that can't wait until thread_free().
1185  */
1186 void
1187 exitctx(kthread_t *t)
1188 {
1189         struct ctxop *ctx;
1190 
1191         for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1192                 if (ctx->exit_op != NULL)
1193                         (ctx->exit_op)(t);
1194 }
1195 
1196 /*
1197  * freectx is called from thread_free() and exec() to get
1198  * rid of old thread context ops.
1199  */
1200 void
1201 freectx(kthread_t *t, int isexec)
1202 {
1203         struct ctxop *ctx;
1204 
1205         kpreempt_disable();
1206         while ((ctx = t->t_ctx) != NULL) {
1207                 t->t_ctx = ctx->next;
1208                 if (ctx->free_op != NULL)
1209                         (ctx->free_op)(ctx->arg, isexec);
1210                 kmem_free(ctx, sizeof (struct ctxop));
1211         }
1212         kpreempt_enable();
1213 }
1214 
1215 /*
1216  * freectx_ctx is called from lwp_create() when lwp is reused from
1217  * lwp_deathrow and its thread structure is added to thread_deathrow.
1218  * The thread structure to which this ctx was attached may be already
1219  * freed by the thread reaper so free_op implementations shouldn't rely
1220  * on thread structure to which this ctx was attached still being around.
1221  */
1222 void
1223 freectx_ctx(struct ctxop *ctx)
1224 {
1225         struct ctxop *nctx;
1226 
1227         ASSERT(ctx != NULL);
1228 
1229         kpreempt_disable();
1230         do {
1231                 nctx = ctx->next;
1232                 if (ctx->free_op != NULL)
1233                         (ctx->free_op)(ctx->arg, 0);
1234                 kmem_free(ctx, sizeof (struct ctxop));
1235         } while ((ctx = nctx) != NULL);
1236         kpreempt_enable();
1237 }
1238 
1239 /*
1240  * Set the thread running; arrange for it to be swapped in if necessary.
1241  */
1242 void
1243 setrun_locked(kthread_t *t)
1244 {
1245         ASSERT(THREAD_LOCK_HELD(t));
1246         if (t->t_state == TS_SLEEP) {
1247                 /*
1248                  * Take off sleep queue.
1249                  */
1250                 SOBJ_UNSLEEP(t->t_sobj_ops, t);
1251         } else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1252                 /*
1253                  * Already on dispatcher queue.
1254                  */
1255                 return;
1256         } else if (t->t_state == TS_WAIT) {
1257                 waitq_setrun(t);
1258         } else if (t->t_state == TS_STOPPED) {
1259                 /*
1260                  * All of the sending of SIGCONT (TC_XSTART) and /proc
1261                  * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1262                  * requested that the thread be run.
1263                  * Just calling setrun() is not sufficient to set a stopped
1264                  * thread running.  TP_TXSTART is always set if the thread
1265                  * is not stopped by a jobcontrol stop signal.
1266                  * TP_TPSTART is always set if /proc is not controlling it.
1267                  * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1268                  * The thread won't be stopped unless one of these
1269                  * three mechanisms did it.
1270                  *
1271                  * These flags must be set before calling setrun_locked(t).
1272                  * They can't be passed as arguments because the streams
1273                  * code calls setrun() indirectly and the mechanism for
1274                  * doing so admits only one argument.  Note that the
1275                  * thread must be locked in order to change t_schedflags.
1276                  */
1277                 if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1278                         return;
1279                 /*
1280                  * Process is no longer stopped (a thread is running).
1281                  */
1282                 t->t_whystop = 0;
1283                 t->t_whatstop = 0;
1284                 /*
1285                  * Strictly speaking, we do not have to clear these
1286                  * flags here; they are cleared on entry to stop().
1287                  * However, they are confusing when doing kernel
1288                  * debugging or when they are revealed by ps(1).
1289                  */
1290                 t->t_schedflag &= ~TS_ALLSTART;
1291                 THREAD_TRANSITION(t);   /* drop stopped-thread lock */
1292                 ASSERT(t->t_lockp == &transition_lock);
1293                 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1294                 /*
1295                  * Let the class put the process on the dispatcher queue.
1296                  */
1297                 CL_SETRUN(t);
1298         }
1299 }
1300 
1301 void
1302 setrun(kthread_t *t)
1303 {
1304         thread_lock(t);
1305         setrun_locked(t);
1306         thread_unlock(t);
1307 }
1308 
1309 /*
1310  * Unpin an interrupted thread.
1311  *      When an interrupt occurs, the interrupt is handled on the stack
1312  *      of an interrupt thread, taken from a pool linked to the CPU structure.
1313  *
1314  *      When swtch() is switching away from an interrupt thread because it
1315  *      blocked or was preempted, this routine is called to complete the
1316  *      saving of the interrupted thread state, and returns the interrupted
1317  *      thread pointer so it may be resumed.
1318  *
1319  *      Called by swtch() only at high spl.
1320  */
1321 kthread_t *
1322 thread_unpin()
1323 {
1324         kthread_t       *t = curthread; /* current thread */
1325         kthread_t       *itp;           /* interrupted thread */
1326         int             i;              /* interrupt level */
1327         extern int      intr_passivate();
1328 
1329         ASSERT(t->t_intr != NULL);
1330 
1331         itp = t->t_intr;             /* interrupted thread */
1332         t->t_intr = NULL;            /* clear interrupt ptr */
1333 
1334         ht_end_intr();
1335 
1336         /*
1337          * Get state from interrupt thread for the one
1338          * it interrupted.
1339          */
1340 
1341         i = intr_passivate(t, itp);
1342 
1343         TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1344             "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1345             i, t, t, itp, itp);
1346 
1347         /*
1348          * Dissociate the current thread from the interrupted thread's LWP.
1349          */
1350         t->t_lwp = NULL;
1351 
1352         /*
1353          * Interrupt handlers above the level that spinlocks block must
1354          * not block.
1355          */
1356 #if DEBUG
1357         if (i < 0 || i > LOCK_LEVEL)
1358                 cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1359 #endif
1360 
1361         /*
1362          * Compute the CPU's base interrupt level based on the active
1363          * interrupts.
1364          */
1365         ASSERT(CPU->cpu_intr_actv & (1 << i));
1366         set_base_spl();
1367 
1368         return (itp);
1369 }
1370 
1371 /*
1372  * Create and initialize an interrupt thread.
1373  *      Returns non-zero on error.
1374  *      Called at spl7() or better.
1375  */
1376 void
1377 thread_create_intr(struct cpu *cp)
1378 {
1379         kthread_t *tp;
1380 
1381         tp = thread_create(NULL, 0,
1382             (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1383 
1384         /*
1385          * Set the thread in the TS_FREE state.  The state will change
1386          * to TS_ONPROC only while the interrupt is active.  Think of these
1387          * as being on a private free list for the CPU.  Being TS_FREE keeps
1388          * inactive interrupt threads out of debugger thread lists.
1389          *
1390          * We cannot call thread_create with TS_FREE because of the current
1391          * checks there for ONPROC.  Fix this when thread_create takes flags.
1392          */
1393         THREAD_FREEINTR(tp, cp);
1394 
1395         /*
1396          * Nobody should ever reference the credentials of an interrupt
1397          * thread so make it NULL to catch any such references.
1398          */
1399         tp->t_cred = NULL;
1400         tp->t_flag |= T_INTR_THREAD;
1401         tp->t_cpu = cp;
1402         tp->t_bound_cpu = cp;
1403         tp->t_disp_queue = cp->cpu_disp;
1404         tp->t_affinitycnt = 1;
1405         tp->t_preempt = 1;
1406 
1407         /*
1408          * Don't make a user-requested binding on this thread so that
1409          * the processor can be offlined.
1410          */
1411         tp->t_bind_cpu = PBIND_NONE; /* no USER-requested binding */
1412         tp->t_bind_pset = PS_NONE;
1413 
1414 #if defined(__i386) || defined(__amd64)
1415         tp->t_stk -= STACK_ALIGN;
1416         *(tp->t_stk) = 0;            /* terminate intr thread stack */
1417 #endif
1418 
1419         /*
1420          * Link onto CPU's interrupt pool.
1421          */
1422         tp->t_link = cp->cpu_intr_thread;
1423         cp->cpu_intr_thread = tp;
1424 }
1425 
1426 /*
1427  * TSD -- THREAD SPECIFIC DATA
1428  */
1429 static kmutex_t         tsd_mutex;       /* linked list spin lock */
1430 static uint_t           tsd_nkeys;       /* size of destructor array */
1431 /* per-key destructor funcs */
1432 static void             (**tsd_destructor)(void *);
1433 /* list of tsd_thread's */
1434 static struct tsd_thread        *tsd_list;
1435 
1436 /*
1437  * Default destructor
1438  *      Needed because NULL destructor means that the key is unused
1439  */
1440 /* ARGSUSED */
1441 void
1442 tsd_defaultdestructor(void *value)
1443 {}
1444 
1445 /*
1446  * Create a key (index into per thread array)
1447  *      Locks out tsd_create, tsd_destroy, and tsd_exit
1448  *      May allocate memory with lock held
1449  */
1450 void
1451 tsd_create(uint_t *keyp, void (*destructor)(void *))
1452 {
1453         int     i;
1454         uint_t  nkeys;
1455 
1456         /*
1457          * if key is allocated, do nothing
1458          */
1459         mutex_enter(&tsd_mutex);
1460         if (*keyp) {
1461                 mutex_exit(&tsd_mutex);
1462                 return;
1463         }
1464         /*
1465          * find an unused key
1466          */
1467         if (destructor == NULL)
1468                 destructor = tsd_defaultdestructor;
1469 
1470         for (i = 0; i < tsd_nkeys; ++i)
1471                 if (tsd_destructor[i] == NULL)
1472                         break;
1473 
1474         /*
1475          * if no unused keys, increase the size of the destructor array
1476          */
1477         if (i == tsd_nkeys) {
1478                 if ((nkeys = (tsd_nkeys << 1)) == 0)
1479                         nkeys = 1;
1480                 tsd_destructor =
1481                     (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1482                     (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1483                     (size_t)(nkeys * sizeof (void (*)(void *))));
1484                 tsd_nkeys = nkeys;
1485         }
1486 
1487         /*
1488          * allocate the next available unused key
1489          */
1490         tsd_destructor[i] = destructor;
1491         *keyp = i + 1;
1492         mutex_exit(&tsd_mutex);
1493 }
1494 
1495 /*
1496  * Destroy a key -- this is for unloadable modules
1497  *
1498  * Assumes that the caller is preventing tsd_set and tsd_get
1499  * Locks out tsd_create, tsd_destroy, and tsd_exit
1500  * May free memory with lock held
1501  */
1502 void
1503 tsd_destroy(uint_t *keyp)
1504 {
1505         uint_t key;
1506         struct tsd_thread *tsd;
1507 
1508         /*
1509          * protect the key namespace and our destructor lists
1510          */
1511         mutex_enter(&tsd_mutex);
1512         key = *keyp;
1513         *keyp = 0;
1514 
1515         ASSERT(key <= tsd_nkeys);
1516 
1517         /*
1518          * if the key is valid
1519          */
1520         if (key != 0) {
1521                 uint_t k = key - 1;
1522                 /*
1523                  * for every thread with TSD, call key's destructor
1524                  */
1525                 for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1526                         /*
1527                          * no TSD for key in this thread
1528                          */
1529                         if (key > tsd->ts_nkeys)
1530                                 continue;
1531                         /*
1532                          * call destructor for key
1533                          */
1534                         if (tsd->ts_value[k] && tsd_destructor[k])
1535                                 (*tsd_destructor[k])(tsd->ts_value[k]);
1536                         /*
1537                          * reset value for key
1538                          */
1539                         tsd->ts_value[k] = NULL;
1540                 }
1541                 /*
1542                  * actually free the key (NULL destructor == unused)
1543                  */
1544                 tsd_destructor[k] = NULL;
1545         }
1546 
1547         mutex_exit(&tsd_mutex);
1548 }
1549 
1550 /*
1551  * Quickly return the per thread value that was stored with the specified key
1552  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1553  */
1554 void *
1555 tsd_get(uint_t key)
1556 {
1557         return (tsd_agent_get(curthread, key));
1558 }
1559 
1560 /*
1561  * Set a per thread value indexed with the specified key
1562  */
1563 int
1564 tsd_set(uint_t key, void *value)
1565 {
1566         return (tsd_agent_set(curthread, key, value));
1567 }
1568 
1569 /*
1570  * Like tsd_get(), except that the agent lwp can get the tsd of
1571  * another thread in the same process (the agent thread only runs when the
1572  * process is completely stopped by /proc), or syslwp is creating a new lwp.
1573  */
1574 void *
1575 tsd_agent_get(kthread_t *t, uint_t key)
1576 {
1577         struct tsd_thread *tsd = t->t_tsd;
1578 
1579         ASSERT(t == curthread ||
1580             ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1581 
1582         if (key && tsd != NULL && key <= tsd->ts_nkeys)
1583                 return (tsd->ts_value[key - 1]);
1584         return (NULL);
1585 }
1586 
1587 /*
1588  * Like tsd_set(), except that the agent lwp can set the tsd of
1589  * another thread in the same process, or syslwp can set the tsd
1590  * of a thread it's in the middle of creating.
1591  *
1592  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1593  * May lock out tsd_destroy (and tsd_create), may allocate memory with
1594  * lock held
1595  */
1596 int
1597 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1598 {
1599         struct tsd_thread *tsd = t->t_tsd;
1600 
1601         ASSERT(t == curthread ||
1602             ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1603 
1604         if (key == 0)
1605                 return (EINVAL);
1606         if (tsd == NULL)
1607                 tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1608         if (key <= tsd->ts_nkeys) {
1609                 tsd->ts_value[key - 1] = value;
1610                 return (0);
1611         }
1612 
1613         ASSERT(key <= tsd_nkeys);
1614 
1615         /*
1616          * lock out tsd_destroy()
1617          */
1618         mutex_enter(&tsd_mutex);
1619         if (tsd->ts_nkeys == 0) {
1620                 /*
1621                  * Link onto list of threads with TSD
1622                  */
1623                 if ((tsd->ts_next = tsd_list) != NULL)
1624                         tsd_list->ts_prev = tsd;
1625                 tsd_list = tsd;
1626         }
1627 
1628         /*
1629          * Allocate thread local storage and set the value for key
1630          */
1631         tsd->ts_value = tsd_realloc(tsd->ts_value,
1632             tsd->ts_nkeys * sizeof (void *),
1633             key * sizeof (void *));
1634         tsd->ts_nkeys = key;
1635         tsd->ts_value[key - 1] = value;
1636         mutex_exit(&tsd_mutex);
1637 
1638         return (0);
1639 }
1640 
1641 
1642 /*
1643  * Return the per thread value that was stored with the specified key
1644  *      If necessary, create the key and the value
1645  *      Assumes the caller is protecting *keyp from tsd_destroy
1646  */
1647 void *
1648 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1649 {
1650         void *value;
1651         uint_t key = *keyp;
1652         struct tsd_thread *tsd = curthread->t_tsd;
1653 
1654         if (tsd == NULL)
1655                 tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1656         if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1657                 return (value);
1658         if (key == 0)
1659                 tsd_create(keyp, destroy);
1660         (void) tsd_set(*keyp, value = (*allocate)());
1661 
1662         return (value);
1663 }
1664 
1665 /*
1666  * Called from thread_exit() to run the destructor function for each tsd
1667  *      Locks out tsd_create and tsd_destroy
1668  *      Assumes that the destructor *DOES NOT* use tsd
1669  */
1670 void
1671 tsd_exit(void)
1672 {
1673         int i;
1674         struct tsd_thread *tsd = curthread->t_tsd;
1675 
1676         if (tsd == NULL)
1677                 return;
1678 
1679         if (tsd->ts_nkeys == 0) {
1680                 kmem_free(tsd, sizeof (*tsd));
1681                 curthread->t_tsd = NULL;
1682                 return;
1683         }
1684 
1685         /*
1686          * lock out tsd_create and tsd_destroy, call
1687          * the destructor, and mark the value as destroyed.
1688          */
1689         mutex_enter(&tsd_mutex);
1690 
1691         for (i = 0; i < tsd->ts_nkeys; i++) {
1692                 if (tsd->ts_value[i] && tsd_destructor[i])
1693                         (*tsd_destructor[i])(tsd->ts_value[i]);
1694                 tsd->ts_value[i] = NULL;
1695         }
1696 
1697         /*
1698          * remove from linked list of threads with TSD
1699          */
1700         if (tsd->ts_next)
1701                 tsd->ts_next->ts_prev = tsd->ts_prev;
1702         if (tsd->ts_prev)
1703                 tsd->ts_prev->ts_next = tsd->ts_next;
1704         if (tsd_list == tsd)
1705                 tsd_list = tsd->ts_next;
1706 
1707         mutex_exit(&tsd_mutex);
1708 
1709         /*
1710          * free up the TSD
1711          */
1712         kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1713         kmem_free(tsd, sizeof (struct tsd_thread));
1714         curthread->t_tsd = NULL;
1715 }
1716 
1717 /*
1718  * realloc
1719  */
1720 static void *
1721 tsd_realloc(void *old, size_t osize, size_t nsize)
1722 {
1723         void *new;
1724 
1725         new = kmem_zalloc(nsize, KM_SLEEP);
1726         if (old) {
1727                 bcopy(old, new, osize);
1728                 kmem_free(old, osize);
1729         }
1730         return (new);
1731 }
1732 
1733 /*
1734  * Return non-zero if an interrupt is being serviced.
1735  */
1736 int
1737 servicing_interrupt()
1738 {
1739         int onintr = 0;
1740 
1741         /* Are we an interrupt thread */
1742         if (curthread->t_flag & T_INTR_THREAD)
1743                 return (1);
1744         /* Are we servicing a high level interrupt? */
1745         if (CPU_ON_INTR(CPU)) {
1746                 kpreempt_disable();
1747                 onintr = CPU_ON_INTR(CPU);
1748                 kpreempt_enable();
1749         }
1750         return (onintr);
1751 }
1752 
1753 
1754 /*
1755  * Change the dispatch priority of a thread in the system.
1756  * Used when raising or lowering a thread's priority.
1757  * (E.g., priority inheritance)
1758  *
1759  * Since threads are queued according to their priority, we
1760  * we must check the thread's state to determine whether it
1761  * is on a queue somewhere. If it is, we've got to:
1762  *
1763  *      o Dequeue the thread.
1764  *      o Change its effective priority.
1765  *      o Enqueue the thread.
1766  *
1767  * Assumptions: The thread whose priority we wish to change
1768  * must be locked before we call thread_change_(e)pri().
1769  * The thread_change(e)pri() function doesn't drop the thread
1770  * lock--that must be done by its caller.
1771  */
1772 void
1773 thread_change_epri(kthread_t *t, pri_t disp_pri)
1774 {
1775         uint_t  state;
1776 
1777         ASSERT(THREAD_LOCK_HELD(t));
1778 
1779         /*
1780          * If the inherited priority hasn't actually changed,
1781          * just return.
1782          */
1783         if (t->t_epri == disp_pri)
1784                 return;
1785 
1786         state = t->t_state;
1787 
1788         /*
1789          * If it's not on a queue, change the priority with impunity.
1790          */
1791         if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1792                 t->t_epri = disp_pri;
1793                 if (state == TS_ONPROC) {
1794                         cpu_t *cp = t->t_disp_queue->disp_cpu;
1795 
1796                         if (t == cp->cpu_dispthread)
1797                                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1798                 }
1799         } else if (state == TS_SLEEP) {
1800                 /*
1801                  * Take the thread out of its sleep queue.
1802                  * Change the inherited priority.
1803                  * Re-enqueue the thread.
1804                  * Each synchronization object exports a function
1805                  * to do this in an appropriate manner.
1806                  */
1807                 SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1808         } else if (state == TS_WAIT) {
1809                 /*
1810                  * Re-enqueue a thread on the wait queue if its
1811                  * effective priority needs to change.
1812                  */
1813                 if (disp_pri != t->t_epri)
1814                         waitq_change_pri(t, disp_pri);
1815         } else {
1816                 /*
1817                  * The thread is on a run queue.
1818                  * Note: setbackdq() may not put the thread
1819                  * back on the same run queue where it originally
1820                  * resided.
1821                  */
1822                 (void) dispdeq(t);
1823                 t->t_epri = disp_pri;
1824                 setbackdq(t);
1825         }
1826         schedctl_set_cidpri(t);
1827 }
1828 
1829 /*
1830  * Function: Change the t_pri field of a thread.
1831  * Side Effects: Adjust the thread ordering on a run queue
1832  *               or sleep queue, if necessary.
1833  * Returns: 1 if the thread was on a run queue, else 0.
1834  */
1835 int
1836 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1837 {
1838         uint_t  state;
1839         int     on_rq = 0;
1840 
1841         ASSERT(THREAD_LOCK_HELD(t));
1842 
1843         state = t->t_state;
1844         THREAD_WILLCHANGE_PRI(t, disp_pri);
1845 
1846         /*
1847          * If it's not on a queue, change the priority with impunity.
1848          */
1849         if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1850                 t->t_pri = disp_pri;
1851 
1852                 if (state == TS_ONPROC) {
1853                         cpu_t *cp = t->t_disp_queue->disp_cpu;
1854 
1855                         if (t == cp->cpu_dispthread)
1856                                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1857                 }
1858         } else if (state == TS_SLEEP) {
1859                 /*
1860                  * If the priority has changed, take the thread out of
1861                  * its sleep queue and change the priority.
1862                  * Re-enqueue the thread.
1863                  * Each synchronization object exports a function
1864                  * to do this in an appropriate manner.
1865                  */
1866                 if (disp_pri != t->t_pri)
1867                         SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1868         } else if (state == TS_WAIT) {
1869                 /*
1870                  * Re-enqueue a thread on the wait queue if its
1871                  * priority needs to change.
1872                  */
1873                 if (disp_pri != t->t_pri)
1874                         waitq_change_pri(t, disp_pri);
1875         } else {
1876                 /*
1877                  * The thread is on a run queue.
1878                  * Note: setbackdq() may not put the thread
1879                  * back on the same run queue where it originally
1880                  * resided.
1881                  *
1882                  * We still requeue the thread even if the priority
1883                  * is unchanged to preserve round-robin (and other)
1884                  * effects between threads of the same priority.
1885                  */
1886                 on_rq = dispdeq(t);
1887                 ASSERT(on_rq);
1888                 t->t_pri = disp_pri;
1889                 if (front) {
1890                         setfrontdq(t);
1891                 } else {
1892                         setbackdq(t);
1893                 }
1894         }
1895         schedctl_set_cidpri(t);
1896         return (on_rq);
1897 }
1898 
1899 /*
1900  * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
1901  * specific pattern.
1902  */
1903 static void
1904 stkinfo_begin(kthread_t *t)
1905 {
1906         caddr_t start;  /* stack start */
1907         caddr_t end;    /* stack end  */
1908         uint64_t *ptr;  /* pattern pointer */
1909 
1910         /*
1911          * Stack grows up or down, see thread_create(),
1912          * compute stack memory area start and end (start < end).
1913          */
1914         if (t->t_stk > t->t_stkbase) {
1915                 /* stack grows down */
1916                 start = t->t_stkbase;
1917                 end = t->t_stk;
1918         } else {
1919                 /* stack grows up */
1920                 start = t->t_stk;
1921                 end = t->t_stkbase;
1922         }
1923 
1924         /*
1925          * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1926          * alignement for start and end in stack area boundaries
1927          * (protection against corrupt t_stkbase/t_stk data).
1928          */
1929         if ((((uintptr_t)start) & 0x7) != 0) {
1930                 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1931         }
1932         end = (caddr_t)(((uintptr_t)end) & (~0x7));
1933 
1934         if ((end <= start) || (end - start) > (1024 * 1024)) {
1935                 /* negative or stack size > 1 meg, assume bogus */
1936                 return;
1937         }
1938 
1939         /* fill stack area with a pattern (instead of zeros) */
1940         ptr = (uint64_t *)((void *)start);
1941         while (ptr < (uint64_t *)((void *)end)) {
1942                 *ptr++ = KMEM_STKINFO_PATTERN;
1943         }
1944 }
1945 
1946 
1947 /*
1948  * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
1949  * compute the percentage of kernel stack really used, and set in the log
1950  * if it's the latest highest percentage.
1951  */
1952 static void
1953 stkinfo_end(kthread_t *t)
1954 {
1955         caddr_t start;  /* stack start */
1956         caddr_t end;    /* stack end  */
1957         uint64_t *ptr;  /* pattern pointer */
1958         size_t stksz;   /* stack size */
1959         size_t smallest = 0;
1960         size_t percent = 0;
1961         uint_t index = 0;
1962         uint_t i;
1963         static size_t smallest_percent = (size_t)-1;
1964         static uint_t full = 0;
1965 
1966         /* create the stackinfo log, if doesn't already exist */
1967         mutex_enter(&kmem_stkinfo_lock);
1968         if (kmem_stkinfo_log == NULL) {
1969                 kmem_stkinfo_log = (kmem_stkinfo_t *)
1970                     kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
1971                     (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
1972                 if (kmem_stkinfo_log == NULL) {
1973                         mutex_exit(&kmem_stkinfo_lock);
1974                         return;
1975                 }
1976         }
1977         mutex_exit(&kmem_stkinfo_lock);
1978 
1979         /*
1980          * Stack grows up or down, see thread_create(),
1981          * compute stack memory area start and end (start < end).
1982          */
1983         if (t->t_stk > t->t_stkbase) {
1984                 /* stack grows down */
1985                 start = t->t_stkbase;
1986                 end = t->t_stk;
1987         } else {
1988                 /* stack grows up */
1989                 start = t->t_stk;
1990                 end = t->t_stkbase;
1991         }
1992 
1993         /* stack size as found in kthread_t */
1994         stksz = end - start;
1995 
1996         /*
1997          * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1998          * alignement for start and end in stack area boundaries
1999          * (protection against corrupt t_stkbase/t_stk data).
2000          */
2001         if ((((uintptr_t)start) & 0x7) != 0) {
2002                 start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2003         }
2004         end = (caddr_t)(((uintptr_t)end) & (~0x7));
2005 
2006         if ((end <= start) || (end - start) > (1024 * 1024)) {
2007                 /* negative or stack size > 1 meg, assume bogus */
2008                 return;
2009         }
2010 
2011         /* search until no pattern in the stack */
2012         if (t->t_stk > t->t_stkbase) {
2013                 /* stack grows down */
2014 #if defined(__i386) || defined(__amd64)
2015                 /*
2016                  * 6 longs are pushed on stack, see thread_load(). Skip
2017                  * them, so if kthread has never run, percent is zero.
2018                  * 8 bytes alignement is preserved for a 32 bit kernel,
2019                  * 6 x 4 = 24, 24 is a multiple of 8.
2020                  *
2021                  */
2022                 end -= (6 * sizeof (long));
2023 #endif
2024                 ptr = (uint64_t *)((void *)start);
2025                 while (ptr < (uint64_t *)((void *)end)) {
2026                         if (*ptr != KMEM_STKINFO_PATTERN) {
2027                                 percent = stkinfo_percent(end,
2028                                     start, (caddr_t)ptr);
2029                                 break;
2030                         }
2031                         ptr++;
2032                 }
2033         } else {
2034                 /* stack grows up */
2035                 ptr = (uint64_t *)((void *)end);
2036                 ptr--;
2037                 while (ptr >= (uint64_t *)((void *)start)) {
2038                         if (*ptr != KMEM_STKINFO_PATTERN) {
2039                                 percent = stkinfo_percent(start,
2040                                     end, (caddr_t)ptr);
2041                                 break;
2042                         }
2043                         ptr--;
2044                 }
2045         }
2046 
2047         DTRACE_PROBE3(stack__usage, kthread_t *, t,
2048             size_t, stksz, size_t, percent);
2049 
2050         if (percent == 0) {
2051                 return;
2052         }
2053 
2054         mutex_enter(&kmem_stkinfo_lock);
2055         if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2056                 /*
2057                  * The log is full and already contains the highest values
2058                  */
2059                 mutex_exit(&kmem_stkinfo_lock);
2060                 return;
2061         }
2062 
2063         /* keep a log of the highest used stack */
2064         for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2065                 if (kmem_stkinfo_log[i].percent == 0) {
2066                         index = i;
2067                         full++;
2068                         break;
2069                 }
2070                 if (smallest == 0) {
2071                         smallest = kmem_stkinfo_log[i].percent;
2072                         index = i;
2073                         continue;
2074                 }
2075                 if (kmem_stkinfo_log[i].percent < smallest) {
2076                         smallest = kmem_stkinfo_log[i].percent;
2077                         index = i;
2078                 }
2079         }
2080 
2081         if (percent >= kmem_stkinfo_log[index].percent) {
2082                 kmem_stkinfo_log[index].kthread = (caddr_t)t;
2083                 kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2084                 kmem_stkinfo_log[index].start = start;
2085                 kmem_stkinfo_log[index].stksz = stksz;
2086                 kmem_stkinfo_log[index].percent = percent;
2087                 kmem_stkinfo_log[index].t_tid = t->t_tid;
2088                 kmem_stkinfo_log[index].cmd[0] = '\0';
2089                 if (t->t_tid != 0) {
2090                         stksz = strlen((t->t_procp)->p_user.u_comm);
2091                         if (stksz >= KMEM_STKINFO_STR_SIZE) {
2092                                 stksz = KMEM_STKINFO_STR_SIZE - 1;
2093                                 kmem_stkinfo_log[index].cmd[stksz] = '\0';
2094                         } else {
2095                                 stksz += 1;
2096                         }
2097                         (void) memcpy(kmem_stkinfo_log[index].cmd,
2098                             (t->t_procp)->p_user.u_comm, stksz);
2099                 }
2100                 if (percent < smallest_percent) {
2101                         smallest_percent = percent;
2102                 }
2103         }
2104         mutex_exit(&kmem_stkinfo_lock);
2105 }
2106 
2107 /*
2108  * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2109  */
2110 static size_t
2111 stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2112 {
2113         size_t percent;
2114         size_t s;
2115 
2116         if (t_stk > t_stkbase) {
2117                 /* stack grows down */
2118                 if (sp > t_stk) {
2119                         return (0);
2120                 }
2121                 if (sp < t_stkbase) {
2122                         return (100);
2123                 }
2124                 percent = t_stk - sp + 1;
2125                 s = t_stk - t_stkbase + 1;
2126         } else {
2127                 /* stack grows up */
2128                 if (sp < t_stk) {
2129                         return (0);
2130                 }
2131                 if (sp > t_stkbase) {
2132                         return (100);
2133                 }
2134                 percent = sp - t_stk + 1;
2135                 s = t_stkbase - t_stk + 1;
2136         }
2137         percent = ((100 * percent) / s) + 1;
2138         if (percent > 100) {
2139                 percent = 100;
2140         }
2141         return (percent);
2142 }
2143 
2144 /*
2145  * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2146  * long.  It is expected that callers (acting on behalf of userland clients)
2147  * will perform any required checks to return the correct error semantics.
2148  * It is also expected callers on behalf of userland clients have done
2149  * any necessary permission checks.
2150  */
2151 int
2152 thread_setname(kthread_t *t, const char *name)
2153 {
2154         char *buf = NULL;
2155 
2156         /*
2157          * We optimistically assume that a thread's name will only be set
2158          * once and so allocate memory in preparation of setting t_name.
2159          * If it turns out a name has already been set, we just discard (free)
2160          * the buffer we just allocated and reuse the current buffer
2161          * (as all should be THREAD_NAME_MAX large).
2162          *
2163          * Such an arrangement means over the lifetime of a kthread_t, t_name
2164          * is either NULL or has one value (the address of the buffer holding
2165          * the current thread name).   The assumption is that most kthread_t
2166          * instances will not have a name assigned, so dynamically allocating
2167          * the memory should minimize the footprint of this feature, but by
2168          * having the buffer persist for the life of the thread, it simplifies
2169          * usage in highly constrained situations (e.g. dtrace).
2170          */
2171         if (name != NULL && name[0] != '\0') {
2172                 for (size_t i = 0; name[i] != '\0'; i++) {
2173                         if (!isprint(name[i]))
2174                                 return (EINVAL);
2175                 }
2176 
2177                 buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2178                 (void) strlcpy(buf, name, THREAD_NAME_MAX);
2179         }
2180 
2181         mutex_enter(&ttoproc(t)->p_lock);
2182         if (t->t_name == NULL) {
2183                 t->t_name = buf;
2184         } else {
2185                 if (buf != NULL) {
2186                         (void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2187                         kmem_free(buf, THREAD_NAME_MAX);
2188                 } else {
2189                         bzero(t->t_name, THREAD_NAME_MAX);
2190                 }
2191         }
2192         mutex_exit(&ttoproc(t)->p_lock);
2193         return (0);
2194 }
2195 
2196 int
2197 thread_vsetname(kthread_t *t, const char *fmt, ...)
2198 {
2199         char name[THREAD_NAME_MAX];
2200         va_list va;
2201         int rc;
2202 
2203         va_start(va, fmt);
2204         rc = vsnprintf(name, sizeof (name), fmt, va);
2205         va_end(va);
2206 
2207         if (rc < 0)
2208                 return (EINVAL);
2209 
2210         if (rc >= sizeof (name))
2211                 return (ENAMETOOLONG);
2212 
2213         return (thread_setname(t, name));
2214 }
--- EOF ---