1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2016 by Delphix. All rights reserved.
  25  */
  26 /*
  27  * Copyright 2016 Joyent, Inc.
  28  */
  29 
  30 #include "lint.h"
  31 #include "thr_uberdata.h"
  32 #include <pthread.h>
  33 #include <procfs.h>
  34 #include <sys/uio.h>
  35 #include <ctype.h>
  36 #include "libc.h"
  37 
  38 /*
  39  * These symbols should not be exported from libc, but
  40  * /lib/libm.so.2 references _thr_main.  libm needs to be fixed.
  41  * Also, some older versions of the Studio compiler/debugger
  42  * components reference them.  These need to be fixed, too.
  43  */
  44 #pragma weak _thr_main = thr_main
  45 #pragma weak _thr_create = thr_create
  46 #pragma weak _thr_join = thr_join
  47 #pragma weak _thr_self = thr_self
  48 
  49 #undef errno
  50 extern int errno;
  51 
  52 /*
  53  * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
  54  * "we are linked with libthread".  The Sun Workshop 6 update 1 compilation
  55  * system used it illegally (it is a consolidation private symbol).
  56  * To accommodate this and possibly other abusers of the symbol,
  57  * we make it always equal to 1 now that libthread has been folded
  58  * into libc.  The new __libc_threaded symbol is used to indicate
  59  * the new meaning, "more than one thread exists".
  60  */
  61 int __threaded = 1;             /* always equal to 1 */
  62 int __libc_threaded = 0;        /* zero until first thr_create() */
  63 
  64 /*
  65  * thr_concurrency and pthread_concurrency are not used by the library.
  66  * They exist solely to hold and return the values set by calls to
  67  * thr_setconcurrency() and pthread_setconcurrency().
  68  * Because thr_concurrency is affected by the THR_NEW_LWP flag
  69  * to thr_create(), thr_concurrency is protected by link_lock.
  70  */
  71 static  int     thr_concurrency = 1;
  72 static  int     pthread_concurrency;
  73 
  74 #define HASHTBLSZ       1024    /* must be a power of two */
  75 #define TIDHASH(tid, udp)       (tid & (udp)->hash_mask)
  76 
  77 /* initial allocation, just enough for one lwp */
  78 #pragma align 64(init_hash_table)
  79 thr_hash_table_t init_hash_table[1] = {
  80         { DEFAULTMUTEX, DEFAULTCV, NULL },
  81 };
  82 
  83 extern const Lc_interface rtld_funcs[];
  84 
  85 /*
  86  * The weak version is known to libc_db and mdb.
  87  */
  88 #pragma weak _uberdata = __uberdata
  89 uberdata_t __uberdata = {
  90         { DEFAULTMUTEX, NULL, 0 },      /* link_lock */
  91         { RECURSIVEMUTEX, NULL, 0 },    /* ld_lock */
  92         { RECURSIVEMUTEX, NULL, 0 },    /* fork_lock */
  93         { RECURSIVEMUTEX, NULL, 0 },    /* atfork_lock */
  94         { RECURSIVEMUTEX, NULL, 0 },    /* callout_lock */
  95         { DEFAULTMUTEX, NULL, 0 },      /* tdb_hash_lock */
  96         { 0, },                         /* tdb_hash_lock_stats */
  97         { { 0 }, },                     /* siguaction[NSIG] */
  98         {{ DEFAULTMUTEX, NULL, 0 },             /* bucket[NBUCKETS] */
  99         { DEFAULTMUTEX, NULL, 0 },
 100         { DEFAULTMUTEX, NULL, 0 },
 101         { DEFAULTMUTEX, NULL, 0 },
 102         { DEFAULTMUTEX, NULL, 0 },
 103         { DEFAULTMUTEX, NULL, 0 },
 104         { DEFAULTMUTEX, NULL, 0 },
 105         { DEFAULTMUTEX, NULL, 0 },
 106         { DEFAULTMUTEX, NULL, 0 },
 107         { DEFAULTMUTEX, NULL, 0 }},
 108         { RECURSIVEMUTEX, NULL, NULL },         /* atexit_root */
 109         { RECURSIVEMUTEX, NULL },               /* quickexit_root */
 110         { DEFAULTMUTEX, 0, 0, NULL },           /* tsd_metadata */
 111         { DEFAULTMUTEX, {0, 0}, {0, 0} },       /* tls_metadata */
 112         0,                      /* primary_map */
 113         0,                      /* bucket_init */
 114         0,                      /* pad[0] */
 115         0,                      /* pad[1] */
 116         { 0 },                  /* uberflags */
 117         NULL,                   /* queue_head */
 118         init_hash_table,        /* thr_hash_table */
 119         1,                      /* hash_size: size of the hash table */
 120         0,                      /* hash_mask: hash_size - 1 */
 121         NULL,                   /* ulwp_one */
 122         NULL,                   /* all_lwps */
 123         NULL,                   /* all_zombies */
 124         0,                      /* nthreads */
 125         0,                      /* nzombies */
 126         0,                      /* ndaemons */
 127         0,                      /* pid */
 128         sigacthandler,          /* sigacthandler */
 129         NULL,                   /* lwp_stacks */
 130         NULL,                   /* lwp_laststack */
 131         0,                      /* nfreestack */
 132         10,                     /* thread_stack_cache */
 133         NULL,                   /* ulwp_freelist */
 134         NULL,                   /* ulwp_lastfree */
 135         NULL,                   /* ulwp_replace_free */
 136         NULL,                   /* ulwp_replace_last */
 137         NULL,                   /* atforklist */
 138         NULL,                   /* robustlocks */
 139         NULL,                   /* robustlist */
 140         NULL,                   /* progname */
 141         NULL,                   /* ub_comm_page */
 142         NULL,                   /* __tdb_bootstrap */
 143         {                       /* tdb */
 144                 NULL,           /* tdb_sync_addr_hash */
 145                 0,              /* tdb_register_count */
 146                 0,              /* tdb_hash_alloc_failed */
 147                 NULL,           /* tdb_sync_addr_free */
 148                 NULL,           /* tdb_sync_addr_last */
 149                 0,              /* tdb_sync_alloc */
 150                 { 0, 0 },       /* tdb_ev_global_mask */
 151                 tdb_events,     /* tdb_events array */
 152         },
 153 };
 154 
 155 /*
 156  * The weak version is known to libc_db and mdb.
 157  */
 158 #pragma weak _tdb_bootstrap = __tdb_bootstrap
 159 uberdata_t **__tdb_bootstrap = NULL;
 160 
 161 int     thread_queue_fifo = 4;
 162 int     thread_queue_dump = 0;
 163 int     thread_cond_wait_defer = 0;
 164 int     thread_error_detection = 0;
 165 int     thread_async_safe = 0;
 166 int     thread_stack_cache = 10;
 167 int     thread_door_noreserve = 0;
 168 int     thread_locks_misaligned = 0;
 169 
 170 static  ulwp_t  *ulwp_alloc(void);
 171 static  void    ulwp_free(ulwp_t *);
 172 
 173 /*
 174  * Insert the lwp into the hash table.
 175  */
 176 void
 177 hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
 178 {
 179         ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
 180         udp->thr_hash_table[ix].hash_bucket = ulwp;
 181         ulwp->ul_ix = ix;
 182 }
 183 
 184 void
 185 hash_in(ulwp_t *ulwp, uberdata_t *udp)
 186 {
 187         int ix = TIDHASH(ulwp->ul_lwpid, udp);
 188         mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 189 
 190         lmutex_lock(mp);
 191         hash_in_unlocked(ulwp, ix, udp);
 192         lmutex_unlock(mp);
 193 }
 194 
 195 /*
 196  * Delete the lwp from the hash table.
 197  */
 198 void
 199 hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
 200 {
 201         ulwp_t **ulwpp;
 202 
 203         for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
 204             ulwp != *ulwpp;
 205             ulwpp = &(*ulwpp)->ul_hash)
 206                 ;
 207         *ulwpp = ulwp->ul_hash;
 208         ulwp->ul_hash = NULL;
 209         ulwp->ul_ix = -1;
 210 }
 211 
 212 void
 213 hash_out(ulwp_t *ulwp, uberdata_t *udp)
 214 {
 215         int ix;
 216 
 217         if ((ix = ulwp->ul_ix) >= 0) {
 218                 mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 219 
 220                 lmutex_lock(mp);
 221                 hash_out_unlocked(ulwp, ix, udp);
 222                 lmutex_unlock(mp);
 223         }
 224 }
 225 
 226 /*
 227  * Retain stack information for thread structures that are being recycled for
 228  * new threads.  All other members of the thread structure should be zeroed.
 229  */
 230 static void
 231 ulwp_clean(ulwp_t *ulwp)
 232 {
 233         caddr_t stk = ulwp->ul_stk;
 234         size_t mapsiz = ulwp->ul_mapsiz;
 235         size_t guardsize = ulwp->ul_guardsize;
 236         uintptr_t stktop = ulwp->ul_stktop;
 237         size_t stksiz = ulwp->ul_stksiz;
 238 
 239         (void) memset(ulwp, 0, sizeof (*ulwp));
 240 
 241         ulwp->ul_stk = stk;
 242         ulwp->ul_mapsiz = mapsiz;
 243         ulwp->ul_guardsize = guardsize;
 244         ulwp->ul_stktop = stktop;
 245         ulwp->ul_stksiz = stksiz;
 246 }
 247 
 248 static int stackprot;
 249 
 250 /*
 251  * Answer the question, "Is the lwp in question really dead?"
 252  * We must inquire of the operating system to be really sure
 253  * because the lwp may have called lwp_exit() but it has not
 254  * yet completed the exit.
 255  */
 256 static int
 257 dead_and_buried(ulwp_t *ulwp)
 258 {
 259         if (ulwp->ul_lwpid == (lwpid_t)(-1))
 260                 return (1);
 261         if (ulwp->ul_dead && ulwp->ul_detached &&
 262             _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
 263                 ulwp->ul_lwpid = (lwpid_t)(-1);
 264                 return (1);
 265         }
 266         return (0);
 267 }
 268 
 269 /*
 270  * Attempt to keep the stack cache within the specified cache limit.
 271  */
 272 static void
 273 trim_stack_cache(int cache_limit)
 274 {
 275         ulwp_t *self = curthread;
 276         uberdata_t *udp = self->ul_uberdata;
 277         ulwp_t *prev = NULL;
 278         ulwp_t **ulwpp = &udp->lwp_stacks;
 279         ulwp_t *ulwp;
 280 
 281         ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));
 282 
 283         while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
 284                 if (dead_and_buried(ulwp)) {
 285                         *ulwpp = ulwp->ul_next;
 286                         if (ulwp == udp->lwp_laststack)
 287                                 udp->lwp_laststack = prev;
 288                         hash_out(ulwp, udp);
 289                         udp->nfreestack--;
 290                         (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
 291                         /*
 292                          * Now put the free ulwp on the ulwp freelist.
 293                          */
 294                         ulwp->ul_mapsiz = 0;
 295                         ulwp->ul_next = NULL;
 296                         if (udp->ulwp_freelist == NULL)
 297                                 udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
 298                         else {
 299                                 udp->ulwp_lastfree->ul_next = ulwp;
 300                                 udp->ulwp_lastfree = ulwp;
 301                         }
 302                 } else {
 303                         prev = ulwp;
 304                         ulwpp = &ulwp->ul_next;
 305                 }
 306         }
 307 }
 308 
 309 /*
 310  * Find an unused stack of the requested size
 311  * or create a new stack of the requested size.
 312  * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
 313  * thr_exit() stores 1 in the ul_dead member.
 314  * thr_join() stores -1 in the ul_lwpid member.
 315  */
 316 static ulwp_t *
 317 find_stack(size_t stksize, size_t guardsize)
 318 {
 319         static size_t pagesize = 0;
 320 
 321         uberdata_t *udp = curthread->ul_uberdata;
 322         size_t mapsize;
 323         ulwp_t *prev;
 324         ulwp_t *ulwp;
 325         ulwp_t **ulwpp;
 326         void *stk;
 327 
 328         /*
 329          * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
 330          * unless overridden by the system's configuration.
 331          */
 332         if (stackprot == 0) {   /* do this once */
 333                 long lprot = _sysconf(_SC_STACK_PROT);
 334                 if (lprot <= 0)
 335                         lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
 336                 stackprot = (int)lprot;
 337         }
 338         if (pagesize == 0)      /* do this once */
 339                 pagesize = _sysconf(_SC_PAGESIZE);
 340 
 341         /*
 342          * One megabyte stacks by default, but subtract off
 343          * two pages for the system-created red zones.
 344          * Round up a non-zero stack size to a pagesize multiple.
 345          */
 346         if (stksize == 0)
 347                 stksize = DEFAULTSTACK - 2 * pagesize;
 348         else
 349                 stksize = ((stksize + pagesize - 1) & -pagesize);
 350 
 351         /*
 352          * Round up the mapping size to a multiple of pagesize.
 353          * Note: mmap() provides at least one page of red zone
 354          * so we deduct that from the value of guardsize.
 355          */
 356         if (guardsize != 0)
 357                 guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
 358         mapsize = stksize + guardsize;
 359 
 360         lmutex_lock(&udp->link_lock);
 361         for (prev = NULL, ulwpp = &udp->lwp_stacks;
 362             (ulwp = *ulwpp) != NULL;
 363             prev = ulwp, ulwpp = &ulwp->ul_next) {
 364                 if (ulwp->ul_mapsiz == mapsize &&
 365                     ulwp->ul_guardsize == guardsize &&
 366                     dead_and_buried(ulwp)) {
 367                         /*
 368                          * The previous lwp is gone; reuse the stack.
 369                          * Remove the ulwp from the stack list.
 370                          */
 371                         *ulwpp = ulwp->ul_next;
 372                         ulwp->ul_next = NULL;
 373                         if (ulwp == udp->lwp_laststack)
 374                                 udp->lwp_laststack = prev;
 375                         hash_out(ulwp, udp);
 376                         udp->nfreestack--;
 377                         lmutex_unlock(&udp->link_lock);
 378                         ulwp_clean(ulwp);
 379                         return (ulwp);
 380                 }
 381         }
 382 
 383         /*
 384          * None of the cached stacks matched our mapping size.
 385          * Reduce the stack cache to get rid of possibly
 386          * very old stacks that will never be reused.
 387          */
 388         if (udp->nfreestack > udp->thread_stack_cache)
 389                 trim_stack_cache(udp->thread_stack_cache);
 390         else if (udp->nfreestack > 0)
 391                 trim_stack_cache(udp->nfreestack - 1);
 392         lmutex_unlock(&udp->link_lock);
 393 
 394         /*
 395          * Create a new stack.
 396          */
 397         if ((stk = mmap(NULL, mapsize, stackprot,
 398             MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
 399                 /*
 400                  * We have allocated our stack.  Now allocate the ulwp.
 401                  */
 402                 ulwp = ulwp_alloc();
 403                 if (ulwp == NULL)
 404                         (void) munmap(stk, mapsize);
 405                 else {
 406                         ulwp->ul_stk = stk;
 407                         ulwp->ul_mapsiz = mapsize;
 408                         ulwp->ul_guardsize = guardsize;
 409                         ulwp->ul_stktop = (uintptr_t)stk + mapsize;
 410                         ulwp->ul_stksiz = stksize;
 411                         if (guardsize)  /* protect the extra red zone */
 412                                 (void) mprotect(stk, guardsize, PROT_NONE);
 413                 }
 414         }
 415         return (ulwp);
 416 }
 417 
 418 /*
 419  * Get a ulwp_t structure from the free list or allocate a new one.
 420  * Such ulwp_t's do not have a stack allocated by the library.
 421  */
 422 static ulwp_t *
 423 ulwp_alloc(void)
 424 {
 425         ulwp_t *self = curthread;
 426         uberdata_t *udp = self->ul_uberdata;
 427         size_t tls_size;
 428         ulwp_t *prev;
 429         ulwp_t *ulwp;
 430         ulwp_t **ulwpp;
 431         caddr_t data;
 432 
 433         lmutex_lock(&udp->link_lock);
 434         for (prev = NULL, ulwpp = &udp->ulwp_freelist;
 435             (ulwp = *ulwpp) != NULL;
 436             prev = ulwp, ulwpp = &ulwp->ul_next) {
 437                 if (dead_and_buried(ulwp)) {
 438                         *ulwpp = ulwp->ul_next;
 439                         ulwp->ul_next = NULL;
 440                         if (ulwp == udp->ulwp_lastfree)
 441                                 udp->ulwp_lastfree = prev;
 442                         hash_out(ulwp, udp);
 443                         lmutex_unlock(&udp->link_lock);
 444                         ulwp_clean(ulwp);
 445                         return (ulwp);
 446                 }
 447         }
 448         lmutex_unlock(&udp->link_lock);
 449 
 450         tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
 451         data = lmalloc(sizeof (*ulwp) + tls_size);
 452         if (data != NULL) {
 453                 /* LINTED pointer cast may result in improper alignment */
 454                 ulwp = (ulwp_t *)(data + tls_size);
 455         }
 456         return (ulwp);
 457 }
 458 
 459 /*
 460  * Free a ulwp structure.
 461  * If there is an associated stack, put it on the stack list and
 462  * munmap() previously freed stacks up to the residual cache limit.
 463  * Else put it on the ulwp free list and never call lfree() on it.
 464  */
 465 static void
 466 ulwp_free(ulwp_t *ulwp)
 467 {
 468         uberdata_t *udp = curthread->ul_uberdata;
 469 
 470         ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
 471         ulwp->ul_next = NULL;
 472         if (ulwp == udp->ulwp_one)   /* don't reuse the primoridal stack */
 473                 /*EMPTY*/;
 474         else if (ulwp->ul_mapsiz != 0) {
 475                 if (udp->lwp_stacks == NULL)
 476                         udp->lwp_stacks = udp->lwp_laststack = ulwp;
 477                 else {
 478                         udp->lwp_laststack->ul_next = ulwp;
 479                         udp->lwp_laststack = ulwp;
 480                 }
 481                 if (++udp->nfreestack > udp->thread_stack_cache)
 482                         trim_stack_cache(udp->thread_stack_cache);
 483         } else {
 484                 if (udp->ulwp_freelist == NULL)
 485                         udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
 486                 else {
 487                         udp->ulwp_lastfree->ul_next = ulwp;
 488                         udp->ulwp_lastfree = ulwp;
 489                 }
 490         }
 491 }
 492 
 493 /*
 494  * Find a named lwp and return a pointer to its hash list location.
 495  * On success, returns with the hash lock held.
 496  */
 497 ulwp_t **
 498 find_lwpp(thread_t tid)
 499 {
 500         uberdata_t *udp = curthread->ul_uberdata;
 501         int ix = TIDHASH(tid, udp);
 502         mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
 503         ulwp_t *ulwp;
 504         ulwp_t **ulwpp;
 505 
 506         if (tid == 0)
 507                 return (NULL);
 508 
 509         lmutex_lock(mp);
 510         for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
 511             (ulwp = *ulwpp) != NULL;
 512             ulwpp = &ulwp->ul_hash) {
 513                 if (ulwp->ul_lwpid == tid)
 514                         return (ulwpp);
 515         }
 516         lmutex_unlock(mp);
 517         return (NULL);
 518 }
 519 
 520 /*
 521  * Wake up all lwps waiting on this lwp for some reason.
 522  */
 523 void
 524 ulwp_broadcast(ulwp_t *ulwp)
 525 {
 526         ulwp_t *self = curthread;
 527         uberdata_t *udp = self->ul_uberdata;
 528 
 529         ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
 530         (void) cond_broadcast(ulwp_condvar(ulwp, udp));
 531 }
 532 
 533 /*
 534  * Find a named lwp and return a pointer to it.
 535  * Returns with the hash lock held.
 536  */
 537 ulwp_t *
 538 find_lwp(thread_t tid)
 539 {
 540         ulwp_t *self = curthread;
 541         uberdata_t *udp = self->ul_uberdata;
 542         ulwp_t *ulwp = NULL;
 543         ulwp_t **ulwpp;
 544 
 545         if (self->ul_lwpid == tid) {
 546                 ulwp = self;
 547                 ulwp_lock(ulwp, udp);
 548         } else if ((ulwpp = find_lwpp(tid)) != NULL) {
 549                 ulwp = *ulwpp;
 550         }
 551 
 552         if (ulwp && ulwp->ul_dead) {
 553                 ulwp_unlock(ulwp, udp);
 554                 ulwp = NULL;
 555         }
 556 
 557         return (ulwp);
 558 }
 559 
 560 int
 561 _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
 562     long flags, thread_t *new_thread, size_t guardsize)
 563 {
 564         ulwp_t *self = curthread;
 565         uberdata_t *udp = self->ul_uberdata;
 566         ucontext_t uc;
 567         uint_t lwp_flags;
 568         thread_t tid;
 569         int error;
 570         ulwp_t *ulwp;
 571 
 572         /*
 573          * Enforce the restriction of not creating any threads
 574          * until the primary link map has been initialized.
 575          * Also, disallow thread creation to a child of vfork().
 576          */
 577         if (!self->ul_primarymap || self->ul_vfork)
 578                 return (ENOTSUP);
 579 
 580         if (udp->hash_size == 1)
 581                 finish_init();
 582 
 583         if ((stk || stksize) && stksize < MINSTACK)
 584                 return (EINVAL);
 585 
 586         if (stk == NULL) {
 587                 if ((ulwp = find_stack(stksize, guardsize)) == NULL)
 588                         return (ENOMEM);
 589                 stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
 590         } else {
 591                 /* initialize the private stack */
 592                 if ((ulwp = ulwp_alloc()) == NULL)
 593                         return (ENOMEM);
 594                 ulwp->ul_stk = stk;
 595                 ulwp->ul_stktop = (uintptr_t)stk + stksize;
 596                 ulwp->ul_stksiz = stksize;
 597         }
 598         /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
 599         ulwp->ul_ix = -1;
 600         ulwp->ul_errnop = &ulwp->ul_errno;
 601 
 602         lwp_flags = LWP_SUSPENDED;
 603         if (flags & (THR_DETACHED|THR_DAEMON)) {
 604                 flags |= THR_DETACHED;
 605                 lwp_flags |= LWP_DETACHED;
 606         }
 607         if (flags & THR_DAEMON)
 608                 lwp_flags |= LWP_DAEMON;
 609 
 610         /* creating a thread: enforce mt-correctness in mutex_lock() */
 611         self->ul_async_safe = 1;
 612 
 613         /* per-thread copies of global variables, for speed */
 614         ulwp->ul_queue_fifo = self->ul_queue_fifo;
 615         ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
 616         ulwp->ul_error_detection = self->ul_error_detection;
 617         ulwp->ul_async_safe = self->ul_async_safe;
 618         ulwp->ul_max_spinners = self->ul_max_spinners;
 619         ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
 620         ulwp->ul_queue_spin = self->ul_queue_spin;
 621         ulwp->ul_door_noreserve = self->ul_door_noreserve;
 622         ulwp->ul_misaligned = self->ul_misaligned;
 623 
 624         /* new thread inherits creating thread's scheduling parameters */
 625         ulwp->ul_policy = self->ul_policy;
 626         ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
 627         ulwp->ul_cid = self->ul_cid;
 628         ulwp->ul_rtclassid = self->ul_rtclassid;
 629 
 630         ulwp->ul_primarymap = self->ul_primarymap;
 631         ulwp->ul_self = ulwp;
 632         ulwp->ul_uberdata = udp;
 633 
 634         /* debugger support */
 635         ulwp->ul_usropts = flags;
 636 
 637 #ifdef __sparc
 638         /*
 639          * We cache several instructions in the thread structure for use
 640          * by the fasttrap DTrace provider. When changing this, read the
 641          * comment in fasttrap.h for the all the other places that must
 642          * be changed.
 643          */
 644         ulwp->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
 645         ulwp->ul_drestore = 0x81e80000;      /* restore %g0, %g0, %g0 */
 646         ulwp->ul_dftret = 0x91d0203a;        /* ta 0x3a */
 647         ulwp->ul_dreturn = 0x81ca0000;       /* return %o0 */
 648 #endif
 649 
 650         ulwp->ul_startpc = func;
 651         ulwp->ul_startarg = arg;
 652         _fpinherit(ulwp);
 653         /*
 654          * Defer signals on the new thread until its TLS constructors
 655          * have been called.  _thrp_setup() will call sigon() after
 656          * it has called tls_setup().
 657          */
 658         ulwp->ul_sigdefer = 1;
 659 
 660         error = setup_context(&uc, _thrp_setup, ulwp,
 661             (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
 662         if (error != 0 && stk != NULL)  /* inaccessible stack */
 663                 error = EFAULT;
 664 
 665         /*
 666          * Call enter_critical() to avoid being suspended until we
 667          * have linked the new thread into the proper lists.
 668          * This is necessary because forkall() and fork1() must
 669          * suspend all threads and they must see a complete list.
 670          */
 671         enter_critical(self);
 672         uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
 673         if (error != 0 ||
 674             (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
 675                 exit_critical(self);
 676                 ulwp->ul_lwpid = (lwpid_t)(-1);
 677                 ulwp->ul_dead = 1;
 678                 ulwp->ul_detached = 1;
 679                 lmutex_lock(&udp->link_lock);
 680                 ulwp_free(ulwp);
 681                 lmutex_unlock(&udp->link_lock);
 682                 return (error);
 683         }
 684         self->ul_nocancel = 0;       /* cancellation is now possible */
 685         udp->uberflags.uf_mt = 1;
 686         if (new_thread)
 687                 *new_thread = tid;
 688         if (flags & THR_DETACHED)
 689                 ulwp->ul_detached = 1;
 690         ulwp->ul_lwpid = tid;
 691         ulwp->ul_stop = TSTP_REGULAR;
 692         if (flags & THR_SUSPENDED)
 693                 ulwp->ul_created = 1;
 694 
 695         lmutex_lock(&udp->link_lock);
 696         ulwp->ul_forw = udp->all_lwps;
 697         ulwp->ul_back = udp->all_lwps->ul_back;
 698         ulwp->ul_back->ul_forw = ulwp;
 699         ulwp->ul_forw->ul_back = ulwp;
 700         hash_in(ulwp, udp);
 701         udp->nthreads++;
 702         if (flags & THR_DAEMON)
 703                 udp->ndaemons++;
 704         if (flags & THR_NEW_LWP)
 705                 thr_concurrency++;
 706         __libc_threaded = 1;            /* inform stdio */
 707         lmutex_unlock(&udp->link_lock);
 708 
 709         if (__td_event_report(self, TD_CREATE, udp)) {
 710                 self->ul_td_evbuf.eventnum = TD_CREATE;
 711                 self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
 712                 tdb_event(TD_CREATE, udp);
 713         }
 714 
 715         exit_critical(self);
 716 
 717         if (!(flags & THR_SUSPENDED))
 718                 (void) _thrp_continue(tid, TSTP_REGULAR);
 719 
 720         return (0);
 721 }
 722 
 723 int
 724 thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
 725     long flags, thread_t *new_thread)
 726 {
 727         return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0));
 728 }
 729 
 730 /*
 731  * A special cancellation cleanup hook for DCE.
 732  * cleanuphndlr, when it is not NULL, will contain a callback
 733  * function to be called before a thread is terminated in
 734  * thr_exit() as a result of being cancelled.
 735  */
 736 static void (*cleanuphndlr)(void) = NULL;
 737 
 738 /*
 739  * _pthread_setcleanupinit: sets the cleanup hook.
 740  */
 741 int
 742 _pthread_setcleanupinit(void (*func)(void))
 743 {
 744         cleanuphndlr = func;
 745         return (0);
 746 }
 747 
 748 void
 749 _thrp_exit()
 750 {
 751         ulwp_t *self = curthread;
 752         uberdata_t *udp = self->ul_uberdata;
 753         ulwp_t *replace = NULL;
 754 
 755         if (__td_event_report(self, TD_DEATH, udp)) {
 756                 self->ul_td_evbuf.eventnum = TD_DEATH;
 757                 tdb_event(TD_DEATH, udp);
 758         }
 759 
 760         ASSERT(self->ul_sigdefer != 0);
 761 
 762         lmutex_lock(&udp->link_lock);
 763         udp->nthreads--;
 764         if (self->ul_usropts & THR_NEW_LWP)
 765                 thr_concurrency--;
 766         if (self->ul_usropts & THR_DAEMON)
 767                 udp->ndaemons--;
 768         else if (udp->nthreads == udp->ndaemons) {
 769                 /*
 770                  * We are the last non-daemon thread exiting.
 771                  * Exit the process.  We retain our TSD and TLS so
 772                  * that atexit() application functions can use them.
 773                  */
 774                 lmutex_unlock(&udp->link_lock);
 775                 exit(0);
 776                 thr_panic("_thrp_exit(): exit(0) returned");
 777         }
 778         lmutex_unlock(&udp->link_lock);
 779 
 780         tmem_exit();            /* deallocate tmem allocations */
 781         tsd_exit();             /* deallocate thread-specific data */
 782         tls_exit();             /* deallocate thread-local storage */
 783         heldlock_exit();        /* deal with left-over held locks */
 784 
 785         /* block all signals to finish exiting */
 786         block_all_signals(self);
 787         /* also prevent ourself from being suspended */
 788         enter_critical(self);
 789         rwl_free(self);
 790         lmutex_lock(&udp->link_lock);
 791         ulwp_free(self);
 792         (void) ulwp_lock(self, udp);
 793 
 794         if (self->ul_mapsiz && !self->ul_detached) {
 795                 /*
 796                  * We want to free the stack for reuse but must keep
 797                  * the ulwp_t struct for the benefit of thr_join().
 798                  * For this purpose we allocate a replacement ulwp_t.
 799                  */
 800                 if ((replace = udp->ulwp_replace_free) == NULL)
 801                         replace = lmalloc(REPLACEMENT_SIZE);
 802                 else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
 803                         udp->ulwp_replace_last = NULL;
 804         }
 805 
 806         if (udp->all_lwps == self)
 807                 udp->all_lwps = self->ul_forw;
 808         if (udp->all_lwps == self)
 809                 udp->all_lwps = NULL;
 810         else {
 811                 self->ul_forw->ul_back = self->ul_back;
 812                 self->ul_back->ul_forw = self->ul_forw;
 813         }
 814         self->ul_forw = self->ul_back = NULL;
 815 #if defined(THREAD_DEBUG)
 816         /* collect queue lock statistics before marking ourself dead */
 817         record_spin_locks(self);
 818 #endif
 819         self->ul_dead = 1;
 820         self->ul_pleasestop = 0;
 821         if (replace != NULL) {
 822                 int ix = self->ul_ix;                /* the hash index */
 823                 (void) memcpy(replace, self, REPLACEMENT_SIZE);
 824                 replace->ul_self = replace;
 825                 replace->ul_next = NULL;     /* clone not on stack list */
 826                 replace->ul_mapsiz = 0;              /* allows clone to be freed */
 827                 replace->ul_replace = 1;     /* requires clone to be freed */
 828                 hash_out_unlocked(self, ix, udp);
 829                 hash_in_unlocked(replace, ix, udp);
 830                 ASSERT(!(self->ul_detached));
 831                 self->ul_detached = 1;               /* this frees the stack */
 832                 self->ul_schedctl = NULL;
 833                 self->ul_schedctl_called = &udp->uberflags;
 834                 set_curthread(self = replace);
 835                 /*
 836                  * Having just changed the address of curthread, we
 837                  * must reset the ownership of the locks we hold so
 838                  * that assertions will not fire when we release them.
 839                  */
 840                 udp->link_lock.mutex_owner = (uintptr_t)self;
 841                 ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
 842                 /*
 843                  * NOTE:
 844                  * On i386, %gs still references the original, not the
 845                  * replacement, ulwp structure.  Fetching the replacement
 846                  * curthread pointer via %gs:0 works correctly since the
 847                  * original ulwp structure will not be reallocated until
 848                  * this lwp has completed its lwp_exit() system call (see
 849                  * dead_and_buried()), but from here on out, we must make
 850                  * no references to %gs:<offset> other than %gs:0.
 851                  */
 852         }
 853         /*
 854          * Put non-detached terminated threads in the all_zombies list.
 855          */
 856         if (!self->ul_detached) {
 857                 udp->nzombies++;
 858                 if (udp->all_zombies == NULL) {
 859                         ASSERT(udp->nzombies == 1);
 860                         udp->all_zombies = self->ul_forw = self->ul_back = self;
 861                 } else {
 862                         self->ul_forw = udp->all_zombies;
 863                         self->ul_back = udp->all_zombies->ul_back;
 864                         self->ul_back->ul_forw = self;
 865                         self->ul_forw->ul_back = self;
 866                 }
 867         }
 868         /*
 869          * Notify everyone waiting for this thread.
 870          */
 871         ulwp_broadcast(self);
 872         (void) ulwp_unlock(self, udp);
 873         /*
 874          * Prevent any more references to the schedctl data.
 875          * We are exiting and continue_fork() may not find us.
 876          * Do this just before dropping link_lock, since fork
 877          * serializes on link_lock.
 878          */
 879         self->ul_schedctl = NULL;
 880         self->ul_schedctl_called = &udp->uberflags;
 881         lmutex_unlock(&udp->link_lock);
 882 
 883         ASSERT(self->ul_critical == 1);
 884         ASSERT(self->ul_preempt == 0);
 885         _lwp_terminate();       /* never returns */
 886         thr_panic("_thrp_exit(): _lwp_terminate() returned");
 887 }
 888 
 889 #if defined(THREAD_DEBUG)
 890 void
 891 collect_queue_statistics()
 892 {
 893         uberdata_t *udp = curthread->ul_uberdata;
 894         ulwp_t *ulwp;
 895 
 896         if (thread_queue_dump) {
 897                 lmutex_lock(&udp->link_lock);
 898                 if ((ulwp = udp->all_lwps) != NULL) {
 899                         do {
 900                                 record_spin_locks(ulwp);
 901                         } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
 902                 }
 903                 lmutex_unlock(&udp->link_lock);
 904         }
 905 }
 906 #endif
 907 
 908 static void __NORETURN
 909 _thrp_exit_common(void *status, int unwind)
 910 {
 911         ulwp_t *self = curthread;
 912         int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);
 913 
 914         ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);
 915 
 916         /*
 917          * Disable cancellation and call the special DCE cancellation
 918          * cleanup hook if it is enabled.  Do nothing else before calling
 919          * the DCE cancellation cleanup hook; it may call longjmp() and
 920          * never return here.
 921          */
 922         self->ul_cancel_disabled = 1;
 923         self->ul_cancel_async = 0;
 924         self->ul_save_async = 0;
 925         self->ul_cancelable = 0;
 926         self->ul_cancel_pending = 0;
 927         set_cancel_pending_flag(self, 1);
 928         if (cancelled && cleanuphndlr != NULL)
 929                 (*cleanuphndlr)();
 930 
 931         /*
 932          * Block application signals while we are exiting.
 933          * We call out to C++, TSD, and TLS destructors while exiting
 934          * and these are application-defined, so we cannot be assured
 935          * that they won't reset the signal mask.  We use sigoff() to
 936          * defer any signals that may be received as a result of this
 937          * bad behavior.  Such signals will be lost to the process
 938          * when the thread finishes exiting.
 939          */
 940         (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
 941         sigoff(self);
 942 
 943         self->ul_rval = status;
 944 
 945         /*
 946          * If thr_exit is being called from the places where
 947          * C++ destructors are to be called such as cancellation
 948          * points, then set this flag. It is checked in _t_cancel()
 949          * to decide whether _ex_unwind() is to be called or not.
 950          */
 951         if (unwind)
 952                 self->ul_unwind = 1;
 953 
 954         /*
 955          * _thrp_unwind() will eventually call _thrp_exit().
 956          * It never returns.
 957          */
 958         _thrp_unwind(NULL);
 959         thr_panic("_thrp_exit_common(): _thrp_unwind() returned");
 960 
 961         for (;;)        /* to shut the compiler up about __NORETURN */
 962                 continue;
 963 }
 964 
 965 /*
 966  * Called when a thread returns from its start function.
 967  * We are at the top of the stack; no unwinding is necessary.
 968  */
 969 void
 970 _thrp_terminate(void *status)
 971 {
 972         _thrp_exit_common(status, 0);
 973 }
 974 
 975 #pragma weak pthread_exit = thr_exit
 976 #pragma weak _thr_exit = thr_exit
 977 void
 978 thr_exit(void *status)
 979 {
 980         _thrp_exit_common(status, 1);
 981 }
 982 
 983 int
 984 _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
 985 {
 986         uberdata_t *udp = curthread->ul_uberdata;
 987         mutex_t *mp;
 988         void *rval;
 989         thread_t found;
 990         ulwp_t *ulwp;
 991         ulwp_t **ulwpp;
 992         int replace;
 993         int error;
 994 
 995         if (do_cancel)
 996                 error = lwp_wait(tid, &found);
 997         else {
 998                 while ((error = __lwp_wait(tid, &found)) == EINTR)
 999                         ;
1000         }
1001         if (error)
1002                 return (error);
1003 
1004         /*
1005          * We must hold link_lock to avoid a race condition with find_stack().
1006          */
1007         lmutex_lock(&udp->link_lock);
1008         if ((ulwpp = find_lwpp(found)) == NULL) {
1009                 /*
1010                  * lwp_wait() found an lwp that the library doesn't know
1011                  * about.  It must have been created with _lwp_create().
1012                  * Just return its lwpid; we can't know its status.
1013                  */
1014                 lmutex_unlock(&udp->link_lock);
1015                 rval = NULL;
1016         } else {
1017                 /*
1018                  * Remove ulwp from the hash table.
1019                  */
1020                 ulwp = *ulwpp;
1021                 *ulwpp = ulwp->ul_hash;
1022                 ulwp->ul_hash = NULL;
1023                 /*
1024                  * Remove ulwp from all_zombies list.
1025                  */
1026                 ASSERT(udp->nzombies >= 1);
1027                 if (udp->all_zombies == ulwp)
1028                         udp->all_zombies = ulwp->ul_forw;
1029                 if (udp->all_zombies == ulwp)
1030                         udp->all_zombies = NULL;
1031                 else {
1032                         ulwp->ul_forw->ul_back = ulwp->ul_back;
1033                         ulwp->ul_back->ul_forw = ulwp->ul_forw;
1034                 }
1035                 ulwp->ul_forw = ulwp->ul_back = NULL;
1036                 udp->nzombies--;
1037                 ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
1038                     !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
1039                 /*
1040                  * We can't call ulwp_unlock(ulwp) after we set
1041                  * ulwp->ul_ix = -1 so we have to get a pointer to the
1042                  * ulwp's hash table mutex now in order to unlock it below.
1043                  */
1044                 mp = ulwp_mutex(ulwp, udp);
1045                 ulwp->ul_lwpid = (lwpid_t)(-1);
1046                 ulwp->ul_ix = -1;
1047                 rval = ulwp->ul_rval;
1048                 replace = ulwp->ul_replace;
1049                 lmutex_unlock(mp);
1050                 if (replace) {
1051                         ulwp->ul_next = NULL;
1052                         if (udp->ulwp_replace_free == NULL)
1053                                 udp->ulwp_replace_free =
1054                                     udp->ulwp_replace_last = ulwp;
1055                         else {
1056                                 udp->ulwp_replace_last->ul_next = ulwp;
1057                                 udp->ulwp_replace_last = ulwp;
1058                         }
1059                 }
1060                 lmutex_unlock(&udp->link_lock);
1061         }
1062 
1063         if (departed != NULL)
1064                 *departed = found;
1065         if (status != NULL)
1066                 *status = rval;
1067         return (0);
1068 }
1069 
1070 int
1071 thr_join(thread_t tid, thread_t *departed, void **status)
1072 {
1073         int error = _thrp_join(tid, departed, status, 1);
1074         return ((error == EINVAL)? ESRCH : error);
1075 }
1076 
1077 /*
1078  * pthread_join() differs from Solaris thr_join():
1079  * It does not return the departed thread's id
1080  * and hence does not have a "departed" argument.
1081  * It returns EINVAL if tid refers to a detached thread.
1082  */
1083 #pragma weak _pthread_join = pthread_join
1084 int
1085 pthread_join(pthread_t tid, void **status)
1086 {
1087         return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
1088 }
1089 
1090 int
1091 pthread_detach(pthread_t tid)
1092 {
1093         uberdata_t *udp = curthread->ul_uberdata;
1094         ulwp_t *ulwp;
1095         ulwp_t **ulwpp;
1096         int error = 0;
1097 
1098         if ((ulwpp = find_lwpp(tid)) == NULL)
1099                 return (ESRCH);
1100         ulwp = *ulwpp;
1101 
1102         if (ulwp->ul_dead) {
1103                 ulwp_unlock(ulwp, udp);
1104                 error = _thrp_join(tid, NULL, NULL, 0);
1105         } else {
1106                 error = __lwp_detach(tid);
1107                 ulwp->ul_detached = 1;
1108                 ulwp->ul_usropts |= THR_DETACHED;
1109                 ulwp_unlock(ulwp, udp);
1110         }
1111         return (error);
1112 }
1113 
1114 static const char *
1115 ematch(const char *ev, const char *match)
1116 {
1117         int c;
1118 
1119         while ((c = *match++) != '\0') {
1120                 if (*ev++ != c)
1121                         return (NULL);
1122         }
1123         if (*ev++ != '=')
1124                 return (NULL);
1125         return (ev);
1126 }
1127 
1128 static int
1129 envvar(const char *ev, const char *match, int limit)
1130 {
1131         int val = -1;
1132         const char *ename;
1133 
1134         if ((ename = ematch(ev, match)) != NULL) {
1135                 int c;
1136                 for (val = 0; (c = *ename) != '\0'; ename++) {
1137                         if (!isdigit(c)) {
1138                                 val = -1;
1139                                 break;
1140                         }
1141                         val = val * 10 + (c - '0');
1142                         if (val > limit) {
1143                                 val = limit;
1144                                 break;
1145                         }
1146                 }
1147         }
1148         return (val);
1149 }
1150 
1151 static void
1152 etest(const char *ev)
1153 {
1154         int value;
1155 
1156         if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
1157                 thread_queue_spin = value;
1158         if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
1159                 thread_adaptive_spin = value;
1160         if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
1161                 thread_max_spinners = value;
1162         if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
1163                 thread_queue_fifo = value;
1164 #if defined(THREAD_DEBUG)
1165         if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
1166                 thread_queue_verify = value;
1167         if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
1168                 thread_queue_dump = value;
1169 #endif
1170         if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
1171                 thread_stack_cache = value;
1172         if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
1173                 thread_cond_wait_defer = value;
1174         if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
1175                 thread_error_detection = value;
1176         if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
1177                 thread_async_safe = value;
1178         if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
1179                 thread_door_noreserve = value;
1180         if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
1181                 thread_locks_misaligned = value;
1182 }
1183 
1184 /*
1185  * Look for and evaluate environment variables of the form "_THREAD_*".
1186  * For compatibility with the past, we also look for environment
1187  * names of the form "LIBTHREAD_*".
1188  */
1189 static void
1190 set_thread_vars()
1191 {
1192         extern const char **_environ;
1193         const char **pev;
1194         const char *ev;
1195         char c;
1196 
1197         if ((pev = _environ) == NULL)
1198                 return;
1199         while ((ev = *pev++) != NULL) {
1200                 c = *ev;
1201                 if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
1202                         etest(ev + 8);
1203                 if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
1204                         etest(ev + 10);
1205         }
1206 }
1207 
1208 /* PROBE_SUPPORT begin */
1209 #pragma weak __tnf_probe_notify
1210 extern void __tnf_probe_notify(void);
1211 /* PROBE_SUPPORT end */
1212 
1213 /* same as atexit() but private to the library */
1214 extern int _atexit(void (*)(void));
1215 
1216 /* same as _cleanup() but private to the library */
1217 extern void __cleanup(void);
1218 
1219 extern void atfork_init(void);
1220 
1221 #ifdef __amd64
1222 extern void __proc64id(void);
1223 #endif
1224 
1225 static void
1226 init_auxv_data(uberdata_t *udp)
1227 {
1228         Dl_argsinfo_t args;
1229 
1230         udp->ub_comm_page = NULL;
1231         if (dlinfo(RTLD_SELF, RTLD_DI_ARGSINFO, &args) < 0)
1232                 return;
1233 
1234         while (args.dla_auxv->a_type != AT_NULL) {
1235                 if (args.dla_auxv->a_type == AT_SUN_COMMPAGE) {
1236                         udp->ub_comm_page = args.dla_auxv->a_un.a_ptr;
1237                 }
1238                 args.dla_auxv++;
1239         }
1240 }
1241 
1242 /*
1243  * libc_init() is called by ld.so.1 for library initialization.
1244  * We perform minimal initialization; enough to work with the main thread.
1245  */
1246 void
1247 libc_init(void)
1248 {
1249         uberdata_t *udp = &__uberdata;
1250         ulwp_t *oldself = __curthread();
1251         ucontext_t uc;
1252         ulwp_t *self;
1253         struct rlimit rl;
1254         caddr_t data;
1255         size_t tls_size;
1256         int setmask;
1257 
1258         /*
1259          * For the initial stage of initialization, we must be careful
1260          * not to call any function that could possibly call _cerror().
1261          * For this purpose, we call only the raw system call wrappers.
1262          */
1263 
1264 #ifdef __amd64
1265         /*
1266          * Gather information about cache layouts for optimized
1267          * AMD and Intel assembler strfoo() and memfoo() functions.
1268          */
1269         __proc64id();
1270 #endif
1271 
1272         /*
1273          * Every libc, regardless of which link map, must register __cleanup().
1274          */
1275         (void) _atexit(__cleanup);
1276 
1277         /*
1278          * Every libc, regardless of link map, needs to go through and check
1279          * its aux vectors.  Doing so will indicate whether or not this has
1280          * been given a comm page (to optimize certain system actions).
1281          */
1282         init_auxv_data(udp);
1283 
1284         /*
1285          * We keep our uberdata on one of (a) the first alternate link map
1286          * or (b) the primary link map.  We switch to the primary link map
1287          * and stay there once we see it.  All intermediate link maps are
1288          * subject to being unloaded at any time.
1289          */
1290         if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
1291                 __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
1292                 mutex_setup();
1293                 atfork_init();  /* every link map needs atfork() processing */
1294                 init_progname();
1295                 return;
1296         }
1297 
1298         /*
1299          * To establish the main stack information, we have to get our context.
1300          * This is also convenient to use for getting our signal mask.
1301          */
1302         uc.uc_flags = UC_ALL;
1303         (void) __getcontext(&uc);
1304         ASSERT(uc.uc_link == NULL);
1305 
1306         tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
1307         ASSERT(primary_link_map || tls_size == 0);
1308         data = lmalloc(sizeof (ulwp_t) + tls_size);
1309         if (data == NULL)
1310                 thr_panic("cannot allocate thread structure for main thread");
1311         /* LINTED pointer cast may result in improper alignment */
1312         self = (ulwp_t *)(data + tls_size);
1313         init_hash_table[0].hash_bucket = self;
1314 
1315         self->ul_sigmask = uc.uc_sigmask;
1316         delete_reserved_signals(&self->ul_sigmask);
1317         /*
1318          * Are the old and new sets different?
1319          * (This can happen if we are currently blocking SIGCANCEL.)
1320          * If so, we must explicitly set our signal mask, below.
1321          */
1322         setmask =
1323             ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
1324             (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
1325             (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
1326             (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));
1327 
1328 #ifdef __sparc
1329         /*
1330          * We cache several instructions in the thread structure for use
1331          * by the fasttrap DTrace provider. When changing this, read the
1332          * comment in fasttrap.h for the all the other places that must
1333          * be changed.
1334          */
1335         self->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
1336         self->ul_drestore = 0x81e80000;      /* restore %g0, %g0, %g0 */
1337         self->ul_dftret = 0x91d0203a;        /* ta 0x3a */
1338         self->ul_dreturn = 0x81ca0000;       /* return %o0 */
1339 #endif
1340 
1341         self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
1342         (void) getrlimit(RLIMIT_STACK, &rl);
1343         self->ul_stksiz = rl.rlim_cur;
1344         self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
1345 
1346         self->ul_forw = self->ul_back = self;
1347         self->ul_hash = NULL;
1348         self->ul_ix = 0;
1349         self->ul_lwpid = 1; /* _lwp_self() */
1350         self->ul_main = 1;
1351         self->ul_self = self;
1352         self->ul_policy = -1;                /* initialize only when needed */
1353         self->ul_pri = 0;
1354         self->ul_cid = 0;
1355         self->ul_rtclassid = -1;
1356         self->ul_uberdata = udp;
1357         if (oldself != NULL) {
1358                 int i;
1359 
1360                 ASSERT(primary_link_map);
1361                 ASSERT(oldself->ul_main == 1);
1362                 self->ul_stsd = oldself->ul_stsd;
1363                 for (i = 0; i < TSD_NFAST; i++)
1364                         self->ul_ftsd[i] = oldself->ul_ftsd[i];
1365                 self->ul_tls = oldself->ul_tls;
1366                 /*
1367                  * Retrieve all pointers to uberdata allocated
1368                  * while running on previous link maps.
1369                  * We would like to do a structure assignment here, but
1370                  * gcc turns structure assignments into calls to memcpy(),
1371                  * a function exported from libc.  We can't call any such
1372                  * external functions until we establish curthread, below,
1373                  * so we just call our private version of memcpy().
1374                  */
1375                 (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
1376                 /*
1377                  * These items point to global data on the primary link map.
1378                  */
1379                 udp->thr_hash_table = init_hash_table;
1380                 udp->sigacthandler = sigacthandler;
1381                 udp->tdb.tdb_events = tdb_events;
1382                 ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
1383                 ASSERT(udp->lwp_stacks == NULL);
1384                 ASSERT(udp->ulwp_freelist == NULL);
1385                 ASSERT(udp->ulwp_replace_free == NULL);
1386                 ASSERT(udp->hash_size == 1);
1387         }
1388         udp->all_lwps = self;
1389         udp->ulwp_one = self;
1390         udp->pid = getpid();
1391         udp->nthreads = 1;
1392         /*
1393          * In every link map, tdb_bootstrap points to the same piece of
1394          * allocated memory.  When the primary link map is initialized,
1395          * the allocated memory is assigned a pointer to the one true
1396          * uberdata.  This allows libc_db to initialize itself regardless
1397          * of which instance of libc it finds in the address space.
1398          */
1399         if (udp->tdb_bootstrap == NULL)
1400                 udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
1401         __tdb_bootstrap = udp->tdb_bootstrap;
1402         if (primary_link_map) {
1403                 self->ul_primarymap = 1;
1404                 udp->primary_map = 1;
1405                 *udp->tdb_bootstrap = udp;
1406         }
1407         /*
1408          * Cancellation can't happen until:
1409          *      pthread_cancel() is called
1410          * or:
1411          *      another thread is created
1412          * For now, as a single-threaded process, set the flag that tells
1413          * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
1414          */
1415         self->ul_nocancel = 1;
1416 
1417 #if defined(__amd64)
1418         (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
1419 #elif defined(__i386)
1420         (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
1421 #endif  /* __i386 || __amd64 */
1422         set_curthread(self);            /* redundant on i386 */
1423         /*
1424          * Now curthread is established and it is safe to call any
1425          * function in libc except one that uses thread-local storage.
1426          */
1427         self->ul_errnop = &errno;
1428         if (oldself != NULL) {
1429                 /* tls_size was zero when oldself was allocated */
1430                 lfree(oldself, sizeof (ulwp_t));
1431         }
1432         mutex_setup();
1433         atfork_init();
1434         signal_init();
1435 
1436         /*
1437          * If the stack is unlimited, we set the size to zero to disable
1438          * stack checking.
1439          * XXX: Work harder here.  Get the stack size from /proc/self/rmap
1440          */
1441         if (self->ul_stksiz == RLIM_INFINITY) {
1442                 self->ul_ustack.ss_sp = (void *)self->ul_stktop;
1443                 self->ul_ustack.ss_size = 0;
1444         } else {
1445                 self->ul_ustack.ss_sp = self->ul_stk;
1446                 self->ul_ustack.ss_size = self->ul_stksiz;
1447         }
1448         self->ul_ustack.ss_flags = 0;
1449         (void) setustack(&self->ul_ustack);
1450 
1451         /*
1452          * Get the variables that affect thread behavior from the environment.
1453          */
1454         set_thread_vars();
1455         udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
1456         udp->thread_stack_cache = thread_stack_cache;
1457 
1458         /*
1459          * Make per-thread copies of global variables, for speed.
1460          */
1461         self->ul_queue_fifo = (char)thread_queue_fifo;
1462         self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
1463         self->ul_error_detection = (char)thread_error_detection;
1464         self->ul_async_safe = (char)thread_async_safe;
1465         self->ul_door_noreserve = (char)thread_door_noreserve;
1466         self->ul_misaligned = (char)thread_locks_misaligned;
1467         self->ul_max_spinners = (uint8_t)thread_max_spinners;
1468         self->ul_adaptive_spin = thread_adaptive_spin;
1469         self->ul_queue_spin = thread_queue_spin;
1470 
1471 #if defined(__sparc) && !defined(_LP64)
1472         if (self->ul_misaligned) {
1473                 /*
1474                  * Tell the kernel to fix up ldx/stx instructions that
1475                  * refer to non-8-byte aligned data instead of giving
1476                  * the process an alignment trap and generating SIGBUS.
1477                  *
1478                  * Programs compiled for 32-bit sparc with the Studio SS12
1479                  * compiler get this done for them automatically (in _init()).
1480                  * We do it here for the benefit of programs compiled with
1481                  * other compilers, like gcc.
1482                  *
1483                  * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
1484                  * environment variable horrible hack to work.
1485                  */
1486                 extern void _do_fix_align(void);
1487                 _do_fix_align();
1488         }
1489 #endif
1490 
1491         /*
1492          * When we have initialized the primary link map, inform
1493          * the dynamic linker about our interface functions.
1494          * Set up our pointer to the program name.
1495          */
1496         if (self->ul_primarymap)
1497                 _ld_libc((void *)rtld_funcs);
1498         init_progname();
1499 
1500         /*
1501          * Defer signals until TLS constructors have been called.
1502          */
1503         sigoff(self);
1504         tls_setup();
1505         sigon(self);
1506         if (setmask)
1507                 (void) restore_signals(self);
1508 
1509         /*
1510          * Make private copies of __xpg4 and __xpg6 so libc can test
1511          * them after this point without invoking the dynamic linker.
1512          */
1513         libc__xpg4 = __xpg4;
1514         libc__xpg6 = __xpg6;
1515 
1516         /* PROBE_SUPPORT begin */
1517         if (self->ul_primarymap && __tnf_probe_notify != NULL)
1518                 __tnf_probe_notify();
1519         /* PROBE_SUPPORT end */
1520 
1521         init_sigev_thread();
1522         init_aio();
1523 
1524         /*
1525          * We need to reset __threaded dynamically at runtime, so that
1526          * __threaded can be bound to __threaded outside libc which may not
1527          * have initial value of 1 (without a copy relocation in a.out).
1528          */
1529         __threaded = 1;
1530 }
1531 
1532 #pragma fini(libc_fini)
1533 void
1534 libc_fini()
1535 {
1536         /*
1537          * If we are doing fini processing for the instance of libc
1538          * on the first alternate link map (this happens only when
1539          * the dynamic linker rejects a bad audit library), then clear
1540          * __curthread().  We abandon whatever memory was allocated by
1541          * lmalloc() while running on this alternate link-map but we
1542          * don't care (and can't find the memory in any case); we just
1543          * want to protect the application from this bad audit library.
1544          * No fini processing is done by libc in the normal case.
1545          */
1546 
1547         uberdata_t *udp = curthread->ul_uberdata;
1548 
1549         if (udp->primary_map == 0 && udp == &__uberdata)
1550                 set_curthread(NULL);
1551 }
1552 
1553 /*
1554  * finish_init is called when we are about to become multi-threaded,
1555  * that is, on the first call to thr_create().
1556  */
1557 void
1558 finish_init()
1559 {
1560         ulwp_t *self = curthread;
1561         uberdata_t *udp = self->ul_uberdata;
1562         thr_hash_table_t *htp;
1563         void *data;
1564         int i;
1565 
1566         /*
1567          * No locks needed here; we are single-threaded on the first call.
1568          * We can be called only after the primary link map has been set up.
1569          */
1570         ASSERT(self->ul_primarymap);
1571         ASSERT(self == udp->ulwp_one);
1572         ASSERT(!udp->uberflags.uf_mt);
1573         ASSERT(udp->hash_size == 1);
1574 
1575         /*
1576          * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
1577          */
1578         update_sched(self);
1579 
1580         /*
1581          * Allocate the queue_head array if not already allocated.
1582          */
1583         if (udp->queue_head == NULL)
1584                 queue_alloc();
1585 
1586         /*
1587          * Now allocate the thread hash table.
1588          */
1589         if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
1590             PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
1591             == MAP_FAILED)
1592                 thr_panic("cannot allocate thread hash table");
1593 
1594         udp->thr_hash_table = htp = (thr_hash_table_t *)data;
1595         udp->hash_size = HASHTBLSZ;
1596         udp->hash_mask = HASHTBLSZ - 1;
1597 
1598         for (i = 0; i < HASHTBLSZ; i++, htp++) {
1599                 htp->hash_lock.mutex_flag = LOCK_INITED;
1600                 htp->hash_lock.mutex_magic = MUTEX_MAGIC;
1601                 htp->hash_cond.cond_magic = COND_MAGIC;
1602         }
1603         hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1604 
1605         /*
1606          * Set up the SIGCANCEL handler for threads cancellation.
1607          */
1608         setup_cancelsig(SIGCANCEL);
1609 
1610         /*
1611          * Arrange to do special things on exit --
1612          * - collect queue statistics from all remaining active threads.
1613          * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
1614          * - grab assert_lock to ensure that assertion failures
1615          *   and a core dump take precedence over _exit().
1616          * (Functions are called in the reverse order of their registration.)
1617          */
1618         (void) _atexit(grab_assert_lock);
1619 #if defined(THREAD_DEBUG)
1620         (void) _atexit(dump_queue_statistics);
1621         (void) _atexit(collect_queue_statistics);
1622 #endif
1623 }
1624 
1625 /*
1626  * Used only by postfork1_child(), below.
1627  */
1628 static void
1629 mark_dead_and_buried(ulwp_t *ulwp)
1630 {
1631         ulwp->ul_dead = 1;
1632         ulwp->ul_lwpid = (lwpid_t)(-1);
1633         ulwp->ul_hash = NULL;
1634         ulwp->ul_ix = -1;
1635         ulwp->ul_schedctl = NULL;
1636         ulwp->ul_schedctl_called = NULL;
1637 }
1638 
1639 /*
1640  * This is called from fork1() in the child.
1641  * Reset our data structures to reflect one lwp.
1642  */
1643 void
1644 postfork1_child()
1645 {
1646         ulwp_t *self = curthread;
1647         uberdata_t *udp = self->ul_uberdata;
1648         queue_head_t *qp;
1649         ulwp_t *next;
1650         ulwp_t *ulwp;
1651         int i;
1652 
1653         /* daemon threads shouldn't call fork1(), but oh well... */
1654         self->ul_usropts &= ~THR_DAEMON;
1655         udp->nthreads = 1;
1656         udp->ndaemons = 0;
1657         udp->uberflags.uf_mt = 0;
1658         __libc_threaded = 0;
1659         for (i = 0; i < udp->hash_size; i++)
1660                 udp->thr_hash_table[i].hash_bucket = NULL;
1661         self->ul_lwpid = _lwp_self();
1662         hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
1663 
1664         /*
1665          * Some thread in the parent might have been suspended
1666          * while holding udp->callout_lock or udp->ld_lock.
1667          * Reinitialize the child's copies.
1668          */
1669         (void) mutex_init(&udp->callout_lock,
1670             USYNC_THREAD | LOCK_RECURSIVE, NULL);
1671         (void) mutex_init(&udp->ld_lock,
1672             USYNC_THREAD | LOCK_RECURSIVE, NULL);
1673 
1674         /* no one in the child is on a sleep queue; reinitialize */
1675         if ((qp = udp->queue_head) != NULL) {
1676                 (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
1677                 for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
1678                         qp->qh_type = (i < QHASHSIZE)? MX : CV;
1679                         qp->qh_lock.mutex_flag = LOCK_INITED;
1680                         qp->qh_lock.mutex_magic = MUTEX_MAGIC;
1681                         qp->qh_hlist = &qp->qh_def_root;
1682 #if defined(THREAD_DEBUG)
1683                         qp->qh_hlen = 1;
1684                         qp->qh_hmax = 1;
1685 #endif
1686                 }
1687         }
1688 
1689         /*
1690          * Do post-fork1 processing for subsystems that need it.
1691          * We need to do this before unmapping all of the abandoned
1692          * threads' stacks, below(), because the post-fork1 actions
1693          * might require access to those stacks.
1694          */
1695         postfork1_child_sigev_aio();
1696         postfork1_child_sigev_mq();
1697         postfork1_child_sigev_timer();
1698         postfork1_child_aio();
1699         /*
1700          * The above subsystems use thread pools, so this action
1701          * must be performed after those actions.
1702          */
1703         postfork1_child_tpool();
1704 
1705         /*
1706          * All lwps except ourself are gone.  Mark them so.
1707          * First mark all of the lwps that have already been freed.
1708          * Then mark and free all of the active lwps except ourself.
1709          * Since we are single-threaded, no locks are required here.
1710          */
1711         for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
1712                 mark_dead_and_buried(ulwp);
1713         for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
1714                 mark_dead_and_buried(ulwp);
1715         for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
1716                 next = ulwp->ul_forw;
1717                 ulwp->ul_forw = ulwp->ul_back = NULL;
1718                 mark_dead_and_buried(ulwp);
1719                 tsd_free(ulwp);
1720                 tls_free(ulwp);
1721                 rwl_free(ulwp);
1722                 heldlock_free(ulwp);
1723                 ulwp_free(ulwp);
1724         }
1725         self->ul_forw = self->ul_back = udp->all_lwps = self;
1726         if (self != udp->ulwp_one)
1727                 mark_dead_and_buried(udp->ulwp_one);
1728         if ((ulwp = udp->all_zombies) != NULL) {
1729                 ASSERT(udp->nzombies != 0);
1730                 do {
1731                         next = ulwp->ul_forw;
1732                         ulwp->ul_forw = ulwp->ul_back = NULL;
1733                         mark_dead_and_buried(ulwp);
1734                         udp->nzombies--;
1735                         if (ulwp->ul_replace) {
1736                                 ulwp->ul_next = NULL;
1737                                 if (udp->ulwp_replace_free == NULL) {
1738                                         udp->ulwp_replace_free =
1739                                             udp->ulwp_replace_last = ulwp;
1740                                 } else {
1741                                         udp->ulwp_replace_last->ul_next = ulwp;
1742                                         udp->ulwp_replace_last = ulwp;
1743                                 }
1744                         }
1745                 } while ((ulwp = next) != udp->all_zombies);
1746                 ASSERT(udp->nzombies == 0);
1747                 udp->all_zombies = NULL;
1748                 udp->nzombies = 0;
1749         }
1750         trim_stack_cache(0);
1751 }
1752 
1753 lwpid_t
1754 lwp_self(void)
1755 {
1756         return (curthread->ul_lwpid);
1757 }
1758 
1759 #pragma weak _ti_thr_self = thr_self
1760 #pragma weak pthread_self = thr_self
1761 thread_t
1762 thr_self()
1763 {
1764         return (curthread->ul_lwpid);
1765 }
1766 
1767 int
1768 thr_main()
1769 {
1770         ulwp_t *self = __curthread();
1771 
1772         return ((self == NULL)? -1 : self->ul_main);
1773 }
1774 
1775 int
1776 _thrp_cancelled(void)
1777 {
1778         return (curthread->ul_rval == PTHREAD_CANCELED);
1779 }
1780 
1781 int
1782 _thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
1783 {
1784         stk->ss_sp = (void *)ulwp->ul_stktop;
1785         stk->ss_size = ulwp->ul_stksiz;
1786         stk->ss_flags = 0;
1787         return (0);
1788 }
1789 
1790 #pragma weak _thr_stksegment = thr_stksegment
1791 int
1792 thr_stksegment(stack_t *stk)
1793 {
1794         return (_thrp_stksegment(curthread, stk));
1795 }
1796 
1797 void
1798 force_continue(ulwp_t *ulwp)
1799 {
1800 #if defined(THREAD_DEBUG)
1801         ulwp_t *self = curthread;
1802         uberdata_t *udp = self->ul_uberdata;
1803 #endif
1804         int error;
1805         timespec_t ts;
1806 
1807         ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1808         ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
1809 
1810         for (;;) {
1811                 error = _lwp_continue(ulwp->ul_lwpid);
1812                 if (error != 0 && error != EINTR)
1813                         break;
1814                 error = 0;
1815                 if (ulwp->ul_stopping) {     /* it is stopping itsself */
1816                         ts.tv_sec = 0;          /* give it a chance to run */
1817                         ts.tv_nsec = 100000;    /* 100 usecs or clock tick */
1818                         (void) __nanosleep(&ts, NULL);
1819                 }
1820                 if (!ulwp->ul_stopping)              /* it is running now */
1821                         break;                  /* so we are done */
1822                 /*
1823                  * It is marked as being in the process of stopping
1824                  * itself.  Loop around and continue it again.
1825                  * It may not have been stopped the first time.
1826                  */
1827         }
1828 }
1829 
1830 /*
1831  * Suspend an lwp with lwp_suspend(), then move it to a safe point,
1832  * that is, to a point where ul_critical and ul_rtld are both zero.
1833  * On return, the ulwp_lock() is dropped as with ulwp_unlock().
1834  * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
1835  * If we have to drop link_lock, we store 1 through link_dropped.
1836  * If the lwp exits before it can be suspended, we return ESRCH.
1837  */
1838 int
1839 safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
1840 {
1841         ulwp_t *self = curthread;
1842         uberdata_t *udp = self->ul_uberdata;
1843         cond_t *cvp = ulwp_condvar(ulwp, udp);
1844         mutex_t *mp = ulwp_mutex(ulwp, udp);
1845         thread_t tid = ulwp->ul_lwpid;
1846         int ix = ulwp->ul_ix;
1847         int error = 0;
1848 
1849         ASSERT(whystopped == TSTP_REGULAR ||
1850             whystopped == TSTP_MUTATOR ||
1851             whystopped == TSTP_FORK);
1852         ASSERT(ulwp != self);
1853         ASSERT(!ulwp->ul_stop);
1854         ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
1855         ASSERT(MUTEX_OWNED(mp, self));
1856 
1857         if (link_dropped != NULL)
1858                 *link_dropped = 0;
1859 
1860         /*
1861          * We must grab the target's spin lock before suspending it.
1862          * See the comments below and in _thrp_suspend() for why.
1863          */
1864         spin_lock_set(&ulwp->ul_spinlock);
1865         (void) ___lwp_suspend(tid);
1866         spin_lock_clear(&ulwp->ul_spinlock);
1867 
1868 top:
1869         if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
1870             ulwp->ul_stopping) {
1871                 /* thread is already safe */
1872                 ulwp->ul_stop |= whystopped;
1873         } else {
1874                 /*
1875                  * Setting ul_pleasestop causes the target thread to stop
1876                  * itself in _thrp_suspend(), below, after we drop its lock.
1877                  * We must continue the critical thread before dropping
1878                  * link_lock because the critical thread may be holding
1879                  * the queue lock for link_lock.  This is delicate.
1880                  */
1881                 ulwp->ul_pleasestop |= whystopped;
1882                 force_continue(ulwp);
1883                 if (link_dropped != NULL) {
1884                         *link_dropped = 1;
1885                         lmutex_unlock(&udp->link_lock);
1886                         /* be sure to drop link_lock only once */
1887                         link_dropped = NULL;
1888                 }
1889 
1890                 /*
1891                  * The thread may disappear by calling thr_exit() so we
1892                  * cannot rely on the ulwp pointer after dropping the lock.
1893                  * Instead, we search the hash table to find it again.
1894                  * When we return, we may find that the thread has been
1895                  * continued by some other thread.  The suspend/continue
1896                  * interfaces are prone to such race conditions by design.
1897                  */
1898                 while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
1899                     (ulwp->ul_pleasestop & whystopped)) {
1900                         (void) __cond_wait(cvp, mp);
1901                         for (ulwp = udp->thr_hash_table[ix].hash_bucket;
1902                             ulwp != NULL; ulwp = ulwp->ul_hash) {
1903                                 if (ulwp->ul_lwpid == tid)
1904                                         break;
1905                         }
1906                 }
1907 
1908                 if (ulwp == NULL || ulwp->ul_dead)
1909                         error = ESRCH;
1910                 else {
1911                         /*
1912                          * Do another lwp_suspend() to make sure we don't
1913                          * return until the target thread is fully stopped
1914                          * in the kernel.  Don't apply lwp_suspend() until
1915                          * we know that the target is not holding any
1916                          * queue locks, that is, that it has completed
1917                          * ulwp_unlock(self) and has, or at least is
1918                          * about to, call lwp_suspend() on itself.  We do
1919                          * this by grabbing the target's spin lock.
1920                          */
1921                         ASSERT(ulwp->ul_lwpid == tid);
1922                         spin_lock_set(&ulwp->ul_spinlock);
1923                         (void) ___lwp_suspend(tid);
1924                         spin_lock_clear(&ulwp->ul_spinlock);
1925                         /*
1926                          * If some other thread did a thr_continue()
1927                          * on the target thread we have to start over.
1928                          */
1929                         if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
1930                                 goto top;
1931                 }
1932         }
1933 
1934         (void) cond_broadcast(cvp);
1935         lmutex_unlock(mp);
1936         return (error);
1937 }
1938 
1939 int
1940 _thrp_suspend(thread_t tid, uchar_t whystopped)
1941 {
1942         ulwp_t *self = curthread;
1943         uberdata_t *udp = self->ul_uberdata;
1944         ulwp_t *ulwp;
1945         int error = 0;
1946 
1947         ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
1948         ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);
1949 
1950         /*
1951          * We can't suspend anyone except ourself while
1952          * some other thread is performing a fork.
1953          * This also allows only one suspension at a time.
1954          */
1955         if (tid != self->ul_lwpid)
1956                 fork_lock_enter();
1957 
1958         if ((ulwp = find_lwp(tid)) == NULL)
1959                 error = ESRCH;
1960         else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
1961                 ulwp_unlock(ulwp, udp);
1962                 error = EINVAL;
1963         } else if (ulwp->ul_stop) {  /* already stopped */
1964                 ulwp->ul_stop |= whystopped;
1965                 ulwp_broadcast(ulwp);
1966                 ulwp_unlock(ulwp, udp);
1967         } else if (ulwp != self) {
1968                 /*
1969                  * After suspending the other thread, move it out of a
1970                  * critical section and deal with the schedctl mappings.
1971                  * safe_suspend() suspends the other thread, calls
1972                  * ulwp_broadcast(ulwp) and drops the ulwp lock.
1973                  */
1974                 error = safe_suspend(ulwp, whystopped, NULL);
1975         } else {
1976                 int schedctl_after_fork = 0;
1977 
1978                 /*
1979                  * We are suspending ourself.  We must not take a signal
1980                  * until we return from lwp_suspend() and clear ul_stopping.
1981                  * This is to guard against siglongjmp().
1982                  */
1983                 enter_critical(self);
1984                 self->ul_sp = stkptr();
1985                 _flush_windows();       /* sparc */
1986                 self->ul_pleasestop = 0;
1987                 self->ul_stop |= whystopped;
1988                 /*
1989                  * Grab our spin lock before dropping ulwp_mutex(self).
1990                  * This prevents the suspending thread from applying
1991                  * lwp_suspend() to us before we emerge from
1992                  * lmutex_unlock(mp) and have dropped mp's queue lock.
1993                  */
1994                 spin_lock_set(&self->ul_spinlock);
1995                 self->ul_stopping = 1;
1996                 ulwp_broadcast(self);
1997                 ulwp_unlock(self, udp);
1998                 /*
1999                  * From this point until we return from lwp_suspend(),
2000                  * we must not call any function that might invoke the
2001                  * dynamic linker, that is, we can only call functions
2002                  * private to the library.
2003                  *
2004                  * Also, this is a nasty race condition for a process
2005                  * that is undergoing a forkall() operation:
2006                  * Once we clear our spinlock (below), we are vulnerable
2007                  * to being suspended by the forkall() thread before
2008                  * we manage to suspend ourself in ___lwp_suspend().
2009                  * See safe_suspend() and force_continue().
2010                  *
2011                  * To avoid a SIGSEGV due to the disappearance
2012                  * of the schedctl mappings in the child process,
2013                  * which can happen in spin_lock_clear() if we
2014                  * are suspended while we are in the middle of
2015                  * its call to preempt(), we preemptively clear
2016                  * our own schedctl pointer before dropping our
2017                  * spinlock.  We reinstate it, in both the parent
2018                  * and (if this really is a forkall()) the child.
2019                  */
2020                 if (whystopped & TSTP_FORK) {
2021                         schedctl_after_fork = 1;
2022                         self->ul_schedctl = NULL;
2023                         self->ul_schedctl_called = &udp->uberflags;
2024                 }
2025                 spin_lock_clear(&self->ul_spinlock);
2026                 (void) ___lwp_suspend(tid);
2027                 /*
2028                  * Somebody else continued us.
2029                  * We can't grab ulwp_lock(self)
2030                  * until after clearing ul_stopping.
2031                  * force_continue() relies on this.
2032                  */
2033                 self->ul_stopping = 0;
2034                 self->ul_sp = 0;
2035                 if (schedctl_after_fork) {
2036                         self->ul_schedctl_called = NULL;
2037                         self->ul_schedctl = NULL;
2038                         (void) setup_schedctl();
2039                 }
2040                 ulwp_lock(self, udp);
2041                 ulwp_broadcast(self);
2042                 ulwp_unlock(self, udp);
2043                 exit_critical(self);
2044         }
2045 
2046         if (tid != self->ul_lwpid)
2047                 fork_lock_exit();
2048 
2049         return (error);
2050 }
2051 
2052 /*
2053  * Suspend all lwps other than ourself in preparation for fork.
2054  */
2055 void
2056 suspend_fork()
2057 {
2058         ulwp_t *self = curthread;
2059         uberdata_t *udp = self->ul_uberdata;
2060         ulwp_t *ulwp;
2061         int link_dropped;
2062 
2063         ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2064 top:
2065         lmutex_lock(&udp->link_lock);
2066 
2067         for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2068                 ulwp_lock(ulwp, udp);
2069                 if (ulwp->ul_stop) { /* already stopped */
2070                         ulwp->ul_stop |= TSTP_FORK;
2071                         ulwp_broadcast(ulwp);
2072                         ulwp_unlock(ulwp, udp);
2073                 } else {
2074                         /*
2075                          * Move the stopped lwp out of a critical section.
2076                          */
2077                         if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
2078                             link_dropped)
2079                                 goto top;
2080                 }
2081         }
2082 
2083         lmutex_unlock(&udp->link_lock);
2084 }
2085 
2086 void
2087 continue_fork(int child)
2088 {
2089         ulwp_t *self = curthread;
2090         uberdata_t *udp = self->ul_uberdata;
2091         ulwp_t *ulwp;
2092 
2093         ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
2094 
2095         /*
2096          * Clear the schedctl pointers in the child of forkall().
2097          */
2098         if (child) {
2099                 for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2100                         ulwp->ul_schedctl_called =
2101                             ulwp->ul_dead? &udp->uberflags : NULL;
2102                         ulwp->ul_schedctl = NULL;
2103                 }
2104         }
2105 
2106         /*
2107          * Set all lwps that were stopped for fork() running again.
2108          */
2109         lmutex_lock(&udp->link_lock);
2110         for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2111                 mutex_t *mp = ulwp_mutex(ulwp, udp);
2112                 lmutex_lock(mp);
2113                 ASSERT(ulwp->ul_stop & TSTP_FORK);
2114                 ulwp->ul_stop &= ~TSTP_FORK;
2115                 ulwp_broadcast(ulwp);
2116                 if (!ulwp->ul_stop)
2117                         force_continue(ulwp);
2118                 lmutex_unlock(mp);
2119         }
2120         lmutex_unlock(&udp->link_lock);
2121 }
2122 
2123 int
2124 _thrp_continue(thread_t tid, uchar_t whystopped)
2125 {
2126         uberdata_t *udp = curthread->ul_uberdata;
2127         ulwp_t *ulwp;
2128         mutex_t *mp;
2129         int error = 0;
2130 
2131         ASSERT(whystopped == TSTP_REGULAR ||
2132             whystopped == TSTP_MUTATOR);
2133 
2134         /*
2135          * We single-thread the entire thread suspend/continue mechanism.
2136          */
2137         fork_lock_enter();
2138 
2139         if ((ulwp = find_lwp(tid)) == NULL) {
2140                 fork_lock_exit();
2141                 return (ESRCH);
2142         }
2143 
2144         mp = ulwp_mutex(ulwp, udp);
2145         if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
2146                 error = EINVAL;
2147         } else if (ulwp->ul_stop & whystopped) {
2148                 ulwp->ul_stop &= ~whystopped;
2149                 ulwp_broadcast(ulwp);
2150                 if (!ulwp->ul_stop) {
2151                         if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
2152                                 ulwp->ul_sp = 0;
2153                                 ulwp->ul_created = 0;
2154                         }
2155                         force_continue(ulwp);
2156                 }
2157         }
2158         lmutex_unlock(mp);
2159 
2160         fork_lock_exit();
2161         return (error);
2162 }
2163 
2164 int
2165 thr_suspend(thread_t tid)
2166 {
2167         return (_thrp_suspend(tid, TSTP_REGULAR));
2168 }
2169 
2170 int
2171 thr_continue(thread_t tid)
2172 {
2173         return (_thrp_continue(tid, TSTP_REGULAR));
2174 }
2175 
2176 void
2177 thr_yield()
2178 {
2179         yield();
2180 }
2181 
2182 #pragma weak pthread_kill = thr_kill
2183 #pragma weak _thr_kill = thr_kill
2184 int
2185 thr_kill(thread_t tid, int sig)
2186 {
2187         if (sig == SIGCANCEL)
2188                 return (EINVAL);
2189         return (_lwp_kill(tid, sig));
2190 }
2191 
2192 /*
2193  * Exit a critical section, take deferred actions if necessary.
2194  * Called from exit_critical() and from sigon().
2195  */
2196 void
2197 do_exit_critical()
2198 {
2199         ulwp_t *self = curthread;
2200         int sig;
2201 
2202         ASSERT(self->ul_critical == 0);
2203 
2204         /*
2205          * Don't suspend ourself or take a deferred signal while dying
2206          * or while executing inside the dynamic linker (ld.so.1).
2207          */
2208         if (self->ul_dead || self->ul_rtld)
2209                 return;
2210 
2211         while (self->ul_pleasestop ||
2212             (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
2213                 /*
2214                  * Avoid a recursive call to exit_critical() in _thrp_suspend()
2215                  * by keeping self->ul_critical == 1 here.
2216                  */
2217                 self->ul_critical++;
2218                 while (self->ul_pleasestop) {
2219                         /*
2220                          * Guard against suspending ourself while on a sleep
2221                          * queue.  See the comments in call_user_handler().
2222                          */
2223                         unsleep_self();
2224                         set_parking_flag(self, 0);
2225                         (void) _thrp_suspend(self->ul_lwpid,
2226                             self->ul_pleasestop);
2227                 }
2228                 self->ul_critical--;
2229 
2230                 if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
2231                         /*
2232                          * Clear ul_cursig before proceeding.
2233                          * This protects us from the dynamic linker's
2234                          * calls to bind_guard()/bind_clear() in the
2235                          * event that it is invoked to resolve a symbol
2236                          * like take_deferred_signal() below.
2237                          */
2238                         self->ul_cursig = 0;
2239                         take_deferred_signal(sig);
2240                         ASSERT(self->ul_cursig == 0);
2241                 }
2242         }
2243         ASSERT(self->ul_critical == 0);
2244 }
2245 
2246 /*
2247  * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
2248  * (ld.so.1) when it has do do something, like resolve a symbol to be called
2249  * by the application or one of its libraries.  _ti_bind_guard() is called
2250  * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
2251  * application.  The dynamic linker gets special dispensation from libc to
2252  * run in a critical region (all signals deferred and no thread suspension
2253  * or forking allowed), and to be immune from cancellation for the duration.
2254  */
2255 int
2256 _ti_bind_guard(int flags)
2257 {
2258         ulwp_t *self = curthread;
2259         uberdata_t *udp = self->ul_uberdata;
2260         int bindflag = (flags & THR_FLG_RTLD);
2261 
2262         if ((self->ul_bindflags & bindflag) == bindflag)
2263                 return (0);
2264         self->ul_bindflags |= bindflag;
2265         if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2266                 sigoff(self);   /* see no signals while holding ld_lock */
2267                 self->ul_rtld++;     /* don't suspend while in ld.so.1 */
2268                 (void) mutex_lock(&udp->ld_lock);
2269         }
2270         enter_critical(self);
2271         self->ul_save_state = self->ul_cancel_disabled;
2272         self->ul_cancel_disabled = 1;
2273         set_cancel_pending_flag(self, 0);
2274         return (1);
2275 }
2276 
2277 int
2278 _ti_bind_clear(int flags)
2279 {
2280         ulwp_t *self = curthread;
2281         uberdata_t *udp = self->ul_uberdata;
2282         int bindflag = (flags & THR_FLG_RTLD);
2283 
2284         if ((self->ul_bindflags & bindflag) == 0)
2285                 return (self->ul_bindflags);
2286         self->ul_bindflags &= ~bindflag;
2287         self->ul_cancel_disabled = self->ul_save_state;
2288         set_cancel_pending_flag(self, 0);
2289         exit_critical(self);
2290         if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
2291                 if (MUTEX_OWNED(&udp->ld_lock, self)) {
2292                         (void) mutex_unlock(&udp->ld_lock);
2293                         self->ul_rtld--;
2294                         sigon(self);    /* reenable signals */
2295                 }
2296         }
2297         return (self->ul_bindflags);
2298 }
2299 
2300 /*
2301  * Tell the dynamic linker (ld.so.1) whether or not it was entered from
2302  * a critical region in libc.  Return zero if not, else return non-zero.
2303  */
2304 int
2305 _ti_critical(void)
2306 {
2307         ulwp_t *self = curthread;
2308         int level = self->ul_critical;
2309 
2310         if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
2311                 return (level); /* ld.so.1 hasn't (yet) called enter() */
2312         return (level - 1);
2313 }
2314 
2315 /*
2316  * sigoff() and sigon() enable cond_wait() to behave (optionally) like
2317  * it does in the old libthread (see the comments in cond_wait_queue()).
2318  * Also, signals are deferred at thread startup until TLS constructors
2319  * have all been called, at which time _thrp_setup() calls sigon().
2320  *
2321  * _sigoff() and _sigon() are external consolidation-private interfaces to
2322  * sigoff() and sigon(), respectively, in libc.  These are used in libnsl.
2323  * Also, _sigoff() and _sigon() are called from dbx's run-time checking
2324  * (librtc.so) to defer signals during its critical sections (not to be
2325  * confused with libc critical sections [see exit_critical() above]).
2326  */
2327 void
2328 _sigoff(void)
2329 {
2330         ulwp_t *self = curthread;
2331 
2332         sigoff(self);
2333 }
2334 
2335 void
2336 _sigon(void)
2337 {
2338         ulwp_t *self = curthread;
2339 
2340         ASSERT(self->ul_sigdefer > 0);
2341         sigon(self);
2342 }
2343 
2344 int
2345 thr_getconcurrency()
2346 {
2347         return (thr_concurrency);
2348 }
2349 
2350 int
2351 pthread_getconcurrency()
2352 {
2353         return (pthread_concurrency);
2354 }
2355 
2356 int
2357 thr_setconcurrency(int new_level)
2358 {
2359         uberdata_t *udp = curthread->ul_uberdata;
2360 
2361         if (new_level < 0)
2362                 return (EINVAL);
2363         if (new_level > 65536)               /* 65536 is totally arbitrary */
2364                 return (EAGAIN);
2365         lmutex_lock(&udp->link_lock);
2366         if (new_level > thr_concurrency)
2367                 thr_concurrency = new_level;
2368         lmutex_unlock(&udp->link_lock);
2369         return (0);
2370 }
2371 
2372 int
2373 pthread_setconcurrency(int new_level)
2374 {
2375         if (new_level < 0)
2376                 return (EINVAL);
2377         if (new_level > 65536)               /* 65536 is totally arbitrary */
2378                 return (EAGAIN);
2379         pthread_concurrency = new_level;
2380         return (0);
2381 }
2382 
2383 size_t
2384 thr_min_stack(void)
2385 {
2386         return (MINSTACK);
2387 }
2388 
2389 int
2390 __nthreads(void)
2391 {
2392         return (curthread->ul_uberdata->nthreads);
2393 }
2394 
2395 /*
2396  * XXX
2397  * The remainder of this file implements the private interfaces to java for
2398  * garbage collection.  It is no longer used, at least by java 1.2.
2399  * It can all go away once all old JVMs have disappeared.
2400  */
2401 
2402 int     suspendingallmutators;  /* when non-zero, suspending all mutators. */
2403 int     suspendedallmutators;   /* when non-zero, all mutators suspended. */
2404 int     mutatorsbarrier;        /* when non-zero, mutators barrier imposed. */
2405 mutex_t mutatorslock = DEFAULTMUTEX;    /* used to enforce mutators barrier. */
2406 cond_t  mutatorscv = DEFAULTCV;         /* where non-mutators sleep. */
2407 
2408 /*
2409  * Get the available register state for the target thread.
2410  * Return non-volatile registers: TRS_NONVOLATILE
2411  */
2412 #pragma weak _thr_getstate = thr_getstate
2413 int
2414 thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
2415 {
2416         ulwp_t *self = curthread;
2417         uberdata_t *udp = self->ul_uberdata;
2418         ulwp_t **ulwpp;
2419         ulwp_t *ulwp;
2420         int error = 0;
2421         int trs_flag = TRS_LWPID;
2422 
2423         if (tid == 0 || self->ul_lwpid == tid) {
2424                 ulwp = self;
2425                 ulwp_lock(ulwp, udp);
2426         } else if ((ulwpp = find_lwpp(tid)) != NULL) {
2427                 ulwp = *ulwpp;
2428         } else {
2429                 if (flag)
2430                         *flag = TRS_INVALID;
2431                 return (ESRCH);
2432         }
2433 
2434         if (ulwp->ul_dead) {
2435                 trs_flag = TRS_INVALID;
2436         } else if (!ulwp->ul_stop && !suspendedallmutators) {
2437                 error = EINVAL;
2438                 trs_flag = TRS_INVALID;
2439         } else if (ulwp->ul_stop) {
2440                 trs_flag = TRS_NONVOLATILE;
2441                 getgregs(ulwp, rs);
2442         }
2443 
2444         if (flag)
2445                 *flag = trs_flag;
2446         if (lwp)
2447                 *lwp = tid;
2448         if (ss != NULL)
2449                 (void) _thrp_stksegment(ulwp, ss);
2450 
2451         ulwp_unlock(ulwp, udp);
2452         return (error);
2453 }
2454 
2455 /*
2456  * Set the appropriate register state for the target thread.
2457  * This is not used by java.  It exists solely for the MSTC test suite.
2458  */
2459 #pragma weak _thr_setstate = thr_setstate
2460 int
2461 thr_setstate(thread_t tid, int flag, gregset_t rs)
2462 {
2463         uberdata_t *udp = curthread->ul_uberdata;
2464         ulwp_t *ulwp;
2465         int error = 0;
2466 
2467         if ((ulwp = find_lwp(tid)) == NULL)
2468                 return (ESRCH);
2469 
2470         if (!ulwp->ul_stop && !suspendedallmutators)
2471                 error = EINVAL;
2472         else if (rs != NULL) {
2473                 switch (flag) {
2474                 case TRS_NONVOLATILE:
2475                         /* do /proc stuff here? */
2476                         if (ulwp->ul_stop)
2477                                 setgregs(ulwp, rs);
2478                         else
2479                                 error = EINVAL;
2480                         break;
2481                 case TRS_LWPID:         /* do /proc stuff here? */
2482                 default:
2483                         error = EINVAL;
2484                         break;
2485                 }
2486         }
2487 
2488         ulwp_unlock(ulwp, udp);
2489         return (error);
2490 }
2491 
2492 int
2493 getlwpstatus(thread_t tid, struct lwpstatus *sp)
2494 {
2495         extern ssize_t __pread(int, void *, size_t, off_t);
2496         char buf[100];
2497         int fd;
2498 
2499         /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
2500         (void) strcpy(buf, "/proc/self/lwp/");
2501         ultos((uint64_t)tid, 10, buf + strlen(buf));
2502         (void) strcat(buf, "/lwpstatus");
2503         if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
2504                 while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
2505                         if (sp->pr_flags & PR_STOPPED) {
2506                                 (void) __close(fd);
2507                                 return (0);
2508                         }
2509                         yield();        /* give it a chance to stop */
2510                 }
2511                 (void) __close(fd);
2512         }
2513         return (-1);
2514 }
2515 
2516 int
2517 putlwpregs(thread_t tid, prgregset_t prp)
2518 {
2519         extern ssize_t __writev(int, const struct iovec *, int);
2520         char buf[100];
2521         int fd;
2522         long dstop_sreg[2];
2523         long run_null[2];
2524         iovec_t iov[3];
2525 
2526         /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
2527         (void) strcpy(buf, "/proc/self/lwp/");
2528         ultos((uint64_t)tid, 10, buf + strlen(buf));
2529         (void) strcat(buf, "/lwpctl");
2530         if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
2531                 dstop_sreg[0] = PCDSTOP;        /* direct it to stop */
2532                 dstop_sreg[1] = PCSREG;         /* set the registers */
2533                 iov[0].iov_base = (caddr_t)dstop_sreg;
2534                 iov[0].iov_len = sizeof (dstop_sreg);
2535                 iov[1].iov_base = (caddr_t)prp; /* from the register set */
2536                 iov[1].iov_len = sizeof (prgregset_t);
2537                 run_null[0] = PCRUN;            /* make it runnable again */
2538                 run_null[1] = 0;
2539                 iov[2].iov_base = (caddr_t)run_null;
2540                 iov[2].iov_len = sizeof (run_null);
2541                 if (__writev(fd, iov, 3) >= 0) {
2542                         (void) __close(fd);
2543                         return (0);
2544                 }
2545                 (void) __close(fd);
2546         }
2547         return (-1);
2548 }
2549 
2550 static ulong_t
2551 gettsp_slow(thread_t tid)
2552 {
2553         char buf[100];
2554         struct lwpstatus status;
2555 
2556         if (getlwpstatus(tid, &status) != 0) {
2557                 /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
2558                 (void) strcpy(buf, "__gettsp(");
2559                 ultos((uint64_t)tid, 10, buf + strlen(buf));
2560                 (void) strcat(buf, "): can't read lwpstatus");
2561                 thr_panic(buf);
2562         }
2563         return (status.pr_reg[R_SP]);
2564 }
2565 
2566 ulong_t
2567 __gettsp(thread_t tid)
2568 {
2569         uberdata_t *udp = curthread->ul_uberdata;
2570         ulwp_t *ulwp;
2571         ulong_t result;
2572 
2573         if ((ulwp = find_lwp(tid)) == NULL)
2574                 return (0);
2575 
2576         if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
2577                 ulwp_unlock(ulwp, udp);
2578                 return (result);
2579         }
2580 
2581         result = gettsp_slow(tid);
2582         ulwp_unlock(ulwp, udp);
2583         return (result);
2584 }
2585 
2586 /*
2587  * This tells java stack walkers how to find the ucontext
2588  * structure passed to signal handlers.
2589  */
2590 #pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
2591 void
2592 thr_sighndlrinfo(void (**func)(), int *funcsize)
2593 {
2594         *func = &__sighndlr;
2595         *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
2596 }
2597 
2598 /*
2599  * Mark a thread a mutator or reset a mutator to being a default,
2600  * non-mutator thread.
2601  */
2602 #pragma weak _thr_setmutator = thr_setmutator
2603 int
2604 thr_setmutator(thread_t tid, int enabled)
2605 {
2606         ulwp_t *self = curthread;
2607         uberdata_t *udp = self->ul_uberdata;
2608         ulwp_t *ulwp;
2609         int error;
2610         int cancel_state;
2611 
2612         enabled = enabled? 1 : 0;
2613 top:
2614         if (tid == 0) {
2615                 ulwp = self;
2616                 ulwp_lock(ulwp, udp);
2617         } else if ((ulwp = find_lwp(tid)) == NULL) {
2618                 return (ESRCH);
2619         }
2620 
2621         /*
2622          * The target thread should be the caller itself or a suspended thread.
2623          * This prevents the target from also changing its ul_mutator field.
2624          */
2625         error = 0;
2626         if (ulwp != self && !ulwp->ul_stop && enabled)
2627                 error = EINVAL;
2628         else if (ulwp->ul_mutator != enabled) {
2629                 lmutex_lock(&mutatorslock);
2630                 if (mutatorsbarrier) {
2631                         ulwp_unlock(ulwp, udp);
2632                         (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
2633                             &cancel_state);
2634                         while (mutatorsbarrier)
2635                                 (void) cond_wait(&mutatorscv, &mutatorslock);
2636                         (void) pthread_setcancelstate(cancel_state, NULL);
2637                         lmutex_unlock(&mutatorslock);
2638                         goto top;
2639                 }
2640                 ulwp->ul_mutator = enabled;
2641                 lmutex_unlock(&mutatorslock);
2642         }
2643 
2644         ulwp_unlock(ulwp, udp);
2645         return (error);
2646 }
2647 
2648 /*
2649  * Establish a barrier against new mutators.  Any non-mutator trying
2650  * to become a mutator is suspended until the barrier is removed.
2651  */
2652 #pragma weak _thr_mutators_barrier = thr_mutators_barrier
2653 void
2654 thr_mutators_barrier(int enabled)
2655 {
2656         int oldvalue;
2657         int cancel_state;
2658 
2659         lmutex_lock(&mutatorslock);
2660 
2661         /*
2662          * Wait if trying to set the barrier while it is already set.
2663          */
2664         (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2665         while (mutatorsbarrier && enabled)
2666                 (void) cond_wait(&mutatorscv, &mutatorslock);
2667         (void) pthread_setcancelstate(cancel_state, NULL);
2668 
2669         oldvalue = mutatorsbarrier;
2670         mutatorsbarrier = enabled;
2671         /*
2672          * Wakeup any blocked non-mutators when barrier is removed.
2673          */
2674         if (oldvalue && !enabled)
2675                 (void) cond_broadcast(&mutatorscv);
2676         lmutex_unlock(&mutatorslock);
2677 }
2678 
2679 /*
2680  * Suspend the set of all mutators except for the caller.  The list
2681  * of actively running threads is searched and only the mutators
2682  * in this list are suspended.  Actively running non-mutators remain
2683  * running.  Any other thread is suspended.
2684  */
2685 #pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
2686 int
2687 thr_suspend_allmutators(void)
2688 {
2689         ulwp_t *self = curthread;
2690         uberdata_t *udp = self->ul_uberdata;
2691         ulwp_t *ulwp;
2692         int link_dropped;
2693 
2694         /*
2695          * We single-thread the entire thread suspend/continue mechanism.
2696          */
2697         fork_lock_enter();
2698 
2699 top:
2700         lmutex_lock(&udp->link_lock);
2701 
2702         if (suspendingallmutators || suspendedallmutators) {
2703                 lmutex_unlock(&udp->link_lock);
2704                 fork_lock_exit();
2705                 return (EINVAL);
2706         }
2707         suspendingallmutators = 1;
2708 
2709         for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2710                 ulwp_lock(ulwp, udp);
2711                 if (!ulwp->ul_mutator) {
2712                         ulwp_unlock(ulwp, udp);
2713                 } else if (ulwp->ul_stop) {  /* already stopped */
2714                         ulwp->ul_stop |= TSTP_MUTATOR;
2715                         ulwp_broadcast(ulwp);
2716                         ulwp_unlock(ulwp, udp);
2717                 } else {
2718                         /*
2719                          * Move the stopped lwp out of a critical section.
2720                          */
2721                         if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
2722                             link_dropped) {
2723                                 suspendingallmutators = 0;
2724                                 goto top;
2725                         }
2726                 }
2727         }
2728 
2729         suspendedallmutators = 1;
2730         suspendingallmutators = 0;
2731         lmutex_unlock(&udp->link_lock);
2732         fork_lock_exit();
2733         return (0);
2734 }
2735 
2736 /*
2737  * Suspend the target mutator.  The caller is permitted to suspend
2738  * itself.  If a mutator barrier is enabled, the caller will suspend
2739  * itself as though it had been suspended by thr_suspend_allmutators().
2740  * When the barrier is removed, this thread will be resumed.  Any
2741  * suspended mutator, whether suspended by thr_suspend_mutator(), or by
2742  * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
2743  */
2744 #pragma weak _thr_suspend_mutator = thr_suspend_mutator
2745 int
2746 thr_suspend_mutator(thread_t tid)
2747 {
2748         if (tid == 0)
2749                 tid = curthread->ul_lwpid;
2750         return (_thrp_suspend(tid, TSTP_MUTATOR));
2751 }
2752 
2753 /*
2754  * Resume the set of all suspended mutators.
2755  */
2756 #pragma weak _thr_continue_allmutators = thr_continue_allmutators
2757 int
2758 thr_continue_allmutators()
2759 {
2760         ulwp_t *self = curthread;
2761         uberdata_t *udp = self->ul_uberdata;
2762         ulwp_t *ulwp;
2763 
2764         /*
2765          * We single-thread the entire thread suspend/continue mechanism.
2766          */
2767         fork_lock_enter();
2768 
2769         lmutex_lock(&udp->link_lock);
2770         if (!suspendedallmutators) {
2771                 lmutex_unlock(&udp->link_lock);
2772                 fork_lock_exit();
2773                 return (EINVAL);
2774         }
2775         suspendedallmutators = 0;
2776 
2777         for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
2778                 mutex_t *mp = ulwp_mutex(ulwp, udp);
2779                 lmutex_lock(mp);
2780                 if (ulwp->ul_stop & TSTP_MUTATOR) {
2781                         ulwp->ul_stop &= ~TSTP_MUTATOR;
2782                         ulwp_broadcast(ulwp);
2783                         if (!ulwp->ul_stop)
2784                                 force_continue(ulwp);
2785                 }
2786                 lmutex_unlock(mp);
2787         }
2788 
2789         lmutex_unlock(&udp->link_lock);
2790         fork_lock_exit();
2791         return (0);
2792 }
2793 
2794 /*
2795  * Resume a suspended mutator.
2796  */
2797 #pragma weak _thr_continue_mutator = thr_continue_mutator
2798 int
2799 thr_continue_mutator(thread_t tid)
2800 {
2801         return (_thrp_continue(tid, TSTP_MUTATOR));
2802 }
2803 
2804 #pragma weak _thr_wait_mutator = thr_wait_mutator
2805 int
2806 thr_wait_mutator(thread_t tid, int dontwait)
2807 {
2808         uberdata_t *udp = curthread->ul_uberdata;
2809         ulwp_t *ulwp;
2810         int cancel_state;
2811         int error = 0;
2812 
2813         (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
2814 top:
2815         if ((ulwp = find_lwp(tid)) == NULL) {
2816                 (void) pthread_setcancelstate(cancel_state, NULL);
2817                 return (ESRCH);
2818         }
2819 
2820         if (!ulwp->ul_mutator)
2821                 error = EINVAL;
2822         else if (dontwait) {
2823                 if (!(ulwp->ul_stop & TSTP_MUTATOR))
2824                         error = EWOULDBLOCK;
2825         } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
2826                 cond_t *cvp = ulwp_condvar(ulwp, udp);
2827                 mutex_t *mp = ulwp_mutex(ulwp, udp);
2828 
2829                 (void) cond_wait(cvp, mp);
2830                 (void) lmutex_unlock(mp);
2831                 goto top;
2832         }
2833 
2834         ulwp_unlock(ulwp, udp);
2835         (void) pthread_setcancelstate(cancel_state, NULL);
2836         return (error);
2837 }
2838 
2839 /* PROBE_SUPPORT begin */
2840 
2841 void
2842 thr_probe_setup(void *data)
2843 {
2844         curthread->ul_tpdp = data;
2845 }
2846 
2847 static void *
2848 _thread_probe_getfunc()
2849 {
2850         return (curthread->ul_tpdp);
2851 }
2852 
2853 void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;
2854 
2855 /* ARGSUSED */
2856 void
2857 _resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
2858 {
2859         /* never called */
2860 }
2861 
2862 /* ARGSUSED */
2863 void
2864 _resume_ret(ulwp_t *oldlwp)
2865 {
2866         /* never called */
2867 }
2868 
2869 /* PROBE_SUPPORT end */