1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 #ifndef _THR_UBERDATA_H
  27 #define _THR_UBERDATA_H
  28 
  29 #include <stdlib.h>
  30 #include <unistd.h>
  31 #include <sys/types.h>
  32 #include <fcntl.h>
  33 #include <string.h>
  34 #include <signal.h>
  35 #include <ucontext.h>
  36 #include <thread.h>
  37 #include <pthread.h>
  38 #include <atomic.h>
  39 #include <link.h>
  40 #include <sys/resource.h>
  41 #include <sys/lwp.h>
  42 #include <errno.h>
  43 #include <sys/asm_linkage.h>
  44 #include <sys/regset.h>
  45 #include <sys/fcntl.h>
  46 #include <sys/mman.h>
  47 #include <synch.h>
  48 #include <door.h>
  49 #include <limits.h>
  50 #include <sys/synch32.h>
  51 #include <schedctl.h>
  52 #include <sys/priocntl.h>
  53 #include <thread_db.h>
  54 #include <setjmp.h>
  55 #include "libc_int.h"
  56 #include "tdb_agent.h"
  57 #include "thr_debug.h"
  58 
  59 /*
  60  * This is an implementation-specific include file for threading support.
  61  * It is not to be seen by the clients of the library.
  62  *
  63  * This file also describes uberdata in libc.
  64  *
  65  * The term "uberdata" refers to data that is unique and visible across
  66  * all link maps.  The name is meant to imply that such data is truly
  67  * global, not just locally global to a particular link map.
  68  *
  69  * See the Linker and Libraries Guide for a full description of alternate
  70  * link maps and how they are set up and used.
  71  *
  72  * Alternate link maps implement multiple global namespaces within a single
  73  * process.  There may be multiple instances of identical dynamic libraries
  74  * loaded in a process's address space at the same time, each on a different
  75  * link map (as determined by the dynamic linker), each with its own set of
  76  * global variables.  Which particular instance of a global variable is seen
  77  * by a thread running in the process is determined by the link map on which
  78  * the thread happens to be executing at the time.
  79  *
  80  * However, there are aspects of a process that are unique across all
  81  * link maps, in particular the structures used to implement threads
  82  * of control (in Sparc terminology, there is only one %g7 regardless
  83  * of the link map on which the thread is executing).
  84  *
  85  * All uberdata is referenced from a base pointer in the thread's ulwp_t
  86  * structure (which is also uberdata).  All allocations and deallocations
  87  * of uberdata are made via the uberdata-aware lmalloc() and lfree()
  88  * interfaces (malloc() and free() are simply locally-global).
  89  */
  90 
  91 /*
  92  * Special libc-private access to errno.
  93  * We do this so that references to errno do not invoke the dynamic linker.
  94  */
  95 #undef errno
  96 #define errno (*curthread->ul_errnop)
  97 
  98 /*
  99  * See <sys/synch32.h> for the reasons for these values
 100  * and why they are different for sparc and intel.
 101  */
 102 #if defined(__sparc)
 103 
 104 /* lock.lock64.pad[x]      4 5 6 7 */
 105 #define LOCKMASK        0xff000000
 106 #define WAITERMASK      0x000000ff
 107 #define SPINNERMASK     0x00ff0000
 108 #define SPINNERSHIFT    16
 109 #define WAITER          0x00000001
 110 #define LOCKSET         0xff
 111 #define LOCKCLEAR       0
 112 
 113 #define PIDSHIFT        32
 114 #define LOCKMASK64      0xffffffffff000000ULL
 115 #define LOCKBYTE64      0x00000000ff000000ULL
 116 #define WAITERMASK64    0x00000000000000ffULL
 117 #define SPINNERMASK64   0x0000000000ff0000ULL
 118 
 119 #elif defined(__x86)
 120 
 121 /* lock.lock64.pad[x]      7 6 5 4 */
 122 #define LOCKMASK        0xff000000
 123 #define WAITERMASK      0x00ff0000
 124 #define SPINNERMASK     0x0000ff00
 125 #define SPINNERSHIFT    8
 126 #define WAITER          0x00010000
 127 #define LOCKSET         0x01
 128 #define LOCKCLEAR       0
 129 
 130 #define PIDSHIFT        0
 131 #define LOCKMASK64      0xff000000ffffffffULL
 132 #define LOCKBYTE64      0x0100000000000000ULL
 133 #define WAITERMASK64    0x00ff000000000000ULL
 134 #define SPINNERMASK64   0x0000ff0000000000ULL
 135 
 136 #else
 137 #error "neither __sparc nor __x86 is defined"
 138 #endif
 139 
 140 /*
 141  * Fetch the owner of a USYNC_THREAD mutex.
 142  * Don't use this with process-shared mutexes;
 143  * the owing thread may be in a different process.
 144  */
 145 #define MUTEX_OWNER(mp) ((ulwp_t *)(uintptr_t)(mp)->mutex_owner)
 146 
 147 /*
 148  * Test if a thread owns a process-private (USYNC_THREAD) mutex.
 149  * This is inappropriate for a process-shared (USYNC_PROCESS) mutex.
 150  * The 'mp' argument must not have side-effects since it is evaluated twice.
 151  */
 152 #define MUTEX_OWNED(mp, thrp)   \
 153         ((mp)->mutex_lockw != 0 && MUTEX_OWNER(mp) == thrp)
 154 
 155 
 156 /*
 157  * uberflags.uf_tdb_register_sync is an interface with libc_db to enable the
 158  * collection of lock statistics by a debugger or other collecting tool.
 159  *
 160  * uberflags.uf_thread_error_detection is set by an environment variable:
 161  *      _THREAD_ERROR_DETECTION
 162  *              0 == no detection of locking primitive errors.
 163  *              1 == detect errors and issue a warning message.
 164  *              2 == detect errors, issue a warning message, and dump core.
 165  *
 166  * We bundle these together in uberflags.uf_trs_ted to make a test of either
 167  * being non-zero a single memory reference (for speed of mutex_lock(), etc).
 168  *
 169  * uberflags.uf_mt is set non-zero when the first thread (in addition
 170  * to the main thread) is created.
 171  *
 172  * We bundle all these flags together in uberflags.uf_all to make a test
 173  * of any being non-zero a single memory reference (again, for speed).
 174  */
 175 typedef union {
 176         int     uf_all;                 /* combined all flags */
 177         struct {
 178                 short   h_pad;
 179                 short   h_trs_ted;      /* combined reg sync & error detect */
 180         } uf_h;
 181         struct {
 182                 char    x_mt;
 183                 char    x_pad;
 184                 char    x_tdb_register_sync;
 185                 char    x_thread_error_detection;
 186         } uf_x;
 187 } uberflags_t;
 188 
 189 #define uf_mt                           uf_x.x_mt
 190 #define uf_tdb_register_sync            uf_x.x_tdb_register_sync
 191 #define uf_thread_error_detection       uf_x.x_thread_error_detection
 192 #define uf_trs_ted                      uf_h.h_trs_ted  /* both of the above */
 193 
 194 /*
 195  * NOTE WELL:
 196  * To enable further optimization, the "ul_schedctl_called" member
 197  * of the ulwp_t structure (below) serves double-duty:
 198  *      1. If NULL, it means that the thread must call __schedctl()
 199  *         to set up its schedctl mappings before acquiring a mutex.
 200  *         This is required by the implementation of adaptive mutex locking.
 201  *      2. If non-NULL, it points to uberdata.uberflags, so that tests of
 202  *         uberflags can be made without additional memory references.
 203  * This allows the common case of _mutex_lock() and _mutex_unlock() for
 204  * USYNC_THREAD mutexes with no error detection and no lock statistics
 205  * to be optimized for speed.
 206  */
 207 
 208 /* double the default stack size for 64-bit processes */
 209 #ifdef _LP64
 210 #define MINSTACK        (8 * 1024)
 211 #define DEFAULTSTACK    (2 * 1024 * 1024)
 212 #else
 213 #define MINSTACK        (4 * 1024)
 214 #define DEFAULTSTACK    (1024 * 1024)
 215 #endif
 216 
 217 #define MUTEX_TRY       0
 218 #define MUTEX_LOCK      1
 219 #define MUTEX_NOCEIL    0x40
 220 
 221 #if defined(__x86)
 222 
 223 typedef struct {        /* structure returned by fnstenv */
 224         int     fctrl;          /* control word */
 225         int     fstat;          /* status word (flags, etc) */
 226         int     ftag;           /* tag of which regs busy */
 227         int     misc[4];        /* other stuff, 28 bytes total */
 228 } fpuenv_t;
 229 
 230 #ifdef _SYSCALL32
 231 typedef fpuenv_t fpuenv32_t;
 232 #endif  /* _SYSCALL32 */
 233 
 234 #elif defined(__sparc)
 235 
 236 typedef struct {        /* fp state structure */
 237         greg_t  fsr;
 238         greg_t  fpu_en;
 239 } fpuenv_t;
 240 
 241 #ifdef _SYSCALL32
 242 typedef struct {
 243         greg32_t        fsr;
 244         greg32_t        fpu_en;
 245 } fpuenv32_t;
 246 #endif  /* _SYSCALL32 */
 247 
 248 #endif  /* __x86 */
 249 
 250 #if defined(__x86)
 251 extern  void    ht_pause(void);         /* "pause" instruction */
 252 #define SMT_PAUSE()     ht_pause()
 253 #elif defined(SMT_PAUSE_FUNCTION)
 254 extern  void    SMT_PAUSE_FUNCTION(void);
 255 #define SMT_PAUSE()     SMT_PAUSE_FUNCTION()
 256 #else
 257 #define SMT_PAUSE()     smt_pause()
 258 #endif  /* __x86 */
 259 
 260 /*
 261  * Cleanup handler related data.
 262  * This structure is exported as _cleanup_t in pthread.h.
 263  * pthread.h exports only the size of this structure, so check
 264  * _cleanup_t in pthread.h before making any change here.
 265  */
 266 typedef struct __cleanup {
 267         struct __cleanup *next;         /* pointer to next handler */
 268         caddr_t fp;                     /* current frame pointer */
 269         void    (*func)(void *);        /* cleanup handler address */
 270         void    *arg;                   /* handler's argument */
 271 } __cleanup_t;
 272 
 273 /*
 274  * Thread-Specific Data (TSD)
 275  * TSD_NFAST includes the invalid key zero, so there
 276  * are really only (TSD_NFAST - 1) fast key slots.
 277  */
 278 typedef void (*PFrV)(void *);
 279 #define TSD_UNALLOCATED ((PFrV)1)
 280 #define TSD_NFAST       9
 281 
 282 /*
 283  * The tsd union is designed to burn a little memory (9 words) to make
 284  * lookups blindingly fast.  Note that tsd_nalloc could be placed at the
 285  * end of the pad region to increase the likelihood that it falls on the
 286  * same cache line as the data.
 287  */
 288 typedef union tsd {
 289         uint_t tsd_nalloc;              /* Amount of allocated storage */
 290         void *tsd_pad[TSD_NFAST];
 291         void *tsd_data[1];
 292 } tsd_t;
 293 
 294 typedef struct {
 295         mutex_t tsdm_lock;              /* Lock protecting the data */
 296         uint_t tsdm_nkeys;              /* Number of allocated keys */
 297         uint_t tsdm_nused;              /* Number of used keys */
 298         PFrV *tsdm_destro;              /* Per-key destructors */
 299         char tsdm_pad[64 -              /* pad to 64 bytes */
 300                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (PFrV *))];
 301 } tsd_metadata_t;
 302 
 303 #ifdef _SYSCALL32
 304 typedef union tsd32 {
 305         uint_t tsd_nalloc;              /* Amount of allocated storage */
 306         caddr32_t tsd_pad[TSD_NFAST];
 307         caddr32_t tsd_data[1];
 308 } tsd32_t;
 309 
 310 typedef struct {
 311         mutex_t tsdm_lock;              /* Lock protecting the data */
 312         uint_t tsdm_nkeys;              /* Number of allocated keys */
 313         uint_t tsdm_nused;              /* Number of used keys */
 314         caddr32_t tsdm_destro;          /* Per-key destructors */
 315         char tsdm_pad[64 -              /* pad to 64 bytes */
 316                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (caddr32_t))];
 317 } tsd_metadata32_t;
 318 #endif  /* _SYSCALL32 */
 319 
 320 
 321 /*
 322  * Thread-Local Storage (TLS)
 323  */
 324 typedef struct {
 325         void            *tls_data;
 326         size_t          tls_size;
 327 } tls_t;
 328 
 329 typedef struct {
 330         mutex_t tls_lock;               /* Lock protecting the data */
 331         tls_t   tls_modinfo;            /* Root of all TLS_modinfo data */
 332         tls_t   static_tls;             /* Template for static TLS */
 333         char    tls_pad[64 -            /* pad to 64 bytes */
 334                 (sizeof (mutex_t) + 2 * sizeof (tls_t))];
 335 } tls_metadata_t;
 336 
 337 #ifdef _SYSCALL32
 338 typedef struct {
 339         caddr32_t       tls_data;
 340         size32_t        tls_size;
 341 } tls32_t;
 342 
 343 typedef struct {
 344         mutex_t tls_lock;               /* Lock protecting the data */
 345         tls32_t tls_modinfo;            /* Root of all TLS_modinfo data */
 346         tls32_t static_tls;             /* Template for static TLS */
 347         char    tls_pad[64 -            /* pad to 64 bytes */
 348                 (sizeof (mutex_t) + 2 * sizeof (tls32_t))];
 349 } tls_metadata32_t;
 350 #endif  /* _SYSCALL32 */
 351 
 352 
 353 /*
 354  * Sleep queue root for USYNC_THREAD condvars and mutexes.
 355  * There is a default queue root for each queue head (see below).
 356  * Also, each ulwp_t contains a queue root that can be used
 357  * when the thread is enqueued on the queue, if necessary
 358  * (when more than one wchan hashes to the same queue head).
 359  */
 360 typedef struct queue_root {
 361         struct queue_root       *qr_next;
 362         struct queue_root       *qr_prev;
 363         struct ulwp             *qr_head;
 364         struct ulwp             *qr_tail;
 365         void                    *qr_wchan;
 366         uint32_t                qr_rtcount;
 367         uint32_t                qr_qlen;
 368         uint32_t                qr_qmax;
 369 } queue_root_t;
 370 
 371 #ifdef _SYSCALL32
 372 typedef struct queue_root32 {
 373         caddr32_t               qr_next;
 374         caddr32_t               qr_prev;
 375         caddr32_t               qr_head;
 376         caddr32_t               qr_tail;
 377         caddr32_t               qr_wchan;
 378         uint32_t                qr_rtcount;
 379         uint32_t                qr_qlen;
 380         uint32_t                qr_qmax;
 381 } queue_root32_t;
 382 #endif
 383 
 384 /*
 385  * Sleep queue heads for USYNC_THREAD condvars and mutexes.
 386  * The size and alignment is 128 bytes to reduce cache conflicts.
 387  * Each queue head points to a list of queue roots, defined above.
 388  * Each queue head contains a default queue root for use when only one
 389  * is needed.  It is always at the tail of the queue root hash chain.
 390  */
 391 typedef union {
 392         uint64_t                qh_64[16];
 393         struct {
 394                 mutex_t         q_lock;
 395                 uint8_t         q_qcnt;
 396                 uint8_t         q_type;         /* MX or CV */
 397                 uint8_t         q_pad1[2];
 398                 uint32_t        q_lockcount;
 399                 uint32_t        q_qlen;
 400                 uint32_t        q_qmax;
 401                 void            *q_wchan;       /* valid only while locked */
 402                 struct queue_root *q_root;      /* valid only while locked */
 403                 struct queue_root *q_hlist;
 404 #if !defined(_LP64)
 405                 caddr_t         q_pad2[3];
 406 #endif
 407                 queue_root_t    q_def_root;
 408                 uint32_t        q_hlen;
 409                 uint32_t        q_hmax;
 410         } qh_qh;
 411 } queue_head_t;
 412 
 413 #define qh_lock         qh_qh.q_lock
 414 #define qh_qcnt         qh_qh.q_qcnt
 415 #define qh_type         qh_qh.q_type
 416 #if defined(THREAD_DEBUG)
 417 #define qh_lockcount    qh_qh.q_lockcount
 418 #define qh_qlen         qh_qh.q_qlen
 419 #define qh_qmax         qh_qh.q_qmax
 420 #endif
 421 #define qh_wchan        qh_qh.q_wchan
 422 #define qh_root         qh_qh.q_root
 423 #define qh_hlist        qh_qh.q_hlist
 424 #define qh_def_root     qh_qh.q_def_root
 425 #define qh_hlen         qh_qh.q_hlen
 426 #define qh_hmax         qh_qh.q_hmax
 427 
 428 /* queue types passed to queue_lock() */
 429 #define MX      0
 430 #define CV      1
 431 #define QHASHSHIFT      9                       /* number of hashing bits */
 432 #define QHASHSIZE       (1 << QHASHSHIFT) /* power of 2 (1<<9 == 512) */
 433 #define QUEUE_HASH(wchan, type) ((uint_t)                       \
 434         ((((uintptr_t)(wchan) >> 3)                               \
 435         ^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3)))               \
 436         & (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE))
 437 
 438 extern  queue_head_t    *queue_lock(void *, int);
 439 extern  void            queue_unlock(queue_head_t *);
 440 extern  void            enqueue(queue_head_t *, struct ulwp *, int);
 441 extern  struct ulwp     *dequeue(queue_head_t *, int *);
 442 extern  struct ulwp     **queue_slot(queue_head_t *, struct ulwp **, int *);
 443 extern  struct ulwp     *queue_waiter(queue_head_t *);
 444 extern  int             dequeue_self(queue_head_t *);
 445 extern  void            queue_unlink(queue_head_t *,
 446                                 struct ulwp **, struct ulwp *);
 447 extern  void            unsleep_self(void);
 448 extern  void            spin_lock_set(mutex_t *);
 449 extern  void            spin_lock_clear(mutex_t *);
 450 
 451 /*
 452  * Scheduling class information structure.
 453  */
 454 typedef struct {
 455         short           pcc_state;
 456         short           pcc_policy;
 457         pri_t           pcc_primin;
 458         pri_t           pcc_primax;
 459         pcinfo_t        pcc_info;
 460 } pcclass_t;
 461 
 462 /*
 463  * Memory block for chain of owned ceiling mutexes.
 464  */
 465 typedef struct mxchain {
 466         struct mxchain  *mxchain_next;
 467         mutex_t         *mxchain_mx;
 468 } mxchain_t;
 469 
 470 /*
 471  * Pointer to an rwlock that is held for reading.
 472  * Used in rw_rdlock() to allow a thread that already holds a read
 473  * lock to acquire another read lock on the same rwlock even if
 474  * there are writers waiting.  This to avoid deadlock when acquiring
 475  * a read lock more than once in the presence of pending writers.
 476  * POSIX mandates this behavior.
 477  */
 478 typedef struct {
 479         void    *rd_rwlock;     /* the rwlock held for reading */
 480         size_t  rd_count;       /* count of read locks applied */
 481 } readlock_t;
 482 
 483 #ifdef _SYSCALL32
 484 typedef struct {
 485         caddr32_t       rd_rwlock;
 486         size32_t        rd_count;
 487 } readlock32_t;
 488 #endif  /* _SYSCALL32 */
 489 
 490 /*
 491  * Maximum number of read locks allowed for one thread on one rwlock.
 492  * This could be as large as INT_MAX, but the SUSV3 test suite would
 493  * take an inordinately long time to complete.  This is big enough.
 494  */
 495 #define READ_LOCK_MAX   100000
 496 
 497 #define ul_tlsent       ul_tls.tls_data /* array of pointers to dynamic TLS */
 498 #define ul_ntlsent      ul_tls.tls_size /* number of entries in ul_tlsent */
 499 
 500 /*
 501  * Round up an integral value to a multiple of 64
 502  */
 503 #define roundup64(x)    (-(-(x) & -64))
 504 
 505 /*
 506  * NOTE:  Whatever changes are made to ulwp_t must be
 507  * reflected in $SRC/cmd/mdb/common/modules/libc/libc.c
 508  *
 509  * NOTE: ul_self *must* be the first member of ulwp_t on x86
 510  * Low-level x86 code relies on this.
 511  */
 512 typedef struct ulwp {
 513         /*
 514          * These members always need to come first on sparc.
 515          * For dtrace, a ulwp_t must be aligned on a 64-byte boundary.
 516          */
 517 #if defined(__sparc)
 518         uint32_t        ul_dinstr;      /* scratch space for dtrace */
 519         uint32_t        ul_padsparc0[15];
 520         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
 521         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
 522         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
 523         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
 524 #endif
 525         struct ulwp     *ul_self;       /* pointer to self */
 526 #if defined(__i386)
 527         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
 528 #elif defined(__amd64)
 529         uint8_t         ul_dinstr[56];  /* scratch space for dtrace */
 530 #endif
 531         struct uberdata *ul_uberdata;   /* uber (super-global) data */
 532         tls_t           ul_tls;         /* dynamic thread-local storage base */
 533         struct ulwp     *ul_forw;       /* forw, back all_lwps list, */
 534         struct ulwp     *ul_back;       /* protected by link_lock */
 535         struct ulwp     *ul_next;       /* list to keep track of stacks */
 536         struct ulwp     *ul_hash;       /* hash chain linked list */
 537         void            *ul_rval;       /* return value from thr_exit() */
 538         caddr_t         ul_stk;         /* mapping base of the stack */
 539         size_t          ul_mapsiz;      /* mapping size of the stack */
 540         size_t          ul_guardsize;   /* normally _lpagesize */
 541         uintptr_t       ul_stktop;      /* broken thr_stksegment() interface */
 542         size_t          ul_stksiz;      /* broken thr_stksegment() interface */
 543         stack_t         ul_ustack;      /* current stack boundaries */
 544         int             ul_ix;          /* hash index */
 545         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
 546         pri_t           ul_pri;         /* scheduling priority */
 547         pri_t           ul_epri;        /* real-time ceiling priority */
 548         char            ul_policy;      /* scheduling policy */
 549         char            ul_cid;         /* scheduling class id */
 550         union {
 551                 struct {
 552                         char    cursig; /* deferred signal number */
 553                         char    pleasestop; /* lwp requested to stop itself */
 554                 } s;
 555                 short   curplease;      /* for testing both at once */
 556         } ul_cp;
 557         char            ul_stop;        /* reason for stopping */
 558         char            ul_signalled;   /* this lwp was cond_signal()d */
 559         char            ul_dead;        /* this lwp has called thr_exit */
 560         char            ul_unwind;      /* posix: unwind C++ stack */
 561         char            ul_detached;    /* THR_DETACHED at thread_create() */
 562                                         /* or pthread_detach() was called */
 563         char            ul_writer;      /* sleeping in rw_wrlock() */
 564         char            ul_stopping;    /* set by curthread: stopping self */
 565         char            ul_cancel_prologue;     /* for _cancel_prologue() */
 566         short           ul_preempt;     /* no_preempt()/preempt() */
 567         short           ul_savpreempt;  /* pre-existing preempt value */
 568         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
 569         char            ul_main;        /* thread is the main thread */
 570         char            ul_fork;        /* thread is performing a fork */
 571         char            ul_primarymap;  /* primary link-map is initialized */
 572         /* per-thread copies of the corresponding global variables */
 573         uint8_t         ul_max_spinners;        /* thread_max_spinners */
 574         char            ul_door_noreserve;      /* thread_door_noreserve */
 575         char            ul_queue_fifo;          /* thread_queue_fifo */
 576         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
 577         char            ul_error_detection;     /* thread_error_detection */
 578         char            ul_async_safe;          /* thread_async_safe */
 579         char            ul_rt;                  /* found on an RT queue */
 580         char            ul_rtqueued;            /* was RT when queued */
 581         char            ul_misaligned;          /* thread_locks_misaligned */
 582         char            ul_pad[3];
 583         int             ul_adaptive_spin;       /* thread_adaptive_spin */
 584         int             ul_queue_spin;          /* thread_queue_spin */
 585         volatile int    ul_critical;    /* non-zero == in a critical region */
 586         int             ul_sigdefer;    /* non-zero == defer signals */
 587         int             ul_vfork;       /* thread is the child of vfork() */
 588         int             ul_cancelable;  /* _cancelon()/_canceloff() */
 589         char            ul_cancel_pending;  /* pthread_cancel() was called */
 590         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
 591         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
 592         char            ul_save_async;  /* saved copy of ul_cancel_async */
 593         char            ul_mutator;     /* lwp is a mutator (java interface) */
 594         char            ul_created;     /* created suspended */
 595         char            ul_replace;     /* replacement; must be free()d */
 596         uchar_t         ul_nocancel;    /* cancellation can't happen */
 597         int             ul_errno;       /* per-thread errno */
 598         int             *ul_errnop;     /* pointer to errno or self->ul_errno */
 599         __cleanup_t     *ul_clnup_hdr;  /* head of cleanup handlers list */
 600         uberflags_t     *ul_schedctl_called;    /* ul_schedctl is set up */
 601         volatile sc_shared_t *ul_schedctl;      /* schedctl data */
 602         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
 603         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
 604         tsd_t           *ul_stsd;       /* slow TLS for keys >= TSD_NFAST */
 605         void            *ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
 606         td_evbuf_t      ul_td_evbuf;    /* event buffer */
 607         char            ul_td_events_enable;    /* event mechanism enabled */
 608         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
 609         char            ul_qtype;       /* MX or CV */
 610         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
 611         int             ul_rtld;        /* thread is running inside ld.so.1 */
 612         int             ul_usropts;     /* flags given to thr_create() */
 613         void            *(*ul_startpc)(void *); /* start func (thr_create()) */
 614         void            *ul_startarg;   /* argument for start function */
 615         void            *ul_wchan;      /* synch object when sleeping */
 616         struct ulwp     *ul_link;       /* sleep queue link */
 617         queue_head_t    *ul_sleepq;     /* sleep queue thread is waiting on */
 618         mutex_t         *ul_cvmutex;    /* mutex dropped when waiting on a cv */
 619         mxchain_t       *ul_mxchain;    /* chain of owned ceiling mutexes */
 620         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
 621         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
 622                                 /* 0 means there is but a single entry */
 623         union {                         /* single entry or pointer to array */
 624                 readlock_t      single;
 625                 readlock_t      *array;
 626         } ul_readlock;
 627         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
 628                                 /* 0 means there is but a single entry */
 629         union {                         /* single entry or pointer to array */
 630                 mutex_t         *single;
 631                 mutex_t         **array;
 632         } ul_heldlocks;
 633         /* PROBE_SUPPORT begin */
 634         void            *ul_tpdp;
 635         /* PROBE_SUPPORT end */
 636         ucontext_t      *ul_siglink;    /* pointer to previous context */
 637         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
 638         uint_t          ul_spin_lock_spin2;
 639         uint_t          ul_spin_lock_sleep;
 640         uint_t          ul_spin_lock_wakeup;
 641         queue_root_t    ul_queue_root;  /* root of a sleep queue */
 642         id_t            ul_rtclassid;   /* real-time class id */
 643         uint_t          ul_pilocks;     /* count of PI locks held */
 644                 /* the following members *must* be last in the structure */
 645                 /* they are discarded when ulwp is replaced on thr_exit() */
 646         sigset_t        ul_sigmask;     /* thread's current signal mask */
 647         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
 648         siginfo_t       ul_siginfo;     /* deferred siginfo */
 649         mutex_t         ul_spinlock;    /* used when suspending/continuing */
 650         fpuenv_t        ul_fpuenv;      /* floating point state */
 651         uintptr_t       ul_sp;          /* stack pointer when blocked */
 652         void            *ul_ex_unwind;  /* address of _ex_unwind() or -1 */
 653 #if defined(sparc)
 654         void            *ul_unwind_ret; /* used only by _ex_clnup_handler() */
 655 #endif
 656 } ulwp_t;
 657 
 658 #define ul_cursig       ul_cp.s.cursig          /* deferred signal number */
 659 #define ul_pleasestop   ul_cp.s.pleasestop      /* lwp requested to stop */
 660 #define ul_curplease    ul_cp.curplease         /* for testing both at once */
 661 
 662 /*
 663  * This is the size of a replacement ulwp, retained only for the benefit
 664  * of thr_join().  The trailing members are unneeded for this purpose.
 665  */
 666 #define REPLACEMENT_SIZE        ((size_t)&((ulwp_t *)NULL)->ul_sigmask)
 667 
 668 /*
 669  * Definitions for static initialization of signal sets,
 670  * plus some sneaky optimizations in various places.
 671  */
 672 
 673 #define SIGMASK(sig)    ((uint32_t)1 << (((sig) - 1) & (32 - 1)))
 674 
 675 #if (MAXSIG > (2 * 32) && MAXSIG <= (3 * 32))
 676 #define FILLSET0        0xffffffffu
 677 #define FILLSET1        0xffffffffu
 678 #define FILLSET2        ((1u << (MAXSIG - 64)) - 1)
 679 #define FILLSET3        0
 680 #else
 681 #error "fix me: MAXSIG out of bounds"
 682 #endif
 683 
 684 #define CANTMASK0       (SIGMASK(SIGKILL) | SIGMASK(SIGSTOP))
 685 #define CANTMASK1       0
 686 #define CANTMASK2       0
 687 #define CANTMASK3       0
 688 
 689 #define MASKSET0        (FILLSET0 & ~CANTMASK0)
 690 #define MASKSET1        (FILLSET1 & ~CANTMASK1)
 691 #define MASKSET2        (FILLSET2 & ~CANTMASK2)
 692 #define MASKSET3        (FILLSET3 & ~CANTMASK3)
 693 
 694 extern  const sigset_t maskset;         /* set of all maskable signals */
 695 
 696 extern  int     thread_adaptive_spin;
 697 extern  uint_t  thread_max_spinners;
 698 extern  int     thread_queue_spin;
 699 extern  int     thread_queue_fifo;
 700 extern  int     thread_queue_dump;
 701 extern  int     thread_cond_wait_defer;
 702 extern  int     thread_async_safe;
 703 extern  int     thread_queue_verify;
 704 
 705 /*
 706  * pthread_atfork() related data, used to store atfork handlers.
 707  */
 708 typedef struct atfork {
 709         struct atfork *forw;            /* forward pointer */
 710         struct atfork *back;            /* backward pointer */
 711         void (*prepare)(void);          /* pre-fork handler */
 712         void (*parent)(void);           /* post-fork parent handler */
 713         void (*child)(void);            /* post-fork child handler */
 714 } atfork_t;
 715 
 716 /*
 717  * Element in the table and in the list of registered process
 718  * robust locks.  We keep track of these to make sure that we
 719  * only call ___lwp_mutex_register() once for each such lock
 720  * after it is first mapped in (or newly mapped in).
 721  */
 722 typedef struct robust {
 723         struct robust   *robust_next;   /* hash table list */
 724         struct robust   *robust_list;   /* global list */
 725         mutex_t         *robust_lock;
 726 } robust_t;
 727 
 728 /*
 729  * Invalid address, used to mark an unused element in the hash table.
 730  */
 731 #define INVALID_ADDR    ((void *)(uintptr_t)(-1L))
 732 
 733 /*
 734  * Parameters of the lock registration hash table.
 735  */
 736 #define LOCKSHIFT       15                      /* number of hashing bits */
 737 #define LOCKHASHSZ      (1 << LOCKSHIFT)  /* power of 2 (1<<15 == 32K) */
 738 #define LOCK_HASH(addr) (uint_t)                        \
 739         ((((uintptr_t)(addr) >> 3)                        \
 740         ^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3))) \
 741         & (LOCKHASHSZ - 1))
 742 
 743 /*
 744  * Make our hot locks reside on private cache lines (64 bytes).
 745  */
 746 typedef struct {
 747         mutex_t pad_lock;
 748         char    pad_pad[64 - sizeof (mutex_t)];
 749 } pad_lock_t;
 750 
 751 /*
 752  * Make our semi-hot locks reside on semi-private cache lines (32 bytes).
 753  */
 754 typedef struct {
 755         mutex_t pad_lock;
 756         char    pad_pad[32 - sizeof (mutex_t)];
 757 } pad32_lock_t;
 758 
 759 /*
 760  * The threads hash table is used for fast lookup and locking of an active
 761  * thread structure (ulwp_t) given a thread-id.  It is an N-element array of
 762  * thr_hash_table_t structures, where N == 1 before the main thread creates
 763  * the first additional thread and N == 1024 afterwards.  Each element of the
 764  * table is 64 bytes in size and alignment to reduce cache conflicts.
 765  */
 766 typedef struct {
 767         mutex_t hash_lock;      /* lock per bucket */
 768         cond_t  hash_cond;      /* convar per bucket */
 769         ulwp_t  *hash_bucket;   /* hash bucket points to the list of ulwps */
 770         char    hash_pad[64 -   /* pad out to 64 bytes */
 771                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (ulwp_t *))];
 772 } thr_hash_table_t;
 773 
 774 #ifdef _SYSCALL32
 775 typedef struct {
 776         mutex_t hash_lock;
 777         cond_t  hash_cond;
 778         caddr32_t hash_bucket;
 779         char    hash_pad[64 -
 780                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (caddr32_t))];
 781 } thr_hash_table32_t;
 782 #endif  /* _SYSCALL32 */
 783 
 784 
 785 /*
 786  * siguaction members have 128-byte size and 64-byte alignment.
 787  * We know that sizeof (struct sigaction) is 32 bytes for both
 788  * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes.
 789  */
 790 typedef struct {
 791         rwlock_t        sig_lock;
 792         struct sigaction sig_uaction;
 793         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)];
 794 } siguaction_t;
 795 
 796 #ifdef _SYSCALL32
 797 typedef struct {
 798         rwlock_t        sig_lock;
 799         struct sigaction32 sig_uaction;
 800         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)];
 801 } siguaction32_t;
 802 #endif  /* _SYSCALL32 */
 803 
 804 
 805 /*
 806  * Bucket structures, used by lmalloc()/lfree().
 807  * See port/threads/alloc.c for details.
 808  * A bucket's size and alignment is 64 bytes.
 809  */
 810 typedef struct {
 811         mutex_t bucket_lock;    /* protects the free list allocations */
 812         void    *free_list;     /* LIFO list of blocks to allocate/free */
 813         size_t  chunks;         /* number of 64K blocks mmap()ed last time */
 814         char    pad64[64 -      /* pad out to 64 bytes */
 815                 (sizeof (mutex_t) + sizeof (void *) + sizeof (size_t))];
 816 } bucket_t;
 817 
 818 #ifdef _SYSCALL32
 819 typedef struct {
 820         mutex_t         bucket_lock;
 821         caddr32_t       free_list;
 822         size32_t        chunks;
 823         char    pad64[64 -      /* pad out to 64 bytes */
 824                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (size32_t))];
 825 } bucket32_t;
 826 #endif  /* _SYSCALL32 */
 827 
 828 #define NBUCKETS        10      /* sizes ranging from 64 to 32768 */
 829 
 830 
 831 /*
 832  * atexit() data structures.
 833  * See port/gen/atexit.c for details.
 834  */
 835 typedef void (*_exithdlr_func_t) (void);
 836 
 837 typedef struct _exthdlr {
 838         struct _exthdlr         *next;  /* next in handler list */
 839         _exithdlr_func_t        hdlr;   /* handler itself */
 840 } _exthdlr_t;
 841 
 842 typedef struct {
 843         mutex_t         exitfns_lock;
 844         _exthdlr_t      *head;
 845         void            *exit_frame_monitor;
 846         char            exit_pad[64 -   /* pad out to 64 bytes */
 847                 (sizeof (mutex_t) + sizeof (_exthdlr_t *) + sizeof (void *))];
 848 } atexit_root_t;
 849 
 850 #ifdef _SYSCALL32
 851 typedef struct {
 852         mutex_t         exitfns_lock;
 853         caddr32_t       head;
 854         caddr32_t       exit_frame_monitor;
 855         char            exit_pad[64 -   /* pad out to 64 bytes */
 856                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (caddr32_t))];
 857 } atexit_root32_t;
 858 #endif  /* _SYSCALL32 */
 859 
 860 
 861 /*
 862  * This is data that is global to all link maps (uberdata, aka super-global).
 863  * Note: When changing this, please be sure to keep the 32-bit variant of
 864  * this in sync.  (see uberdata32_t below)
 865  */
 866 typedef struct uberdata {
 867         pad_lock_t      _link_lock;
 868         pad_lock_t      _ld_lock;
 869         pad_lock_t      _fork_lock;
 870         pad_lock_t      _atfork_lock;
 871         pad32_lock_t    _callout_lock;
 872         pad32_lock_t    _tdb_hash_lock;
 873         tdb_sync_stats_t tdb_hash_lock_stats;
 874         siguaction_t    siguaction[NSIG];
 875         bucket_t        bucket[NBUCKETS];
 876         atexit_root_t   atexit_root;
 877         tsd_metadata_t  tsd_metadata;
 878         tls_metadata_t  tls_metadata;
 879         /*
 880          * Every object before this point has size and alignment of 64 bytes.
 881          * Don't add any other type of data before this point.
 882          */
 883         char    primary_map;    /* set when primary link map is initialized */
 884         char    bucket_init;    /* set when bucket[NBUCKETS] is initialized */
 885         char    pad[2];
 886         uberflags_t     uberflags;
 887         queue_head_t    *queue_head;
 888         thr_hash_table_t *thr_hash_table;
 889         uint_t          hash_size;      /* # of entries in thr_hash_table[] */
 890         uint_t          hash_mask;      /* hash_size - 1 */
 891         ulwp_t  *ulwp_one;      /* main thread */
 892         ulwp_t  *all_lwps;      /* circular ul_forw/ul_back list of live lwps */
 893         ulwp_t  *all_zombies;   /* circular ul_forw/ul_back list of zombies */
 894         int     nthreads;       /* total number of live threads/lwps */
 895         int     nzombies;       /* total number of zombie threads */
 896         int     ndaemons;       /* total number of THR_DAEMON threads/lwps */
 897         pid_t   pid;            /* the current process's pid */
 898         void    (*sigacthandler)(int, siginfo_t *, void *);
 899         ulwp_t  *lwp_stacks;
 900         ulwp_t  *lwp_laststack;
 901         int     nfreestack;
 902         int     thread_stack_cache;
 903         ulwp_t  *ulwp_freelist;
 904         ulwp_t  *ulwp_lastfree;
 905         ulwp_t  *ulwp_replace_free;
 906         ulwp_t  *ulwp_replace_last;
 907         atfork_t        *atforklist;    /* circular Q for fork handlers */
 908         robust_t        **robustlocks;  /* table of registered robust locks */
 909         robust_t        *robustlist;    /* list of registered robust locks */
 910         char    *progname;      /* the basename of the program, from argv[0] */
 911         struct uberdata **tdb_bootstrap;
 912         tdb_t   tdb;            /* thread debug interfaces (for libc_db) */
 913 } uberdata_t;
 914 
 915 #define link_lock       _link_lock.pad_lock
 916 #define ld_lock         _ld_lock.pad_lock
 917 #define fork_lock       _fork_lock.pad_lock
 918 #define atfork_lock     _atfork_lock.pad_lock
 919 #define callout_lock    _callout_lock.pad_lock
 920 #define tdb_hash_lock   _tdb_hash_lock.pad_lock
 921 
 922 #pragma align 64(__uberdata)
 923 extern  uberdata_t      __uberdata;
 924 extern  uberdata_t      **__tdb_bootstrap;      /* known to libc_db and mdb */
 925 extern  int             primary_link_map;
 926 
 927 #define ulwp_mutex(ulwp, udp)   \
 928         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_lock)
 929 #define ulwp_condvar(ulwp, udp) \
 930         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_cond)
 931 
 932 /*
 933  * Grab and release the hash table lock for the specified lwp.
 934  */
 935 #define ulwp_lock(ulwp, udp)    lmutex_lock(ulwp_mutex(ulwp, udp))
 936 #define ulwp_unlock(ulwp, udp)  lmutex_unlock(ulwp_mutex(ulwp, udp))
 937 
 938 #ifdef _SYSCALL32       /* needed by libc_db */
 939 
 940 typedef struct ulwp32 {
 941 #if defined(__sparc)
 942         uint32_t        ul_dinstr;      /* scratch space for dtrace */
 943         uint32_t        ul_padsparc0[15];
 944         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
 945         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
 946         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
 947         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
 948 #endif
 949         caddr32_t       ul_self;        /* pointer to self */
 950 #if defined(__x86)
 951         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
 952 #endif
 953         caddr32_t       ul_uberdata;    /* uber (super-global) data */
 954         tls32_t         ul_tls;         /* dynamic thread-local storage base */
 955         caddr32_t       ul_forw;        /* forw, back all_lwps list, */
 956         caddr32_t       ul_back;        /* protected by link_lock */
 957         caddr32_t       ul_next;        /* list to keep track of stacks */
 958         caddr32_t       ul_hash;        /* hash chain linked list */
 959         caddr32_t       ul_rval;        /* return value from thr_exit() */
 960         caddr32_t       ul_stk;         /* mapping base of the stack */
 961         size32_t        ul_mapsiz;      /* mapping size of the stack */
 962         size32_t        ul_guardsize;   /* normally _lpagesize */
 963         caddr32_t       ul_stktop;      /* broken thr_stksegment() interface */
 964         size32_t        ul_stksiz;      /* broken thr_stksegment() interface */
 965         stack32_t       ul_ustack;      /* current stack boundaries */
 966         int             ul_ix;          /* hash index */
 967         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
 968         pri_t           ul_pri;         /* scheduling priority */
 969         pri_t           ul_epri;        /* real-time ceiling priority */
 970         char            ul_policy;      /* scheduling policy */
 971         char            ul_cid;         /* scheduling class id */
 972         union {
 973                 struct {
 974                         char    cursig; /* deferred signal number */
 975                         char    pleasestop; /* lwp requested to stop itself */
 976                 } s;
 977                 short   curplease;      /* for testing both at once */
 978         } ul_cp;
 979         char            ul_stop;        /* reason for stopping */
 980         char            ul_signalled;   /* this lwp was cond_signal()d */
 981         char            ul_dead;        /* this lwp has called thr_exit */
 982         char            ul_unwind;      /* posix: unwind C++ stack */
 983         char            ul_detached;    /* THR_DETACHED at thread_create() */
 984                                         /* or pthread_detach() was called */
 985         char            ul_writer;      /* sleeping in rw_wrlock() */
 986         char            ul_stopping;    /* set by curthread: stopping self */
 987         char            ul_cancel_prologue;     /* for _cancel_prologue() */
 988         short           ul_preempt;     /* no_preempt()/preempt() */
 989         short           ul_savpreempt;  /* pre-existing preempt value */
 990         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
 991         char            ul_main;        /* thread is the main thread */
 992         char            ul_fork;        /* thread is performing a fork */
 993         char            ul_primarymap;  /* primary link-map is initialized */
 994         /* per-thread copies of the corresponding global variables */
 995         uint8_t         ul_max_spinners;        /* thread_max_spinners */
 996         char            ul_door_noreserve;      /* thread_door_noreserve */
 997         char            ul_queue_fifo;          /* thread_queue_fifo */
 998         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
 999         char            ul_error_detection;     /* thread_error_detection */
1000         char            ul_async_safe;          /* thread_async_safe */
1001         char            ul_rt;                  /* found on an RT queue */
1002         char            ul_rtqueued;            /* was RT when queued */
1003         char            ul_misaligned;          /* thread_locks_misaligned */
1004         char            ul_pad[3];
1005         int             ul_adaptive_spin;       /* thread_adaptive_spin */
1006         int             ul_queue_spin;          /* thread_queue_spin */
1007         int             ul_critical;    /* non-zero == in a critical region */
1008         int             ul_sigdefer;    /* non-zero == defer signals */
1009         int             ul_vfork;       /* thread is the child of vfork() */
1010         int             ul_cancelable;  /* _cancelon()/_canceloff() */
1011         char            ul_cancel_pending;  /* pthread_cancel() was called */
1012         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
1013         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
1014         char            ul_save_async;  /* saved copy of ul_cancel_async */
1015         char            ul_mutator;     /* lwp is a mutator (java interface) */
1016         char            ul_created;     /* created suspended */
1017         char            ul_replace;     /* replacement; must be free()d */
1018         uchar_t         ul_nocancel;    /* cancellation can't happen */
1019         int             ul_errno;       /* per-thread errno */
1020         caddr32_t       ul_errnop;      /* pointer to errno or self->ul_errno */
1021         caddr32_t       ul_clnup_hdr;   /* head of cleanup handlers list */
1022         caddr32_t       ul_schedctl_called; /* ul_schedctl is set up */
1023         caddr32_t       ul_schedctl;    /* schedctl data */
1024         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
1025         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
1026         caddr32_t       ul_stsd;        /* slow TLS for keys >= TSD_NFAST */
1027         caddr32_t       ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
1028         td_evbuf32_t    ul_td_evbuf;    /* event buffer */
1029         char            ul_td_events_enable;    /* event mechanism enabled */
1030         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
1031         char            ul_qtype;       /* MX or CV */
1032         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
1033         int             ul_rtld;        /* thread is running inside ld.so.1 */
1034         int             ul_usropts;     /* flags given to thr_create() */
1035         caddr32_t       ul_startpc;     /* start func (thr_create()) */
1036         caddr32_t       ul_startarg;    /* argument for start function */
1037         caddr32_t       ul_wchan;       /* synch object when sleeping */
1038         caddr32_t       ul_link;        /* sleep queue link */
1039         caddr32_t       ul_sleepq;      /* sleep queue thread is waiting on */
1040         caddr32_t       ul_cvmutex;     /* mutex dropped when waiting on a cv */
1041         caddr32_t       ul_mxchain;     /* chain of owned ceiling mutexes */
1042         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
1043         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
1044                                 /* 0 means there is but a single entry */
1045         union {                         /* single entry or pointer to array */
1046                 readlock32_t    single;
1047                 caddr32_t       array;
1048         } ul_readlock;
1049         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
1050                                 /* 0 means there is but a single entry */
1051         union {                         /* single entry or pointer to array */
1052                 caddr32_t       single;
1053                 caddr32_t       array;
1054         } ul_heldlocks;
1055         /* PROBE_SUPPORT begin */
1056         caddr32_t       ul_tpdp;
1057         /* PROBE_SUPPORT end */
1058         caddr32_t       ul_siglink;     /* pointer to previous context */
1059         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
1060         uint_t          ul_spin_lock_spin2;
1061         uint_t          ul_spin_lock_sleep;
1062         uint_t          ul_spin_lock_wakeup;
1063         queue_root32_t  ul_queue_root;  /* root of a sleep queue */
1064         id_t            ul_rtclassid;   /* real-time class id */
1065         uint_t          ul_pilocks;     /* count of PI locks held */
1066                 /* the following members *must* be last in the structure */
1067                 /* they are discarded when ulwp is replaced on thr_exit() */
1068         sigset_t        ul_sigmask;     /* thread's current signal mask */
1069         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
1070         siginfo32_t     ul_siginfo;     /* deferred siginfo */
1071         mutex_t         ul_spinlock;    /* used when suspending/continuing */
1072         fpuenv32_t      ul_fpuenv;      /* floating point state */
1073         caddr32_t       ul_sp;          /* stack pointer when blocked */
1074 #if defined(sparc)
1075         caddr32_t       ul_unwind_ret;  /* used only by _ex_clnup_handler() */
1076 #endif
1077 } ulwp32_t;
1078 
1079 #define REPLACEMENT_SIZE32      ((size_t)&((ulwp32_t *)NULL)->ul_sigmask)
1080 
1081 typedef struct uberdata32 {
1082         pad_lock_t      _link_lock;
1083         pad_lock_t      _ld_lock;
1084         pad_lock_t      _fork_lock;
1085         pad_lock_t      _atfork_lock;
1086         pad32_lock_t    _callout_lock;
1087         pad32_lock_t    _tdb_hash_lock;
1088         tdb_sync_stats_t tdb_hash_lock_stats;
1089         siguaction32_t  siguaction[NSIG];
1090         bucket32_t      bucket[NBUCKETS];
1091         atexit_root32_t atexit_root;
1092         tsd_metadata32_t tsd_metadata;
1093         tls_metadata32_t tls_metadata;
1094         char            primary_map;
1095         char            bucket_init;
1096         char            pad[2];
1097         uberflags_t     uberflags;
1098         caddr32_t       queue_head;
1099         caddr32_t       thr_hash_table;
1100         uint_t          hash_size;
1101         uint_t          hash_mask;
1102         caddr32_t       ulwp_one;
1103         caddr32_t       all_lwps;
1104         caddr32_t       all_zombies;
1105         int             nthreads;
1106         int             nzombies;
1107         int             ndaemons;
1108         int             pid;
1109         caddr32_t       sigacthandler;
1110         caddr32_t       lwp_stacks;
1111         caddr32_t       lwp_laststack;
1112         int             nfreestack;
1113         int             thread_stack_cache;
1114         caddr32_t       ulwp_freelist;
1115         caddr32_t       ulwp_lastfree;
1116         caddr32_t       ulwp_replace_free;
1117         caddr32_t       ulwp_replace_last;
1118         caddr32_t       atforklist;
1119         caddr32_t       robustlocks;
1120         caddr32_t       robustlist;
1121         caddr32_t       progname;
1122         caddr32_t       tdb_bootstrap;
1123         tdb32_t         tdb;
1124 } uberdata32_t;
1125 
1126 #endif  /* _SYSCALL32 */
1127 
1128 /* ul_stop values */
1129 #define TSTP_REGULAR    0x01    /* Stopped by thr_suspend() */
1130 #define TSTP_MUTATOR    0x08    /* stopped by thr_suspend_*mutator*() */
1131 #define TSTP_FORK       0x20    /* stopped by suspend_fork() */
1132 
1133 /*
1134  * Implementation-specific attribute types for pthread_mutexattr_init() etc.
1135  */
1136 
1137 typedef struct  _cvattr {
1138         int     pshared;
1139         clockid_t clockid;
1140 } cvattr_t;
1141 
1142 typedef struct  _mattr {
1143         int     pshared;
1144         int     protocol;
1145         int     prioceiling;
1146         int     type;
1147         int     robustness;
1148 } mattr_t;
1149 
1150 typedef struct  _thrattr {
1151         size_t  stksize;
1152         void    *stkaddr;
1153         int     detachstate;
1154         int     daemonstate;
1155         int     scope;
1156         int     prio;
1157         int     policy;
1158         int     inherit;
1159         size_t  guardsize;
1160 } thrattr_t;
1161 
1162 typedef struct  _rwlattr {
1163         int     pshared;
1164 } rwlattr_t;
1165 
1166 /* _curthread() is inline for speed */
1167 extern  ulwp_t          *_curthread(void);
1168 #define curthread       (_curthread())
1169 
1170 /* this version (also inline) can be tested for NULL */
1171 extern  ulwp_t          *__curthread(void);
1172 
1173 /* get the current stack pointer (also inline) */
1174 extern  greg_t          stkptr(void);
1175 
1176 /*
1177  * Suppress __attribute__((...)) if we are not compiling with gcc
1178  */
1179 #if !defined(__GNUC__)
1180 #define __attribute__(string)
1181 #endif
1182 
1183 /* Fetch the dispatch (kernel) priority of a thread */
1184 #define real_priority(ulwp)     \
1185         ((ulwp)->ul_schedctl? (ulwp)->ul_schedctl->sc_priority : 0)
1186 
1187 /*
1188  * Implementation functions.  Not visible outside of the library itself.
1189  */
1190 extern  int     __nanosleep(const timespec_t *, timespec_t *);
1191 extern  void    getgregs(ulwp_t *, gregset_t);
1192 extern  void    setgregs(ulwp_t *, gregset_t);
1193 extern  void    thr_panic(const char *);
1194 #pragma rarely_called(thr_panic)
1195 extern  ulwp_t  *find_lwp(thread_t);
1196 extern  void    finish_init(void);
1197 extern  void    update_sched(ulwp_t *);
1198 extern  void    queue_alloc(void);
1199 extern  void    tsd_exit(void);
1200 extern  void    tsd_free(ulwp_t *);
1201 extern  void    tls_setup(void);
1202 extern  void    tls_exit(void);
1203 extern  void    tls_free(ulwp_t *);
1204 extern  void    rwl_free(ulwp_t *);
1205 extern  void    heldlock_exit(void);
1206 extern  void    heldlock_free(ulwp_t *);
1207 extern  void    sigacthandler(int, siginfo_t *, void *);
1208 extern  void    signal_init(void);
1209 extern  int     sigequalset(const sigset_t *, const sigset_t *);
1210 extern  void    mutex_setup(void);
1211 extern  void    take_deferred_signal(int);
1212 extern  void    *setup_top_frame(void *, size_t, ulwp_t *);
1213 extern  int     setup_context(ucontext_t *, void *(*func)(ulwp_t *),
1214                         ulwp_t *ulwp, caddr_t stk, size_t stksize);
1215 extern  volatile sc_shared_t *setup_schedctl(void);
1216 extern  void    *lmalloc(size_t);
1217 extern  void    lfree(void *, size_t);
1218 extern  void    *libc_malloc(size_t);
1219 extern  void    *libc_realloc(void *, size_t);
1220 extern  void    libc_free(void *);
1221 extern  char    *libc_strdup(const char *);
1222 extern  void    ultos(uint64_t, int, char *);
1223 extern  void    lock_error(const mutex_t *, const char *, void *, const char *);
1224 extern  void    rwlock_error(const rwlock_t *, const char *, const char *);
1225 extern  void    thread_error(const char *);
1226 extern  void    grab_assert_lock(void);
1227 extern  void    dump_queue_statistics(void);
1228 extern  void    collect_queue_statistics(void);
1229 extern  void    record_spin_locks(ulwp_t *);
1230 extern  void    remember_lock(mutex_t *);
1231 extern  void    forget_lock(mutex_t *);
1232 extern  void    register_lock(mutex_t *);
1233 extern  void    unregister_locks(void);
1234 #if defined(__sparc)
1235 extern  void    _flush_windows(void);
1236 #else
1237 #define _flush_windows()
1238 #endif
1239 extern  void    set_curthread(void *);
1240 
1241 /*
1242  * Utility function used when waking up many threads (more than MAXLWPS)
1243  * all at once.  See mutex_wakeup_all(), cond_broadcast(), and rw_unlock().
1244  */
1245 #define MAXLWPS 128     /* max remembered lwpids before overflow */
1246 #define NEWLWPS 2048    /* max remembered lwpids at first overflow */
1247 extern  lwpid_t *alloc_lwpids(lwpid_t *, int *, int *);
1248 
1249 /* enter a critical section */
1250 #define enter_critical(self)    (self->ul_critical++)
1251 
1252 /* exit a critical section, take deferred actions if necessary */
1253 extern  void    do_exit_critical(void);
1254 #define exit_critical(self)                                     \
1255         (void) (self->ul_critical--,                         \
1256             ((self->ul_curplease && self->ul_critical == 0)?      \
1257             (do_exit_critical(), 0) : 0))
1258 
1259 /*
1260  * Like enter_critical()/exit_critical() but just for deferring signals.
1261  * Unlike enter_critical()/exit_critical(), ul_sigdefer may be set while
1262  * calling application functions like constructors and destructors.
1263  * Care must be taken if the application function attempts to set
1264  * the signal mask while a deferred signal is present; the setting
1265  * of the signal mask must also be deferred.
1266  */
1267 #define sigoff(self)    (self->ul_sigdefer++)
1268 #define sigon(self)                                             \
1269         (void) ((--self->ul_sigdefer == 0 &&                 \
1270             self->ul_curplease && self->ul_critical == 0)?        \
1271             (do_exit_critical(), 0) : 0)
1272 
1273 /* these are exported functions */
1274 extern  void    _sigoff(void);
1275 extern  void    _sigon(void);
1276 
1277 #define sigorset(s1, s2)                                \
1278         (((s1)->__sigbits[0] |= (s2)->__sigbits[0]),      \
1279         ((s1)->__sigbits[1] |= (s2)->__sigbits[1]),       \
1280         ((s1)->__sigbits[2] |= (s2)->__sigbits[2]),       \
1281         ((s1)->__sigbits[3] |= (s2)->__sigbits[3]))
1282 
1283 #define sigandset(s1, s2)                               \
1284         (((s1)->__sigbits[0] &= (s2)->__sigbits[0]),  \
1285         ((s1)->__sigbits[1] &= (s2)->__sigbits[1]),   \
1286         ((s1)->__sigbits[2] &= (s2)->__sigbits[2]),   \
1287         ((s1)->__sigbits[3] &= (s2)->__sigbits[3]))
1288 
1289 #define sigdiffset(s1, s2)                              \
1290         (((s1)->__sigbits[0] &= ~(s2)->__sigbits[0]), \
1291         ((s1)->__sigbits[1] &= ~(s2)->__sigbits[1]),  \
1292         ((s1)->__sigbits[2] &= ~(s2)->__sigbits[2]),  \
1293         ((s1)->__sigbits[3] &= ~(s2)->__sigbits[3]))
1294 
1295 #define delete_reserved_signals(s)                      \
1296         (((s)->__sigbits[0] &= MASKSET0),                \
1297         ((s)->__sigbits[1] &= (MASKSET1 & ~SIGMASK(SIGCANCEL))),\
1298         ((s)->__sigbits[2] &= MASKSET2),         \
1299         ((s)->__sigbits[3] &= MASKSET3))
1300 
1301 extern  void    block_all_signals(ulwp_t *self);
1302 
1303 /*
1304  * When restoring the signal mask after having previously called
1305  * block_all_signals(), if we have a deferred signal present then
1306  * do nothing other than ASSERT() that we are in a critical region.
1307  * The signal mask will be set when we emerge from the critical region
1308  * and call take_deferred_signal().  There is no race condition here
1309  * because the kernel currently has all signals blocked for this thread.
1310  */
1311 #define restore_signals(self)                                           \
1312         ((void) ((self)->ul_cursig?                                  \
1313         (ASSERT((self)->ul_critical + (self)->ul_sigdefer != 0), 0) :     \
1314         __lwp_sigmask(SIG_SETMASK, &(self)->ul_sigmask)))
1315 
1316 extern  void    set_cancel_pending_flag(ulwp_t *, int);
1317 extern  void    set_cancel_eintr_flag(ulwp_t *);
1318 extern  void    set_parking_flag(ulwp_t *, int);
1319 extern  int     cancel_active(void);
1320 
1321 extern  void    *_thrp_setup(ulwp_t *);
1322 extern  void    _fpinherit(ulwp_t *);
1323 extern  void    _lwp_start(void);
1324 extern  void    _lwp_terminate(void);
1325 extern  void    lmutex_lock(mutex_t *);
1326 extern  void    lmutex_unlock(mutex_t *);
1327 extern  void    lrw_rdlock(rwlock_t *);
1328 extern  void    lrw_wrlock(rwlock_t *);
1329 extern  void    lrw_unlock(rwlock_t *);
1330 extern  void    sig_mutex_lock(mutex_t *);
1331 extern  void    sig_mutex_unlock(mutex_t *);
1332 extern  int     sig_mutex_trylock(mutex_t *);
1333 extern  int     sig_cond_wait(cond_t *, mutex_t *);
1334 extern  int     sig_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1335 extern  void    cancel_safe_mutex_lock(mutex_t *);
1336 extern  void    cancel_safe_mutex_unlock(mutex_t *);
1337 extern  int     cancel_safe_mutex_trylock(mutex_t *);
1338 extern  void    _prefork_handler(void);
1339 extern  void    _postfork_parent_handler(void);
1340 extern  void    _postfork_child_handler(void);
1341 extern  void    postfork1_child(void);
1342 extern  void    postfork1_child_aio(void);
1343 extern  void    postfork1_child_sigev_aio(void);
1344 extern  void    postfork1_child_sigev_mq(void);
1345 extern  void    postfork1_child_sigev_timer(void);
1346 extern  void    postfork1_child_tpool(void);
1347 extern  void    fork_lock_enter(void);
1348 extern  void    fork_lock_exit(void);
1349 extern  void    suspend_fork(void);
1350 extern  void    continue_fork(int);
1351 extern  void    do_sigcancel(void);
1352 extern  void    setup_cancelsig(int);
1353 extern  void    init_sigev_thread(void);
1354 extern  void    init_aio(void);
1355 extern  void    init_progname(void);
1356 extern  void    _cancelon(void);
1357 extern  void    _canceloff(void);
1358 extern  void    _canceloff_nocancel(void);
1359 extern  void    _cancel_prologue(void);
1360 extern  void    _cancel_epilogue(void);
1361 extern  void    no_preempt(ulwp_t *);
1362 extern  void    preempt(ulwp_t *);
1363 extern  void    _thrp_unwind(void *);
1364 
1365 extern  pid_t   __forkx(int);
1366 extern  pid_t   __forkallx(int);
1367 extern  int     __open(const char *, int, mode_t);
1368 extern  int     __open64(const char *, int, mode_t);
1369 extern  int     __openat(int, const char *, int, mode_t);
1370 extern  int     __openat64(int, const char *, int, mode_t);
1371 extern  int     __close(int);
1372 extern  ssize_t __read(int, void *, size_t);
1373 extern  ssize_t __write(int, const void *, size_t);
1374 extern  int     __fcntl(int, int, ...);
1375 extern  int     __lwp_continue(lwpid_t);
1376 extern  int     __lwp_create(ucontext_t *, uint_t, lwpid_t *);
1377 extern  int     ___lwp_suspend(lwpid_t);
1378 extern  int     lwp_wait(lwpid_t, lwpid_t *);
1379 extern  int     __lwp_wait(lwpid_t, lwpid_t *);
1380 extern  int     __lwp_detach(lwpid_t);
1381 extern  sc_shared_t *__schedctl(void);
1382 
1383 /* actual system call traps */
1384 extern  int     __setcontext(const ucontext_t *);
1385 extern  int     __getcontext(ucontext_t *);
1386 extern  int     __clock_gettime(clockid_t, timespec_t *);
1387 extern  void    abstime_to_reltime(clockid_t, const timespec_t *, timespec_t *);
1388 extern  void    hrt2ts(hrtime_t, timespec_t *);
1389 
1390 extern  int     __sigaction(int, const struct sigaction *, struct sigaction *);
1391 extern  int     __sigprocmask(int, const sigset_t *, sigset_t *);
1392 extern  int     __lwp_sigmask(int, const sigset_t *);
1393 extern  void    __sighndlr(int, siginfo_t *, ucontext_t *, void (*)());
1394 extern  caddr_t __sighndlrend;
1395 #pragma unknown_control_flow(__sighndlr)
1396 
1397 /* belongs in <pthread.h> */
1398 #define PTHREAD_CREATE_DAEMON_NP        0x100   /* = THR_DAEMON */
1399 #define PTHREAD_CREATE_NONDAEMON_NP     0
1400 extern  int     pthread_attr_setdaemonstate_np(pthread_attr_t *, int);
1401 extern  int     pthread_attr_getdaemonstate_np(const pthread_attr_t *, int *);
1402 
1403 extern  int     mutex_held(mutex_t *);
1404 extern  int     mutex_lock_internal(mutex_t *, timespec_t *, int);
1405 extern  int     mutex_unlock_internal(mutex_t *, int);
1406 
1407 /* not cancellation points: */
1408 extern  int     __cond_wait(cond_t *, mutex_t *);
1409 extern  int     __cond_timedwait(cond_t *, mutex_t *, const timespec_t *);
1410 extern  int     __cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1411 
1412 extern  int     rw_read_held(rwlock_t *);
1413 extern  int     rw_write_held(rwlock_t *);
1414 
1415 extern  int     _thrp_create(void *, size_t, void *(*)(void *), void *, long,
1416                         thread_t *, size_t);
1417 extern  int     _thrp_suspend(thread_t, uchar_t);
1418 extern  int     _thrp_continue(thread_t, uchar_t);
1419 
1420 extern  void    _thrp_terminate(void *);
1421 extern  void    _thrp_exit(void);
1422 
1423 extern  const pcclass_t *get_info_by_class(id_t);
1424 extern  const pcclass_t *get_info_by_policy(int);
1425 extern  const thrattr_t *def_thrattr(void);
1426 extern  id_t    setparam(idtype_t, id_t, int, int);
1427 extern  id_t    setprio(idtype_t, id_t, int, int *);
1428 extern  id_t    getparam(idtype_t, id_t, int *, struct sched_param *);
1429 
1430 /*
1431  * System call wrappers (direct interfaces to the kernel)
1432  */
1433 extern  int     ___lwp_mutex_register(mutex_t *, mutex_t **);
1434 extern  int     ___lwp_mutex_trylock(mutex_t *, ulwp_t *);
1435 extern  int     ___lwp_mutex_timedlock(mutex_t *, timespec_t *, ulwp_t *);
1436 extern  int     ___lwp_mutex_unlock(mutex_t *);
1437 extern  int     ___lwp_mutex_wakeup(mutex_t *, int);
1438 extern  int     ___lwp_cond_wait(cond_t *, mutex_t *, timespec_t *, int);
1439 extern  int     ___lwp_sema_timedwait(lwp_sema_t *, timespec_t *, int);
1440 extern  int     __lwp_rwlock_rdlock(rwlock_t *, timespec_t *);
1441 extern  int     __lwp_rwlock_wrlock(rwlock_t *, timespec_t *);
1442 extern  int     __lwp_rwlock_tryrdlock(rwlock_t *);
1443 extern  int     __lwp_rwlock_trywrlock(rwlock_t *);
1444 extern  int     __lwp_rwlock_unlock(rwlock_t *);
1445 extern  int     __lwp_park(timespec_t *, lwpid_t);
1446 extern  int     __lwp_unpark(lwpid_t);
1447 extern  int     __lwp_unpark_all(lwpid_t *, int);
1448 #if defined(__x86)
1449 extern  int     ___lwp_private(int, int, void *);
1450 #endif  /* __x86 */
1451 
1452 /*
1453  * inlines
1454  */
1455 extern  int             set_lock_byte(volatile uint8_t *);
1456 extern  uint32_t        atomic_swap_32(volatile uint32_t *, uint32_t);
1457 extern  uint32_t        atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
1458 extern  void            atomic_inc_32(volatile uint32_t *);
1459 extern  void            atomic_dec_32(volatile uint32_t *);
1460 extern  void            atomic_and_32(volatile uint32_t *, uint32_t);
1461 extern  void            atomic_or_32(volatile uint32_t *, uint32_t);
1462 #if defined(__sparc)
1463 extern  ulong_t         caller(void);
1464 extern  ulong_t         getfp(void);
1465 #endif  /* __sparc */
1466 
1467 #include "thr_inlines.h"
1468 
1469 #endif  /* _THR_UBERDATA_H */