1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * Copyright 2016 Joyent, Inc.
  28  * Copyright 2018 Nexenta Systems, Inc.
  29  */
  30 
  31 #ifndef _THR_UBERDATA_H
  32 #define _THR_UBERDATA_H
  33 
  34 #include <stdlib.h>
  35 #include <unistd.h>
  36 #include <sys/types.h>
  37 #include <fcntl.h>
  38 #include <string.h>
  39 #include <signal.h>
  40 #include <ucontext.h>
  41 #include <thread.h>
  42 #include <pthread.h>
  43 #include <atomic.h>
  44 #include <link.h>
  45 #include <sys/resource.h>
  46 #include <sys/lwp.h>
  47 #include <errno.h>
  48 #include <sys/asm_linkage.h>
  49 #include <sys/regset.h>
  50 #include <sys/fcntl.h>
  51 #include <sys/mman.h>
  52 #include <synch.h>
  53 #include <door.h>
  54 #include <limits.h>
  55 #include <sys/synch32.h>
  56 #include <schedctl.h>
  57 #include <sys/priocntl.h>
  58 #include <thread_db.h>
  59 #include <setjmp.h>
  60 #include <sys/thread.h>
  61 #include "libc_int.h"
  62 #include "tdb_agent.h"
  63 #include "thr_debug.h"
  64 
  65 /*
  66  * This is an implementation-specific include file for threading support.
  67  * It is not to be seen by the clients of the library.
  68  *
  69  * This file also describes uberdata in libc.
  70  *
  71  * The term "uberdata" refers to data that is unique and visible across
  72  * all link maps.  The name is meant to imply that such data is truly
  73  * global, not just locally global to a particular link map.
  74  *
  75  * See the Linker and Libraries Guide for a full description of alternate
  76  * link maps and how they are set up and used.
  77  *
  78  * Alternate link maps implement multiple global namespaces within a single
  79  * process.  There may be multiple instances of identical dynamic libraries
  80  * loaded in a process's address space at the same time, each on a different
  81  * link map (as determined by the dynamic linker), each with its own set of
  82  * global variables.  Which particular instance of a global variable is seen
  83  * by a thread running in the process is determined by the link map on which
  84  * the thread happens to be executing at the time.
  85  *
  86  * However, there are aspects of a process that are unique across all
  87  * link maps, in particular the structures used to implement threads
  88  * of control (in Sparc terminology, there is only one %g7 regardless
  89  * of the link map on which the thread is executing).
  90  *
  91  * All uberdata is referenced from a base pointer in the thread's ulwp_t
  92  * structure (which is also uberdata).  All allocations and deallocations
  93  * of uberdata are made via the uberdata-aware lmalloc() and lfree()
  94  * interfaces (malloc() and free() are simply locally-global).
  95  */
  96 
  97 /*
  98  * Special libc-private access to errno.
  99  * We do this so that references to errno do not invoke the dynamic linker.
 100  */
 101 #undef errno
 102 #define errno (*curthread->ul_errnop)
 103 
 104 /*
 105  * See <sys/synch32.h> for the reasons for these values
 106  * and why they are different for sparc and intel.
 107  */
 108 #if defined(__sparc)
 109 
 110 /* lock.lock64.pad[x]      4 5 6 7 */
 111 #define LOCKMASK        0xff000000
 112 #define WAITERMASK      0x000000ff
 113 #define SPINNERMASK     0x00ff0000
 114 #define SPINNERSHIFT    16
 115 #define WAITER          0x00000001
 116 #define LOCKSET         0xff
 117 #define LOCKCLEAR       0
 118 
 119 #define PIDSHIFT        32
 120 #define LOCKMASK64      0xffffffffff000000ULL
 121 #define LOCKBYTE64      0x00000000ff000000ULL
 122 #define WAITERMASK64    0x00000000000000ffULL
 123 #define SPINNERMASK64   0x0000000000ff0000ULL
 124 
 125 #elif defined(__x86)
 126 
 127 /* lock.lock64.pad[x]      7 6 5 4 */
 128 #define LOCKMASK        0xff000000
 129 #define WAITERMASK      0x00ff0000
 130 #define SPINNERMASK     0x0000ff00
 131 #define SPINNERSHIFT    8
 132 #define WAITER          0x00010000
 133 #define LOCKSET         0x01
 134 #define LOCKCLEAR       0
 135 
 136 #define PIDSHIFT        0
 137 #define LOCKMASK64      0xff000000ffffffffULL
 138 #define LOCKBYTE64      0x0100000000000000ULL
 139 #define WAITERMASK64    0x00ff000000000000ULL
 140 #define SPINNERMASK64   0x0000ff0000000000ULL
 141 
 142 #else
 143 #error "neither __sparc nor __x86 is defined"
 144 #endif
 145 
 146 /*
 147  * Fetch the owner of a USYNC_THREAD mutex.
 148  * Don't use this with process-shared mutexes;
 149  * the owing thread may be in a different process.
 150  */
 151 #define MUTEX_OWNER(mp) ((ulwp_t *)(uintptr_t)(mp)->mutex_owner)
 152 
 153 /*
 154  * Test if a thread owns a process-private (USYNC_THREAD) mutex.
 155  * This is inappropriate for a process-shared (USYNC_PROCESS) mutex.
 156  * The 'mp' argument must not have side-effects since it is evaluated twice.
 157  */
 158 #define MUTEX_OWNED(mp, thrp)   \
 159         ((mp)->mutex_lockw != 0 && MUTEX_OWNER(mp) == thrp)
 160 
 161 
 162 /*
 163  * uberflags.uf_tdb_register_sync is an interface with libc_db to enable the
 164  * collection of lock statistics by a debugger or other collecting tool.
 165  *
 166  * uberflags.uf_thread_error_detection is set by an environment variable:
 167  *      _THREAD_ERROR_DETECTION
 168  *              0 == no detection of locking primitive errors.
 169  *              1 == detect errors and issue a warning message.
 170  *              2 == detect errors, issue a warning message, and dump core.
 171  *
 172  * We bundle these together in uberflags.uf_trs_ted to make a test of either
 173  * being non-zero a single memory reference (for speed of mutex_lock(), etc).
 174  *
 175  * uberflags.uf_mt is set non-zero when the first thread (in addition
 176  * to the main thread) is created.
 177  *
 178  * We bundle all these flags together in uberflags.uf_all to make a test
 179  * of any being non-zero a single memory reference (again, for speed).
 180  */
 181 typedef union {
 182         int     uf_all;                 /* combined all flags */
 183         struct {
 184                 short   h_pad;
 185                 short   h_trs_ted;      /* combined reg sync & error detect */
 186         } uf_h;
 187         struct {
 188                 char    x_mt;
 189                 char    x_pad;
 190                 char    x_tdb_register_sync;
 191                 char    x_thread_error_detection;
 192         } uf_x;
 193 } uberflags_t;
 194 
 195 #define uf_mt                           uf_x.x_mt
 196 #define uf_tdb_register_sync            uf_x.x_tdb_register_sync
 197 #define uf_thread_error_detection       uf_x.x_thread_error_detection
 198 #define uf_trs_ted                      uf_h.h_trs_ted  /* both of the above */
 199 
 200 /*
 201  * NOTE WELL:
 202  * To enable further optimization, the "ul_schedctl_called" member
 203  * of the ulwp_t structure (below) serves double-duty:
 204  *      1. If NULL, it means that the thread must call __schedctl()
 205  *         to set up its schedctl mappings before acquiring a mutex.
 206  *         This is required by the implementation of adaptive mutex locking.
 207  *      2. If non-NULL, it points to uberdata.uberflags, so that tests of
 208  *         uberflags can be made without additional memory references.
 209  * This allows the common case of _mutex_lock() and _mutex_unlock() for
 210  * USYNC_THREAD mutexes with no error detection and no lock statistics
 211  * to be optimized for speed.
 212  */
 213 
 214 /* double the default stack size for 64-bit processes */
 215 #ifdef _LP64
 216 #define MINSTACK        (8 * 1024)
 217 #define DEFAULTSTACK    (2 * 1024 * 1024)
 218 #else
 219 #define MINSTACK        (4 * 1024)
 220 #define DEFAULTSTACK    (1024 * 1024)
 221 #endif
 222 
 223 #define MUTEX_TRY       0
 224 #define MUTEX_LOCK      1
 225 #define MUTEX_NOCEIL    0x40
 226 
 227 #if defined(__x86)
 228 
 229 typedef struct {        /* structure returned by fnstenv */
 230         int     fctrl;          /* control word */
 231         int     fstat;          /* status word (flags, etc) */
 232         int     ftag;           /* tag of which regs busy */
 233         int     misc[4];        /* other stuff, 28 bytes total */
 234 } fpuenv_t;
 235 
 236 #ifdef _SYSCALL32
 237 typedef fpuenv_t fpuenv32_t;
 238 #endif  /* _SYSCALL32 */
 239 
 240 #elif defined(__sparc)
 241 
 242 typedef struct {        /* fp state structure */
 243         greg_t  fsr;
 244         greg_t  fpu_en;
 245 } fpuenv_t;
 246 
 247 #ifdef _SYSCALL32
 248 typedef struct {
 249         greg32_t        fsr;
 250         greg32_t        fpu_en;
 251 } fpuenv32_t;
 252 #endif  /* _SYSCALL32 */
 253 
 254 #endif  /* __x86 */
 255 
 256 #if defined(__x86)
 257 extern  void    ht_pause(void);         /* "pause" instruction */
 258 #define SMT_PAUSE()     ht_pause()
 259 #elif defined(SMT_PAUSE_FUNCTION)
 260 extern  void    SMT_PAUSE_FUNCTION(void);
 261 #define SMT_PAUSE()     SMT_PAUSE_FUNCTION()
 262 #else
 263 #define SMT_PAUSE()     smt_pause()
 264 #endif  /* __x86 */
 265 
 266 /*
 267  * Cleanup handler related data.
 268  * This structure is exported as _cleanup_t in pthread.h.
 269  * pthread.h exports only the size of this structure, so check
 270  * _cleanup_t in pthread.h before making any change here.
 271  */
 272 typedef struct __cleanup {
 273         struct __cleanup *next;         /* pointer to next handler */
 274         caddr_t fp;                     /* current frame pointer */
 275         void    (*func)(void *);        /* cleanup handler address */
 276         void    *arg;                   /* handler's argument */
 277 } __cleanup_t;
 278 
 279 /*
 280  * Thread-Specific Data (TSD)
 281  * TSD_NFAST includes the invalid key zero, so there
 282  * are really only (TSD_NFAST - 1) fast key slots.
 283  */
 284 typedef void (*PFrV)(void *);
 285 #define TSD_UNALLOCATED ((PFrV)1)
 286 #define TSD_NFAST       9
 287 
 288 /*
 289  * The tsd union is designed to burn a little memory (9 words) to make
 290  * lookups blindingly fast.  Note that tsd_nalloc could be placed at the
 291  * end of the pad region to increase the likelihood that it falls on the
 292  * same cache line as the data.
 293  */
 294 typedef union tsd {
 295         uint_t tsd_nalloc;              /* Amount of allocated storage */
 296         void *tsd_pad[TSD_NFAST];
 297         void *tsd_data[1];
 298 } tsd_t;
 299 
 300 typedef struct {
 301         mutex_t tsdm_lock;              /* Lock protecting the data */
 302         uint_t tsdm_nkeys;              /* Number of allocated keys */
 303         uint_t tsdm_nused;              /* Number of used keys */
 304         PFrV *tsdm_destro;              /* Per-key destructors */
 305         char tsdm_pad[64 -              /* pad to 64 bytes */
 306                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (PFrV *))];
 307 } tsd_metadata_t;
 308 
 309 #ifdef _SYSCALL32
 310 typedef union tsd32 {
 311         uint_t tsd_nalloc;              /* Amount of allocated storage */
 312         caddr32_t tsd_pad[TSD_NFAST];
 313         caddr32_t tsd_data[1];
 314 } tsd32_t;
 315 
 316 typedef struct {
 317         mutex_t tsdm_lock;              /* Lock protecting the data */
 318         uint_t tsdm_nkeys;              /* Number of allocated keys */
 319         uint_t tsdm_nused;              /* Number of used keys */
 320         caddr32_t tsdm_destro;          /* Per-key destructors */
 321         char tsdm_pad[64 -              /* pad to 64 bytes */
 322                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (caddr32_t))];
 323 } tsd_metadata32_t;
 324 #endif  /* _SYSCALL32 */
 325 
 326 
 327 /*
 328  * Thread-Local Storage (TLS)
 329  */
 330 typedef struct {
 331         void            *tls_data;
 332         size_t          tls_size;
 333 } tls_t;
 334 
 335 typedef struct {
 336         mutex_t tls_lock;               /* Lock protecting the data */
 337         tls_t   tls_modinfo;            /* Root of all TLS_modinfo data */
 338         tls_t   static_tls;             /* Template for static TLS */
 339         char    tls_pad[64 -            /* pad to 64 bytes */
 340                 (sizeof (mutex_t) + 2 * sizeof (tls_t))];
 341 } tls_metadata_t;
 342 
 343 #ifdef _SYSCALL32
 344 typedef struct {
 345         caddr32_t       tls_data;
 346         size32_t        tls_size;
 347 } tls32_t;
 348 
 349 typedef struct {
 350         mutex_t tls_lock;               /* Lock protecting the data */
 351         tls32_t tls_modinfo;            /* Root of all TLS_modinfo data */
 352         tls32_t static_tls;             /* Template for static TLS */
 353         char    tls_pad[64 -            /* pad to 64 bytes */
 354                 (sizeof (mutex_t) + 2 * sizeof (tls32_t))];
 355 } tls_metadata32_t;
 356 #endif  /* _SYSCALL32 */
 357 
 358 
 359 /*
 360  * Sleep queue root for USYNC_THREAD condvars and mutexes.
 361  * There is a default queue root for each queue head (see below).
 362  * Also, each ulwp_t contains a queue root that can be used
 363  * when the thread is enqueued on the queue, if necessary
 364  * (when more than one wchan hashes to the same queue head).
 365  */
 366 typedef struct queue_root {
 367         struct queue_root       *qr_next;
 368         struct queue_root       *qr_prev;
 369         struct ulwp             *qr_head;
 370         struct ulwp             *qr_tail;
 371         void                    *qr_wchan;
 372         uint32_t                qr_rtcount;
 373         uint32_t                qr_qlen;
 374         uint32_t                qr_qmax;
 375 } queue_root_t;
 376 
 377 #ifdef _SYSCALL32
 378 typedef struct queue_root32 {
 379         caddr32_t               qr_next;
 380         caddr32_t               qr_prev;
 381         caddr32_t               qr_head;
 382         caddr32_t               qr_tail;
 383         caddr32_t               qr_wchan;
 384         uint32_t                qr_rtcount;
 385         uint32_t                qr_qlen;
 386         uint32_t                qr_qmax;
 387 } queue_root32_t;
 388 #endif
 389 
 390 /*
 391  * Sleep queue heads for USYNC_THREAD condvars and mutexes.
 392  * The size and alignment is 128 bytes to reduce cache conflicts.
 393  * Each queue head points to a list of queue roots, defined above.
 394  * Each queue head contains a default queue root for use when only one
 395  * is needed.  It is always at the tail of the queue root hash chain.
 396  */
 397 typedef union {
 398         uint64_t                qh_64[16];
 399         struct {
 400                 mutex_t         q_lock;
 401                 uint8_t         q_qcnt;
 402                 uint8_t         q_type;         /* MX or CV */
 403                 uint8_t         q_pad1[2];
 404                 uint32_t        q_lockcount;
 405                 uint32_t        q_qlen;
 406                 uint32_t        q_qmax;
 407                 void            *q_wchan;       /* valid only while locked */
 408                 struct queue_root *q_root;      /* valid only while locked */
 409                 struct queue_root *q_hlist;
 410 #if !defined(_LP64)
 411                 caddr_t         q_pad2[3];
 412 #endif
 413                 queue_root_t    q_def_root;
 414                 uint32_t        q_hlen;
 415                 uint32_t        q_hmax;
 416         } qh_qh;
 417 } queue_head_t;
 418 
 419 #define qh_lock         qh_qh.q_lock
 420 #define qh_qcnt         qh_qh.q_qcnt
 421 #define qh_type         qh_qh.q_type
 422 #if defined(THREAD_DEBUG)
 423 #define qh_lockcount    qh_qh.q_lockcount
 424 #define qh_qlen         qh_qh.q_qlen
 425 #define qh_qmax         qh_qh.q_qmax
 426 #endif
 427 #define qh_wchan        qh_qh.q_wchan
 428 #define qh_root         qh_qh.q_root
 429 #define qh_hlist        qh_qh.q_hlist
 430 #define qh_def_root     qh_qh.q_def_root
 431 #define qh_hlen         qh_qh.q_hlen
 432 #define qh_hmax         qh_qh.q_hmax
 433 
 434 /* queue types passed to queue_lock() */
 435 #define MX      0
 436 #define CV      1
 437 #define QHASHSHIFT      9                       /* number of hashing bits */
 438 #define QHASHSIZE       (1 << QHASHSHIFT) /* power of 2 (1<<9 == 512) */
 439 #define QUEUE_HASH(wchan, type) ((uint_t)                       \
 440         ((((uintptr_t)(wchan) >> 3)                               \
 441         ^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3)))               \
 442         & (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE))
 443 
 444 extern  queue_head_t    *queue_lock(void *, int);
 445 extern  void            queue_unlock(queue_head_t *);
 446 extern  void            enqueue(queue_head_t *, struct ulwp *, int);
 447 extern  struct ulwp     *dequeue(queue_head_t *, int *);
 448 extern  struct ulwp     **queue_slot(queue_head_t *, struct ulwp **, int *);
 449 extern  struct ulwp     *queue_waiter(queue_head_t *);
 450 extern  int             dequeue_self(queue_head_t *);
 451 extern  void            queue_unlink(queue_head_t *,
 452                                 struct ulwp **, struct ulwp *);
 453 extern  void            unsleep_self(void);
 454 extern  void            spin_lock_set(mutex_t *);
 455 extern  void            spin_lock_clear(mutex_t *);
 456 
 457 /*
 458  * Scheduling class information structure.
 459  */
 460 typedef struct {
 461         short           pcc_state;
 462         short           pcc_policy;
 463         pri_t           pcc_primin;
 464         pri_t           pcc_primax;
 465         pcinfo_t        pcc_info;
 466 } pcclass_t;
 467 
 468 /*
 469  * Memory block for chain of owned ceiling mutexes.
 470  */
 471 typedef struct mxchain {
 472         struct mxchain  *mxchain_next;
 473         mutex_t         *mxchain_mx;
 474 } mxchain_t;
 475 
 476 /*
 477  * Pointer to an rwlock that is held for reading.
 478  * Used in rw_rdlock() to allow a thread that already holds a read
 479  * lock to acquire another read lock on the same rwlock even if
 480  * there are writers waiting.  This to avoid deadlock when acquiring
 481  * a read lock more than once in the presence of pending writers.
 482  * POSIX mandates this behavior.
 483  */
 484 typedef struct {
 485         void    *rd_rwlock;     /* the rwlock held for reading */
 486         size_t  rd_count;       /* count of read locks applied */
 487 } readlock_t;
 488 
 489 #ifdef _SYSCALL32
 490 typedef struct {
 491         caddr32_t       rd_rwlock;
 492         size32_t        rd_count;
 493 } readlock32_t;
 494 #endif  /* _SYSCALL32 */
 495 
 496 /*
 497  * As part of per-thread caching libumem (ptcumem), we add a small amount to the
 498  * thread's uberdata to facilitate it. The tm_roots are the roots of linked
 499  * lists which is used by libumem to chain together allocations. tm_size is used
 500  * to track the total amount of data stored across those linked lists. For more
 501  * information, see libumem's big theory statement.
 502  */
 503 #define NTMEMBASE       16
 504 
 505 typedef struct {
 506         size_t          tm_size;
 507         void            *tm_roots[NTMEMBASE];
 508 } tumem_t;
 509 
 510 #ifdef _SYSCALL32
 511 typedef struct {
 512         uint32_t        tm_size;
 513         caddr32_t       tm_roots[NTMEMBASE];
 514 } tumem32_t;
 515 #endif
 516 
 517 typedef void (*tmem_func_t)(void *, int);
 518 
 519 /*
 520  * Maximum number of read locks allowed for one thread on one rwlock.
 521  * This could be as large as INT_MAX, but the SUSV3 test suite would
 522  * take an inordinately long time to complete.  This is big enough.
 523  */
 524 #define READ_LOCK_MAX   100000
 525 
 526 #define ul_tlsent       ul_tls.tls_data /* array of pointers to dynamic TLS */
 527 #define ul_ntlsent      ul_tls.tls_size /* number of entries in ul_tlsent */
 528 
 529 /*
 530  * Round up an integral value to a multiple of 64
 531  */
 532 #define roundup64(x)    (-(-(x) & -64))
 533 
 534 /*
 535  * NOTE:  Whatever changes are made to ulwp_t must be
 536  * reflected in $SRC/cmd/mdb/common/modules/libc/libc.c
 537  *
 538  * NOTE: ul_self *must* be the first member of ulwp_t on x86
 539  * Low-level x86 code relies on this.
 540  */
 541 typedef struct ulwp {
 542         /*
 543          * These members always need to come first on sparc.
 544          * For dtrace, a ulwp_t must be aligned on a 64-byte boundary.
 545          */
 546 #if defined(__sparc)
 547         uint32_t        ul_dinstr;      /* scratch space for dtrace */
 548         uint32_t        ul_padsparc0[15];
 549         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
 550         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
 551         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
 552         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
 553 #endif
 554         struct ulwp     *ul_self;       /* pointer to self */
 555 #if defined(__i386)
 556         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
 557 #elif defined(__amd64)
 558         uint8_t         ul_dinstr[56];  /* scratch space for dtrace */
 559 #endif
 560         struct uberdata *ul_uberdata;   /* uber (super-global) data */
 561         tls_t           ul_tls;         /* dynamic thread-local storage base */
 562         struct ulwp     *ul_forw;       /* forw, back all_lwps list, */
 563         struct ulwp     *ul_back;       /* protected by link_lock */
 564         struct ulwp     *ul_next;       /* list to keep track of stacks */
 565         struct ulwp     *ul_hash;       /* hash chain linked list */
 566         void            *ul_rval;       /* return value from thr_exit() */
 567         caddr_t         ul_stk;         /* mapping base of the stack */
 568         size_t          ul_mapsiz;      /* mapping size of the stack */
 569         size_t          ul_guardsize;   /* normally _lpagesize */
 570         uintptr_t       ul_stktop;      /* broken thr_stksegment() interface */
 571         size_t          ul_stksiz;      /* broken thr_stksegment() interface */
 572         stack_t         ul_ustack;      /* current stack boundaries */
 573         int             ul_ix;          /* hash index */
 574         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
 575         pri_t           ul_pri;         /* scheduling priority */
 576         pri_t           ul_epri;        /* real-time ceiling priority */
 577         char            ul_policy;      /* scheduling policy */
 578         char            ul_cid;         /* scheduling class id */
 579         union {
 580                 struct {
 581                         char    cursig; /* deferred signal number */
 582                         char    pleasestop; /* lwp requested to stop itself */
 583                 } s;
 584                 short   curplease;      /* for testing both at once */
 585         } ul_cp;
 586         char            ul_stop;        /* reason for stopping */
 587         char            ul_signalled;   /* this lwp was cond_signal()d */
 588         char            ul_dead;        /* this lwp has called thr_exit */
 589         char            ul_unwind;      /* posix: unwind C++ stack */
 590         char            ul_detached;    /* THR_DETACHED at thread_create() */
 591                                         /* or pthread_detach() was called */
 592         char            ul_writer;      /* sleeping in rw_wrlock() */
 593         char            ul_stopping;    /* set by curthread: stopping self */
 594         char            ul_cancel_prologue;     /* for _cancel_prologue() */
 595         short           ul_preempt;     /* no_preempt()/preempt() */
 596         short           ul_savpreempt;  /* pre-existing preempt value */
 597         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
 598         char            ul_main;        /* thread is the main thread */
 599         char            ul_fork;        /* thread is performing a fork */
 600         char            ul_primarymap;  /* primary link-map is initialized */
 601         /* per-thread copies of the corresponding global variables */
 602         uint8_t         ul_max_spinners;        /* thread_max_spinners */
 603         char            ul_door_noreserve;      /* thread_door_noreserve */
 604         char            ul_queue_fifo;          /* thread_queue_fifo */
 605         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
 606         char            ul_error_detection;     /* thread_error_detection */
 607         char            ul_async_safe;          /* thread_async_safe */
 608         char            ul_rt;                  /* found on an RT queue */
 609         char            ul_rtqueued;            /* was RT when queued */
 610         char            ul_misaligned;          /* thread_locks_misaligned */
 611         char            ul_pad[3];
 612         int             ul_adaptive_spin;       /* thread_adaptive_spin */
 613         int             ul_queue_spin;          /* thread_queue_spin */
 614         volatile int    ul_critical;    /* non-zero == in a critical region */
 615         int             ul_sigdefer;    /* non-zero == defer signals */
 616         int             ul_vfork;       /* thread is the child of vfork() */
 617         int             ul_cancelable;  /* _cancelon()/_canceloff() */
 618         char            ul_cancel_pending;  /* pthread_cancel() was called */
 619         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
 620         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
 621         char            ul_save_async;  /* saved copy of ul_cancel_async */
 622         char            ul_mutator;     /* lwp is a mutator (java interface) */
 623         char            ul_created;     /* created suspended */
 624         char            ul_replace;     /* replacement; must be free()d */
 625         uchar_t         ul_nocancel;    /* cancellation can't happen */
 626         int             ul_errno;       /* per-thread errno */
 627         int             *ul_errnop;     /* pointer to errno or self->ul_errno */
 628         __cleanup_t     *ul_clnup_hdr;  /* head of cleanup handlers list */
 629         uberflags_t     *ul_schedctl_called;    /* ul_schedctl is set up */
 630         volatile sc_shared_t *ul_schedctl;      /* schedctl data */
 631         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
 632         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
 633         tsd_t           *ul_stsd;       /* slow TLS for keys >= TSD_NFAST */
 634         void            *ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
 635         td_evbuf_t      ul_td_evbuf;    /* event buffer */
 636         char            ul_td_events_enable;    /* event mechanism enabled */
 637         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
 638         char            ul_qtype;       /* MX or CV */
 639         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
 640         int             ul_rtld;        /* thread is running inside ld.so.1 */
 641         int             ul_usropts;     /* flags given to thr_create() */
 642         void            *(*ul_startpc)(void *); /* start func (thr_create()) */
 643         void            *ul_startarg;   /* argument for start function */
 644         void            *ul_wchan;      /* synch object when sleeping */
 645         struct ulwp     *ul_link;       /* sleep queue link */
 646         queue_head_t    *ul_sleepq;     /* sleep queue thread is waiting on */
 647         mutex_t         *ul_cvmutex;    /* mutex dropped when waiting on a cv */
 648         mxchain_t       *ul_mxchain;    /* chain of owned ceiling mutexes */
 649         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
 650         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
 651                                 /* 0 means there is but a single entry */
 652         union {                         /* single entry or pointer to array */
 653                 readlock_t      single;
 654                 readlock_t      *array;
 655         } ul_readlock;
 656         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
 657                                 /* 0 means there is but a single entry */
 658         union {                         /* single entry or pointer to array */
 659                 mutex_t         *single;
 660                 mutex_t         **array;
 661         } ul_heldlocks;
 662         /* PROBE_SUPPORT begin */
 663         void            *ul_tpdp;
 664         /* PROBE_SUPPORT end */
 665         ucontext_t      *ul_siglink;    /* pointer to previous context */
 666         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
 667         uint_t          ul_spin_lock_spin2;
 668         uint_t          ul_spin_lock_sleep;
 669         uint_t          ul_spin_lock_wakeup;
 670         queue_root_t    ul_queue_root;  /* root of a sleep queue */
 671         id_t            ul_rtclassid;   /* real-time class id */
 672         uint_t          ul_pilocks;     /* count of PI locks held */
 673                 /* the following members *must* be last in the structure */
 674                 /* they are discarded when ulwp is replaced on thr_exit() */
 675         sigset_t        ul_sigmask;     /* thread's current signal mask */
 676         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
 677         siginfo_t       ul_siginfo;     /* deferred siginfo */
 678         mutex_t         ul_spinlock;    /* used when suspending/continuing */
 679         fpuenv_t        ul_fpuenv;      /* floating point state */
 680         uintptr_t       ul_sp;          /* stack pointer when blocked */
 681         void            *ul_ex_unwind;  /* address of _ex_unwind() or -1 */
 682 #if defined(sparc)
 683         void            *ul_unwind_ret; /* used only by _ex_clnup_handler() */
 684 #endif
 685         tumem_t         ul_tmem;        /* used only by umem */
 686         uint_t          ul_ptinherit;   /* pthreads sched inherit value */
 687         char            ul_ntoabuf[18]; /* thread-specific inet_ntoa buffer */
 688 } ulwp_t;
 689 
 690 #define ul_cursig       ul_cp.s.cursig          /* deferred signal number */
 691 #define ul_pleasestop   ul_cp.s.pleasestop      /* lwp requested to stop */
 692 #define ul_curplease    ul_cp.curplease         /* for testing both at once */
 693 
 694 /*
 695  * This is the size of a replacement ulwp, retained only for the benefit
 696  * of thr_join().  The trailing members are unneeded for this purpose.
 697  */
 698 #define REPLACEMENT_SIZE        ((size_t)&((ulwp_t *)NULL)->ul_sigmask)
 699 
 700 /*
 701  * Definitions for static initialization of signal sets,
 702  * plus some sneaky optimizations in various places.
 703  */
 704 
 705 #define SIGMASK(sig)    ((uint32_t)1 << (((sig) - 1) & (32 - 1)))
 706 
 707 #if (MAXSIG > (2 * 32) && MAXSIG <= (3 * 32))
 708 #define FILLSET0        0xffffffffu
 709 #define FILLSET1        0xffffffffu
 710 #define FILLSET2        ((1u << (MAXSIG - 64)) - 1)
 711 #define FILLSET3        0
 712 #else
 713 #error "fix me: MAXSIG out of bounds"
 714 #endif
 715 
 716 #define CANTMASK0       (SIGMASK(SIGKILL) | SIGMASK(SIGSTOP))
 717 #define CANTMASK1       0
 718 #define CANTMASK2       0
 719 #define CANTMASK3       0
 720 
 721 #define MASKSET0        (FILLSET0 & ~CANTMASK0)
 722 #define MASKSET1        (FILLSET1 & ~CANTMASK1)
 723 #define MASKSET2        (FILLSET2 & ~CANTMASK2)
 724 #define MASKSET3        (FILLSET3 & ~CANTMASK3)
 725 
 726 extern  const sigset_t maskset;         /* set of all maskable signals */
 727 
 728 extern  int     thread_adaptive_spin;
 729 extern  uint_t  thread_max_spinners;
 730 extern  int     thread_queue_spin;
 731 extern  int     thread_queue_fifo;
 732 extern  int     thread_queue_dump;
 733 extern  int     thread_cond_wait_defer;
 734 extern  int     thread_async_safe;
 735 extern  int     thread_queue_verify;
 736 
 737 /*
 738  * pthread_atfork() related data, used to store atfork handlers.
 739  */
 740 typedef struct atfork {
 741         struct atfork *forw;            /* forward pointer */
 742         struct atfork *back;            /* backward pointer */
 743         void (*prepare)(void);          /* pre-fork handler */
 744         void (*parent)(void);           /* post-fork parent handler */
 745         void (*child)(void);            /* post-fork child handler */
 746 } atfork_t;
 747 
 748 /*
 749  * Element in the table and in the list of registered process
 750  * robust locks.  We keep track of these to make sure that we
 751  * only call ___lwp_mutex_register() once for each such lock
 752  * after it is first mapped in (or newly mapped in).
 753  */
 754 typedef struct robust {
 755         struct robust   *robust_next;   /* hash table list */
 756         struct robust   *robust_list;   /* global list */
 757         mutex_t         *robust_lock;
 758 } robust_t;
 759 
 760 /*
 761  * Invalid address, used to mark an unused element in the hash table.
 762  */
 763 #define INVALID_ADDR    ((void *)(uintptr_t)(-1L))
 764 
 765 /*
 766  * Parameters of the lock registration hash table.
 767  */
 768 #define LOCKSHIFT       15                      /* number of hashing bits */
 769 #define LOCKHASHSZ      (1 << LOCKSHIFT)  /* power of 2 (1<<15 == 32K) */
 770 #define LOCK_HASH(addr) (uint_t)                        \
 771         ((((uintptr_t)(addr) >> 3)                        \
 772         ^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3))) \
 773         & (LOCKHASHSZ - 1))
 774 
 775 /*
 776  * Make our hot locks reside on private cache lines (64 bytes).
 777  */
 778 typedef struct {
 779         mutex_t pad_lock;
 780         char    pad_pad[64 - sizeof (mutex_t)];
 781 } pad_lock_t;
 782 
 783 /*
 784  * Make our semi-hot locks reside on semi-private cache lines (32 bytes).
 785  */
 786 typedef struct {
 787         mutex_t pad_lock;
 788         char    pad_pad[32 - sizeof (mutex_t)];
 789 } pad32_lock_t;
 790 
 791 /*
 792  * The threads hash table is used for fast lookup and locking of an active
 793  * thread structure (ulwp_t) given a thread-id.  It is an N-element array of
 794  * thr_hash_table_t structures, where N == 1 before the main thread creates
 795  * the first additional thread and N == 1024 afterwards.  Each element of the
 796  * table is 64 bytes in size and alignment to reduce cache conflicts.
 797  */
 798 typedef struct {
 799         mutex_t hash_lock;      /* lock per bucket */
 800         cond_t  hash_cond;      /* convar per bucket */
 801         ulwp_t  *hash_bucket;   /* hash bucket points to the list of ulwps */
 802         char    hash_pad[64 -   /* pad out to 64 bytes */
 803                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (ulwp_t *))];
 804 } thr_hash_table_t;
 805 
 806 #ifdef _SYSCALL32
 807 typedef struct {
 808         mutex_t hash_lock;
 809         cond_t  hash_cond;
 810         caddr32_t hash_bucket;
 811         char    hash_pad[64 -
 812                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (caddr32_t))];
 813 } thr_hash_table32_t;
 814 #endif  /* _SYSCALL32 */
 815 
 816 
 817 /*
 818  * siguaction members have 128-byte size and 64-byte alignment.
 819  * We know that sizeof (struct sigaction) is 32 bytes for both
 820  * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes.
 821  */
 822 typedef struct {
 823         rwlock_t        sig_lock;
 824         struct sigaction sig_uaction;
 825         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)];
 826 } siguaction_t;
 827 
 828 #ifdef _SYSCALL32
 829 typedef struct {
 830         rwlock_t        sig_lock;
 831         struct sigaction32 sig_uaction;
 832         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)];
 833 } siguaction32_t;
 834 #endif  /* _SYSCALL32 */
 835 
 836 
 837 /*
 838  * Bucket structures, used by lmalloc()/lfree().
 839  * See port/threads/alloc.c for details.
 840  * A bucket's size and alignment is 64 bytes.
 841  */
 842 typedef struct {
 843         mutex_t bucket_lock;    /* protects the free list allocations */
 844         void    *free_list;     /* LIFO list of blocks to allocate/free */
 845         size_t  chunks;         /* number of 64K blocks mmap()ed last time */
 846         char    pad64[64 -      /* pad out to 64 bytes */
 847                 (sizeof (mutex_t) + sizeof (void *) + sizeof (size_t))];
 848 } bucket_t;
 849 
 850 #ifdef _SYSCALL32
 851 typedef struct {
 852         mutex_t         bucket_lock;
 853         caddr32_t       free_list;
 854         size32_t        chunks;
 855         char    pad64[64 -      /* pad out to 64 bytes */
 856                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (size32_t))];
 857 } bucket32_t;
 858 #endif  /* _SYSCALL32 */
 859 
 860 #define NBUCKETS        10      /* sizes ranging from 64 to 32768 */
 861 
 862 
 863 /*
 864  * atexit() data structures.
 865  * See port/gen/atexit.c for details.
 866  */
 867 typedef void (*_exithdlr_func_t) (void*);
 868 
 869 typedef struct _exthdlr {
 870         struct _exthdlr         *next;  /* next in handler list */
 871         _exithdlr_func_t        hdlr;   /* handler itself */
 872         void                    *arg;   /* argument to handler */
 873         void                    *dso;   /* DSO associated with handler */
 874 } _exthdlr_t;
 875 
 876 typedef struct {
 877         mutex_t         exitfns_lock;
 878         _exthdlr_t      *head;
 879         /*
 880          * exit_frame_monitor is part of a private contract between libc and
 881          * the Sun C++ runtime.
 882          *
 883          * It should be NULL until exit() is called, and thereafter hold the
 884          * frame pointer of the function implementing our exit processing.
 885          */
 886         void            *exit_frame_monitor;
 887         char            exit_pad[64 -   /* pad out to 64 bytes */
 888                 (sizeof (mutex_t) + sizeof (_exthdlr_t *) + sizeof (void *))];
 889 } atexit_root_t;
 890 
 891 #ifdef _SYSCALL32
 892 typedef struct {
 893         mutex_t         exitfns_lock;
 894         caddr32_t       head;
 895         caddr32_t       exit_frame_monitor;
 896         char            exit_pad[64 -   /* pad out to 64 bytes */
 897                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (caddr32_t))];
 898 } atexit_root32_t;
 899 #endif  /* _SYSCALL32 */
 900 
 901 /*
 902  * at_quick_exit() and quick_exit() data structures. The ISO/IEC C11 odd
 903  * siblings of atexit()
 904  */
 905 typedef void (*_quick_exithdlr_func_t)(void);
 906 
 907 typedef struct _qexthdlr {
 908         struct _qexthdlr        *next;  /* next in handler list */
 909         _quick_exithdlr_func_t  hdlr;   /* handler itself */
 910 } _qexthdlr_t;
 911 
 912 /*
 913  * We add a pad on 32-bit systems to allow us to always have the structure size
 914  * be 32-bytes which helps us deal with the compiler's alignment when building
 915  * in ILP32 / LP64 systems.
 916  */
 917 typedef struct {
 918         mutex_t         exitfns_lock;
 919         _qexthdlr_t     *head;
 920 #if !defined(_LP64)
 921         uint32_t        pad;
 922 #endif
 923 } quickexit_root_t;
 924 
 925 #ifdef _SYSCALL32
 926 typedef struct {
 927         mutex_t         exitfns_lock;
 928         caddr32_t       head;
 929         uint32_t        pad;
 930 } quickexit_root32_t;
 931 #endif /* _SYSCALL32 */
 932 
 933 /*
 934  * This is data that is global to all link maps (uberdata, aka super-global).
 935  * Note: When changing this, please be sure to keep the 32-bit variant of
 936  * this in sync.  (see uberdata32_t below)
 937  */
 938 typedef struct uberdata {
 939         pad_lock_t      _link_lock;
 940         pad_lock_t      _ld_lock;
 941         pad_lock_t      _fork_lock;
 942         pad_lock_t      _atfork_lock;
 943         pad32_lock_t    _callout_lock;
 944         pad32_lock_t    _tdb_hash_lock;
 945         tdb_sync_stats_t tdb_hash_lock_stats;
 946         siguaction_t    siguaction[NSIG];
 947         bucket_t        bucket[NBUCKETS];
 948         atexit_root_t   atexit_root;
 949         quickexit_root_t quickexit_root;
 950         tsd_metadata_t  tsd_metadata;
 951         tls_metadata_t  tls_metadata;
 952         /*
 953          * Every object before this point has size and alignment of 64 bytes.
 954          * Don't add any other type of data before this point.
 955          */
 956         char    primary_map;    /* set when primary link map is initialized */
 957         char    bucket_init;    /* set when bucket[NBUCKETS] is initialized */
 958         char    pad[2];
 959         uberflags_t     uberflags;
 960         queue_head_t    *queue_head;
 961         thr_hash_table_t *thr_hash_table;
 962         uint_t          hash_size;      /* # of entries in thr_hash_table[] */
 963         uint_t          hash_mask;      /* hash_size - 1 */
 964         ulwp_t  *ulwp_one;      /* main thread */
 965         ulwp_t  *all_lwps;      /* circular ul_forw/ul_back list of live lwps */
 966         ulwp_t  *all_zombies;   /* circular ul_forw/ul_back list of zombies */
 967         int     nthreads;       /* total number of live threads/lwps */
 968         int     nzombies;       /* total number of zombie threads */
 969         int     ndaemons;       /* total number of THR_DAEMON threads/lwps */
 970         pid_t   pid;            /* the current process's pid */
 971         void    (*sigacthandler)(int, siginfo_t *, void *);
 972         ulwp_t  *lwp_stacks;
 973         ulwp_t  *lwp_laststack;
 974         int     nfreestack;
 975         int     thread_stack_cache;
 976         ulwp_t  *ulwp_freelist;
 977         ulwp_t  *ulwp_lastfree;
 978         ulwp_t  *ulwp_replace_free;
 979         ulwp_t  *ulwp_replace_last;
 980         atfork_t        *atforklist;    /* circular Q for fork handlers */
 981         robust_t        **robustlocks;  /* table of registered robust locks */
 982         robust_t        *robustlist;    /* list of registered robust locks */
 983         char    *progname;      /* the basename of the program, from argv[0] */
 984         void    *ub_comm_page;  /* arch-specific comm page of kernel data */
 985         struct uberdata **tdb_bootstrap;
 986         tdb_t   tdb;            /* thread debug interfaces (for libc_db) */
 987 } uberdata_t;
 988 
 989 #define link_lock       _link_lock.pad_lock
 990 #define ld_lock         _ld_lock.pad_lock
 991 #define fork_lock       _fork_lock.pad_lock
 992 #define atfork_lock     _atfork_lock.pad_lock
 993 #define callout_lock    _callout_lock.pad_lock
 994 #define tdb_hash_lock   _tdb_hash_lock.pad_lock
 995 
 996 #pragma align 64(__uberdata)
 997 extern  uberdata_t      __uberdata;
 998 extern  uberdata_t      **__tdb_bootstrap;      /* known to libc_db and mdb */
 999 extern  int             primary_link_map;
1000 
1001 #define ulwp_mutex(ulwp, udp)   \
1002         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_lock)
1003 #define ulwp_condvar(ulwp, udp) \
1004         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_cond)
1005 
1006 /*
1007  * Grab and release the hash table lock for the specified lwp.
1008  */
1009 #define ulwp_lock(ulwp, udp)    lmutex_lock(ulwp_mutex(ulwp, udp))
1010 #define ulwp_unlock(ulwp, udp)  lmutex_unlock(ulwp_mutex(ulwp, udp))
1011 
1012 #ifdef _SYSCALL32       /* needed by libc_db */
1013 
1014 typedef struct ulwp32 {
1015 #if defined(__sparc)
1016         uint32_t        ul_dinstr;      /* scratch space for dtrace */
1017         uint32_t        ul_padsparc0[15];
1018         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
1019         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
1020         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
1021         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
1022 #endif
1023         caddr32_t       ul_self;        /* pointer to self */
1024 #if defined(__x86)
1025         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
1026 #endif
1027         caddr32_t       ul_uberdata;    /* uber (super-global) data */
1028         tls32_t         ul_tls;         /* dynamic thread-local storage base */
1029         caddr32_t       ul_forw;        /* forw, back all_lwps list, */
1030         caddr32_t       ul_back;        /* protected by link_lock */
1031         caddr32_t       ul_next;        /* list to keep track of stacks */
1032         caddr32_t       ul_hash;        /* hash chain linked list */
1033         caddr32_t       ul_rval;        /* return value from thr_exit() */
1034         caddr32_t       ul_stk;         /* mapping base of the stack */
1035         size32_t        ul_mapsiz;      /* mapping size of the stack */
1036         size32_t        ul_guardsize;   /* normally _lpagesize */
1037         caddr32_t       ul_stktop;      /* broken thr_stksegment() interface */
1038         size32_t        ul_stksiz;      /* broken thr_stksegment() interface */
1039         stack32_t       ul_ustack;      /* current stack boundaries */
1040         int             ul_ix;          /* hash index */
1041         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
1042         pri_t           ul_pri;         /* scheduling priority */
1043         pri_t           ul_epri;        /* real-time ceiling priority */
1044         char            ul_policy;      /* scheduling policy */
1045         char            ul_cid;         /* scheduling class id */
1046         union {
1047                 struct {
1048                         char    cursig; /* deferred signal number */
1049                         char    pleasestop; /* lwp requested to stop itself */
1050                 } s;
1051                 short   curplease;      /* for testing both at once */
1052         } ul_cp;
1053         char            ul_stop;        /* reason for stopping */
1054         char            ul_signalled;   /* this lwp was cond_signal()d */
1055         char            ul_dead;        /* this lwp has called thr_exit */
1056         char            ul_unwind;      /* posix: unwind C++ stack */
1057         char            ul_detached;    /* THR_DETACHED at thread_create() */
1058                                         /* or pthread_detach() was called */
1059         char            ul_writer;      /* sleeping in rw_wrlock() */
1060         char            ul_stopping;    /* set by curthread: stopping self */
1061         char            ul_cancel_prologue;     /* for _cancel_prologue() */
1062         short           ul_preempt;     /* no_preempt()/preempt() */
1063         short           ul_savpreempt;  /* pre-existing preempt value */
1064         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
1065         char            ul_main;        /* thread is the main thread */
1066         char            ul_fork;        /* thread is performing a fork */
1067         char            ul_primarymap;  /* primary link-map is initialized */
1068         /* per-thread copies of the corresponding global variables */
1069         uint8_t         ul_max_spinners;        /* thread_max_spinners */
1070         char            ul_door_noreserve;      /* thread_door_noreserve */
1071         char            ul_queue_fifo;          /* thread_queue_fifo */
1072         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
1073         char            ul_error_detection;     /* thread_error_detection */
1074         char            ul_async_safe;          /* thread_async_safe */
1075         char            ul_rt;                  /* found on an RT queue */
1076         char            ul_rtqueued;            /* was RT when queued */
1077         char            ul_misaligned;          /* thread_locks_misaligned */
1078         char            ul_pad[3];
1079         int             ul_adaptive_spin;       /* thread_adaptive_spin */
1080         int             ul_queue_spin;          /* thread_queue_spin */
1081         int             ul_critical;    /* non-zero == in a critical region */
1082         int             ul_sigdefer;    /* non-zero == defer signals */
1083         int             ul_vfork;       /* thread is the child of vfork() */
1084         int             ul_cancelable;  /* _cancelon()/_canceloff() */
1085         char            ul_cancel_pending;  /* pthread_cancel() was called */
1086         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
1087         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
1088         char            ul_save_async;  /* saved copy of ul_cancel_async */
1089         char            ul_mutator;     /* lwp is a mutator (java interface) */
1090         char            ul_created;     /* created suspended */
1091         char            ul_replace;     /* replacement; must be free()d */
1092         uchar_t         ul_nocancel;    /* cancellation can't happen */
1093         int             ul_errno;       /* per-thread errno */
1094         caddr32_t       ul_errnop;      /* pointer to errno or self->ul_errno */
1095         caddr32_t       ul_clnup_hdr;   /* head of cleanup handlers list */
1096         caddr32_t       ul_schedctl_called; /* ul_schedctl is set up */
1097         caddr32_t       ul_schedctl;    /* schedctl data */
1098         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
1099         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
1100         caddr32_t       ul_stsd;        /* slow TLS for keys >= TSD_NFAST */
1101         caddr32_t       ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
1102         td_evbuf32_t    ul_td_evbuf;    /* event buffer */
1103         char            ul_td_events_enable;    /* event mechanism enabled */
1104         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
1105         char            ul_qtype;       /* MX or CV */
1106         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
1107         int             ul_rtld;        /* thread is running inside ld.so.1 */
1108         int             ul_usropts;     /* flags given to thr_create() */
1109         caddr32_t       ul_startpc;     /* start func (thr_create()) */
1110         caddr32_t       ul_startarg;    /* argument for start function */
1111         caddr32_t       ul_wchan;       /* synch object when sleeping */
1112         caddr32_t       ul_link;        /* sleep queue link */
1113         caddr32_t       ul_sleepq;      /* sleep queue thread is waiting on */
1114         caddr32_t       ul_cvmutex;     /* mutex dropped when waiting on a cv */
1115         caddr32_t       ul_mxchain;     /* chain of owned ceiling mutexes */
1116         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
1117         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
1118                                 /* 0 means there is but a single entry */
1119         union {                         /* single entry or pointer to array */
1120                 readlock32_t    single;
1121                 caddr32_t       array;
1122         } ul_readlock;
1123         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
1124                                 /* 0 means there is but a single entry */
1125         union {                         /* single entry or pointer to array */
1126                 caddr32_t       single;
1127                 caddr32_t       array;
1128         } ul_heldlocks;
1129         /* PROBE_SUPPORT begin */
1130         caddr32_t       ul_tpdp;
1131         /* PROBE_SUPPORT end */
1132         caddr32_t       ul_siglink;     /* pointer to previous context */
1133         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
1134         uint_t          ul_spin_lock_spin2;
1135         uint_t          ul_spin_lock_sleep;
1136         uint_t          ul_spin_lock_wakeup;
1137         queue_root32_t  ul_queue_root;  /* root of a sleep queue */
1138         id_t            ul_rtclassid;   /* real-time class id */
1139         uint_t          ul_pilocks;     /* count of PI locks held */
1140                 /* the following members *must* be last in the structure */
1141                 /* they are discarded when ulwp is replaced on thr_exit() */
1142         sigset_t        ul_sigmask;     /* thread's current signal mask */
1143         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
1144         siginfo32_t     ul_siginfo;     /* deferred siginfo */
1145         mutex_t         ul_spinlock;    /* used when suspending/continuing */
1146         fpuenv32_t      ul_fpuenv;      /* floating point state */
1147         caddr32_t       ul_sp;          /* stack pointer when blocked */
1148 #if defined(sparc)
1149         caddr32_t       ul_unwind_ret;  /* used only by _ex_clnup_handler() */
1150 #endif
1151         tumem32_t       ul_tmem;        /* used only by umem */
1152 } ulwp32_t;
1153 
1154 #define REPLACEMENT_SIZE32      ((size_t)&((ulwp32_t *)NULL)->ul_sigmask)
1155 
1156 typedef struct uberdata32 {
1157         pad_lock_t      _link_lock;
1158         pad_lock_t      _ld_lock;
1159         pad_lock_t      _fork_lock;
1160         pad_lock_t      _atfork_lock;
1161         pad32_lock_t    _callout_lock;
1162         pad32_lock_t    _tdb_hash_lock;
1163         tdb_sync_stats_t tdb_hash_lock_stats;
1164         siguaction32_t  siguaction[NSIG];
1165         bucket32_t      bucket[NBUCKETS];
1166         atexit_root32_t atexit_root;
1167         quickexit_root32_t quickexit_root;
1168         tsd_metadata32_t tsd_metadata;
1169         tls_metadata32_t tls_metadata;
1170         char            primary_map;
1171         char            bucket_init;
1172         char            pad[2];
1173         uberflags_t     uberflags;
1174         caddr32_t       queue_head;
1175         caddr32_t       thr_hash_table;
1176         uint_t          hash_size;
1177         uint_t          hash_mask;
1178         caddr32_t       ulwp_one;
1179         caddr32_t       all_lwps;
1180         caddr32_t       all_zombies;
1181         int             nthreads;
1182         int             nzombies;
1183         int             ndaemons;
1184         int             pid;
1185         caddr32_t       sigacthandler;
1186         caddr32_t       lwp_stacks;
1187         caddr32_t       lwp_laststack;
1188         int             nfreestack;
1189         int             thread_stack_cache;
1190         caddr32_t       ulwp_freelist;
1191         caddr32_t       ulwp_lastfree;
1192         caddr32_t       ulwp_replace_free;
1193         caddr32_t       ulwp_replace_last;
1194         caddr32_t       atforklist;
1195         caddr32_t       robustlocks;
1196         caddr32_t       robustlist;
1197         caddr32_t       progname;
1198         caddr32_t       ub_comm_page;
1199         caddr32_t       tdb_bootstrap;
1200         tdb32_t         tdb;
1201 } uberdata32_t;
1202 
1203 #endif  /* _SYSCALL32 */
1204 
1205 /* ul_stop values */
1206 #define TSTP_REGULAR    0x01    /* Stopped by thr_suspend() */
1207 #define TSTP_MUTATOR    0x08    /* stopped by thr_suspend_*mutator*() */
1208 #define TSTP_FORK       0x20    /* stopped by suspend_fork() */
1209 
1210 /*
1211  * Implementation-specific attribute types for pthread_mutexattr_init() etc.
1212  */
1213 
1214 typedef struct  _cvattr {
1215         int     pshared;
1216         clockid_t clockid;
1217 } cvattr_t;
1218 
1219 typedef struct  _mattr {
1220         int     pshared;
1221         int     protocol;
1222         int     prioceiling;
1223         int     type;
1224         int     robustness;
1225 } mattr_t;
1226 
1227 typedef struct  _thrattr {
1228         size_t  stksize;
1229         void    *stkaddr;
1230         int     detachstate;
1231         int     daemonstate;
1232         int     scope;
1233         int     prio;
1234         int     policy;
1235         int     inherit;
1236         size_t  guardsize;
1237         char    name[THREAD_NAME_MAX];
1238 } thrattr_t;
1239 
1240 typedef struct  _rwlattr {
1241         int     pshared;
1242 } rwlattr_t;
1243 
1244 /* _curthread() is inline for speed */
1245 extern  ulwp_t          *_curthread(void);
1246 #define curthread       (_curthread())
1247 
1248 /* this version (also inline) can be tested for NULL */
1249 extern  ulwp_t          *__curthread(void);
1250 
1251 /* get the current stack pointer (also inline) */
1252 extern  greg_t          stkptr(void);
1253 
1254 /*
1255  * Suppress __attribute__((...)) if we are not compiling with gcc
1256  */
1257 #if !defined(__GNUC__)
1258 #define __attribute__(string)
1259 #endif
1260 
1261 /* Fetch the dispatch (kernel) priority of a thread */
1262 #define real_priority(ulwp)     \
1263         ((ulwp)->ul_schedctl? (ulwp)->ul_schedctl->sc_priority : 0)
1264 
1265 /*
1266  * Implementation functions.  Not visible outside of the library itself.
1267  */
1268 extern  int     __nanosleep(const timespec_t *, timespec_t *);
1269 extern  void    getgregs(ulwp_t *, gregset_t);
1270 extern  void    setgregs(ulwp_t *, gregset_t);
1271 extern  void    thr_panic(const char *);
1272 #pragma rarely_called(thr_panic)
1273 extern  void    mutex_panic(mutex_t *, const char *);
1274 #pragma rarely_called(mutex_panic)
1275 extern  ulwp_t  *find_lwp(thread_t);
1276 extern  void    finish_init(void);
1277 extern  void    update_sched(ulwp_t *);
1278 extern  void    queue_alloc(void);
1279 extern  void    tmem_exit(void);
1280 extern  void    tsd_exit(void);
1281 extern  void    tsd_free(ulwp_t *);
1282 extern  void    tls_setup(void);
1283 extern  void    tls_exit(void);
1284 extern  void    tls_free(ulwp_t *);
1285 extern  void    rwl_free(ulwp_t *);
1286 extern  void    heldlock_exit(void);
1287 extern  void    heldlock_free(ulwp_t *);
1288 extern  void    sigacthandler(int, siginfo_t *, void *);
1289 extern  void    signal_init(void);
1290 extern  int     sigequalset(const sigset_t *, const sigset_t *);
1291 extern  void    mutex_setup(void);
1292 extern  void    take_deferred_signal(int);
1293 extern  void    *setup_top_frame(void *, size_t, ulwp_t *);
1294 extern  int     setup_context(ucontext_t *, void *(*func)(ulwp_t *),
1295                         ulwp_t *ulwp, caddr_t stk, size_t stksize);
1296 extern  volatile sc_shared_t *setup_schedctl(void);
1297 extern  void    *lmalloc(size_t);
1298 extern  void    lfree(void *, size_t);
1299 extern  void    *libc_malloc(size_t);
1300 extern  void    *libc_realloc(void *, size_t);
1301 extern  void    libc_free(void *);
1302 extern  char    *libc_strdup(const char *);
1303 extern  void    ultos(uint64_t, int, char *);
1304 extern  void    lock_error(const mutex_t *, const char *, void *, const char *);
1305 extern  void    rwlock_error(const rwlock_t *, const char *, const char *);
1306 extern  void    thread_error(const char *);
1307 extern  void    grab_assert_lock(void);
1308 extern  void    dump_queue_statistics(void);
1309 extern  void    collect_queue_statistics(void);
1310 extern  void    record_spin_locks(ulwp_t *);
1311 extern  void    remember_lock(mutex_t *);
1312 extern  void    forget_lock(mutex_t *);
1313 extern  void    register_lock(mutex_t *);
1314 extern  void    unregister_locks(void);
1315 #if defined(__sparc)
1316 extern  void    _flush_windows(void);
1317 #else
1318 #define _flush_windows()
1319 #endif
1320 extern  void    set_curthread(void *);
1321 
1322 /*
1323  * Utility function used when waking up many threads (more than MAXLWPS)
1324  * all at once.  See mutex_wakeup_all(), cond_broadcast(), and rw_unlock().
1325  */
1326 #define MAXLWPS 128     /* max remembered lwpids before overflow */
1327 #define NEWLWPS 2048    /* max remembered lwpids at first overflow */
1328 extern  lwpid_t *alloc_lwpids(lwpid_t *, int *, int *);
1329 
1330 /* enter a critical section */
1331 #define enter_critical(self)    (self->ul_critical++)
1332 
1333 /* exit a critical section, take deferred actions if necessary */
1334 extern  void    do_exit_critical(void);
1335 #define exit_critical(self)                                     \
1336         (void) (self->ul_critical--,                         \
1337             ((self->ul_curplease && self->ul_critical == 0)?      \
1338             (do_exit_critical(), 0) : 0))
1339 
1340 /*
1341  * Like enter_critical()/exit_critical() but just for deferring signals.
1342  * Unlike enter_critical()/exit_critical(), ul_sigdefer may be set while
1343  * calling application functions like constructors and destructors.
1344  * Care must be taken if the application function attempts to set
1345  * the signal mask while a deferred signal is present; the setting
1346  * of the signal mask must also be deferred.
1347  */
1348 #define sigoff(self)    (self->ul_sigdefer++)
1349 #define sigon(self)                                             \
1350         (void) ((--self->ul_sigdefer == 0 &&                 \
1351             self->ul_curplease && self->ul_critical == 0)?        \
1352             (do_exit_critical(), 0) : 0)
1353 
1354 /* these are exported functions */
1355 extern  void    _sigoff(void);
1356 extern  void    _sigon(void);
1357 
1358 #define sigorset(s1, s2)                                \
1359         (((s1)->__sigbits[0] |= (s2)->__sigbits[0]),      \
1360         ((s1)->__sigbits[1] |= (s2)->__sigbits[1]),       \
1361         ((s1)->__sigbits[2] |= (s2)->__sigbits[2]),       \
1362         ((s1)->__sigbits[3] |= (s2)->__sigbits[3]))
1363 
1364 #define sigandset(s1, s2)                               \
1365         (((s1)->__sigbits[0] &= (s2)->__sigbits[0]),  \
1366         ((s1)->__sigbits[1] &= (s2)->__sigbits[1]),   \
1367         ((s1)->__sigbits[2] &= (s2)->__sigbits[2]),   \
1368         ((s1)->__sigbits[3] &= (s2)->__sigbits[3]))
1369 
1370 #define sigdiffset(s1, s2)                              \
1371         (((s1)->__sigbits[0] &= ~(s2)->__sigbits[0]), \
1372         ((s1)->__sigbits[1] &= ~(s2)->__sigbits[1]),  \
1373         ((s1)->__sigbits[2] &= ~(s2)->__sigbits[2]),  \
1374         ((s1)->__sigbits[3] &= ~(s2)->__sigbits[3]))
1375 
1376 #define delete_reserved_signals(s)                      \
1377         (((s)->__sigbits[0] &= MASKSET0),                \
1378         ((s)->__sigbits[1] &= (MASKSET1 & ~SIGMASK(SIGCANCEL))),\
1379         ((s)->__sigbits[2] &= MASKSET2),         \
1380         ((s)->__sigbits[3] &= MASKSET3))
1381 
1382 extern  void    block_all_signals(ulwp_t *self);
1383 
1384 /*
1385  * When restoring the signal mask after having previously called
1386  * block_all_signals(), if we have a deferred signal present then
1387  * do nothing other than ASSERT() that we are in a critical region.
1388  * The signal mask will be set when we emerge from the critical region
1389  * and call take_deferred_signal().  There is no race condition here
1390  * because the kernel currently has all signals blocked for this thread.
1391  */
1392 #define restore_signals(self)                                           \
1393         ((void) ((self)->ul_cursig?                                  \
1394         (ASSERT((self)->ul_critical + (self)->ul_sigdefer != 0), 0) :     \
1395         __lwp_sigmask(SIG_SETMASK, &(self)->ul_sigmask)))
1396 
1397 extern  void    set_cancel_pending_flag(ulwp_t *, int);
1398 extern  void    set_cancel_eintr_flag(ulwp_t *);
1399 extern  void    set_parking_flag(ulwp_t *, int);
1400 extern  int     cancel_active(void);
1401 
1402 extern  void    *_thrp_setup(ulwp_t *);
1403 extern  void    _fpinherit(ulwp_t *);
1404 extern  void    _lwp_start(void);
1405 extern  void    _lwp_terminate(void);
1406 extern  void    lmutex_lock(mutex_t *);
1407 extern  void    lmutex_unlock(mutex_t *);
1408 extern  void    lrw_rdlock(rwlock_t *);
1409 extern  void    lrw_wrlock(rwlock_t *);
1410 extern  void    lrw_unlock(rwlock_t *);
1411 extern  void    sig_mutex_lock(mutex_t *);
1412 extern  void    sig_mutex_unlock(mutex_t *);
1413 extern  int     sig_mutex_trylock(mutex_t *);
1414 extern  int     sig_cond_wait(cond_t *, mutex_t *);
1415 extern  int     sig_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1416 extern  void    cancel_safe_mutex_lock(mutex_t *);
1417 extern  void    cancel_safe_mutex_unlock(mutex_t *);
1418 extern  int     cancel_safe_mutex_trylock(mutex_t *);
1419 extern  void    _prefork_handler(void);
1420 extern  void    _postfork_parent_handler(void);
1421 extern  void    _postfork_child_handler(void);
1422 extern  void    postfork1_child(void);
1423 extern  void    postfork1_child_aio(void);
1424 extern  void    postfork1_child_sigev_aio(void);
1425 extern  void    postfork1_child_sigev_mq(void);
1426 extern  void    postfork1_child_sigev_timer(void);
1427 extern  void    postfork1_child_tpool(void);
1428 extern  void    fork_lock_enter(void);
1429 extern  void    fork_lock_exit(void);
1430 extern  void    suspend_fork(void);
1431 extern  void    continue_fork(int);
1432 extern  void    do_sigcancel(void);
1433 extern  void    setup_cancelsig(int);
1434 extern  void    init_sigev_thread(void);
1435 extern  void    init_aio(void);
1436 extern  void    init_progname(void);
1437 extern  void    _cancelon(void);
1438 extern  void    _canceloff(void);
1439 extern  void    _canceloff_nocancel(void);
1440 extern  void    _cancel_prologue(void);
1441 extern  void    _cancel_epilogue(void);
1442 extern  void    no_preempt(ulwp_t *);
1443 extern  void    preempt(ulwp_t *);
1444 extern  void    _thrp_unwind(void *);
1445 
1446 extern  pid_t   __forkx(int);
1447 extern  pid_t   __forkallx(int);
1448 extern  int     __open(const char *, int, mode_t);
1449 extern  int     __open64(const char *, int, mode_t);
1450 extern  int     __openat(int, const char *, int, mode_t);
1451 extern  int     __openat64(int, const char *, int, mode_t);
1452 extern  int     __close(int);
1453 extern  ssize_t __read(int, void *, size_t);
1454 extern  ssize_t __write(int, const void *, size_t);
1455 extern  int     __fcntl(int, int, ...);
1456 extern  int     __lwp_continue(lwpid_t);
1457 extern  int     __lwp_create(ucontext_t *, uint_t, lwpid_t *);
1458 extern  int     ___lwp_suspend(lwpid_t);
1459 extern  int     lwp_wait(lwpid_t, lwpid_t *);
1460 extern  int     __lwp_wait(lwpid_t, lwpid_t *);
1461 extern  int     __lwp_detach(lwpid_t);
1462 extern  sc_shared_t *__schedctl(void);
1463 
1464 /* actual system call traps */
1465 extern  int     __setcontext(const ucontext_t *);
1466 extern  int     __getcontext(ucontext_t *);
1467 extern  int     __clock_gettime(clockid_t, timespec_t *);
1468 extern  void    abstime_to_reltime(clockid_t, const timespec_t *, timespec_t *);
1469 extern  void    hrt2ts(hrtime_t, timespec_t *);
1470 
1471 extern  int     __sigaction(int, const struct sigaction *, struct sigaction *);
1472 extern  int     __sigprocmask(int, const sigset_t *, sigset_t *);
1473 extern  int     __lwp_sigmask(int, const sigset_t *);
1474 extern  void    __sighndlr(int, siginfo_t *, ucontext_t *, void (*)());
1475 extern  caddr_t __sighndlrend;
1476 #pragma unknown_control_flow(__sighndlr)
1477 
1478 /* belongs in <pthread.h> */
1479 #define PTHREAD_CREATE_DAEMON_NP        0x100   /* = THR_DAEMON */
1480 #define PTHREAD_CREATE_NONDAEMON_NP     0
1481 extern  int     pthread_attr_setdaemonstate_np(pthread_attr_t *, int);
1482 extern  int     pthread_attr_getdaemonstate_np(const pthread_attr_t *, int *);
1483 
1484 extern  int     mutex_held(mutex_t *);
1485 extern  int     mutex_lock_internal(mutex_t *, timespec_t *, int);
1486 extern  int     mutex_unlock_internal(mutex_t *, int);
1487 
1488 /* not cancellation points: */
1489 extern  int     __cond_wait(cond_t *, mutex_t *);
1490 extern  int     __cond_timedwait(cond_t *, mutex_t *, const timespec_t *);
1491 extern  int     __cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1492 
1493 extern  int     rw_read_held(rwlock_t *);
1494 extern  int     rw_write_held(rwlock_t *);
1495 
1496 extern  int     _thrp_create(void *, size_t, void *(*)(void *), void *, long,
1497                         thread_t *, size_t, const char *);
1498 extern  int     _thrp_suspend(thread_t, uchar_t);
1499 extern  int     _thrp_continue(thread_t, uchar_t);
1500 
1501 extern  void    _thrp_terminate(void *);
1502 extern  void    _thrp_exit(void);
1503 
1504 extern  const pcclass_t *get_info_by_class(id_t);
1505 extern  const pcclass_t *get_info_by_policy(int);
1506 extern  const thrattr_t *def_thrattr(void);
1507 extern  id_t    setparam(idtype_t, id_t, int, int);
1508 extern  id_t    setprio(idtype_t, id_t, int, int *);
1509 extern  id_t    getparam(idtype_t, id_t, int *, struct sched_param *);
1510 
1511 /*
1512  * System call wrappers (direct interfaces to the kernel)
1513  */
1514 extern  int     ___lwp_mutex_register(mutex_t *, mutex_t **);
1515 extern  int     ___lwp_mutex_trylock(mutex_t *, ulwp_t *);
1516 extern  int     ___lwp_mutex_timedlock(mutex_t *, timespec_t *, ulwp_t *);
1517 extern  int     ___lwp_mutex_unlock(mutex_t *);
1518 extern  int     ___lwp_mutex_wakeup(mutex_t *, int);
1519 extern  int     ___lwp_cond_wait(cond_t *, mutex_t *, timespec_t *, int);
1520 extern  int     ___lwp_sema_timedwait(lwp_sema_t *, timespec_t *, int);
1521 extern  int     __lwp_rwlock_rdlock(rwlock_t *, timespec_t *);
1522 extern  int     __lwp_rwlock_wrlock(rwlock_t *, timespec_t *);
1523 extern  int     __lwp_rwlock_tryrdlock(rwlock_t *);
1524 extern  int     __lwp_rwlock_trywrlock(rwlock_t *);
1525 extern  int     __lwp_rwlock_unlock(rwlock_t *);
1526 extern  int     __lwp_park(timespec_t *, lwpid_t);
1527 extern  int     __lwp_unpark(lwpid_t);
1528 extern  int     __lwp_unpark_all(lwpid_t *, int);
1529 #if defined(__x86)
1530 extern  int     ___lwp_private(int, int, void *);
1531 #endif  /* __x86 */
1532 
1533 /*
1534  * inlines
1535  */
1536 extern  int             set_lock_byte(volatile uint8_t *);
1537 extern  uint32_t        atomic_swap_32(volatile uint32_t *, uint32_t);
1538 extern  uint32_t        atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
1539 extern  void            atomic_inc_32(volatile uint32_t *);
1540 extern  void            atomic_dec_32(volatile uint32_t *);
1541 extern  void            atomic_and_32(volatile uint32_t *, uint32_t);
1542 extern  void            atomic_or_32(volatile uint32_t *, uint32_t);
1543 #if defined(__sparc)
1544 extern  ulong_t         caller(void);
1545 extern  ulong_t         getfp(void);
1546 #endif  /* __sparc */
1547 
1548 #include "thr_inlines.h"
1549 
1550 #endif  /* _THR_UBERDATA_H */