1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * Copyright 2016 Joyent, Inc.
  28  * Copyright 2018 Nexenta Systems, Inc.
  29  */
  30 
  31 #ifndef _THR_UBERDATA_H
  32 #define _THR_UBERDATA_H
  33 
  34 #include <stdlib.h>
  35 #include <unistd.h>
  36 #include <sys/types.h>
  37 #include <fcntl.h>
  38 #include <string.h>
  39 #include <signal.h>
  40 #include <ucontext.h>
  41 #include <thread.h>
  42 #include <pthread.h>
  43 #include <atomic.h>
  44 #include <link.h>
  45 #include <sys/resource.h>
  46 #include <sys/lwp.h>
  47 #include <errno.h>
  48 #include <sys/asm_linkage.h>
  49 #include <sys/regset.h>
  50 #include <sys/fcntl.h>
  51 #include <sys/mman.h>
  52 #include <synch.h>
  53 #include <door.h>
  54 #include <limits.h>
  55 #include <sys/synch32.h>
  56 #include <schedctl.h>
  57 #include <sys/priocntl.h>
  58 #include <thread_db.h>
  59 #include <setjmp.h>
  60 #include "libc_int.h"
  61 #include "tdb_agent.h"
  62 #include "thr_debug.h"
  63 
  64 /*
  65  * This is an implementation-specific include file for threading support.
  66  * It is not to be seen by the clients of the library.
  67  *
  68  * This file also describes uberdata in libc.
  69  *
  70  * The term "uberdata" refers to data that is unique and visible across
  71  * all link maps.  The name is meant to imply that such data is truly
  72  * global, not just locally global to a particular link map.
  73  *
  74  * See the Linker and Libraries Guide for a full description of alternate
  75  * link maps and how they are set up and used.
  76  *
  77  * Alternate link maps implement multiple global namespaces within a single
  78  * process.  There may be multiple instances of identical dynamic libraries
  79  * loaded in a process's address space at the same time, each on a different
  80  * link map (as determined by the dynamic linker), each with its own set of
  81  * global variables.  Which particular instance of a global variable is seen
  82  * by a thread running in the process is determined by the link map on which
  83  * the thread happens to be executing at the time.
  84  *
  85  * However, there are aspects of a process that are unique across all
  86  * link maps, in particular the structures used to implement threads
  87  * of control (in Sparc terminology, there is only one %g7 regardless
  88  * of the link map on which the thread is executing).
  89  *
  90  * All uberdata is referenced from a base pointer in the thread's ulwp_t
  91  * structure (which is also uberdata).  All allocations and deallocations
  92  * of uberdata are made via the uberdata-aware lmalloc() and lfree()
  93  * interfaces (malloc() and free() are simply locally-global).
  94  */
  95 
  96 /*
  97  * Special libc-private access to errno.
  98  * We do this so that references to errno do not invoke the dynamic linker.
  99  */
 100 #undef errno
 101 #define errno (*curthread->ul_errnop)
 102 
 103 /*
 104  * See <sys/synch32.h> for the reasons for these values
 105  * and why they are different for sparc and intel.
 106  */
 107 #if defined(__sparc)
 108 
 109 /* lock.lock64.pad[x]      4 5 6 7 */
 110 #define LOCKMASK        0xff000000
 111 #define WAITERMASK      0x000000ff
 112 #define SPINNERMASK     0x00ff0000
 113 #define SPINNERSHIFT    16
 114 #define WAITER          0x00000001
 115 #define LOCKSET         0xff
 116 #define LOCKCLEAR       0
 117 
 118 #define PIDSHIFT        32
 119 #define LOCKMASK64      0xffffffffff000000ULL
 120 #define LOCKBYTE64      0x00000000ff000000ULL
 121 #define WAITERMASK64    0x00000000000000ffULL
 122 #define SPINNERMASK64   0x0000000000ff0000ULL
 123 
 124 #elif defined(__x86)
 125 
 126 /* lock.lock64.pad[x]      7 6 5 4 */
 127 #define LOCKMASK        0xff000000
 128 #define WAITERMASK      0x00ff0000
 129 #define SPINNERMASK     0x0000ff00
 130 #define SPINNERSHIFT    8
 131 #define WAITER          0x00010000
 132 #define LOCKSET         0x01
 133 #define LOCKCLEAR       0
 134 
 135 #define PIDSHIFT        0
 136 #define LOCKMASK64      0xff000000ffffffffULL
 137 #define LOCKBYTE64      0x0100000000000000ULL
 138 #define WAITERMASK64    0x00ff000000000000ULL
 139 #define SPINNERMASK64   0x0000ff0000000000ULL
 140 
 141 #else
 142 #error "neither __sparc nor __x86 is defined"
 143 #endif
 144 
 145 /*
 146  * Fetch the owner of a USYNC_THREAD mutex.
 147  * Don't use this with process-shared mutexes;
 148  * the owing thread may be in a different process.
 149  */
 150 #define MUTEX_OWNER(mp) ((ulwp_t *)(uintptr_t)(mp)->mutex_owner)
 151 
 152 /*
 153  * Test if a thread owns a process-private (USYNC_THREAD) mutex.
 154  * This is inappropriate for a process-shared (USYNC_PROCESS) mutex.
 155  * The 'mp' argument must not have side-effects since it is evaluated twice.
 156  */
 157 #define MUTEX_OWNED(mp, thrp)   \
 158         ((mp)->mutex_lockw != 0 && MUTEX_OWNER(mp) == thrp)
 159 
 160 
 161 /*
 162  * uberflags.uf_tdb_register_sync is an interface with libc_db to enable the
 163  * collection of lock statistics by a debugger or other collecting tool.
 164  *
 165  * uberflags.uf_thread_error_detection is set by an environment variable:
 166  *      _THREAD_ERROR_DETECTION
 167  *              0 == no detection of locking primitive errors.
 168  *              1 == detect errors and issue a warning message.
 169  *              2 == detect errors, issue a warning message, and dump core.
 170  *
 171  * We bundle these together in uberflags.uf_trs_ted to make a test of either
 172  * being non-zero a single memory reference (for speed of mutex_lock(), etc).
 173  *
 174  * uberflags.uf_mt is set non-zero when the first thread (in addition
 175  * to the main thread) is created.
 176  *
 177  * We bundle all these flags together in uberflags.uf_all to make a test
 178  * of any being non-zero a single memory reference (again, for speed).
 179  */
 180 typedef union {
 181         int     uf_all;                 /* combined all flags */
 182         struct {
 183                 short   h_pad;
 184                 short   h_trs_ted;      /* combined reg sync & error detect */
 185         } uf_h;
 186         struct {
 187                 char    x_mt;
 188                 char    x_pad;
 189                 char    x_tdb_register_sync;
 190                 char    x_thread_error_detection;
 191         } uf_x;
 192 } uberflags_t;
 193 
 194 #define uf_mt                           uf_x.x_mt
 195 #define uf_tdb_register_sync            uf_x.x_tdb_register_sync
 196 #define uf_thread_error_detection       uf_x.x_thread_error_detection
 197 #define uf_trs_ted                      uf_h.h_trs_ted  /* both of the above */
 198 
 199 /*
 200  * NOTE WELL:
 201  * To enable further optimization, the "ul_schedctl_called" member
 202  * of the ulwp_t structure (below) serves double-duty:
 203  *      1. If NULL, it means that the thread must call __schedctl()
 204  *         to set up its schedctl mappings before acquiring a mutex.
 205  *         This is required by the implementation of adaptive mutex locking.
 206  *      2. If non-NULL, it points to uberdata.uberflags, so that tests of
 207  *         uberflags can be made without additional memory references.
 208  * This allows the common case of _mutex_lock() and _mutex_unlock() for
 209  * USYNC_THREAD mutexes with no error detection and no lock statistics
 210  * to be optimized for speed.
 211  */
 212 
 213 /* double the default stack size for 64-bit processes */
 214 #ifdef _LP64
 215 #define MINSTACK        (8 * 1024)
 216 #define DEFAULTSTACK    (2 * 1024 * 1024)
 217 #else
 218 #define MINSTACK        (4 * 1024)
 219 #define DEFAULTSTACK    (1024 * 1024)
 220 #endif
 221 
 222 #define MUTEX_TRY       0
 223 #define MUTEX_LOCK      1
 224 #define MUTEX_NOCEIL    0x40
 225 
 226 #if defined(__x86)
 227 
 228 typedef struct {        /* structure returned by fnstenv */
 229         int     fctrl;          /* control word */
 230         int     fstat;          /* status word (flags, etc) */
 231         int     ftag;           /* tag of which regs busy */
 232         int     misc[4];        /* other stuff, 28 bytes total */
 233 } fpuenv_t;
 234 
 235 #ifdef _SYSCALL32
 236 typedef fpuenv_t fpuenv32_t;
 237 #endif  /* _SYSCALL32 */
 238 
 239 #elif defined(__sparc)
 240 
 241 typedef struct {        /* fp state structure */
 242         greg_t  fsr;
 243         greg_t  fpu_en;
 244 } fpuenv_t;
 245 
 246 #ifdef _SYSCALL32
 247 typedef struct {
 248         greg32_t        fsr;
 249         greg32_t        fpu_en;
 250 } fpuenv32_t;
 251 #endif  /* _SYSCALL32 */
 252 
 253 #endif  /* __x86 */
 254 
 255 #if defined(__x86)
 256 extern  void    ht_pause(void);         /* "pause" instruction */
 257 #define SMT_PAUSE()     ht_pause()
 258 #elif defined(SMT_PAUSE_FUNCTION)
 259 extern  void    SMT_PAUSE_FUNCTION(void);
 260 #define SMT_PAUSE()     SMT_PAUSE_FUNCTION()
 261 #else
 262 #define SMT_PAUSE()     smt_pause()
 263 #endif  /* __x86 */
 264 
 265 /*
 266  * Cleanup handler related data.
 267  * This structure is exported as _cleanup_t in pthread.h.
 268  * pthread.h exports only the size of this structure, so check
 269  * _cleanup_t in pthread.h before making any change here.
 270  */
 271 typedef struct __cleanup {
 272         struct __cleanup *next;         /* pointer to next handler */
 273         caddr_t fp;                     /* current frame pointer */
 274         void    (*func)(void *);        /* cleanup handler address */
 275         void    *arg;                   /* handler's argument */
 276 } __cleanup_t;
 277 
 278 /*
 279  * Thread-Specific Data (TSD)
 280  * TSD_NFAST includes the invalid key zero, so there
 281  * are really only (TSD_NFAST - 1) fast key slots.
 282  */
 283 typedef void (*PFrV)(void *);
 284 #define TSD_UNALLOCATED ((PFrV)1)
 285 #define TSD_NFAST       9
 286 
 287 /*
 288  * The tsd union is designed to burn a little memory (9 words) to make
 289  * lookups blindingly fast.  Note that tsd_nalloc could be placed at the
 290  * end of the pad region to increase the likelihood that it falls on the
 291  * same cache line as the data.
 292  */
 293 typedef union tsd {
 294         uint_t tsd_nalloc;              /* Amount of allocated storage */
 295         void *tsd_pad[TSD_NFAST];
 296         void *tsd_data[1];
 297 } tsd_t;
 298 
 299 typedef struct {
 300         mutex_t tsdm_lock;              /* Lock protecting the data */
 301         uint_t tsdm_nkeys;              /* Number of allocated keys */
 302         uint_t tsdm_nused;              /* Number of used keys */
 303         PFrV *tsdm_destro;              /* Per-key destructors */
 304         char tsdm_pad[64 -              /* pad to 64 bytes */
 305                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (PFrV *))];
 306 } tsd_metadata_t;
 307 
 308 #ifdef _SYSCALL32
 309 typedef union tsd32 {
 310         uint_t tsd_nalloc;              /* Amount of allocated storage */
 311         caddr32_t tsd_pad[TSD_NFAST];
 312         caddr32_t tsd_data[1];
 313 } tsd32_t;
 314 
 315 typedef struct {
 316         mutex_t tsdm_lock;              /* Lock protecting the data */
 317         uint_t tsdm_nkeys;              /* Number of allocated keys */
 318         uint_t tsdm_nused;              /* Number of used keys */
 319         caddr32_t tsdm_destro;          /* Per-key destructors */
 320         char tsdm_pad[64 -              /* pad to 64 bytes */
 321                 (sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (caddr32_t))];
 322 } tsd_metadata32_t;
 323 #endif  /* _SYSCALL32 */
 324 
 325 
 326 /*
 327  * Thread-Local Storage (TLS)
 328  */
 329 typedef struct {
 330         void            *tls_data;
 331         size_t          tls_size;
 332 } tls_t;
 333 
 334 typedef struct {
 335         mutex_t tls_lock;               /* Lock protecting the data */
 336         tls_t   tls_modinfo;            /* Root of all TLS_modinfo data */
 337         tls_t   static_tls;             /* Template for static TLS */
 338         char    tls_pad[64 -            /* pad to 64 bytes */
 339                 (sizeof (mutex_t) + 2 * sizeof (tls_t))];
 340 } tls_metadata_t;
 341 
 342 #ifdef _SYSCALL32
 343 typedef struct {
 344         caddr32_t       tls_data;
 345         size32_t        tls_size;
 346 } tls32_t;
 347 
 348 typedef struct {
 349         mutex_t tls_lock;               /* Lock protecting the data */
 350         tls32_t tls_modinfo;            /* Root of all TLS_modinfo data */
 351         tls32_t static_tls;             /* Template for static TLS */
 352         char    tls_pad[64 -            /* pad to 64 bytes */
 353                 (sizeof (mutex_t) + 2 * sizeof (tls32_t))];
 354 } tls_metadata32_t;
 355 #endif  /* _SYSCALL32 */
 356 
 357 
 358 /*
 359  * Sleep queue root for USYNC_THREAD condvars and mutexes.
 360  * There is a default queue root for each queue head (see below).
 361  * Also, each ulwp_t contains a queue root that can be used
 362  * when the thread is enqueued on the queue, if necessary
 363  * (when more than one wchan hashes to the same queue head).
 364  */
 365 typedef struct queue_root {
 366         struct queue_root       *qr_next;
 367         struct queue_root       *qr_prev;
 368         struct ulwp             *qr_head;
 369         struct ulwp             *qr_tail;
 370         void                    *qr_wchan;
 371         uint32_t                qr_rtcount;
 372         uint32_t                qr_qlen;
 373         uint32_t                qr_qmax;
 374 } queue_root_t;
 375 
 376 #ifdef _SYSCALL32
 377 typedef struct queue_root32 {
 378         caddr32_t               qr_next;
 379         caddr32_t               qr_prev;
 380         caddr32_t               qr_head;
 381         caddr32_t               qr_tail;
 382         caddr32_t               qr_wchan;
 383         uint32_t                qr_rtcount;
 384         uint32_t                qr_qlen;
 385         uint32_t                qr_qmax;
 386 } queue_root32_t;
 387 #endif
 388 
 389 /*
 390  * Sleep queue heads for USYNC_THREAD condvars and mutexes.
 391  * The size and alignment is 128 bytes to reduce cache conflicts.
 392  * Each queue head points to a list of queue roots, defined above.
 393  * Each queue head contains a default queue root for use when only one
 394  * is needed.  It is always at the tail of the queue root hash chain.
 395  */
 396 typedef union {
 397         uint64_t                qh_64[16];
 398         struct {
 399                 mutex_t         q_lock;
 400                 uint8_t         q_qcnt;
 401                 uint8_t         q_type;         /* MX or CV */
 402                 uint8_t         q_pad1[2];
 403                 uint32_t        q_lockcount;
 404                 uint32_t        q_qlen;
 405                 uint32_t        q_qmax;
 406                 void            *q_wchan;       /* valid only while locked */
 407                 struct queue_root *q_root;      /* valid only while locked */
 408                 struct queue_root *q_hlist;
 409 #if !defined(_LP64)
 410                 caddr_t         q_pad2[3];
 411 #endif
 412                 queue_root_t    q_def_root;
 413                 uint32_t        q_hlen;
 414                 uint32_t        q_hmax;
 415         } qh_qh;
 416 } queue_head_t;
 417 
 418 #define qh_lock         qh_qh.q_lock
 419 #define qh_qcnt         qh_qh.q_qcnt
 420 #define qh_type         qh_qh.q_type
 421 #if defined(THREAD_DEBUG)
 422 #define qh_lockcount    qh_qh.q_lockcount
 423 #define qh_qlen         qh_qh.q_qlen
 424 #define qh_qmax         qh_qh.q_qmax
 425 #endif
 426 #define qh_wchan        qh_qh.q_wchan
 427 #define qh_root         qh_qh.q_root
 428 #define qh_hlist        qh_qh.q_hlist
 429 #define qh_def_root     qh_qh.q_def_root
 430 #define qh_hlen         qh_qh.q_hlen
 431 #define qh_hmax         qh_qh.q_hmax
 432 
 433 /* queue types passed to queue_lock() */
 434 #define MX      0
 435 #define CV      1
 436 #define QHASHSHIFT      9                       /* number of hashing bits */
 437 #define QHASHSIZE       (1 << QHASHSHIFT) /* power of 2 (1<<9 == 512) */
 438 #define QUEUE_HASH(wchan, type) ((uint_t)                       \
 439         ((((uintptr_t)(wchan) >> 3)                               \
 440         ^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3)))               \
 441         & (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE))
 442 
 443 extern  queue_head_t    *queue_lock(void *, int);
 444 extern  void            queue_unlock(queue_head_t *);
 445 extern  void            enqueue(queue_head_t *, struct ulwp *, int);
 446 extern  struct ulwp     *dequeue(queue_head_t *, int *);
 447 extern  struct ulwp     **queue_slot(queue_head_t *, struct ulwp **, int *);
 448 extern  struct ulwp     *queue_waiter(queue_head_t *);
 449 extern  int             dequeue_self(queue_head_t *);
 450 extern  void            queue_unlink(queue_head_t *,
 451                                 struct ulwp **, struct ulwp *);
 452 extern  void            unsleep_self(void);
 453 extern  void            spin_lock_set(mutex_t *);
 454 extern  void            spin_lock_clear(mutex_t *);
 455 
 456 /*
 457  * Scheduling class information structure.
 458  */
 459 typedef struct {
 460         short           pcc_state;
 461         short           pcc_policy;
 462         pri_t           pcc_primin;
 463         pri_t           pcc_primax;
 464         pcinfo_t        pcc_info;
 465 } pcclass_t;
 466 
 467 /*
 468  * Memory block for chain of owned ceiling mutexes.
 469  */
 470 typedef struct mxchain {
 471         struct mxchain  *mxchain_next;
 472         mutex_t         *mxchain_mx;
 473 } mxchain_t;
 474 
 475 /*
 476  * Pointer to an rwlock that is held for reading.
 477  * Used in rw_rdlock() to allow a thread that already holds a read
 478  * lock to acquire another read lock on the same rwlock even if
 479  * there are writers waiting.  This to avoid deadlock when acquiring
 480  * a read lock more than once in the presence of pending writers.
 481  * POSIX mandates this behavior.
 482  */
 483 typedef struct {
 484         void    *rd_rwlock;     /* the rwlock held for reading */
 485         size_t  rd_count;       /* count of read locks applied */
 486 } readlock_t;
 487 
 488 #ifdef _SYSCALL32
 489 typedef struct {
 490         caddr32_t       rd_rwlock;
 491         size32_t        rd_count;
 492 } readlock32_t;
 493 #endif  /* _SYSCALL32 */
 494 
 495 /*
 496  * As part of per-thread caching libumem (ptcumem), we add a small amount to the
 497  * thread's uberdata to facilitate it. The tm_roots are the roots of linked
 498  * lists which is used by libumem to chain together allocations. tm_size is used
 499  * to track the total amount of data stored across those linked lists. For more
 500  * information, see libumem's big theory statement.
 501  */
 502 #define NTMEMBASE       16
 503 
 504 typedef struct {
 505         size_t          tm_size;
 506         void            *tm_roots[NTMEMBASE];
 507 } tumem_t;
 508 
 509 #ifdef _SYSCALL32
 510 typedef struct {
 511         uint32_t        tm_size;
 512         caddr32_t       tm_roots[NTMEMBASE];
 513 } tumem32_t;
 514 #endif
 515 
 516 typedef void (*tmem_func_t)(void *, int);
 517 
 518 /*
 519  * Maximum number of read locks allowed for one thread on one rwlock.
 520  * This could be as large as INT_MAX, but the SUSV3 test suite would
 521  * take an inordinately long time to complete.  This is big enough.
 522  */
 523 #define READ_LOCK_MAX   100000
 524 
 525 #define ul_tlsent       ul_tls.tls_data /* array of pointers to dynamic TLS */
 526 #define ul_ntlsent      ul_tls.tls_size /* number of entries in ul_tlsent */
 527 
 528 /*
 529  * Round up an integral value to a multiple of 64
 530  */
 531 #define roundup64(x)    (-(-(x) & -64))
 532 
 533 /*
 534  * NOTE:  Whatever changes are made to ulwp_t must be
 535  * reflected in $SRC/cmd/mdb/common/modules/libc/libc.c
 536  *
 537  * NOTE: ul_self *must* be the first member of ulwp_t on x86
 538  * Low-level x86 code relies on this.
 539  */
 540 typedef struct ulwp {
 541         /*
 542          * These members always need to come first on sparc.
 543          * For dtrace, a ulwp_t must be aligned on a 64-byte boundary.
 544          */
 545 #if defined(__sparc)
 546         uint32_t        ul_dinstr;      /* scratch space for dtrace */
 547         uint32_t        ul_padsparc0[15];
 548         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
 549         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
 550         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
 551         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
 552 #endif
 553         struct ulwp     *ul_self;       /* pointer to self */
 554 #if defined(__i386)
 555         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
 556 #elif defined(__amd64)
 557         uint8_t         ul_dinstr[56];  /* scratch space for dtrace */
 558 #endif
 559         struct uberdata *ul_uberdata;   /* uber (super-global) data */
 560         tls_t           ul_tls;         /* dynamic thread-local storage base */
 561         struct ulwp     *ul_forw;       /* forw, back all_lwps list, */
 562         struct ulwp     *ul_back;       /* protected by link_lock */
 563         struct ulwp     *ul_next;       /* list to keep track of stacks */
 564         struct ulwp     *ul_hash;       /* hash chain linked list */
 565         void            *ul_rval;       /* return value from thr_exit() */
 566         caddr_t         ul_stk;         /* mapping base of the stack */
 567         size_t          ul_mapsiz;      /* mapping size of the stack */
 568         size_t          ul_guardsize;   /* normally _lpagesize */
 569         uintptr_t       ul_stktop;      /* broken thr_stksegment() interface */
 570         size_t          ul_stksiz;      /* broken thr_stksegment() interface */
 571         stack_t         ul_ustack;      /* current stack boundaries */
 572         int             ul_ix;          /* hash index */
 573         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
 574         pri_t           ul_pri;         /* scheduling priority */
 575         pri_t           ul_epri;        /* real-time ceiling priority */
 576         char            ul_policy;      /* scheduling policy */
 577         char            ul_cid;         /* scheduling class id */
 578         union {
 579                 struct {
 580                         char    cursig; /* deferred signal number */
 581                         char    pleasestop; /* lwp requested to stop itself */
 582                 } s;
 583                 short   curplease;      /* for testing both at once */
 584         } ul_cp;
 585         char            ul_stop;        /* reason for stopping */
 586         char            ul_signalled;   /* this lwp was cond_signal()d */
 587         char            ul_dead;        /* this lwp has called thr_exit */
 588         char            ul_unwind;      /* posix: unwind C++ stack */
 589         char            ul_detached;    /* THR_DETACHED at thread_create() */
 590                                         /* or pthread_detach() was called */
 591         char            ul_writer;      /* sleeping in rw_wrlock() */
 592         char            ul_stopping;    /* set by curthread: stopping self */
 593         char            ul_cancel_prologue;     /* for _cancel_prologue() */
 594         short           ul_preempt;     /* no_preempt()/preempt() */
 595         short           ul_savpreempt;  /* pre-existing preempt value */
 596         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
 597         char            ul_main;        /* thread is the main thread */
 598         char            ul_fork;        /* thread is performing a fork */
 599         char            ul_primarymap;  /* primary link-map is initialized */
 600         /* per-thread copies of the corresponding global variables */
 601         uint8_t         ul_max_spinners;        /* thread_max_spinners */
 602         char            ul_door_noreserve;      /* thread_door_noreserve */
 603         char            ul_queue_fifo;          /* thread_queue_fifo */
 604         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
 605         char            ul_error_detection;     /* thread_error_detection */
 606         char            ul_async_safe;          /* thread_async_safe */
 607         char            ul_rt;                  /* found on an RT queue */
 608         char            ul_rtqueued;            /* was RT when queued */
 609         char            ul_misaligned;          /* thread_locks_misaligned */
 610         char            ul_pad[3];
 611         int             ul_adaptive_spin;       /* thread_adaptive_spin */
 612         int             ul_queue_spin;          /* thread_queue_spin */
 613         volatile int    ul_critical;    /* non-zero == in a critical region */
 614         int             ul_sigdefer;    /* non-zero == defer signals */
 615         int             ul_vfork;       /* thread is the child of vfork() */
 616         int             ul_cancelable;  /* _cancelon()/_canceloff() */
 617         char            ul_cancel_pending;  /* pthread_cancel() was called */
 618         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
 619         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
 620         char            ul_save_async;  /* saved copy of ul_cancel_async */
 621         char            ul_mutator;     /* lwp is a mutator (java interface) */
 622         char            ul_created;     /* created suspended */
 623         char            ul_replace;     /* replacement; must be free()d */
 624         uchar_t         ul_nocancel;    /* cancellation can't happen */
 625         int             ul_errno;       /* per-thread errno */
 626         int             *ul_errnop;     /* pointer to errno or self->ul_errno */
 627         __cleanup_t     *ul_clnup_hdr;  /* head of cleanup handlers list */
 628         uberflags_t     *ul_schedctl_called;    /* ul_schedctl is set up */
 629         volatile sc_shared_t *ul_schedctl;      /* schedctl data */
 630         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
 631         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
 632         tsd_t           *ul_stsd;       /* slow TLS for keys >= TSD_NFAST */
 633         void            *ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
 634         td_evbuf_t      ul_td_evbuf;    /* event buffer */
 635         char            ul_td_events_enable;    /* event mechanism enabled */
 636         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
 637         char            ul_qtype;       /* MX or CV */
 638         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
 639         int             ul_rtld;        /* thread is running inside ld.so.1 */
 640         int             ul_usropts;     /* flags given to thr_create() */
 641         void            *(*ul_startpc)(void *); /* start func (thr_create()) */
 642         void            *ul_startarg;   /* argument for start function */
 643         void            *ul_wchan;      /* synch object when sleeping */
 644         struct ulwp     *ul_link;       /* sleep queue link */
 645         queue_head_t    *ul_sleepq;     /* sleep queue thread is waiting on */
 646         mutex_t         *ul_cvmutex;    /* mutex dropped when waiting on a cv */
 647         mxchain_t       *ul_mxchain;    /* chain of owned ceiling mutexes */
 648         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
 649         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
 650                                 /* 0 means there is but a single entry */
 651         union {                         /* single entry or pointer to array */
 652                 readlock_t      single;
 653                 readlock_t      *array;
 654         } ul_readlock;
 655         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
 656                                 /* 0 means there is but a single entry */
 657         union {                         /* single entry or pointer to array */
 658                 mutex_t         *single;
 659                 mutex_t         **array;
 660         } ul_heldlocks;
 661         /* PROBE_SUPPORT begin */
 662         void            *ul_tpdp;
 663         /* PROBE_SUPPORT end */
 664         ucontext_t      *ul_siglink;    /* pointer to previous context */
 665         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
 666         uint_t          ul_spin_lock_spin2;
 667         uint_t          ul_spin_lock_sleep;
 668         uint_t          ul_spin_lock_wakeup;
 669         queue_root_t    ul_queue_root;  /* root of a sleep queue */
 670         id_t            ul_rtclassid;   /* real-time class id */
 671         uint_t          ul_pilocks;     /* count of PI locks held */
 672                 /* the following members *must* be last in the structure */
 673                 /* they are discarded when ulwp is replaced on thr_exit() */
 674         sigset_t        ul_sigmask;     /* thread's current signal mask */
 675         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
 676         siginfo_t       ul_siginfo;     /* deferred siginfo */
 677         mutex_t         ul_spinlock;    /* used when suspending/continuing */
 678         fpuenv_t        ul_fpuenv;      /* floating point state */
 679         uintptr_t       ul_sp;          /* stack pointer when blocked */
 680         void            *ul_ex_unwind;  /* address of _ex_unwind() or -1 */
 681 #if defined(sparc)
 682         void            *ul_unwind_ret; /* used only by _ex_clnup_handler() */
 683 #endif
 684         tumem_t         ul_tmem;        /* used only by umem */
 685         uint_t          ul_ptinherit;   /* pthreads sched inherit value */
 686         char            ul_ntoabuf[18]; /* thread-specific inet_ntoa buffer */
 687 } ulwp_t;
 688 
 689 #define ul_cursig       ul_cp.s.cursig          /* deferred signal number */
 690 #define ul_pleasestop   ul_cp.s.pleasestop      /* lwp requested to stop */
 691 #define ul_curplease    ul_cp.curplease         /* for testing both at once */
 692 
 693 /*
 694  * This is the size of a replacement ulwp, retained only for the benefit
 695  * of thr_join().  The trailing members are unneeded for this purpose.
 696  */
 697 #define REPLACEMENT_SIZE        ((size_t)&((ulwp_t *)NULL)->ul_sigmask)
 698 
 699 /*
 700  * Definitions for static initialization of signal sets,
 701  * plus some sneaky optimizations in various places.
 702  */
 703 
 704 #define SIGMASK(sig)    ((uint32_t)1 << (((sig) - 1) & (32 - 1)))
 705 
 706 #if (MAXSIG > (2 * 32) && MAXSIG <= (3 * 32))
 707 #define FILLSET0        0xffffffffu
 708 #define FILLSET1        0xffffffffu
 709 #define FILLSET2        ((1u << (MAXSIG - 64)) - 1)
 710 #define FILLSET3        0
 711 #else
 712 #error "fix me: MAXSIG out of bounds"
 713 #endif
 714 
 715 #define CANTMASK0       (SIGMASK(SIGKILL) | SIGMASK(SIGSTOP))
 716 #define CANTMASK1       0
 717 #define CANTMASK2       0
 718 #define CANTMASK3       0
 719 
 720 #define MASKSET0        (FILLSET0 & ~CANTMASK0)
 721 #define MASKSET1        (FILLSET1 & ~CANTMASK1)
 722 #define MASKSET2        (FILLSET2 & ~CANTMASK2)
 723 #define MASKSET3        (FILLSET3 & ~CANTMASK3)
 724 
 725 extern  const sigset_t maskset;         /* set of all maskable signals */
 726 
 727 extern  int     thread_adaptive_spin;
 728 extern  uint_t  thread_max_spinners;
 729 extern  int     thread_queue_spin;
 730 extern  int     thread_queue_fifo;
 731 extern  int     thread_queue_dump;
 732 extern  int     thread_cond_wait_defer;
 733 extern  int     thread_async_safe;
 734 extern  int     thread_queue_verify;
 735 
 736 /*
 737  * pthread_atfork() related data, used to store atfork handlers.
 738  */
 739 typedef struct atfork {
 740         struct atfork *forw;            /* forward pointer */
 741         struct atfork *back;            /* backward pointer */
 742         void (*prepare)(void);          /* pre-fork handler */
 743         void (*parent)(void);           /* post-fork parent handler */
 744         void (*child)(void);            /* post-fork child handler */
 745 } atfork_t;
 746 
 747 /*
 748  * Element in the table and in the list of registered process
 749  * robust locks.  We keep track of these to make sure that we
 750  * only call ___lwp_mutex_register() once for each such lock
 751  * after it is first mapped in (or newly mapped in).
 752  */
 753 typedef struct robust {
 754         struct robust   *robust_next;   /* hash table list */
 755         struct robust   *robust_list;   /* global list */
 756         mutex_t         *robust_lock;
 757 } robust_t;
 758 
 759 /*
 760  * Invalid address, used to mark an unused element in the hash table.
 761  */
 762 #define INVALID_ADDR    ((void *)(uintptr_t)(-1L))
 763 
 764 /*
 765  * Parameters of the lock registration hash table.
 766  */
 767 #define LOCKSHIFT       15                      /* number of hashing bits */
 768 #define LOCKHASHSZ      (1 << LOCKSHIFT)  /* power of 2 (1<<15 == 32K) */
 769 #define LOCK_HASH(addr) (uint_t)                        \
 770         ((((uintptr_t)(addr) >> 3)                        \
 771         ^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3))) \
 772         & (LOCKHASHSZ - 1))
 773 
 774 /*
 775  * Make our hot locks reside on private cache lines (64 bytes).
 776  */
 777 typedef struct {
 778         mutex_t pad_lock;
 779         char    pad_pad[64 - sizeof (mutex_t)];
 780 } pad_lock_t;
 781 
 782 /*
 783  * Make our semi-hot locks reside on semi-private cache lines (32 bytes).
 784  */
 785 typedef struct {
 786         mutex_t pad_lock;
 787         char    pad_pad[32 - sizeof (mutex_t)];
 788 } pad32_lock_t;
 789 
 790 /*
 791  * The threads hash table is used for fast lookup and locking of an active
 792  * thread structure (ulwp_t) given a thread-id.  It is an N-element array of
 793  * thr_hash_table_t structures, where N == 1 before the main thread creates
 794  * the first additional thread and N == 1024 afterwards.  Each element of the
 795  * table is 64 bytes in size and alignment to reduce cache conflicts.
 796  */
 797 typedef struct {
 798         mutex_t hash_lock;      /* lock per bucket */
 799         cond_t  hash_cond;      /* convar per bucket */
 800         ulwp_t  *hash_bucket;   /* hash bucket points to the list of ulwps */
 801         char    hash_pad[64 -   /* pad out to 64 bytes */
 802                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (ulwp_t *))];
 803 } thr_hash_table_t;
 804 
 805 #ifdef _SYSCALL32
 806 typedef struct {
 807         mutex_t hash_lock;
 808         cond_t  hash_cond;
 809         caddr32_t hash_bucket;
 810         char    hash_pad[64 -
 811                 (sizeof (mutex_t) + sizeof (cond_t) + sizeof (caddr32_t))];
 812 } thr_hash_table32_t;
 813 #endif  /* _SYSCALL32 */
 814 
 815 
 816 /*
 817  * siguaction members have 128-byte size and 64-byte alignment.
 818  * We know that sizeof (struct sigaction) is 32 bytes for both
 819  * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes.
 820  */
 821 typedef struct {
 822         rwlock_t        sig_lock;
 823         struct sigaction sig_uaction;
 824         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)];
 825 } siguaction_t;
 826 
 827 #ifdef _SYSCALL32
 828 typedef struct {
 829         rwlock_t        sig_lock;
 830         struct sigaction32 sig_uaction;
 831         char    sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)];
 832 } siguaction32_t;
 833 #endif  /* _SYSCALL32 */
 834 
 835 
 836 /*
 837  * Bucket structures, used by lmalloc()/lfree().
 838  * See port/threads/alloc.c for details.
 839  * A bucket's size and alignment is 64 bytes.
 840  */
 841 typedef struct {
 842         mutex_t bucket_lock;    /* protects the free list allocations */
 843         void    *free_list;     /* LIFO list of blocks to allocate/free */
 844         size_t  chunks;         /* number of 64K blocks mmap()ed last time */
 845         char    pad64[64 -      /* pad out to 64 bytes */
 846                 (sizeof (mutex_t) + sizeof (void *) + sizeof (size_t))];
 847 } bucket_t;
 848 
 849 #ifdef _SYSCALL32
 850 typedef struct {
 851         mutex_t         bucket_lock;
 852         caddr32_t       free_list;
 853         size32_t        chunks;
 854         char    pad64[64 -      /* pad out to 64 bytes */
 855                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (size32_t))];
 856 } bucket32_t;
 857 #endif  /* _SYSCALL32 */
 858 
 859 #define NBUCKETS        10      /* sizes ranging from 64 to 32768 */
 860 
 861 
 862 /*
 863  * atexit() data structures.
 864  * See port/gen/atexit.c for details.
 865  */
 866 typedef void (*_exithdlr_func_t) (void*);
 867 
 868 typedef struct _exthdlr {
 869         struct _exthdlr         *next;  /* next in handler list */
 870         _exithdlr_func_t        hdlr;   /* handler itself */
 871         void                    *arg;   /* argument to handler */
 872         void                    *dso;   /* DSO associated with handler */
 873 } _exthdlr_t;
 874 
 875 typedef struct {
 876         mutex_t         exitfns_lock;
 877         _exthdlr_t      *head;
 878         /*
 879          * exit_frame_monitor is part of a private contract between libc and
 880          * the Sun C++ runtime.
 881          *
 882          * It should be NULL until exit() is called, and thereafter hold the
 883          * frame pointer of the function implementing our exit processing.
 884          */
 885         void            *exit_frame_monitor;
 886         char            exit_pad[64 -   /* pad out to 64 bytes */
 887                 (sizeof (mutex_t) + sizeof (_exthdlr_t *) + sizeof (void *))];
 888 } atexit_root_t;
 889 
 890 #ifdef _SYSCALL32
 891 typedef struct {
 892         mutex_t         exitfns_lock;
 893         caddr32_t       head;
 894         caddr32_t       exit_frame_monitor;
 895         char            exit_pad[64 -   /* pad out to 64 bytes */
 896                 (sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (caddr32_t))];
 897 } atexit_root32_t;
 898 #endif  /* _SYSCALL32 */
 899 
 900 /*
 901  * at_quick_exit() and quick_exit() data structures. The ISO/IEC C11 odd
 902  * siblings of atexit()
 903  */
 904 typedef void (*_quick_exithdlr_func_t)(void);
 905 
 906 typedef struct _qexthdlr {
 907         struct _qexthdlr        *next;  /* next in handler list */
 908         _quick_exithdlr_func_t  hdlr;   /* handler itself */
 909 } _qexthdlr_t;
 910 
 911 /*
 912  * We add a pad on 32-bit systems to allow us to always have the structure size
 913  * be 32-bytes which helps us deal with the compiler's alignment when building
 914  * in ILP32 / LP64 systems.
 915  */
 916 typedef struct {
 917         mutex_t         exitfns_lock;
 918         _qexthdlr_t     *head;
 919 #if !defined(_LP64)
 920         uint32_t        pad;
 921 #endif
 922 } quickexit_root_t;
 923 
 924 #ifdef _SYSCALL32
 925 typedef struct {
 926         mutex_t         exitfns_lock;
 927         caddr32_t       head;
 928         uint32_t        pad;
 929 } quickexit_root32_t;
 930 #endif /* _SYSCALL32 */
 931 
 932 /*
 933  * This is data that is global to all link maps (uberdata, aka super-global).
 934  * Note: When changing this, please be sure to keep the 32-bit variant of
 935  * this in sync.  (see uberdata32_t below)
 936  */
 937 typedef struct uberdata {
 938         pad_lock_t      _link_lock;
 939         pad_lock_t      _ld_lock;
 940         pad_lock_t      _fork_lock;
 941         pad_lock_t      _atfork_lock;
 942         pad32_lock_t    _callout_lock;
 943         pad32_lock_t    _tdb_hash_lock;
 944         tdb_sync_stats_t tdb_hash_lock_stats;
 945         siguaction_t    siguaction[NSIG];
 946         bucket_t        bucket[NBUCKETS];
 947         atexit_root_t   atexit_root;
 948         quickexit_root_t quickexit_root;
 949         tsd_metadata_t  tsd_metadata;
 950         tls_metadata_t  tls_metadata;
 951         /*
 952          * Every object before this point has size and alignment of 64 bytes.
 953          * Don't add any other type of data before this point.
 954          */
 955         char    primary_map;    /* set when primary link map is initialized */
 956         char    bucket_init;    /* set when bucket[NBUCKETS] is initialized */
 957         char    pad[2];
 958         uberflags_t     uberflags;
 959         queue_head_t    *queue_head;
 960         thr_hash_table_t *thr_hash_table;
 961         uint_t          hash_size;      /* # of entries in thr_hash_table[] */
 962         uint_t          hash_mask;      /* hash_size - 1 */
 963         ulwp_t  *ulwp_one;      /* main thread */
 964         ulwp_t  *all_lwps;      /* circular ul_forw/ul_back list of live lwps */
 965         ulwp_t  *all_zombies;   /* circular ul_forw/ul_back list of zombies */
 966         int     nthreads;       /* total number of live threads/lwps */
 967         int     nzombies;       /* total number of zombie threads */
 968         int     ndaemons;       /* total number of THR_DAEMON threads/lwps */
 969         pid_t   pid;            /* the current process's pid */
 970         void    (*sigacthandler)(int, siginfo_t *, void *);
 971         ulwp_t  *lwp_stacks;
 972         ulwp_t  *lwp_laststack;
 973         int     nfreestack;
 974         int     thread_stack_cache;
 975         ulwp_t  *ulwp_freelist;
 976         ulwp_t  *ulwp_lastfree;
 977         ulwp_t  *ulwp_replace_free;
 978         ulwp_t  *ulwp_replace_last;
 979         atfork_t        *atforklist;    /* circular Q for fork handlers */
 980         robust_t        **robustlocks;  /* table of registered robust locks */
 981         robust_t        *robustlist;    /* list of registered robust locks */
 982         char    *progname;      /* the basename of the program, from argv[0] */
 983         void    *ub_comm_page;  /* arch-specific comm page of kernel data */
 984         struct uberdata **tdb_bootstrap;
 985         tdb_t   tdb;            /* thread debug interfaces (for libc_db) */
 986 } uberdata_t;
 987 
 988 #define link_lock       _link_lock.pad_lock
 989 #define ld_lock         _ld_lock.pad_lock
 990 #define fork_lock       _fork_lock.pad_lock
 991 #define atfork_lock     _atfork_lock.pad_lock
 992 #define callout_lock    _callout_lock.pad_lock
 993 #define tdb_hash_lock   _tdb_hash_lock.pad_lock
 994 
 995 #pragma align 64(__uberdata)
 996 extern  uberdata_t      __uberdata;
 997 extern  uberdata_t      **__tdb_bootstrap;      /* known to libc_db and mdb */
 998 extern  int             primary_link_map;
 999 
1000 #define ulwp_mutex(ulwp, udp)   \
1001         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_lock)
1002 #define ulwp_condvar(ulwp, udp) \
1003         (&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_cond)
1004 
1005 /*
1006  * Grab and release the hash table lock for the specified lwp.
1007  */
1008 #define ulwp_lock(ulwp, udp)    lmutex_lock(ulwp_mutex(ulwp, udp))
1009 #define ulwp_unlock(ulwp, udp)  lmutex_unlock(ulwp_mutex(ulwp, udp))
1010 
1011 #ifdef _SYSCALL32       /* needed by libc_db */
1012 
1013 typedef struct ulwp32 {
1014 #if defined(__sparc)
1015         uint32_t        ul_dinstr;      /* scratch space for dtrace */
1016         uint32_t        ul_padsparc0[15];
1017         uint32_t        ul_dsave;       /* dtrace: save %g1, %g0, %sp */
1018         uint32_t        ul_drestore;    /* dtrace: restore %g0, %g0, %g0 */
1019         uint32_t        ul_dftret;      /* dtrace: return probe fasttrap */
1020         uint32_t        ul_dreturn;     /* dtrace: return %o0 */
1021 #endif
1022         caddr32_t       ul_self;        /* pointer to self */
1023 #if defined(__x86)
1024         uint8_t         ul_dinstr[40];  /* scratch space for dtrace */
1025 #endif
1026         caddr32_t       ul_uberdata;    /* uber (super-global) data */
1027         tls32_t         ul_tls;         /* dynamic thread-local storage base */
1028         caddr32_t       ul_forw;        /* forw, back all_lwps list, */
1029         caddr32_t       ul_back;        /* protected by link_lock */
1030         caddr32_t       ul_next;        /* list to keep track of stacks */
1031         caddr32_t       ul_hash;        /* hash chain linked list */
1032         caddr32_t       ul_rval;        /* return value from thr_exit() */
1033         caddr32_t       ul_stk;         /* mapping base of the stack */
1034         size32_t        ul_mapsiz;      /* mapping size of the stack */
1035         size32_t        ul_guardsize;   /* normally _lpagesize */
1036         caddr32_t       ul_stktop;      /* broken thr_stksegment() interface */
1037         size32_t        ul_stksiz;      /* broken thr_stksegment() interface */
1038         stack32_t       ul_ustack;      /* current stack boundaries */
1039         int             ul_ix;          /* hash index */
1040         lwpid_t         ul_lwpid;       /* thread id, aka the lwp id */
1041         pri_t           ul_pri;         /* scheduling priority */
1042         pri_t           ul_epri;        /* real-time ceiling priority */
1043         char            ul_policy;      /* scheduling policy */
1044         char            ul_cid;         /* scheduling class id */
1045         union {
1046                 struct {
1047                         char    cursig; /* deferred signal number */
1048                         char    pleasestop; /* lwp requested to stop itself */
1049                 } s;
1050                 short   curplease;      /* for testing both at once */
1051         } ul_cp;
1052         char            ul_stop;        /* reason for stopping */
1053         char            ul_signalled;   /* this lwp was cond_signal()d */
1054         char            ul_dead;        /* this lwp has called thr_exit */
1055         char            ul_unwind;      /* posix: unwind C++ stack */
1056         char            ul_detached;    /* THR_DETACHED at thread_create() */
1057                                         /* or pthread_detach() was called */
1058         char            ul_writer;      /* sleeping in rw_wrlock() */
1059         char            ul_stopping;    /* set by curthread: stopping self */
1060         char            ul_cancel_prologue;     /* for _cancel_prologue() */
1061         short           ul_preempt;     /* no_preempt()/preempt() */
1062         short           ul_savpreempt;  /* pre-existing preempt value */
1063         char            ul_sigsuspend;  /* thread is in sigsuspend/pollsys */
1064         char            ul_main;        /* thread is the main thread */
1065         char            ul_fork;        /* thread is performing a fork */
1066         char            ul_primarymap;  /* primary link-map is initialized */
1067         /* per-thread copies of the corresponding global variables */
1068         uint8_t         ul_max_spinners;        /* thread_max_spinners */
1069         char            ul_door_noreserve;      /* thread_door_noreserve */
1070         char            ul_queue_fifo;          /* thread_queue_fifo */
1071         char            ul_cond_wait_defer;     /* thread_cond_wait_defer */
1072         char            ul_error_detection;     /* thread_error_detection */
1073         char            ul_async_safe;          /* thread_async_safe */
1074         char            ul_rt;                  /* found on an RT queue */
1075         char            ul_rtqueued;            /* was RT when queued */
1076         char            ul_misaligned;          /* thread_locks_misaligned */
1077         char            ul_pad[3];
1078         int             ul_adaptive_spin;       /* thread_adaptive_spin */
1079         int             ul_queue_spin;          /* thread_queue_spin */
1080         int             ul_critical;    /* non-zero == in a critical region */
1081         int             ul_sigdefer;    /* non-zero == defer signals */
1082         int             ul_vfork;       /* thread is the child of vfork() */
1083         int             ul_cancelable;  /* _cancelon()/_canceloff() */
1084         char            ul_cancel_pending;  /* pthread_cancel() was called */
1085         char            ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
1086         char            ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
1087         char            ul_save_async;  /* saved copy of ul_cancel_async */
1088         char            ul_mutator;     /* lwp is a mutator (java interface) */
1089         char            ul_created;     /* created suspended */
1090         char            ul_replace;     /* replacement; must be free()d */
1091         uchar_t         ul_nocancel;    /* cancellation can't happen */
1092         int             ul_errno;       /* per-thread errno */
1093         caddr32_t       ul_errnop;      /* pointer to errno or self->ul_errno */
1094         caddr32_t       ul_clnup_hdr;   /* head of cleanup handlers list */
1095         caddr32_t       ul_schedctl_called; /* ul_schedctl is set up */
1096         caddr32_t       ul_schedctl;    /* schedctl data */
1097         int             ul_bindflags;   /* bind_guard() interface to ld.so.1 */
1098         uint_t          ul_libc_locks;  /* count of cancel_safe_mutex_lock()s */
1099         caddr32_t       ul_stsd;        /* slow TLS for keys >= TSD_NFAST */
1100         caddr32_t       ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
1101         td_evbuf32_t    ul_td_evbuf;    /* event buffer */
1102         char            ul_td_events_enable;    /* event mechanism enabled */
1103         char            ul_sync_obj_reg;        /* tdb_sync_obj_register() */
1104         char            ul_qtype;       /* MX or CV */
1105         char            ul_cv_wake;     /* != 0: just wake up, don't requeue */
1106         int             ul_rtld;        /* thread is running inside ld.so.1 */
1107         int             ul_usropts;     /* flags given to thr_create() */
1108         caddr32_t       ul_startpc;     /* start func (thr_create()) */
1109         caddr32_t       ul_startarg;    /* argument for start function */
1110         caddr32_t       ul_wchan;       /* synch object when sleeping */
1111         caddr32_t       ul_link;        /* sleep queue link */
1112         caddr32_t       ul_sleepq;      /* sleep queue thread is waiting on */
1113         caddr32_t       ul_cvmutex;     /* mutex dropped when waiting on a cv */
1114         caddr32_t       ul_mxchain;     /* chain of owned ceiling mutexes */
1115         int             ul_save_state;  /* bind_guard() interface to ld.so.1 */
1116         uint_t          ul_rdlockcnt;   /* # entries in ul_readlock array */
1117                                 /* 0 means there is but a single entry */
1118         union {                         /* single entry or pointer to array */
1119                 readlock32_t    single;
1120                 caddr32_t       array;
1121         } ul_readlock;
1122         uint_t          ul_heldlockcnt; /* # entries in ul_heldlocks array */
1123                                 /* 0 means there is but a single entry */
1124         union {                         /* single entry or pointer to array */
1125                 caddr32_t       single;
1126                 caddr32_t       array;
1127         } ul_heldlocks;
1128         /* PROBE_SUPPORT begin */
1129         caddr32_t       ul_tpdp;
1130         /* PROBE_SUPPORT end */
1131         caddr32_t       ul_siglink;     /* pointer to previous context */
1132         uint_t          ul_spin_lock_spin;      /* spin lock statistics */
1133         uint_t          ul_spin_lock_spin2;
1134         uint_t          ul_spin_lock_sleep;
1135         uint_t          ul_spin_lock_wakeup;
1136         queue_root32_t  ul_queue_root;  /* root of a sleep queue */
1137         id_t            ul_rtclassid;   /* real-time class id */
1138         uint_t          ul_pilocks;     /* count of PI locks held */
1139                 /* the following members *must* be last in the structure */
1140                 /* they are discarded when ulwp is replaced on thr_exit() */
1141         sigset_t        ul_sigmask;     /* thread's current signal mask */
1142         sigset_t        ul_tmpmask;     /* signal mask for sigsuspend/pollsys */
1143         siginfo32_t     ul_siginfo;     /* deferred siginfo */
1144         mutex_t         ul_spinlock;    /* used when suspending/continuing */
1145         fpuenv32_t      ul_fpuenv;      /* floating point state */
1146         caddr32_t       ul_sp;          /* stack pointer when blocked */
1147 #if defined(sparc)
1148         caddr32_t       ul_unwind_ret;  /* used only by _ex_clnup_handler() */
1149 #endif
1150         tumem32_t       ul_tmem;        /* used only by umem */
1151 } ulwp32_t;
1152 
1153 #define REPLACEMENT_SIZE32      ((size_t)&((ulwp32_t *)NULL)->ul_sigmask)
1154 
1155 typedef struct uberdata32 {
1156         pad_lock_t      _link_lock;
1157         pad_lock_t      _ld_lock;
1158         pad_lock_t      _fork_lock;
1159         pad_lock_t      _atfork_lock;
1160         pad32_lock_t    _callout_lock;
1161         pad32_lock_t    _tdb_hash_lock;
1162         tdb_sync_stats_t tdb_hash_lock_stats;
1163         siguaction32_t  siguaction[NSIG];
1164         bucket32_t      bucket[NBUCKETS];
1165         atexit_root32_t atexit_root;
1166         quickexit_root32_t quickexit_root;
1167         tsd_metadata32_t tsd_metadata;
1168         tls_metadata32_t tls_metadata;
1169         char            primary_map;
1170         char            bucket_init;
1171         char            pad[2];
1172         uberflags_t     uberflags;
1173         caddr32_t       queue_head;
1174         caddr32_t       thr_hash_table;
1175         uint_t          hash_size;
1176         uint_t          hash_mask;
1177         caddr32_t       ulwp_one;
1178         caddr32_t       all_lwps;
1179         caddr32_t       all_zombies;
1180         int             nthreads;
1181         int             nzombies;
1182         int             ndaemons;
1183         int             pid;
1184         caddr32_t       sigacthandler;
1185         caddr32_t       lwp_stacks;
1186         caddr32_t       lwp_laststack;
1187         int             nfreestack;
1188         int             thread_stack_cache;
1189         caddr32_t       ulwp_freelist;
1190         caddr32_t       ulwp_lastfree;
1191         caddr32_t       ulwp_replace_free;
1192         caddr32_t       ulwp_replace_last;
1193         caddr32_t       atforklist;
1194         caddr32_t       robustlocks;
1195         caddr32_t       robustlist;
1196         caddr32_t       progname;
1197         caddr32_t       ub_comm_page;
1198         caddr32_t       tdb_bootstrap;
1199         tdb32_t         tdb;
1200 } uberdata32_t;
1201 
1202 #endif  /* _SYSCALL32 */
1203 
1204 /* ul_stop values */
1205 #define TSTP_REGULAR    0x01    /* Stopped by thr_suspend() */
1206 #define TSTP_MUTATOR    0x08    /* stopped by thr_suspend_*mutator*() */
1207 #define TSTP_FORK       0x20    /* stopped by suspend_fork() */
1208 
1209 /*
1210  * Implementation-specific attribute types for pthread_mutexattr_init() etc.
1211  */
1212 
1213 typedef struct  _cvattr {
1214         int     pshared;
1215         clockid_t clockid;
1216 } cvattr_t;
1217 
1218 typedef struct  _mattr {
1219         int     pshared;
1220         int     protocol;
1221         int     prioceiling;
1222         int     type;
1223         int     robustness;
1224 } mattr_t;
1225 
1226 typedef struct  _thrattr {
1227         size_t  stksize;
1228         void    *stkaddr;
1229         int     detachstate;
1230         int     daemonstate;
1231         int     scope;
1232         int     prio;
1233         int     policy;
1234         int     inherit;
1235         size_t  guardsize;
1236 } thrattr_t;
1237 
1238 typedef struct  _rwlattr {
1239         int     pshared;
1240 } rwlattr_t;
1241 
1242 /* _curthread() is inline for speed */
1243 extern  ulwp_t          *_curthread(void);
1244 #define curthread       (_curthread())
1245 
1246 /* this version (also inline) can be tested for NULL */
1247 extern  ulwp_t          *__curthread(void);
1248 
1249 /* get the current stack pointer (also inline) */
1250 extern  greg_t          stkptr(void);
1251 
1252 /*
1253  * Suppress __attribute__((...)) if we are not compiling with gcc
1254  */
1255 #if !defined(__GNUC__)
1256 #define __attribute__(string)
1257 #endif
1258 
1259 /* Fetch the dispatch (kernel) priority of a thread */
1260 #define real_priority(ulwp)     \
1261         ((ulwp)->ul_schedctl? (ulwp)->ul_schedctl->sc_priority : 0)
1262 
1263 /*
1264  * Implementation functions.  Not visible outside of the library itself.
1265  */
1266 extern  int     __nanosleep(const timespec_t *, timespec_t *);
1267 extern  void    getgregs(ulwp_t *, gregset_t);
1268 extern  void    setgregs(ulwp_t *, gregset_t);
1269 extern  void    thr_panic(const char *);
1270 #pragma rarely_called(thr_panic)
1271 extern  void    mutex_panic(mutex_t *, const char *);
1272 #pragma rarely_called(mutex_panic)
1273 extern  ulwp_t  *find_lwp(thread_t);
1274 extern  void    finish_init(void);
1275 extern  void    update_sched(ulwp_t *);
1276 extern  void    queue_alloc(void);
1277 extern  void    tmem_exit(void);
1278 extern  void    tsd_exit(void);
1279 extern  void    tsd_free(ulwp_t *);
1280 extern  void    tls_setup(void);
1281 extern  void    tls_exit(void);
1282 extern  void    tls_free(ulwp_t *);
1283 extern  void    rwl_free(ulwp_t *);
1284 extern  void    heldlock_exit(void);
1285 extern  void    heldlock_free(ulwp_t *);
1286 extern  void    sigacthandler(int, siginfo_t *, void *);
1287 extern  void    signal_init(void);
1288 extern  int     sigequalset(const sigset_t *, const sigset_t *);
1289 extern  void    mutex_setup(void);
1290 extern  void    take_deferred_signal(int);
1291 extern  void    *setup_top_frame(void *, size_t, ulwp_t *);
1292 extern  int     setup_context(ucontext_t *, void *(*func)(ulwp_t *),
1293                         ulwp_t *ulwp, caddr_t stk, size_t stksize);
1294 extern  volatile sc_shared_t *setup_schedctl(void);
1295 extern  void    *lmalloc(size_t);
1296 extern  void    lfree(void *, size_t);
1297 extern  void    *libc_malloc(size_t);
1298 extern  void    *libc_realloc(void *, size_t);
1299 extern  void    libc_free(void *);
1300 extern  char    *libc_strdup(const char *);
1301 extern  void    ultos(uint64_t, int, char *);
1302 extern  void    lock_error(const mutex_t *, const char *, void *, const char *);
1303 extern  void    rwlock_error(const rwlock_t *, const char *, const char *);
1304 extern  void    thread_error(const char *);
1305 extern  void    grab_assert_lock(void);
1306 extern  void    dump_queue_statistics(void);
1307 extern  void    collect_queue_statistics(void);
1308 extern  void    record_spin_locks(ulwp_t *);
1309 extern  void    remember_lock(mutex_t *);
1310 extern  void    forget_lock(mutex_t *);
1311 extern  void    register_lock(mutex_t *);
1312 extern  void    unregister_locks(void);
1313 #if defined(__sparc)
1314 extern  void    _flush_windows(void);
1315 #else
1316 #define _flush_windows()
1317 #endif
1318 extern  void    set_curthread(void *);
1319 
1320 /*
1321  * Utility function used when waking up many threads (more than MAXLWPS)
1322  * all at once.  See mutex_wakeup_all(), cond_broadcast(), and rw_unlock().
1323  */
1324 #define MAXLWPS 128     /* max remembered lwpids before overflow */
1325 #define NEWLWPS 2048    /* max remembered lwpids at first overflow */
1326 extern  lwpid_t *alloc_lwpids(lwpid_t *, int *, int *);
1327 
1328 /* enter a critical section */
1329 #define enter_critical(self)    (self->ul_critical++)
1330 
1331 /* exit a critical section, take deferred actions if necessary */
1332 extern  void    do_exit_critical(void);
1333 #define exit_critical(self)                                     \
1334         (void) (self->ul_critical--,                         \
1335             ((self->ul_curplease && self->ul_critical == 0)?      \
1336             (do_exit_critical(), 0) : 0))
1337 
1338 /*
1339  * Like enter_critical()/exit_critical() but just for deferring signals.
1340  * Unlike enter_critical()/exit_critical(), ul_sigdefer may be set while
1341  * calling application functions like constructors and destructors.
1342  * Care must be taken if the application function attempts to set
1343  * the signal mask while a deferred signal is present; the setting
1344  * of the signal mask must also be deferred.
1345  */
1346 #define sigoff(self)    (self->ul_sigdefer++)
1347 #define sigon(self)                                             \
1348         (void) ((--self->ul_sigdefer == 0 &&                 \
1349             self->ul_curplease && self->ul_critical == 0)?        \
1350             (do_exit_critical(), 0) : 0)
1351 
1352 /* these are exported functions */
1353 extern  void    _sigoff(void);
1354 extern  void    _sigon(void);
1355 
1356 #define sigorset(s1, s2)                                \
1357         (((s1)->__sigbits[0] |= (s2)->__sigbits[0]),      \
1358         ((s1)->__sigbits[1] |= (s2)->__sigbits[1]),       \
1359         ((s1)->__sigbits[2] |= (s2)->__sigbits[2]),       \
1360         ((s1)->__sigbits[3] |= (s2)->__sigbits[3]))
1361 
1362 #define sigandset(s1, s2)                               \
1363         (((s1)->__sigbits[0] &= (s2)->__sigbits[0]),  \
1364         ((s1)->__sigbits[1] &= (s2)->__sigbits[1]),   \
1365         ((s1)->__sigbits[2] &= (s2)->__sigbits[2]),   \
1366         ((s1)->__sigbits[3] &= (s2)->__sigbits[3]))
1367 
1368 #define sigdiffset(s1, s2)                              \
1369         (((s1)->__sigbits[0] &= ~(s2)->__sigbits[0]), \
1370         ((s1)->__sigbits[1] &= ~(s2)->__sigbits[1]),  \
1371         ((s1)->__sigbits[2] &= ~(s2)->__sigbits[2]),  \
1372         ((s1)->__sigbits[3] &= ~(s2)->__sigbits[3]))
1373 
1374 #define delete_reserved_signals(s)                      \
1375         (((s)->__sigbits[0] &= MASKSET0),                \
1376         ((s)->__sigbits[1] &= (MASKSET1 & ~SIGMASK(SIGCANCEL))),\
1377         ((s)->__sigbits[2] &= MASKSET2),         \
1378         ((s)->__sigbits[3] &= MASKSET3))
1379 
1380 extern  void    block_all_signals(ulwp_t *self);
1381 
1382 /*
1383  * When restoring the signal mask after having previously called
1384  * block_all_signals(), if we have a deferred signal present then
1385  * do nothing other than ASSERT() that we are in a critical region.
1386  * The signal mask will be set when we emerge from the critical region
1387  * and call take_deferred_signal().  There is no race condition here
1388  * because the kernel currently has all signals blocked for this thread.
1389  */
1390 #define restore_signals(self)                                           \
1391         ((void) ((self)->ul_cursig?                                  \
1392         (ASSERT((self)->ul_critical + (self)->ul_sigdefer != 0), 0) :     \
1393         __lwp_sigmask(SIG_SETMASK, &(self)->ul_sigmask)))
1394 
1395 extern  void    set_cancel_pending_flag(ulwp_t *, int);
1396 extern  void    set_cancel_eintr_flag(ulwp_t *);
1397 extern  void    set_parking_flag(ulwp_t *, int);
1398 extern  int     cancel_active(void);
1399 
1400 extern  void    *_thrp_setup(ulwp_t *);
1401 extern  void    _fpinherit(ulwp_t *);
1402 extern  void    _lwp_start(void);
1403 extern  void    _lwp_terminate(void);
1404 extern  void    lmutex_lock(mutex_t *);
1405 extern  void    lmutex_unlock(mutex_t *);
1406 extern  void    lrw_rdlock(rwlock_t *);
1407 extern  void    lrw_wrlock(rwlock_t *);
1408 extern  void    lrw_unlock(rwlock_t *);
1409 extern  void    sig_mutex_lock(mutex_t *);
1410 extern  void    sig_mutex_unlock(mutex_t *);
1411 extern  int     sig_mutex_trylock(mutex_t *);
1412 extern  int     sig_cond_wait(cond_t *, mutex_t *);
1413 extern  int     sig_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1414 extern  void    cancel_safe_mutex_lock(mutex_t *);
1415 extern  void    cancel_safe_mutex_unlock(mutex_t *);
1416 extern  int     cancel_safe_mutex_trylock(mutex_t *);
1417 extern  void    _prefork_handler(void);
1418 extern  void    _postfork_parent_handler(void);
1419 extern  void    _postfork_child_handler(void);
1420 extern  void    postfork1_child(void);
1421 extern  void    postfork1_child_aio(void);
1422 extern  void    postfork1_child_sigev_aio(void);
1423 extern  void    postfork1_child_sigev_mq(void);
1424 extern  void    postfork1_child_sigev_timer(void);
1425 extern  void    postfork1_child_tpool(void);
1426 extern  void    fork_lock_enter(void);
1427 extern  void    fork_lock_exit(void);
1428 extern  void    suspend_fork(void);
1429 extern  void    continue_fork(int);
1430 extern  void    do_sigcancel(void);
1431 extern  void    setup_cancelsig(int);
1432 extern  void    init_sigev_thread(void);
1433 extern  void    init_aio(void);
1434 extern  void    init_progname(void);
1435 extern  void    _cancelon(void);
1436 extern  void    _canceloff(void);
1437 extern  void    _canceloff_nocancel(void);
1438 extern  void    _cancel_prologue(void);
1439 extern  void    _cancel_epilogue(void);
1440 extern  void    no_preempt(ulwp_t *);
1441 extern  void    preempt(ulwp_t *);
1442 extern  void    _thrp_unwind(void *);
1443 
1444 extern  pid_t   __forkx(int);
1445 extern  pid_t   __forkallx(int);
1446 extern  int     __open(const char *, int, mode_t);
1447 extern  int     __open64(const char *, int, mode_t);
1448 extern  int     __openat(int, const char *, int, mode_t);
1449 extern  int     __openat64(int, const char *, int, mode_t);
1450 extern  int     __close(int);
1451 extern  ssize_t __read(int, void *, size_t);
1452 extern  ssize_t __write(int, const void *, size_t);
1453 extern  int     __fcntl(int, int, ...);
1454 extern  int     __lwp_continue(lwpid_t);
1455 extern  int     __lwp_create(ucontext_t *, uint_t, lwpid_t *);
1456 extern  int     ___lwp_suspend(lwpid_t);
1457 extern  int     lwp_wait(lwpid_t, lwpid_t *);
1458 extern  int     __lwp_wait(lwpid_t, lwpid_t *);
1459 extern  int     __lwp_detach(lwpid_t);
1460 extern  sc_shared_t *__schedctl(void);
1461 
1462 /* actual system call traps */
1463 extern  int     __setcontext(const ucontext_t *);
1464 extern  int     __getcontext(ucontext_t *);
1465 extern  int     __clock_gettime(clockid_t, timespec_t *);
1466 extern  void    abstime_to_reltime(clockid_t, const timespec_t *, timespec_t *);
1467 extern  void    hrt2ts(hrtime_t, timespec_t *);
1468 
1469 extern  int     __sigaction(int, const struct sigaction *, struct sigaction *);
1470 extern  int     __sigprocmask(int, const sigset_t *, sigset_t *);
1471 extern  int     __lwp_sigmask(int, const sigset_t *);
1472 extern  void    __sighndlr(int, siginfo_t *, ucontext_t *, void (*)());
1473 extern  caddr_t __sighndlrend;
1474 #pragma unknown_control_flow(__sighndlr)
1475 
1476 /* belongs in <pthread.h> */
1477 #define PTHREAD_CREATE_DAEMON_NP        0x100   /* = THR_DAEMON */
1478 #define PTHREAD_CREATE_NONDAEMON_NP     0
1479 extern  int     pthread_attr_setdaemonstate_np(pthread_attr_t *, int);
1480 extern  int     pthread_attr_getdaemonstate_np(const pthread_attr_t *, int *);
1481 
1482 extern  int     mutex_held(mutex_t *);
1483 extern  int     mutex_lock_internal(mutex_t *, timespec_t *, int);
1484 extern  int     mutex_unlock_internal(mutex_t *, int);
1485 
1486 /* not cancellation points: */
1487 extern  int     __cond_wait(cond_t *, mutex_t *);
1488 extern  int     __cond_timedwait(cond_t *, mutex_t *, const timespec_t *);
1489 extern  int     __cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1490 
1491 extern  int     rw_read_held(rwlock_t *);
1492 extern  int     rw_write_held(rwlock_t *);
1493 
1494 extern  int     _thrp_create(void *, size_t, void *(*)(void *), void *, long,
1495                         thread_t *, size_t);
1496 extern  int     _thrp_suspend(thread_t, uchar_t);
1497 extern  int     _thrp_continue(thread_t, uchar_t);
1498 
1499 extern  void    _thrp_terminate(void *);
1500 extern  void    _thrp_exit(void);
1501 
1502 extern  const pcclass_t *get_info_by_class(id_t);
1503 extern  const pcclass_t *get_info_by_policy(int);
1504 extern  const thrattr_t *def_thrattr(void);
1505 extern  id_t    setparam(idtype_t, id_t, int, int);
1506 extern  id_t    setprio(idtype_t, id_t, int, int *);
1507 extern  id_t    getparam(idtype_t, id_t, int *, struct sched_param *);
1508 
1509 /*
1510  * System call wrappers (direct interfaces to the kernel)
1511  */
1512 extern  int     ___lwp_mutex_register(mutex_t *, mutex_t **);
1513 extern  int     ___lwp_mutex_trylock(mutex_t *, ulwp_t *);
1514 extern  int     ___lwp_mutex_timedlock(mutex_t *, timespec_t *, ulwp_t *);
1515 extern  int     ___lwp_mutex_unlock(mutex_t *);
1516 extern  int     ___lwp_mutex_wakeup(mutex_t *, int);
1517 extern  int     ___lwp_cond_wait(cond_t *, mutex_t *, timespec_t *, int);
1518 extern  int     ___lwp_sema_timedwait(lwp_sema_t *, timespec_t *, int);
1519 extern  int     __lwp_rwlock_rdlock(rwlock_t *, timespec_t *);
1520 extern  int     __lwp_rwlock_wrlock(rwlock_t *, timespec_t *);
1521 extern  int     __lwp_rwlock_tryrdlock(rwlock_t *);
1522 extern  int     __lwp_rwlock_trywrlock(rwlock_t *);
1523 extern  int     __lwp_rwlock_unlock(rwlock_t *);
1524 extern  int     __lwp_park(timespec_t *, lwpid_t);
1525 extern  int     __lwp_unpark(lwpid_t);
1526 extern  int     __lwp_unpark_all(lwpid_t *, int);
1527 #if defined(__x86)
1528 extern  int     ___lwp_private(int, int, void *);
1529 #endif  /* __x86 */
1530 
1531 /*
1532  * inlines
1533  */
1534 extern  int             set_lock_byte(volatile uint8_t *);
1535 extern  uint32_t        atomic_swap_32(volatile uint32_t *, uint32_t);
1536 extern  uint32_t        atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
1537 extern  void            atomic_inc_32(volatile uint32_t *);
1538 extern  void            atomic_dec_32(volatile uint32_t *);
1539 extern  void            atomic_and_32(volatile uint32_t *, uint32_t);
1540 extern  void            atomic_or_32(volatile uint32_t *, uint32_t);
1541 #if defined(__sparc)
1542 extern  ulong_t         caller(void);
1543 extern  ulong_t         getfp(void);
1544 #endif  /* __sparc */
1545 
1546 #include "thr_inlines.h"
1547 
1548 #endif  /* _THR_UBERDATA_H */