1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
  29  */
  30 
  31 #ifndef _SYS_DTRACE_IMPL_H
  32 #define _SYS_DTRACE_IMPL_H
  33 
  34 #ifdef  __cplusplus
  35 extern "C" {
  36 #endif
  37 
  38 /*
  39  * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
  40  *
  41  * Note: The contents of this file are private to the implementation of the
  42  * Solaris system and DTrace subsystem and are subject to change at any time
  43  * without notice.  Applications and drivers using these interfaces will fail
  44  * to run on future releases.  These interfaces should not be used for any
  45  * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
  46  * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
  47  */
  48 
  49 #include <sys/dtrace.h>
  50 
  51 /*
  52  * DTrace Implementation Constants and Typedefs
  53  */
  54 #define DTRACE_MAXPROPLEN               128
  55 #define DTRACE_DYNVAR_CHUNKSIZE         256
  56 
  57 struct dtrace_probe;
  58 struct dtrace_ecb;
  59 struct dtrace_predicate;
  60 struct dtrace_action;
  61 struct dtrace_provider;
  62 struct dtrace_state;
  63 
  64 typedef struct dtrace_probe dtrace_probe_t;
  65 typedef struct dtrace_ecb dtrace_ecb_t;
  66 typedef struct dtrace_predicate dtrace_predicate_t;
  67 typedef struct dtrace_action dtrace_action_t;
  68 typedef struct dtrace_provider dtrace_provider_t;
  69 typedef struct dtrace_meta dtrace_meta_t;
  70 typedef struct dtrace_state dtrace_state_t;
  71 typedef uint32_t dtrace_optid_t;
  72 typedef uint32_t dtrace_specid_t;
  73 typedef uint64_t dtrace_genid_t;
  74 
  75 /*
  76  * DTrace Probes
  77  *
  78  * The probe is the fundamental unit of the DTrace architecture.  Probes are
  79  * created by DTrace providers, and managed by the DTrace framework.  A probe
  80  * is identified by a unique <provider, module, function, name> tuple, and has
  81  * a unique probe identifier assigned to it.  (Some probes are not associated
  82  * with a specific point in text; these are called _unanchored probes_ and have
  83  * no module or function associated with them.)  Probes are represented as a
  84  * dtrace_probe structure.  To allow quick lookups based on each element of the
  85  * probe tuple, probes are hashed by each of provider, module, function and
  86  * name.  (If a lookup is performed based on a regular expression, a
  87  * dtrace_probekey is prepared, and a linear search is performed.) Each probe
  88  * is additionally pointed to by a linear array indexed by its identifier.  The
  89  * identifier is the provider's mechanism for indicating to the DTrace
  90  * framework that a probe has fired:  the identifier is passed as the first
  91  * argument to dtrace_probe(), where it is then mapped into the corresponding
  92  * dtrace_probe structure.  From the dtrace_probe structure, dtrace_probe() can
  93  * iterate over the probe's list of enabling control blocks; see "DTrace
  94  * Enabling Control Blocks", below.)
  95  */
  96 struct dtrace_probe {
  97         dtrace_id_t dtpr_id;                    /* probe identifier */
  98         dtrace_ecb_t *dtpr_ecb;                 /* ECB list; see below */
  99         dtrace_ecb_t *dtpr_ecb_last;            /* last ECB in list */
 100         void *dtpr_arg;                         /* provider argument */
 101         dtrace_cacheid_t dtpr_predcache;        /* predicate cache ID */
 102         int dtpr_aframes;                       /* artificial frames */
 103         dtrace_provider_t *dtpr_provider;       /* pointer to provider */
 104         char *dtpr_mod;                         /* probe's module name */
 105         char *dtpr_func;                        /* probe's function name */
 106         char *dtpr_name;                        /* probe's name */
 107         dtrace_probe_t *dtpr_nextmod;           /* next in module hash */
 108         dtrace_probe_t *dtpr_prevmod;           /* previous in module hash */
 109         dtrace_probe_t *dtpr_nextfunc;          /* next in function hash */
 110         dtrace_probe_t *dtpr_prevfunc;          /* previous in function hash */
 111         dtrace_probe_t *dtpr_nextname;          /* next in name hash */
 112         dtrace_probe_t *dtpr_prevname;          /* previous in name hash */
 113         dtrace_genid_t dtpr_gen;                /* probe generation ID */
 114 };
 115 
 116 typedef int dtrace_probekey_f(const char *, const char *, int);
 117 
 118 typedef struct dtrace_probekey {
 119         const char *dtpk_prov;                  /* provider name to match */
 120         dtrace_probekey_f *dtpk_pmatch;         /* provider matching function */
 121         const char *dtpk_mod;                   /* module name to match */
 122         dtrace_probekey_f *dtpk_mmatch;         /* module matching function */
 123         const char *dtpk_func;                  /* func name to match */
 124         dtrace_probekey_f *dtpk_fmatch;         /* func matching function */
 125         const char *dtpk_name;                  /* name to match */
 126         dtrace_probekey_f *dtpk_nmatch;         /* name matching function */
 127         dtrace_id_t dtpk_id;                    /* identifier to match */
 128 } dtrace_probekey_t;
 129 
 130 typedef struct dtrace_hashbucket {
 131         struct dtrace_hashbucket *dthb_next;    /* next on hash chain */
 132         dtrace_probe_t *dthb_chain;             /* chain of probes */
 133         int dthb_len;                           /* number of probes here */
 134 } dtrace_hashbucket_t;
 135 
 136 typedef struct dtrace_hash {
 137         dtrace_hashbucket_t **dth_tab;          /* hash table */
 138         int dth_size;                           /* size of hash table */
 139         int dth_mask;                           /* mask to index into table */
 140         int dth_nbuckets;                       /* total number of buckets */
 141         uintptr_t dth_nextoffs;                 /* offset of next in probe */
 142         uintptr_t dth_prevoffs;                 /* offset of prev in probe */
 143         uintptr_t dth_stroffs;                  /* offset of str in probe */
 144 } dtrace_hash_t;
 145 
 146 /*
 147  * DTrace Enabling Control Blocks
 148  *
 149  * When a provider wishes to fire a probe, it calls into dtrace_probe(),
 150  * passing the probe identifier as the first argument.  As described above,
 151  * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
 152  * structure.  This structure contains information about the probe, and a
 153  * pointer to the list of Enabling Control Blocks (ECBs).  Each ECB points to
 154  * DTrace consumer state, and contains an optional predicate, and a list of
 155  * actions.  (Shown schematically below.)  The ECB abstraction allows a single
 156  * probe to be multiplexed across disjoint consumers, or across disjoint
 157  * enablings of a single probe within one consumer.
 158  *
 159  *   Enabling Control Block
 160  *        dtrace_ecb_t
 161  * +------------------------+
 162  * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
 163  * | dtrace_state_t * ------+--------------> State associated with this ECB
 164  * | dtrace_predicate_t * --+---------+
 165  * | dtrace_action_t * -----+----+    |
 166  * | dtrace_ecb_t * ---+    |    |    |       Predicate (if any)
 167  * +-------------------+----+    |    |       dtrace_predicate_t
 168  *                     |         |    +---> +--------------------+
 169  *                     |         |          | dtrace_difo_t * ---+----> DIFO
 170  *                     |         |          +--------------------+
 171  *                     |         |
 172  *            Next ECB |         |           Action
 173  *            (if any) |         |       dtrace_action_t
 174  *                     :         +--> +-------------------+
 175  *                     :              | dtrace_actkind_t -+------> kind
 176  *                     v              | dtrace_difo_t * --+------> DIFO (if any)
 177  *                                    | dtrace_recdesc_t -+------> record descr.
 178  *                                    | dtrace_action_t * +------+
 179  *                                    +-------------------+      |
 180  *                                                               | Next action
 181  *                               +-------------------------------+  (if any)
 182  *                               |
 183  *                               |           Action
 184  *                               |       dtrace_action_t
 185  *                               +--> +-------------------+
 186  *                                    | dtrace_actkind_t -+------> kind
 187  *                                    | dtrace_difo_t * --+------> DIFO (if any)
 188  *                                    | dtrace_action_t * +------+
 189  *                                    +-------------------+      |
 190  *                                                               | Next action
 191  *                               +-------------------------------+  (if any)
 192  *                               |
 193  *                               :
 194  *                               v
 195  *
 196  *
 197  * dtrace_probe() iterates over the ECB list.  If the ECB needs less space
 198  * than is available in the principal buffer, the ECB is processed:  if the
 199  * predicate is non-NULL, the DIF object is executed.  If the result is
 200  * non-zero, the action list is processed, with each action being executed
 201  * accordingly.  When the action list has been completely executed, processing
 202  * advances to the next ECB.  processing advances to the next ECB.  If the
 203  * result is non-zero; For each ECB, it first determines the The ECB
 204  * abstraction allows disjoint consumers to multiplex on single probes.
 205  */
 206 struct dtrace_ecb {
 207         dtrace_epid_t dte_epid;                 /* enabled probe ID */
 208         uint32_t dte_alignment;                 /* required alignment */
 209         size_t dte_needed;                      /* bytes needed */
 210         size_t dte_size;                        /* total size of payload */
 211         dtrace_predicate_t *dte_predicate;      /* predicate, if any */
 212         dtrace_action_t *dte_action;            /* actions, if any */
 213         dtrace_ecb_t *dte_next;                 /* next ECB on probe */
 214         dtrace_state_t *dte_state;              /* pointer to state */
 215         uint32_t dte_cond;                      /* security condition */
 216         dtrace_probe_t *dte_probe;              /* pointer to probe */
 217         dtrace_action_t *dte_action_last;       /* last action on ECB */
 218         uint64_t dte_uarg;                      /* library argument */
 219 };
 220 
 221 struct dtrace_predicate {
 222         dtrace_difo_t *dtp_difo;                /* DIF object */
 223         dtrace_cacheid_t dtp_cacheid;           /* cache identifier */
 224         int dtp_refcnt;                         /* reference count */
 225 };
 226 
 227 struct dtrace_action {
 228         dtrace_actkind_t dta_kind;              /* kind of action */
 229         uint16_t dta_intuple;                   /* boolean:  in aggregation */
 230         uint32_t dta_refcnt;                    /* reference count */
 231         dtrace_difo_t *dta_difo;                /* pointer to DIFO */
 232         dtrace_recdesc_t dta_rec;               /* record description */
 233         dtrace_action_t *dta_prev;              /* previous action */
 234         dtrace_action_t *dta_next;              /* next action */
 235 };
 236 
 237 typedef struct dtrace_aggregation {
 238         dtrace_action_t dtag_action;            /* action; must be first */
 239         dtrace_aggid_t dtag_id;                 /* identifier */
 240         dtrace_ecb_t *dtag_ecb;                 /* corresponding ECB */
 241         dtrace_action_t *dtag_first;            /* first action in tuple */
 242         uint32_t dtag_base;                     /* base of aggregation */
 243         uint8_t dtag_hasarg;                    /* boolean:  has argument */
 244         uint64_t dtag_initial;                  /* initial value */
 245         void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t);
 246 } dtrace_aggregation_t;
 247 
 248 /*
 249  * DTrace Buffers
 250  *
 251  * Principal buffers, aggregation buffers, and speculative buffers are all
 252  * managed with the dtrace_buffer structure.  By default, this structure
 253  * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
 254  * active and passive buffers, respectively.  For speculative buffers,
 255  * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
 256  * to a scratch buffer.  For all buffer types, the dtrace_buffer structure is
 257  * always allocated on a per-CPU basis; a single dtrace_buffer structure is
 258  * never shared among CPUs.  (That is, there is never true sharing of the
 259  * dtrace_buffer structure; to prevent false sharing of the structure, it must
 260  * always be aligned to the coherence granularity -- generally 64 bytes.)
 261  *
 262  * One of the critical design decisions of DTrace is that a given ECB always
 263  * stores the same quantity and type of data.  This is done to assure that the
 264  * only metadata required for an ECB's traced data is the EPID.  That is, from
 265  * the EPID, the consumer can determine the data layout.  (The data buffer
 266  * layout is shown schematically below.)  By assuring that one can determine
 267  * data layout from the EPID, the metadata stream can be separated from the
 268  * data stream -- simplifying the data stream enormously.
 269  *
 270  *      base of data buffer --->  +------+--------------------+------+
 271  *                                | EPID | data               | EPID |
 272  *                                +------+--------+------+----+------+
 273  *                                | data          | EPID | data      |
 274  *                                +---------------+------+-----------+
 275  *                                | data, cont.                      |
 276  *                                +------+--------------------+------+
 277  *                                | EPID | data               |      |
 278  *                                +------+--------------------+      |
 279  *                                |                ||                |
 280  *                                |                ||                |
 281  *                                |                \/                |
 282  *                                :                                  :
 283  *                                .                                  .
 284  *                                .                                  .
 285  *                                .                                  .
 286  *                                :                                  :
 287  *                                |                                  |
 288  *     limit of data buffer --->  +----------------------------------+
 289  *
 290  * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
 291  * principal buffer (both scratch and payload) exceed the available space.  If
 292  * the ECB's needs exceed available space (and if the principal buffer policy
 293  * is the default "switch" policy), the ECB is dropped, the buffer's drop count
 294  * is incremented, and processing advances to the next ECB.  If the ECB's needs
 295  * can be met with the available space, the ECB is processed, but the offset in
 296  * the principal buffer is only advanced if the ECB completes processing
 297  * without error.
 298  *
 299  * When a buffer is to be switched (either because the buffer is the principal
 300  * buffer with a "switch" policy or because it is an aggregation buffer), a
 301  * cross call is issued to the CPU associated with the buffer.  In the cross
 302  * call context, interrupts are disabled, and the active and the inactive
 303  * buffers are atomically switched.  This involves switching the data pointers,
 304  * copying the various state fields (offset, drops, errors, etc.) into their
 305  * inactive equivalents, and clearing the state fields.  Because interrupts are
 306  * disabled during this procedure, the switch is guaranteed to appear atomic to
 307  * dtrace_probe().
 308  *
 309  * DTrace Ring Buffering
 310  *
 311  * To process a ring buffer correctly, one must know the oldest valid record.
 312  * Processing starts at the oldest record in the buffer and continues until
 313  * the end of the buffer is reached.  Processing then resumes starting with
 314  * the record stored at offset 0 in the buffer, and continues until the
 315  * youngest record is processed.  If trace records are of a fixed-length,
 316  * determining the oldest record is trivial:
 317  *
 318  *   - If the ring buffer has not wrapped, the oldest record is the record
 319  *     stored at offset 0.
 320  *
 321  *   - If the ring buffer has wrapped, the oldest record is the record stored
 322  *     at the current offset.
 323  *
 324  * With variable length records, however, just knowing the current offset
 325  * doesn't suffice for determining the oldest valid record:  assuming that one
 326  * allows for arbitrary data, one has no way of searching forward from the
 327  * current offset to find the oldest valid record.  (That is, one has no way
 328  * of separating data from metadata.) It would be possible to simply refuse to
 329  * process any data in the ring buffer between the current offset and the
 330  * limit, but this leaves (potentially) an enormous amount of otherwise valid
 331  * data unprocessed.
 332  *
 333  * To effect ring buffering, we track two offsets in the buffer:  the current
 334  * offset and the _wrapped_ offset.  If a request is made to reserve some
 335  * amount of data, and the buffer has wrapped, the wrapped offset is
 336  * incremented until the wrapped offset minus the current offset is greater
 337  * than or equal to the reserve request.  This is done by repeatedly looking
 338  * up the ECB corresponding to the EPID at the current wrapped offset, and
 339  * incrementing the wrapped offset by the size of the data payload
 340  * corresponding to that ECB.  If this offset is greater than or equal to the
 341  * limit of the data buffer, the wrapped offset is set to 0.  Thus, the
 342  * current offset effectively "chases" the wrapped offset around the buffer.
 343  * Schematically:
 344  *
 345  *      base of data buffer --->  +------+--------------------+------+
 346  *                                | EPID | data               | EPID |
 347  *                                +------+--------+------+----+------+
 348  *                                | data          | EPID | data      |
 349  *                                +---------------+------+-----------+
 350  *                                | data, cont.                      |
 351  *                                +------+---------------------------+
 352  *                                | EPID | data                      |
 353  *           current offset --->  +------+---------------------------+
 354  *                                | invalid data                     |
 355  *           wrapped offset --->  +------+--------------------+------+
 356  *                                | EPID | data               | EPID |
 357  *                                +------+--------+------+----+------+
 358  *                                | data          | EPID | data      |
 359  *                                +---------------+------+-----------+
 360  *                                :                                  :
 361  *                                .                                  .
 362  *                                .        ... valid data ...        .
 363  *                                .                                  .
 364  *                                :                                  :
 365  *                                +------+-------------+------+------+
 366  *                                | EPID | data        | EPID | data |
 367  *                                +------+------------++------+------+
 368  *                                | data, cont.       | leftover     |
 369  *     limit of data buffer --->  +-------------------+--------------+
 370  *
 371  * If the amount of requested buffer space exceeds the amount of space
 372  * available between the current offset and the end of the buffer:
 373  *
 374  *  (1)  all words in the data buffer between the current offset and the limit
 375  *       of the data buffer (marked "leftover", above) are set to
 376  *       DTRACE_EPIDNONE
 377  *
 378  *  (2)  the wrapped offset is set to zero
 379  *
 380  *  (3)  the iteration process described above occurs until the wrapped offset
 381  *       is greater than the amount of desired space.
 382  *
 383  * The wrapped offset is implemented by (re-)using the inactive offset.
 384  * In a "switch" buffer policy, the inactive offset stores the offset in
 385  * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
 386  * offset.
 387  *
 388  * DTrace Scratch Buffering
 389  *
 390  * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
 391  * To accommodate such requests easily, scratch memory may be allocated in
 392  * the buffer beyond the current offset plus the needed memory of the current
 393  * ECB.  If there isn't sufficient room in the buffer for the requested amount
 394  * of scratch space, the allocation fails and an error is generated.  Scratch
 395  * memory is tracked in the dtrace_mstate_t and is automatically freed when
 396  * the ECB ceases processing.  Note that ring buffers cannot allocate their
 397  * scratch from the principal buffer -- lest they needlessly overwrite older,
 398  * valid data.  Ring buffers therefore have their own dedicated scratch buffer
 399  * from which scratch is allocated.
 400  */
 401 #define DTRACEBUF_RING          0x0001          /* bufpolicy set to "ring" */
 402 #define DTRACEBUF_FILL          0x0002          /* bufpolicy set to "fill" */
 403 #define DTRACEBUF_NOSWITCH      0x0004          /* do not switch buffer */
 404 #define DTRACEBUF_WRAPPED       0x0008          /* ring buffer has wrapped */
 405 #define DTRACEBUF_DROPPED       0x0010          /* drops occurred */
 406 #define DTRACEBUF_ERROR         0x0020          /* errors occurred */
 407 #define DTRACEBUF_FULL          0x0040          /* "fill" buffer is full */
 408 #define DTRACEBUF_CONSUMED      0x0080          /* buffer has been consumed */
 409 #define DTRACEBUF_INACTIVE      0x0100          /* buffer is not yet active */
 410 
 411 typedef struct dtrace_buffer {
 412         uint64_t dtb_offset;                    /* current offset in buffer */
 413         uint64_t dtb_size;                      /* size of buffer */
 414         uint32_t dtb_flags;                     /* flags */
 415         uint32_t dtb_drops;                     /* number of drops */
 416         caddr_t dtb_tomax;                      /* active buffer */
 417         caddr_t dtb_xamot;                      /* inactive buffer */
 418         uint32_t dtb_xamot_flags;               /* inactive flags */
 419         uint32_t dtb_xamot_drops;               /* drops in inactive buffer */
 420         uint64_t dtb_xamot_offset;              /* offset in inactive buffer */
 421         uint32_t dtb_errors;                    /* number of errors */
 422         uint32_t dtb_xamot_errors;              /* errors in inactive buffer */
 423 #ifndef _LP64
 424         uint64_t dtb_pad1;                      /* pad out to 64 bytes */
 425 #endif
 426         uint64_t dtb_switched;                  /* time of last switch */
 427         uint64_t dtb_interval;                  /* observed switch interval */
 428         uint64_t dtb_pad2[6];                   /* pad to avoid false sharing */
 429 } dtrace_buffer_t;
 430 
 431 /*
 432  * DTrace Aggregation Buffers
 433  *
 434  * Aggregation buffers use much of the same mechanism as described above
 435  * ("DTrace Buffers").  However, because an aggregation is fundamentally a
 436  * hash, there exists dynamic metadata associated with an aggregation buffer
 437  * that is not associated with other kinds of buffers.  This aggregation
 438  * metadata is _only_ relevant for the in-kernel implementation of
 439  * aggregations; it is not actually relevant to user-level consumers.  To do
 440  * this, we allocate dynamic aggregation data (hash keys and hash buckets)
 441  * starting below the _limit_ of the buffer, and we allocate data from the
 442  * _base_ of the buffer.  When the aggregation buffer is copied out, _only_ the
 443  * data is copied out; the metadata is simply discarded.  Schematically,
 444  * aggregation buffers look like:
 445  *
 446  *      base of data buffer --->  +-------+------+-----------+-------+
 447  *                                | aggid | key  | value     | aggid |
 448  *                                +-------+------+-----------+-------+
 449  *                                | key                              |
 450  *                                +-------+-------+-----+------------+
 451  *                                | value | aggid | key | value      |
 452  *                                +-------+------++-----+------+-----+
 453  *                                | aggid | key  | value       |     |
 454  *                                +-------+------+-------------+     |
 455  *                                |                ||                |
 456  *                                |                ||                |
 457  *                                |                \/                |
 458  *                                :                                  :
 459  *                                .                                  .
 460  *                                .                                  .
 461  *                                .                                  .
 462  *                                :                                  :
 463  *                                |                /\                |
 464  *                                |                ||   +------------+
 465  *                                |                ||   |            |
 466  *                                +---------------------+            |
 467  *                                | hash keys                        |
 468  *                                | (dtrace_aggkey structures)       |
 469  *                                |                                  |
 470  *                                +----------------------------------+
 471  *                                | hash buckets                     |
 472  *                                | (dtrace_aggbuffer structure)     |
 473  *                                |                                  |
 474  *     limit of data buffer --->  +----------------------------------+
 475  *
 476  *
 477  * As implied above, just as we assure that ECBs always store a constant
 478  * amount of data, we assure that a given aggregation -- identified by its
 479  * aggregation ID -- always stores data of a constant quantity and type.
 480  * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
 481  * given record.
 482  *
 483  * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
 484  * aligned.  (If this the structure changes such that this becomes false, an
 485  * assertion will fail in dtrace_aggregate().)
 486  */
 487 typedef struct dtrace_aggkey {
 488         uint32_t dtak_hashval;                  /* hash value */
 489         uint32_t dtak_action:4;                 /* action -- 4 bits */
 490         uint32_t dtak_size:28;                  /* size -- 28 bits */
 491         caddr_t dtak_data;                      /* data pointer */
 492         struct dtrace_aggkey *dtak_next;        /* next in hash chain */
 493 } dtrace_aggkey_t;
 494 
 495 typedef struct dtrace_aggbuffer {
 496         uintptr_t dtagb_hashsize;               /* number of buckets */
 497         uintptr_t dtagb_free;                   /* free list of keys */
 498         dtrace_aggkey_t **dtagb_hash;           /* hash table */
 499 } dtrace_aggbuffer_t;
 500 
 501 /*
 502  * DTrace Speculations
 503  *
 504  * Speculations have a per-CPU buffer and a global state.  Once a speculation
 505  * buffer has been comitted or discarded, it cannot be reused until all CPUs
 506  * have taken the same action (commit or discard) on their respective
 507  * speculative buffer.  However, because DTrace probes may execute in arbitrary
 508  * context, other CPUs cannot simply be cross-called at probe firing time to
 509  * perform the necessary commit or discard.  The speculation states thus
 510  * optimize for the case that a speculative buffer is only active on one CPU at
 511  * the time of a commit() or discard() -- for if this is the case, other CPUs
 512  * need not take action, and the speculation is immediately available for
 513  * reuse.  If the speculation is active on multiple CPUs, it must be
 514  * asynchronously cleaned -- potentially leading to a higher rate of dirty
 515  * speculative drops.  The speculation states are as follows:
 516  *
 517  *  DTRACESPEC_INACTIVE       <= Initial state; inactive speculation
 518  *  DTRACESPEC_ACTIVE         <= Allocated, but not yet speculatively traced to
 519  *  DTRACESPEC_ACTIVEONE      <= Speculatively traced to on one CPU
 520  *  DTRACESPEC_ACTIVEMANY     <= Speculatively traced to on more than one CPU
 521  *  DTRACESPEC_COMMITTING     <= Currently being commited on one CPU
 522  *  DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
 523  *  DTRACESPEC_DISCARDING     <= Currently being discarded on many CPUs
 524  *
 525  * The state transition diagram is as follows:
 526  *
 527  *     +----------------------------------------------------------+
 528  *     |                                                          |
 529  *     |                      +------------+                      |
 530  *     |  +-------------------| COMMITTING |<-----------------+   |
 531  *     |  |                   +------------+                  |   |
 532  *     |  | copied spec.            ^             commit() on |   | discard() on
 533  *     |  | into principal          |              active CPU |   | active CPU
 534  *     |  |                         | commit()                |   |
 535  *     V  V                         |                         |   |
 536  * +----------+                 +--------+                +-----------+
 537  * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
 538  * +----------+  speculation()  +--------+  speculate()   +-----------+
 539  *     ^  ^                         |                         |   |
 540  *     |  |                         | discard()               |   |
 541  *     |  | asynchronously          |            discard() on |   | speculate()
 542  *     |  | cleaned                 V            inactive CPU |   | on inactive
 543  *     |  |                   +------------+                  |   | CPU
 544  *     |  +-------------------| DISCARDING |<-----------------+   |
 545  *     |                      +------------+                      |
 546  *     | asynchronously             ^                             |
 547  *     | copied spec.               |       discard()             |
 548  *     | into principal             +------------------------+    |
 549  *     |                                                     |    V
 550  *  +----------------+             commit()              +------------+
 551  *  | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
 552  *  +----------------+                                   +------------+
 553  */
 554 typedef enum dtrace_speculation_state {
 555         DTRACESPEC_INACTIVE = 0,
 556         DTRACESPEC_ACTIVE,
 557         DTRACESPEC_ACTIVEONE,
 558         DTRACESPEC_ACTIVEMANY,
 559         DTRACESPEC_COMMITTING,
 560         DTRACESPEC_COMMITTINGMANY,
 561         DTRACESPEC_DISCARDING
 562 } dtrace_speculation_state_t;
 563 
 564 typedef struct dtrace_speculation {
 565         dtrace_speculation_state_t dtsp_state;  /* current speculation state */
 566         int dtsp_cleaning;                      /* non-zero if being cleaned */
 567         dtrace_buffer_t *dtsp_buffer;           /* speculative buffer */
 568 } dtrace_speculation_t;
 569 
 570 /*
 571  * DTrace Dynamic Variables
 572  *
 573  * The dynamic variable problem is obviously decomposed into two subproblems:
 574  * allocating new dynamic storage, and freeing old dynamic storage.  The
 575  * presence of the second problem makes the first much more complicated -- or
 576  * rather, the absence of the second renders the first trivial.  This is the
 577  * case with aggregations, for which there is effectively no deallocation of
 578  * dynamic storage.  (Or more accurately, all dynamic storage is deallocated
 579  * when a snapshot is taken of the aggregation.)  As DTrace dynamic variables
 580  * allow for both dynamic allocation and dynamic deallocation, the
 581  * implementation of dynamic variables is quite a bit more complicated than
 582  * that of their aggregation kin.
 583  *
 584  * We observe that allocating new dynamic storage is tricky only because the
 585  * size can vary -- the allocation problem is much easier if allocation sizes
 586  * are uniform.  We further observe that in D, the size of dynamic variables is
 587  * actually _not_ dynamic -- dynamic variable sizes may be determined by static
 588  * analysis of DIF text.  (This is true even of putatively dynamically-sized
 589  * objects like strings and stacks, the sizes of which are dictated by the
 590  * "stringsize" and "stackframes" variables, respectively.)  We exploit this by
 591  * performing this analysis on all DIF before enabling any probes.  For each
 592  * dynamic load or store, we calculate the dynamically-allocated size plus the
 593  * size of the dtrace_dynvar structure plus the storage required to key the
 594  * data.  For all DIF, we take the largest value and dub it the _chunksize_.
 595  * We then divide dynamic memory into two parts:  a hash table that is wide
 596  * enough to have every chunk in its own bucket, and a larger region of equal
 597  * chunksize units.  Whenever we wish to dynamically allocate a variable, we
 598  * always allocate a single chunk of memory.  Depending on the uniformity of
 599  * allocation, this will waste some amount of memory -- but it eliminates the
 600  * non-determinism inherent in traditional heap fragmentation.
 601  *
 602  * Dynamic objects are allocated by storing a non-zero value to them; they are
 603  * deallocated by storing a zero value to them.  Dynamic variables are
 604  * complicated enormously by being shared between CPUs.  In particular,
 605  * consider the following scenario:
 606  *
 607  *                 CPU A                                 CPU B
 608  *  +---------------------------------+   +---------------------------------+
 609  *  |                                 |   |                                 |
 610  *  | allocates dynamic object a[123] |   |                                 |
 611  *  | by storing the value 345 to it  |   |                                 |
 612  *  |                               --------->                              |
 613  *  |                                 |   | wishing to load from object     |
 614  *  |                                 |   | a[123], performs lookup in      |
 615  *  |                                 |   | dynamic variable space          |
 616  *  |                               <---------                              |
 617  *  | deallocates object a[123] by    |   |                                 |
 618  *  | storing 0 to it                 |   |                                 |
 619  *  |                                 |   |                                 |
 620  *  | allocates dynamic object b[567] |   | performs load from a[123]       |
 621  *  | by storing the value 789 to it  |   |                                 |
 622  *  :                                 :   :                                 :
 623  *  .                                 .   .                                 .
 624  *
 625  * This is obviously a race in the D program, but there are nonetheless only
 626  * two valid values for CPU B's load from a[123]:  345 or 0.  Most importantly,
 627  * CPU B may _not_ see the value 789 for a[123].
 628  *
 629  * There are essentially two ways to deal with this:
 630  *
 631  *  (1)  Explicitly spin-lock variables.  That is, if CPU B wishes to load
 632  *       from a[123], it needs to lock a[123] and hold the lock for the
 633  *       duration that it wishes to manipulate it.
 634  *
 635  *  (2)  Avoid reusing freed chunks until it is known that no CPU is referring
 636  *       to them.
 637  *
 638  * The implementation of (1) is rife with complexity, because it requires the
 639  * user of a dynamic variable to explicitly decree when they are done using it.
 640  * Were all variables by value, this perhaps wouldn't be debilitating -- but
 641  * dynamic variables of non-scalar types are tracked by reference.  That is, if
 642  * a dynamic variable is, say, a string, and that variable is to be traced to,
 643  * say, the principal buffer, the DIF emulation code returns to the main
 644  * dtrace_probe() loop a pointer to the underlying storage, not the contents of
 645  * the storage.  Further, code calling on DIF emulation would have to be aware
 646  * that the DIF emulation has returned a reference to a dynamic variable that
 647  * has been potentially locked.  The variable would have to be unlocked after
 648  * the main dtrace_probe() loop is finished with the variable, and the main
 649  * dtrace_probe() loop would have to be careful to not call any further DIF
 650  * emulation while the variable is locked to avoid deadlock.  More generally,
 651  * if one were to implement (1), DIF emulation code dealing with dynamic
 652  * variables could only deal with one dynamic variable at a time (lest deadlock
 653  * result).  To sum, (1) exports too much subtlety to the users of dynamic
 654  * variables -- increasing maintenance burden and imposing serious constraints
 655  * on future DTrace development.
 656  *
 657  * The implementation of (2) is also complex, but the complexity is more
 658  * manageable.  We need to be sure that when a variable is deallocated, it is
 659  * not placed on a traditional free list, but rather on a _dirty_ list.  Once a
 660  * variable is on a dirty list, it cannot be found by CPUs performing a
 661  * subsequent lookup of the variable -- but it may still be in use by other
 662  * CPUs.  To assure that all CPUs that may be seeing the old variable have
 663  * cleared out of probe context, a dtrace_sync() can be issued.  Once the
 664  * dtrace_sync() has completed, it can be known that all CPUs are done
 665  * manipulating the dynamic variable -- the dirty list can be atomically
 666  * appended to the free list.  Unfortunately, there's a slight hiccup in this
 667  * mechanism:  dtrace_sync() may not be issued from probe context.  The
 668  * dtrace_sync() must be therefore issued asynchronously from non-probe
 669  * context.  For this we rely on the DTrace cleaner, a cyclic that runs at the
 670  * "cleanrate" frequency.  To ease this implementation, we define several chunk
 671  * lists:
 672  *
 673  *   - Dirty.  Deallocated chunks, not yet cleaned.  Not available.
 674  *
 675  *   - Rinsing.  Formerly dirty chunks that are currently being asynchronously
 676  *     cleaned.  Not available, but will be shortly.  Dynamic variable
 677  *     allocation may not spin or block for availability, however.
 678  *
 679  *   - Clean.  Clean chunks, ready for allocation -- but not on the free list.
 680  *
 681  *   - Free.  Available for allocation.
 682  *
 683  * Moreover, to avoid absurd contention, _each_ of these lists is implemented
 684  * on a per-CPU basis.  This is only for performance, not correctness; chunks
 685  * may be allocated from another CPU's free list.  The algorithm for allocation
 686  * then is this:
 687  *
 688  *   (1)  Attempt to atomically allocate from current CPU's free list.  If list
 689  *        is non-empty and allocation is successful, allocation is complete.
 690  *
 691  *   (2)  If the clean list is non-empty, atomically move it to the free list,
 692  *        and reattempt (1).
 693  *
 694  *   (3)  If the dynamic variable space is in the CLEAN state, look for free
 695  *        and clean lists on other CPUs by setting the current CPU to the next
 696  *        CPU, and reattempting (1).  If the next CPU is the current CPU (that
 697  *        is, if all CPUs have been checked), atomically switch the state of
 698  *        the dynamic variable space based on the following:
 699  *
 700  *        - If no free chunks were found and no dirty chunks were found,
 701  *          atomically set the state to EMPTY.
 702  *
 703  *        - If dirty chunks were found, atomically set the state to DIRTY.
 704  *
 705  *        - If rinsing chunks were found, atomically set the state to RINSING.
 706  *
 707  *   (4)  Based on state of dynamic variable space state, increment appropriate
 708  *        counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
 709  *        dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
 710  *        RINSING state).  Fail the allocation.
 711  *
 712  * The cleaning cyclic operates with the following algorithm:  for all CPUs
 713  * with a non-empty dirty list, atomically move the dirty list to the rinsing
 714  * list.  Perform a dtrace_sync().  For all CPUs with a non-empty rinsing list,
 715  * atomically move the rinsing list to the clean list.  Perform another
 716  * dtrace_sync().  By this point, all CPUs have seen the new clean list; the
 717  * state of the dynamic variable space can be restored to CLEAN.
 718  *
 719  * There exist two final races that merit explanation.  The first is a simple
 720  * allocation race:
 721  *
 722  *                 CPU A                                 CPU B
 723  *  +---------------------------------+   +---------------------------------+
 724  *  |                                 |   |                                 |
 725  *  | allocates dynamic object a[123] |   | allocates dynamic object a[123] |
 726  *  | by storing the value 345 to it  |   | by storing the value 567 to it  |
 727  *  |                                 |   |                                 |
 728  *  :                                 :   :                                 :
 729  *  .                                 .   .                                 .
 730  *
 731  * Again, this is a race in the D program.  It can be resolved by having a[123]
 732  * hold the value 345 or a[123] hold the value 567 -- but it must be true that
 733  * a[123] have only _one_ of these values.  (That is, the racing CPUs may not
 734  * put the same element twice on the same hash chain.)  This is resolved
 735  * simply:  before the allocation is undertaken, the start of the new chunk's
 736  * hash chain is noted.  Later, after the allocation is complete, the hash
 737  * chain is atomically switched to point to the new element.  If this fails
 738  * (because of either concurrent allocations or an allocation concurrent with a
 739  * deletion), the newly allocated chunk is deallocated to the dirty list, and
 740  * the whole process of looking up (and potentially allocating) the dynamic
 741  * variable is reattempted.
 742  *
 743  * The final race is a simple deallocation race:
 744  *
 745  *                 CPU A                                 CPU B
 746  *  +---------------------------------+   +---------------------------------+
 747  *  |                                 |   |                                 |
 748  *  | deallocates dynamic object      |   | deallocates dynamic object      |
 749  *  | a[123] by storing the value 0   |   | a[123] by storing the value 0   |
 750  *  | to it                           |   | to it                           |
 751  *  |                                 |   |                                 |
 752  *  :                                 :   :                                 :
 753  *  .                                 .   .                                 .
 754  *
 755  * Once again, this is a race in the D program, but it is one that we must
 756  * handle without corrupting the underlying data structures.  Because
 757  * deallocations require the deletion of a chunk from the middle of a hash
 758  * chain, we cannot use a single-word atomic operation to remove it.  For this,
 759  * we add a spin lock to the hash buckets that is _only_ used for deallocations
 760  * (allocation races are handled as above).  Further, this spin lock is _only_
 761  * held for the duration of the delete; before control is returned to the DIF
 762  * emulation code, the hash bucket is unlocked.
 763  */
 764 typedef struct dtrace_key {
 765         uint64_t dttk_value;                    /* data value or data pointer */
 766         uint64_t dttk_size;                     /* 0 if by-val, >0 if by-ref */
 767 } dtrace_key_t;
 768 
 769 typedef struct dtrace_tuple {
 770         uint32_t dtt_nkeys;                     /* number of keys in tuple */
 771         uint32_t dtt_pad;                       /* padding */
 772         dtrace_key_t dtt_key[1];                /* array of tuple keys */
 773 } dtrace_tuple_t;
 774 
 775 typedef struct dtrace_dynvar {
 776         uint64_t dtdv_hashval;                  /* hash value -- 0 if free */
 777         struct dtrace_dynvar *dtdv_next;        /* next on list or hash chain */
 778         void *dtdv_data;                        /* pointer to data */
 779         dtrace_tuple_t dtdv_tuple;              /* tuple key */
 780 } dtrace_dynvar_t;
 781 
 782 typedef enum dtrace_dynvar_op {
 783         DTRACE_DYNVAR_ALLOC,
 784         DTRACE_DYNVAR_NOALLOC,
 785         DTRACE_DYNVAR_DEALLOC
 786 } dtrace_dynvar_op_t;
 787 
 788 typedef struct dtrace_dynhash {
 789         dtrace_dynvar_t *dtdh_chain;            /* hash chain for this bucket */
 790         uintptr_t dtdh_lock;                    /* deallocation lock */
 791 #ifdef _LP64
 792         uintptr_t dtdh_pad[6];                  /* pad to avoid false sharing */
 793 #else
 794         uintptr_t dtdh_pad[14];                 /* pad to avoid false sharing */
 795 #endif
 796 } dtrace_dynhash_t;
 797 
 798 typedef struct dtrace_dstate_percpu {
 799         dtrace_dynvar_t *dtdsc_free;            /* free list for this CPU */
 800         dtrace_dynvar_t *dtdsc_dirty;           /* dirty list for this CPU */
 801         dtrace_dynvar_t *dtdsc_rinsing;         /* rinsing list for this CPU */
 802         dtrace_dynvar_t *dtdsc_clean;           /* clean list for this CPU */
 803         uint64_t dtdsc_drops;                   /* number of capacity drops */
 804         uint64_t dtdsc_dirty_drops;             /* number of dirty drops */
 805         uint64_t dtdsc_rinsing_drops;           /* number of rinsing drops */
 806 #ifdef _LP64
 807         uint64_t dtdsc_pad;                     /* pad to avoid false sharing */
 808 #else
 809         uint64_t dtdsc_pad[2];                  /* pad to avoid false sharing */
 810 #endif
 811 } dtrace_dstate_percpu_t;
 812 
 813 typedef enum dtrace_dstate_state {
 814         DTRACE_DSTATE_CLEAN = 0,
 815         DTRACE_DSTATE_EMPTY,
 816         DTRACE_DSTATE_DIRTY,
 817         DTRACE_DSTATE_RINSING
 818 } dtrace_dstate_state_t;
 819 
 820 typedef struct dtrace_dstate {
 821         void *dtds_base;                        /* base of dynamic var. space */
 822         size_t dtds_size;                       /* size of dynamic var. space */
 823         size_t dtds_hashsize;                   /* number of buckets in hash */
 824         size_t dtds_chunksize;                  /* size of each chunk */
 825         dtrace_dynhash_t *dtds_hash;            /* pointer to hash table */
 826         dtrace_dstate_state_t dtds_state;       /* current dynamic var. state */
 827         dtrace_dstate_percpu_t *dtds_percpu;    /* per-CPU dyn. var. state */
 828 } dtrace_dstate_t;
 829 
 830 /*
 831  * DTrace Variable State
 832  *
 833  * The DTrace variable state tracks user-defined variables in its dtrace_vstate
 834  * structure.  Each DTrace consumer has exactly one dtrace_vstate structure,
 835  * but some dtrace_vstate structures may exist without a corresponding DTrace
 836  * consumer (see "DTrace Helpers", below).  As described in <sys/dtrace.h>,
 837  * user-defined variables can have one of three scopes:
 838  *
 839  *  DIFV_SCOPE_GLOBAL  =>  global scope
 840  *  DIFV_SCOPE_THREAD  =>  thread-local scope (i.e. "self->" variables)
 841  *  DIFV_SCOPE_LOCAL   =>  clause-local scope (i.e. "this->" variables)
 842  *
 843  * The variable state tracks variables by both their scope and their allocation
 844  * type:
 845  *
 846  *  - The dtvs_globals and dtvs_locals members each point to an array of
 847  *    dtrace_statvar structures.  These structures contain both the variable
 848  *    metadata (dtrace_difv structures) and the underlying storage for all
 849  *    statically allocated variables, including statically allocated
 850  *    DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
 851  *
 852  *  - The dtvs_tlocals member points to an array of dtrace_difv structures for
 853  *    DIFV_SCOPE_THREAD variables.  As such, this array tracks _only_ the
 854  *    variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
 855  *    is allocated out of the dynamic variable space.
 856  *
 857  *  - The dtvs_dynvars member is the dynamic variable state associated with the
 858  *    variable state.  The dynamic variable state (described in "DTrace Dynamic
 859  *    Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
 860  *    dynamically-allocated DIFV_SCOPE_GLOBAL variables.
 861  */
 862 typedef struct dtrace_statvar {
 863         uint64_t dtsv_data;                     /* data or pointer to it */
 864         size_t dtsv_size;                       /* size of pointed-to data */
 865         int dtsv_refcnt;                        /* reference count */
 866         dtrace_difv_t dtsv_var;                 /* variable metadata */
 867 } dtrace_statvar_t;
 868 
 869 typedef struct dtrace_vstate {
 870         dtrace_state_t *dtvs_state;             /* back pointer to state */
 871         dtrace_statvar_t **dtvs_globals;        /* statically-allocated glbls */
 872         int dtvs_nglobals;                      /* number of globals */
 873         dtrace_difv_t *dtvs_tlocals;            /* thread-local metadata */
 874         int dtvs_ntlocals;                      /* number of thread-locals */
 875         dtrace_statvar_t **dtvs_locals;         /* clause-local data */
 876         int dtvs_nlocals;                       /* number of clause-locals */
 877         dtrace_dstate_t dtvs_dynvars;           /* dynamic variable state */
 878 } dtrace_vstate_t;
 879 
 880 /*
 881  * DTrace Machine State
 882  *
 883  * In the process of processing a fired probe, DTrace needs to track and/or
 884  * cache some per-CPU state associated with that particular firing.  This is
 885  * state that is always discarded after the probe firing has completed, and
 886  * much of it is not specific to any DTrace consumer, remaining valid across
 887  * all ECBs.  This state is tracked in the dtrace_mstate structure.
 888  */
 889 #define DTRACE_MSTATE_ARGS              0x00000001
 890 #define DTRACE_MSTATE_PROBE             0x00000002
 891 #define DTRACE_MSTATE_EPID              0x00000004
 892 #define DTRACE_MSTATE_TIMESTAMP         0x00000008
 893 #define DTRACE_MSTATE_STACKDEPTH        0x00000010
 894 #define DTRACE_MSTATE_CALLER            0x00000020
 895 #define DTRACE_MSTATE_IPL               0x00000040
 896 #define DTRACE_MSTATE_FLTOFFS           0x00000080
 897 #define DTRACE_MSTATE_WALLTIMESTAMP     0x00000100
 898 #define DTRACE_MSTATE_USTACKDEPTH       0x00000200
 899 #define DTRACE_MSTATE_UCALLER           0x00000400
 900 
 901 typedef struct dtrace_mstate {
 902         uintptr_t dtms_scratch_base;            /* base of scratch space */
 903         uintptr_t dtms_scratch_ptr;             /* current scratch pointer */
 904         size_t dtms_scratch_size;               /* scratch size */
 905         uint32_t dtms_present;                  /* variables that are present */
 906         uint64_t dtms_arg[5];                   /* cached arguments */
 907         dtrace_epid_t dtms_epid;                /* current EPID */
 908         uint64_t dtms_timestamp;                /* cached timestamp */
 909         hrtime_t dtms_walltimestamp;            /* cached wall timestamp */
 910         int dtms_stackdepth;                    /* cached stackdepth */
 911         int dtms_ustackdepth;                   /* cached ustackdepth */
 912         struct dtrace_probe *dtms_probe;        /* current probe */
 913         uintptr_t dtms_caller;                  /* cached caller */
 914         uint64_t dtms_ucaller;                  /* cached user-level caller */
 915         int dtms_ipl;                           /* cached interrupt pri lev */
 916         int dtms_fltoffs;                       /* faulting DIFO offset */
 917         uintptr_t dtms_strtok;                  /* saved strtok() pointer */
 918         uint32_t dtms_access;                   /* memory access rights */
 919         dtrace_difo_t *dtms_difo;               /* current dif object */
 920         file_t *dtms_getf;                      /* cached rval of getf() */
 921 } dtrace_mstate_t;
 922 
 923 #define DTRACE_COND_OWNER       0x1
 924 #define DTRACE_COND_USERMODE    0x2
 925 #define DTRACE_COND_ZONEOWNER   0x4
 926 
 927 #define DTRACE_PROBEKEY_MAXDEPTH        8       /* max glob recursion depth */
 928 
 929 /*
 930  * Access flag used by dtrace_mstate.dtms_access.
 931  */
 932 #define DTRACE_ACCESS_KERNEL    0x1             /* the priv to read kmem */
 933 #define DTRACE_ACCESS_PROC      0x2             /* the priv for proc state */
 934 #define DTRACE_ACCESS_ARGS      0x4             /* the priv to examine args */
 935 
 936 /*
 937  * DTrace Activity
 938  *
 939  * Each DTrace consumer is in one of several states, which (for purposes of
 940  * avoiding yet-another overloading of the noun "state") we call the current
 941  * _activity_.  The activity transitions on dtrace_go() (from DTRACIOCGO), on
 942  * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action.  Activities may
 943  * only transition in one direction; the activity transition diagram is a
 944  * directed acyclic graph.  The activity transition diagram is as follows:
 945  *
 946  *
 947  * +----------+                   +--------+                   +--------+
 948  * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
 949  * +----------+   dtrace_go(),    +--------+   dtrace_go(),    +--------+
 950  *                before BEGIN        |        after BEGIN       |  |  |
 951  *                                    |                          |  |  |
 952  *                      exit() action |                          |  |  |
 953  *                     from BEGIN ECB |                          |  |  |
 954  *                                    |                          |  |  |
 955  *                                    v                          |  |  |
 956  *                               +----------+     exit() action  |  |  |
 957  * +-----------------------------| DRAINING |<-------------------+  |  |
 958  * |                             +----------+                       |  |
 959  * |                                  |                             |  |
 960  * |                   dtrace_stop(), |                             |  |
 961  * |                     before END   |                             |  |
 962  * |                                  |                             |  |
 963  * |                                  v                             |  |
 964  * | +---------+                 +----------+                       |  |
 965  * | | STOPPED |<----------------| COOLDOWN |<----------------------+  |
 966  * | +---------+  dtrace_stop(), +----------+     dtrace_stop(),       |
 967  * |                after END                       before END         |
 968  * |                                                                   |
 969  * |                              +--------+                           |
 970  * +----------------------------->| KILLED |<--------------------------+
 971  *       deadman timeout or       +--------+     deadman timeout or
 972  *        killed consumer                         killed consumer
 973  *
 974  * Note that once a DTrace consumer has stopped tracing, there is no way to
 975  * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
 976  * the DTrace pseudodevice.
 977  */
 978 typedef enum dtrace_activity {
 979         DTRACE_ACTIVITY_INACTIVE = 0,           /* not yet running */
 980         DTRACE_ACTIVITY_WARMUP,                 /* while starting */
 981         DTRACE_ACTIVITY_ACTIVE,                 /* running */
 982         DTRACE_ACTIVITY_DRAINING,               /* before stopping */
 983         DTRACE_ACTIVITY_COOLDOWN,               /* while stopping */
 984         DTRACE_ACTIVITY_STOPPED,                /* after stopping */
 985         DTRACE_ACTIVITY_KILLED                  /* killed */
 986 } dtrace_activity_t;
 987 
 988 /*
 989  * DTrace Helper Implementation
 990  *
 991  * A description of the helper architecture may be found in <sys/dtrace.h>.
 992  * Each process contains a pointer to its helpers in its p_dtrace_helpers
 993  * member.  This is a pointer to a dtrace_helpers structure, which contains an
 994  * array of pointers to dtrace_helper structures, helper variable state (shared
 995  * among a process's helpers) and a generation count.  (The generation count is
 996  * used to provide an identifier when a helper is added so that it may be
 997  * subsequently removed.)  The dtrace_helper structure is self-explanatory,
 998  * containing pointers to the objects needed to execute the helper.  Note that
 999  * helpers are _duplicated_ across fork(2), and destroyed on exec(2).  No more
1000  * than dtrace_helpers_max are allowed per-process.
1001  */
1002 #define DTRACE_HELPER_ACTION_USTACK     0
1003 #define DTRACE_NHELPER_ACTIONS          1
1004 
1005 typedef struct dtrace_helper_action {
1006         int dtha_generation;                    /* helper action generation */
1007         int dtha_nactions;                      /* number of actions */
1008         dtrace_difo_t *dtha_predicate;          /* helper action predicate */
1009         dtrace_difo_t **dtha_actions;           /* array of actions */
1010         struct dtrace_helper_action *dtha_next; /* next helper action */
1011 } dtrace_helper_action_t;
1012 
1013 typedef struct dtrace_helper_provider {
1014         int dthp_generation;                    /* helper provider generation */
1015         uint32_t dthp_ref;                      /* reference count */
1016         dof_helper_t dthp_prov;                 /* DOF w/ provider and probes */
1017 } dtrace_helper_provider_t;
1018 
1019 typedef struct dtrace_helpers {
1020         dtrace_helper_action_t **dthps_actions; /* array of helper actions */
1021         dtrace_vstate_t dthps_vstate;           /* helper action var. state */
1022         dtrace_helper_provider_t **dthps_provs; /* array of providers */
1023         uint_t dthps_nprovs;                    /* count of providers */
1024         uint_t dthps_maxprovs;                  /* provider array size */
1025         int dthps_generation;                   /* current generation */
1026         pid_t dthps_pid;                        /* pid of associated proc */
1027         int dthps_deferred;                     /* helper in deferred list */
1028         struct dtrace_helpers *dthps_next;      /* next pointer */
1029         struct dtrace_helpers *dthps_prev;      /* prev pointer */
1030 } dtrace_helpers_t;
1031 
1032 /*
1033  * DTrace Helper Action Tracing
1034  *
1035  * Debugging helper actions can be arduous.  To ease the development and
1036  * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
1037  * framework: helper tracing.  If dtrace_helptrace_enabled is non-zero (which
1038  * it is by default on DEBUG kernels), all helper activity will be traced to a
1039  * global, in-kernel ring buffer.  Each entry includes a pointer to the specific
1040  * helper, the location within the helper, and a trace of all local variables.
1041  * The ring buffer may be displayed in a human-readable format with the
1042  * ::dtrace_helptrace mdb(1) dcmd.
1043  */
1044 #define DTRACE_HELPTRACE_NEXT   (-1)
1045 #define DTRACE_HELPTRACE_DONE   (-2)
1046 #define DTRACE_HELPTRACE_ERR    (-3)
1047 
1048 typedef struct dtrace_helptrace {
1049         dtrace_helper_action_t  *dtht_helper;   /* helper action */
1050         int dtht_where;                         /* where in helper action */
1051         int dtht_nlocals;                       /* number of locals */
1052         int dtht_fault;                         /* type of fault (if any) */
1053         int dtht_fltoffs;                       /* DIF offset */
1054         uint64_t dtht_illval;                   /* faulting value */
1055         uint64_t dtht_locals[1];                /* local variables */
1056 } dtrace_helptrace_t;
1057 
1058 /*
1059  * DTrace Credentials
1060  *
1061  * In probe context, we have limited flexibility to examine the credentials
1062  * of the DTrace consumer that created a particular enabling.  We use
1063  * the Least Privilege interfaces to cache the consumer's cred pointer and
1064  * some facts about that credential in a dtrace_cred_t structure. These
1065  * can limit the consumer's breadth of visibility and what actions the
1066  * consumer may take.
1067  */
1068 #define DTRACE_CRV_ALLPROC              0x01
1069 #define DTRACE_CRV_KERNEL               0x02
1070 #define DTRACE_CRV_ALLZONE              0x04
1071 
1072 #define DTRACE_CRV_ALL          (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
1073         DTRACE_CRV_ALLZONE)
1074 
1075 #define DTRACE_CRA_PROC                         0x0001
1076 #define DTRACE_CRA_PROC_CONTROL                 0x0002
1077 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER     0x0004
1078 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE     0x0008
1079 #define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG     0x0010
1080 #define DTRACE_CRA_KERNEL                       0x0020
1081 #define DTRACE_CRA_KERNEL_DESTRUCTIVE           0x0040
1082 
1083 #define DTRACE_CRA_ALL          (DTRACE_CRA_PROC | \
1084         DTRACE_CRA_PROC_CONTROL | \
1085         DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
1086         DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
1087         DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
1088         DTRACE_CRA_KERNEL | \
1089         DTRACE_CRA_KERNEL_DESTRUCTIVE)
1090 
1091 typedef struct dtrace_cred {
1092         cred_t                  *dcr_cred;
1093         uint8_t                 dcr_destructive;
1094         uint8_t                 dcr_visible;
1095         uint16_t                dcr_action;
1096 } dtrace_cred_t;
1097 
1098 /*
1099  * DTrace Consumer State
1100  *
1101  * Each DTrace consumer has an associated dtrace_state structure that contains
1102  * its in-kernel DTrace state -- including options, credentials, statistics and
1103  * pointers to ECBs, buffers, speculations and formats.  A dtrace_state
1104  * structure is also allocated for anonymous enablings.  When anonymous state
1105  * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
1106  * dtrace_state structure.
1107  */
1108 struct dtrace_state {
1109         dev_t dts_dev;                          /* device */
1110         int dts_necbs;                          /* total number of ECBs */
1111         dtrace_ecb_t **dts_ecbs;                /* array of ECBs */
1112         dtrace_epid_t dts_epid;                 /* next EPID to allocate */
1113         size_t dts_needed;                      /* greatest needed space */
1114         struct dtrace_state *dts_anon;          /* anon. state, if grabbed */
1115         dtrace_activity_t dts_activity;         /* current activity */
1116         dtrace_vstate_t dts_vstate;             /* variable state */
1117         dtrace_buffer_t *dts_buffer;            /* principal buffer */
1118         dtrace_buffer_t *dts_aggbuffer;         /* aggregation buffer */
1119         dtrace_speculation_t *dts_speculations; /* speculation array */
1120         int dts_nspeculations;                  /* number of speculations */
1121         int dts_naggregations;                  /* number of aggregations */
1122         dtrace_aggregation_t **dts_aggregations; /* aggregation array */
1123         vmem_t *dts_aggid_arena;                /* arena for aggregation IDs */
1124         uint64_t dts_errors;                    /* total number of errors */
1125         uint32_t dts_speculations_busy;         /* number of spec. busy */
1126         uint32_t dts_speculations_unavail;      /* number of spec unavail */
1127         uint32_t dts_stkstroverflows;           /* stack string tab overflows */
1128         uint32_t dts_dblerrors;                 /* errors in ERROR probes */
1129         uint32_t dts_reserve;                   /* space reserved for END */
1130         hrtime_t dts_laststatus;                /* time of last status */
1131         cyclic_id_t dts_cleaner;                /* cleaning cyclic */
1132         cyclic_id_t dts_deadman;                /* deadman cyclic */
1133         hrtime_t dts_alive;                     /* time last alive */
1134         char dts_speculates;                    /* boolean: has speculations */
1135         char dts_destructive;                   /* boolean: has dest. actions */
1136         int dts_nformats;                       /* number of formats */
1137         char **dts_formats;                     /* format string array */
1138         dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
1139         dtrace_cred_t dts_cred;                 /* credentials */
1140         size_t dts_nretained;                   /* number of retained enabs */
1141         int dts_getf;                           /* number of getf() calls */
1142 };
1143 
1144 struct dtrace_provider {
1145         dtrace_pattr_t dtpv_attr;               /* provider attributes */
1146         dtrace_ppriv_t dtpv_priv;               /* provider privileges */
1147         dtrace_pops_t dtpv_pops;                /* provider operations */
1148         char *dtpv_name;                        /* provider name */
1149         void *dtpv_arg;                         /* provider argument */
1150         hrtime_t dtpv_defunct;                  /* when made defunct */
1151         struct dtrace_provider *dtpv_next;      /* next provider */
1152 };
1153 
1154 struct dtrace_meta {
1155         dtrace_mops_t dtm_mops;                 /* meta provider operations */
1156         char *dtm_name;                         /* meta provider name */
1157         void *dtm_arg;                          /* meta provider user arg */
1158         uint64_t dtm_count;                     /* no. of associated provs. */
1159 };
1160 
1161 /*
1162  * DTrace Enablings
1163  *
1164  * A dtrace_enabling structure is used to track a collection of ECB
1165  * descriptions -- before they have been turned into actual ECBs.  This is
1166  * created as a result of DOF processing, and is generally used to generate
1167  * ECBs immediately thereafter.  However, enablings are also generally
1168  * retained should the probes they describe be created at a later time; as
1169  * each new module or provider registers with the framework, the retained
1170  * enablings are reevaluated, with any new match resulting in new ECBs.  To
1171  * prevent probes from being matched more than once, the enabling tracks the
1172  * last probe generation matched, and only matches probes from subsequent
1173  * generations.
1174  */
1175 typedef struct dtrace_enabling {
1176         dtrace_ecbdesc_t **dten_desc;           /* all ECB descriptions */
1177         int dten_ndesc;                         /* number of ECB descriptions */
1178         int dten_maxdesc;                       /* size of ECB array */
1179         dtrace_vstate_t *dten_vstate;           /* associated variable state */
1180         dtrace_genid_t dten_probegen;           /* matched probe generation */
1181         dtrace_ecbdesc_t *dten_current;         /* current ECB description */
1182         int dten_error;                         /* current error value */
1183         int dten_primed;                        /* boolean: set if primed */
1184         struct dtrace_enabling *dten_prev;      /* previous enabling */
1185         struct dtrace_enabling *dten_next;      /* next enabling */
1186 } dtrace_enabling_t;
1187 
1188 /*
1189  * DTrace Anonymous Enablings
1190  *
1191  * Anonymous enablings are DTrace enablings that are not associated with a
1192  * controlling process, but rather derive their enabling from DOF stored as
1193  * properties in the dtrace.conf file.  If there is an anonymous enabling, a
1194  * DTrace consumer state and enabling are created on attach.  The state may be
1195  * subsequently grabbed by the first consumer specifying the "grabanon"
1196  * option.  As long as an anonymous DTrace enabling exists, dtrace(7D) will
1197  * refuse to unload.
1198  */
1199 typedef struct dtrace_anon {
1200         dtrace_state_t *dta_state;              /* DTrace consumer state */
1201         dtrace_enabling_t *dta_enabling;        /* pointer to enabling */
1202         processorid_t dta_beganon;              /* which CPU BEGIN ran on */
1203 } dtrace_anon_t;
1204 
1205 /*
1206  * DTrace Error Debugging
1207  */
1208 #ifdef DEBUG
1209 #define DTRACE_ERRDEBUG
1210 #endif
1211 
1212 #ifdef DTRACE_ERRDEBUG
1213 
1214 typedef struct dtrace_errhash {
1215         const char      *dter_msg;      /* error message */
1216         int             dter_count;     /* number of times seen */
1217 } dtrace_errhash_t;
1218 
1219 #define DTRACE_ERRHASHSZ        256     /* must be > number of err msgs */
1220 
1221 #endif  /* DTRACE_ERRDEBUG */
1222 
1223 /*
1224  * DTrace Toxic Ranges
1225  *
1226  * DTrace supports safe loads from probe context; if the address turns out to
1227  * be invalid, a bit will be set by the kernel indicating that DTrace
1228  * encountered a memory error, and DTrace will propagate the error to the user
1229  * accordingly.  However, there may exist some regions of memory in which an
1230  * arbitrary load can change system state, and from which it is impossible to
1231  * recover from such a load after it has been attempted.  Examples of this may
1232  * include memory in which programmable I/O registers are mapped (for which a
1233  * read may have some implications for the device) or (in the specific case of
1234  * UltraSPARC-I and -II) the virtual address hole.  The platform is required
1235  * to make DTrace aware of these toxic ranges; DTrace will then check that
1236  * target addresses are not in a toxic range before attempting to issue a
1237  * safe load.
1238  */
1239 typedef struct dtrace_toxrange {
1240         uintptr_t       dtt_base;               /* base of toxic range */
1241         uintptr_t       dtt_limit;              /* limit of toxic range */
1242 } dtrace_toxrange_t;
1243 
1244 extern uint64_t dtrace_getarg(int, int);
1245 extern greg_t dtrace_getfp(void);
1246 extern int dtrace_getipl(void);
1247 extern uintptr_t dtrace_caller(int);
1248 extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
1249 extern void *dtrace_casptr(void *, void *, void *);
1250 extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1251 extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1252 extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1253 extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t,
1254     volatile uint16_t *);
1255 extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
1256 extern ulong_t dtrace_getreg(struct regs *, uint_t);
1257 extern uint64_t dtrace_getvmreg(uint_t, volatile uint16_t *);
1258 extern int dtrace_getstackdepth(int);
1259 extern void dtrace_getupcstack(uint64_t *, int);
1260 extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
1261 extern int dtrace_getustackdepth(void);
1262 extern uintptr_t dtrace_fulword(void *);
1263 extern uint8_t dtrace_fuword8(void *);
1264 extern uint16_t dtrace_fuword16(void *);
1265 extern uint32_t dtrace_fuword32(void *);
1266 extern uint64_t dtrace_fuword64(void *);
1267 extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
1268     int, uintptr_t);
1269 extern int dtrace_assfail(const char *, const char *, int);
1270 extern int dtrace_attached(void);
1271 extern hrtime_t dtrace_gethrestime();
1272 
1273 #ifdef __sparc
1274 extern void dtrace_flush_windows(void);
1275 extern void dtrace_flush_user_windows(void);
1276 extern uint_t dtrace_getotherwin(void);
1277 extern uint_t dtrace_getfprs(void);
1278 #else
1279 extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
1280 extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1281 #endif
1282 
1283 /*
1284  * DTrace Assertions
1285  *
1286  * DTrace calls ASSERT from probe context.  To assure that a failed ASSERT
1287  * does not induce a markedly more catastrophic failure (e.g., one from which
1288  * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
1289  * may safely be called from probe context.  This header file must thus be
1290  * included by any DTrace component that calls ASSERT from probe context, and
1291  * _only_ by those components.  (The only exception to this is kernel
1292  * debugging infrastructure at user-level that doesn't depend on calling
1293  * ASSERT.)
1294  */
1295 #undef ASSERT
1296 #ifdef DEBUG
1297 #define ASSERT(EX)      ((void)((EX) || \
1298                         dtrace_assfail(#EX, __FILE__, __LINE__)))
1299 #else
1300 #define ASSERT(X)       ((void)0)
1301 #endif
1302 
1303 #ifdef  __cplusplus
1304 }
1305 #endif
1306 
1307 #endif /* _SYS_DTRACE_IMPL_H */