1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  25  * Copyright (c) 2012 by Delphix. All rights reserved.
  26  */
  27 
  28 /*
  29  * DTrace - Dynamic Tracing for Solaris
  30  *
  31  * This is the implementation of the Solaris Dynamic Tracing framework
  32  * (DTrace).  The user-visible interface to DTrace is described at length in
  33  * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
  34  * library, the in-kernel DTrace framework, and the DTrace providers are
  35  * described in the block comments in the <sys/dtrace.h> header file.  The
  36  * internal architecture of DTrace is described in the block comments in the
  37  * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
  38  * implementation very much assume mastery of all of these sources; if one has
  39  * an unanswered question about the implementation, one should consult them
  40  * first.
  41  *
  42  * The functions here are ordered roughly as follows:
  43  *
  44  *   - Probe context functions
  45  *   - Probe hashing functions
  46  *   - Non-probe context utility functions
  47  *   - Matching functions
  48  *   - Provider-to-Framework API functions
  49  *   - Probe management functions
  50  *   - DIF object functions
  51  *   - Format functions
  52  *   - Predicate functions
  53  *   - ECB functions
  54  *   - Buffer functions
  55  *   - Enabling functions
  56  *   - DOF functions
  57  *   - Anonymous enabling functions
  58  *   - Consumer state functions
  59  *   - Helper functions
  60  *   - Hook functions
  61  *   - Driver cookbook functions
  62  *
  63  * Each group of functions begins with a block comment labelled the "DTrace
  64  * [Group] Functions", allowing one to find each block by searching forward
  65  * on capital-f functions.
  66  */
  67 #include <sys/errno.h>
  68 #include <sys/stat.h>
  69 #include <sys/modctl.h>
  70 #include <sys/conf.h>
  71 #include <sys/systm.h>
  72 #include <sys/ddi.h>
  73 #include <sys/sunddi.h>
  74 #include <sys/cpuvar.h>
  75 #include <sys/kmem.h>
  76 #include <sys/strsubr.h>
  77 #include <sys/sysmacros.h>
  78 #include <sys/dtrace_impl.h>
  79 #include <sys/atomic.h>
  80 #include <sys/cmn_err.h>
  81 #include <sys/mutex_impl.h>
  82 #include <sys/rwlock_impl.h>
  83 #include <sys/ctf_api.h>
  84 #include <sys/panic.h>
  85 #include <sys/priv_impl.h>
  86 #include <sys/policy.h>
  87 #include <sys/cred_impl.h>
  88 #include <sys/procfs_isa.h>
  89 #include <sys/taskq.h>
  90 #include <sys/mkdev.h>
  91 #include <sys/kdi.h>
  92 #include <sys/zone.h>
  93 #include <sys/socket.h>
  94 #include <netinet/in.h>
  95 #include "strtolctype.h"
  96 
  97 /*
  98  * DTrace Tunable Variables
  99  *
 100  * The following variables may be tuned by adding a line to /etc/system that
 101  * includes both the name of the DTrace module ("dtrace") and the name of the
 102  * variable.  For example:
 103  *
 104  *   set dtrace:dtrace_destructive_disallow = 1
 105  *
 106  * In general, the only variables that one should be tuning this way are those
 107  * that affect system-wide DTrace behavior, and for which the default behavior
 108  * is undesirable.  Most of these variables are tunable on a per-consumer
 109  * basis using DTrace options, and need not be tuned on a system-wide basis.
 110  * When tuning these variables, avoid pathological values; while some attempt
 111  * is made to verify the integrity of these variables, they are not considered
 112  * part of the supported interface to DTrace, and they are therefore not
 113  * checked comprehensively.  Further, these variables should not be tuned
 114  * dynamically via "mdb -kw" or other means; they should only be tuned via
 115  * /etc/system.
 116  */
 117 int             dtrace_destructive_disallow = 0;
 118 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 119 size_t          dtrace_difo_maxsize = (256 * 1024);
 120 dtrace_optval_t dtrace_dof_maxsize = (256 * 1024);
 121 size_t          dtrace_global_maxsize = (16 * 1024);
 122 size_t          dtrace_actions_max = (16 * 1024);
 123 size_t          dtrace_retain_max = 1024;
 124 dtrace_optval_t dtrace_helper_actions_max = 1024;
 125 dtrace_optval_t dtrace_helper_providers_max = 32;
 126 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
 127 size_t          dtrace_strsize_default = 256;
 128 dtrace_optval_t dtrace_cleanrate_default = 9900990;             /* 101 hz */
 129 dtrace_optval_t dtrace_cleanrate_min = 200000;                  /* 5000 hz */
 130 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;  /* 1/minute */
 131 dtrace_optval_t dtrace_aggrate_default = NANOSEC;               /* 1 hz */
 132 dtrace_optval_t dtrace_statusrate_default = NANOSEC;            /* 1 hz */
 133 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;  /* 6/minute */
 134 dtrace_optval_t dtrace_switchrate_default = NANOSEC;            /* 1 hz */
 135 dtrace_optval_t dtrace_nspec_default = 1;
 136 dtrace_optval_t dtrace_specsize_default = 32 * 1024;
 137 dtrace_optval_t dtrace_stackframes_default = 20;
 138 dtrace_optval_t dtrace_ustackframes_default = 20;
 139 dtrace_optval_t dtrace_jstackframes_default = 50;
 140 dtrace_optval_t dtrace_jstackstrsize_default = 512;
 141 int             dtrace_msgdsize_max = 128;
 142 hrtime_t        dtrace_chill_max = 500 * (NANOSEC / MILLISEC);  /* 500 ms */
 143 hrtime_t        dtrace_chill_interval = NANOSEC;                /* 1000 ms */
 144 int             dtrace_devdepth_max = 32;
 145 int             dtrace_err_verbose;
 146 hrtime_t        dtrace_deadman_interval = NANOSEC;
 147 hrtime_t        dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
 148 hrtime_t        dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
 149 hrtime_t        dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
 150 
 151 /*
 152  * DTrace External Variables
 153  *
 154  * As dtrace(7D) is a kernel module, any DTrace variables are obviously
 155  * available to DTrace consumers via the backtick (`) syntax.  One of these,
 156  * dtrace_zero, is made deliberately so:  it is provided as a source of
 157  * well-known, zero-filled memory.  While this variable is not documented,
 158  * it is used by some translators as an implementation detail.
 159  */
 160 const char      dtrace_zero[256] = { 0 };       /* zero-filled memory */
 161 
 162 /*
 163  * DTrace Internal Variables
 164  */
 165 static dev_info_t       *dtrace_devi;           /* device info */
 166 static vmem_t           *dtrace_arena;          /* probe ID arena */
 167 static vmem_t           *dtrace_minor;          /* minor number arena */
 168 static taskq_t          *dtrace_taskq;          /* task queue */
 169 static dtrace_probe_t   **dtrace_probes;        /* array of all probes */
 170 static int              dtrace_nprobes;         /* number of probes */
 171 static dtrace_provider_t *dtrace_provider;      /* provider list */
 172 static dtrace_meta_t    *dtrace_meta_pid;       /* user-land meta provider */
 173 static int              dtrace_opens;           /* number of opens */
 174 static int              dtrace_helpers;         /* number of helpers */
 175 static int              dtrace_getf;            /* number of unpriv getf()s */
 176 static void             *dtrace_softstate;      /* softstate pointer */
 177 static dtrace_hash_t    *dtrace_bymod;          /* probes hashed by module */
 178 static dtrace_hash_t    *dtrace_byfunc;         /* probes hashed by function */
 179 static dtrace_hash_t    *dtrace_byname;         /* probes hashed by name */
 180 static dtrace_toxrange_t *dtrace_toxrange;      /* toxic range array */
 181 static int              dtrace_toxranges;       /* number of toxic ranges */
 182 static int              dtrace_toxranges_max;   /* size of toxic range array */
 183 static dtrace_anon_t    dtrace_anon;            /* anonymous enabling */
 184 static kmem_cache_t     *dtrace_state_cache;    /* cache for dynamic state */
 185 static uint64_t         dtrace_vtime_references; /* number of vtimestamp refs */
 186 static kthread_t        *dtrace_panicked;       /* panicking thread */
 187 static dtrace_ecb_t     *dtrace_ecb_create_cache; /* cached created ECB */
 188 static dtrace_genid_t   dtrace_probegen;        /* current probe generation */
 189 static dtrace_helpers_t *dtrace_deferred_pid;   /* deferred helper list */
 190 static dtrace_enabling_t *dtrace_retained;      /* list of retained enablings */
 191 static dtrace_genid_t   dtrace_retained_gen;    /* current retained enab gen */
 192 static dtrace_dynvar_t  dtrace_dynhash_sink;    /* end of dynamic hash chains */
 193 static int              dtrace_dynvar_failclean; /* dynvars failed to clean */
 194 
 195 /*
 196  * DTrace Locking
 197  * DTrace is protected by three (relatively coarse-grained) locks:
 198  *
 199  * (1) dtrace_lock is required to manipulate essentially any DTrace state,
 200  *     including enabling state, probes, ECBs, consumer state, helper state,
 201  *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
 202  *     probe context is lock-free -- synchronization is handled via the
 203  *     dtrace_sync() cross call mechanism.
 204  *
 205  * (2) dtrace_provider_lock is required when manipulating provider state, or
 206  *     when provider state must be held constant.
 207  *
 208  * (3) dtrace_meta_lock is required when manipulating meta provider state, or
 209  *     when meta provider state must be held constant.
 210  *
 211  * The lock ordering between these three locks is dtrace_meta_lock before
 212  * dtrace_provider_lock before dtrace_lock.  (In particular, there are
 213  * several places where dtrace_provider_lock is held by the framework as it
 214  * calls into the providers -- which then call back into the framework,
 215  * grabbing dtrace_lock.)
 216  *
 217  * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
 218  * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
 219  * role as a coarse-grained lock; it is acquired before both of these locks.
 220  * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
 221  * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
 222  * mod_lock is similar with respect to dtrace_provider_lock in that it must be
 223  * acquired _between_ dtrace_provider_lock and dtrace_lock.
 224  */
 225 static kmutex_t         dtrace_lock;            /* probe state lock */
 226 static kmutex_t         dtrace_provider_lock;   /* provider state lock */
 227 static kmutex_t         dtrace_meta_lock;       /* meta-provider state lock */
 228 
 229 /*
 230  * DTrace Provider Variables
 231  *
 232  * These are the variables relating to DTrace as a provider (that is, the
 233  * provider of the BEGIN, END, and ERROR probes).
 234  */
 235 static dtrace_pattr_t   dtrace_provider_attr = {
 236 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 237 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 238 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 239 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 240 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 241 };
 242 
 243 static void
 244 dtrace_nullop(void)
 245 {}
 246 
 247 static int
 248 dtrace_enable_nullop(void)
 249 {
 250         return (0);
 251 }
 252 
 253 static dtrace_pops_t    dtrace_provider_ops = {
 254         (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
 255         (void (*)(void *, struct modctl *))dtrace_nullop,
 256         (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
 257         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 258         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 259         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 260         NULL,
 261         NULL,
 262         NULL,
 263         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
 264 };
 265 
 266 static dtrace_id_t      dtrace_probeid_begin;   /* special BEGIN probe */
 267 static dtrace_id_t      dtrace_probeid_end;     /* special END probe */
 268 dtrace_id_t             dtrace_probeid_error;   /* special ERROR probe */
 269 
 270 /*
 271  * DTrace Helper Tracing Variables
 272  */
 273 uint32_t dtrace_helptrace_next = 0;
 274 uint32_t dtrace_helptrace_nlocals;
 275 char    *dtrace_helptrace_buffer;
 276 int     dtrace_helptrace_bufsize = 512 * 1024;
 277 
 278 #ifdef DEBUG
 279 int     dtrace_helptrace_enabled = 1;
 280 #else
 281 int     dtrace_helptrace_enabled = 0;
 282 #endif
 283 
 284 /*
 285  * DTrace Error Hashing
 286  *
 287  * On DEBUG kernels, DTrace will track the errors that has seen in a hash
 288  * table.  This is very useful for checking coverage of tests that are
 289  * expected to induce DIF or DOF processing errors, and may be useful for
 290  * debugging problems in the DIF code generator or in DOF generation .  The
 291  * error hash may be examined with the ::dtrace_errhash MDB dcmd.
 292  */
 293 #ifdef DEBUG
 294 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
 295 static const char *dtrace_errlast;
 296 static kthread_t *dtrace_errthread;
 297 static kmutex_t dtrace_errlock;
 298 #endif
 299 
 300 /*
 301  * DTrace Macros and Constants
 302  *
 303  * These are various macros that are useful in various spots in the
 304  * implementation, along with a few random constants that have no meaning
 305  * outside of the implementation.  There is no real structure to this cpp
 306  * mishmash -- but is there ever?
 307  */
 308 #define DTRACE_HASHSTR(hash, probe)     \
 309         dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
 310 
 311 #define DTRACE_HASHNEXT(hash, probe)    \
 312         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
 313 
 314 #define DTRACE_HASHPREV(hash, probe)    \
 315         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
 316 
 317 #define DTRACE_HASHEQ(hash, lhs, rhs)   \
 318         (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
 319             *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
 320 
 321 #define DTRACE_AGGHASHSIZE_SLEW         17
 322 
 323 #define DTRACE_V4MAPPED_OFFSET          (sizeof (uint32_t) * 3)
 324 
 325 /*
 326  * The key for a thread-local variable consists of the lower 61 bits of the
 327  * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
 328  * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
 329  * equal to a variable identifier.  This is necessary (but not sufficient) to
 330  * assure that global associative arrays never collide with thread-local
 331  * variables.  To guarantee that they cannot collide, we must also define the
 332  * order for keying dynamic variables.  That order is:
 333  *
 334  *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
 335  *
 336  * Because the variable-key and the tls-key are in orthogonal spaces, there is
 337  * no way for a global variable key signature to match a thread-local key
 338  * signature.
 339  */
 340 #define DTRACE_TLS_THRKEY(where) { \
 341         uint_t intr = 0; \
 342         uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
 343         for (; actv; actv >>= 1) \
 344                 intr++; \
 345         ASSERT(intr < (1 << 3)); \
 346         (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
 347             (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
 348 }
 349 
 350 #define DT_BSWAP_8(x)   ((x) & 0xff)
 351 #define DT_BSWAP_16(x)  ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
 352 #define DT_BSWAP_32(x)  ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
 353 #define DT_BSWAP_64(x)  ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
 354 
 355 #define DT_MASK_LO 0x00000000FFFFFFFFULL
 356 
 357 #define DTRACE_STORE(type, tomax, offset, what) \
 358         *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
 359 
 360 #ifndef __x86
 361 #define DTRACE_ALIGNCHECK(addr, size, flags)                            \
 362         if (addr & (size - 1)) {                                    \
 363                 *flags |= CPU_DTRACE_BADALIGN;                          \
 364                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 365                 return (0);                                             \
 366         }
 367 #else
 368 #define DTRACE_ALIGNCHECK(addr, size, flags)
 369 #endif
 370 
 371 /*
 372  * Test whether a range of memory starting at testaddr of size testsz falls
 373  * within the range of memory described by addr, sz.  We take care to avoid
 374  * problems with overflow and underflow of the unsigned quantities, and
 375  * disallow all negative sizes.  Ranges of size 0 are allowed.
 376  */
 377 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
 378         ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
 379         (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
 380         (testaddr) + (testsz) >= (testaddr))
 381 
 382 /*
 383  * Test whether alloc_sz bytes will fit in the scratch region.  We isolate
 384  * alloc_sz on the righthand side of the comparison in order to avoid overflow
 385  * or underflow in the comparison with it.  This is simpler than the INRANGE
 386  * check above, because we know that the dtms_scratch_ptr is valid in the
 387  * range.  Allocations of size zero are allowed.
 388  */
 389 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
 390         ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
 391         (mstate)->dtms_scratch_ptr >= (alloc_sz))
 392 
 393 #define DTRACE_LOADFUNC(bits)                                           \
 394 /*CSTYLED*/                                                             \
 395 uint##bits##_t                                                          \
 396 dtrace_load##bits(uintptr_t addr)                                       \
 397 {                                                                       \
 398         size_t size = bits / NBBY;                                      \
 399         /*CSTYLED*/                                                     \
 400         uint##bits##_t rval;                                            \
 401         int i;                                                          \
 402         volatile uint16_t *flags = (volatile uint16_t *)                \
 403             &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;                    \
 404                                                                         \
 405         DTRACE_ALIGNCHECK(addr, size, flags);                           \
 406                                                                         \
 407         for (i = 0; i < dtrace_toxranges; i++) {                     \
 408                 if (addr >= dtrace_toxrange[i].dtt_limit)            \
 409                         continue;                                       \
 410                                                                         \
 411                 if (addr + size <= dtrace_toxrange[i].dtt_base)              \
 412                         continue;                                       \
 413                                                                         \
 414                 /*                                                      \
 415                  * This address falls within a toxic region; return 0.  \
 416                  */                                                     \
 417                 *flags |= CPU_DTRACE_BADADDR;                           \
 418                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 419                 return (0);                                             \
 420         }                                                               \
 421                                                                         \
 422         *flags |= CPU_DTRACE_NOFAULT;                                   \
 423         /*CSTYLED*/                                                     \
 424         rval = *((volatile uint##bits##_t *)addr);                      \
 425         *flags &= ~CPU_DTRACE_NOFAULT;                                      \
 426                                                                         \
 427         return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0);           \
 428 }
 429 
 430 #ifdef _LP64
 431 #define dtrace_loadptr  dtrace_load64
 432 #else
 433 #define dtrace_loadptr  dtrace_load32
 434 #endif
 435 
 436 #define DTRACE_DYNHASH_FREE     0
 437 #define DTRACE_DYNHASH_SINK     1
 438 #define DTRACE_DYNHASH_VALID    2
 439 
 440 #define DTRACE_MATCH_FAIL       -1
 441 #define DTRACE_MATCH_NEXT       0
 442 #define DTRACE_MATCH_DONE       1
 443 #define DTRACE_ANCHORED(probe)  ((probe)->dtpr_func[0] != '\0')
 444 #define DTRACE_STATE_ALIGN      64
 445 
 446 #define DTRACE_FLAGS2FLT(flags)                                         \
 447         (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :               \
 448         ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :            \
 449         ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :                \
 450         ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :            \
 451         ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :            \
 452         ((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :             \
 453         ((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :             \
 454         ((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :   \
 455         ((flags) & CPU_DTRACE_BADSTACK) ?  DTRACEFLT_BADSTACK :             \
 456         DTRACEFLT_UNKNOWN)
 457 
 458 #define DTRACEACT_ISSTRING(act)                                         \
 459         ((act)->dta_kind == DTRACEACT_DIFEXPR &&                     \
 460         (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
 461 
 462 static size_t dtrace_strlen(const char *, size_t);
 463 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
 464 static void dtrace_enabling_provide(dtrace_provider_t *);
 465 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
 466 static void dtrace_enabling_matchall(void);
 467 static void dtrace_enabling_reap(void);
 468 static dtrace_state_t *dtrace_anon_grab(void);
 469 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
 470     dtrace_state_t *, uint64_t, uint64_t);
 471 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
 472 static void dtrace_buffer_drop(dtrace_buffer_t *);
 473 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
 474 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
 475     dtrace_state_t *, dtrace_mstate_t *);
 476 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
 477     dtrace_optval_t);
 478 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
 479 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
 480 static int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *);
 481 static void dtrace_getf_barrier(void);
 482 
 483 /*
 484  * DTrace Probe Context Functions
 485  *
 486  * These functions are called from probe context.  Because probe context is
 487  * any context in which C may be called, arbitrarily locks may be held,
 488  * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
 489  * As a result, functions called from probe context may only call other DTrace
 490  * support functions -- they may not interact at all with the system at large.
 491  * (Note that the ASSERT macro is made probe-context safe by redefining it in
 492  * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
 493  * loads are to be performed from probe context, they _must_ be in terms of
 494  * the safe dtrace_load*() variants.
 495  *
 496  * Some functions in this block are not actually called from probe context;
 497  * for these functions, there will be a comment above the function reading
 498  * "Note:  not called from probe context."
 499  */
 500 void
 501 dtrace_panic(const char *format, ...)
 502 {
 503         va_list alist;
 504 
 505         va_start(alist, format);
 506         dtrace_vpanic(format, alist);
 507         va_end(alist);
 508 }
 509 
 510 int
 511 dtrace_assfail(const char *a, const char *f, int l)
 512 {
 513         dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
 514 
 515         /*
 516          * We just need something here that even the most clever compiler
 517          * cannot optimize away.
 518          */
 519         return (a[(uintptr_t)f]);
 520 }
 521 
 522 /*
 523  * Atomically increment a specified error counter from probe context.
 524  */
 525 static void
 526 dtrace_error(uint32_t *counter)
 527 {
 528         /*
 529          * Most counters stored to in probe context are per-CPU counters.
 530          * However, there are some error conditions that are sufficiently
 531          * arcane that they don't merit per-CPU storage.  If these counters
 532          * are incremented concurrently on different CPUs, scalability will be
 533          * adversely affected -- but we don't expect them to be white-hot in a
 534          * correctly constructed enabling...
 535          */
 536         uint32_t oval, nval;
 537 
 538         do {
 539                 oval = *counter;
 540 
 541                 if ((nval = oval + 1) == 0) {
 542                         /*
 543                          * If the counter would wrap, set it to 1 -- assuring
 544                          * that the counter is never zero when we have seen
 545                          * errors.  (The counter must be 32-bits because we
 546                          * aren't guaranteed a 64-bit compare&swap operation.)
 547                          * To save this code both the infamy of being fingered
 548                          * by a priggish news story and the indignity of being
 549                          * the target of a neo-puritan witch trial, we're
 550                          * carefully avoiding any colorful description of the
 551                          * likelihood of this condition -- but suffice it to
 552                          * say that it is only slightly more likely than the
 553                          * overflow of predicate cache IDs, as discussed in
 554                          * dtrace_predicate_create().
 555                          */
 556                         nval = 1;
 557                 }
 558         } while (dtrace_cas32(counter, oval, nval) != oval);
 559 }
 560 
 561 /*
 562  * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
 563  * uint8_t, a uint16_t, a uint32_t and a uint64_t.
 564  */
 565 DTRACE_LOADFUNC(8)
 566 DTRACE_LOADFUNC(16)
 567 DTRACE_LOADFUNC(32)
 568 DTRACE_LOADFUNC(64)
 569 
 570 static int
 571 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
 572 {
 573         if (dest < mstate->dtms_scratch_base)
 574                 return (0);
 575 
 576         if (dest + size < dest)
 577                 return (0);
 578 
 579         if (dest + size > mstate->dtms_scratch_ptr)
 580                 return (0);
 581 
 582         return (1);
 583 }
 584 
 585 static int
 586 dtrace_canstore_statvar(uint64_t addr, size_t sz,
 587     dtrace_statvar_t **svars, int nsvars)
 588 {
 589         int i;
 590 
 591         for (i = 0; i < nsvars; i++) {
 592                 dtrace_statvar_t *svar = svars[i];
 593 
 594                 if (svar == NULL || svar->dtsv_size == 0)
 595                         continue;
 596 
 597                 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
 598                         return (1);
 599         }
 600 
 601         return (0);
 602 }
 603 
 604 /*
 605  * Check to see if the address is within a memory region to which a store may
 606  * be issued.  This includes the DTrace scratch areas, and any DTrace variable
 607  * region.  The caller of dtrace_canstore() is responsible for performing any
 608  * alignment checks that are needed before stores are actually executed.
 609  */
 610 static int
 611 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 612     dtrace_vstate_t *vstate)
 613 {
 614         /*
 615          * First, check to see if the address is in scratch space...
 616          */
 617         if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
 618             mstate->dtms_scratch_size))
 619                 return (1);
 620 
 621         /*
 622          * Now check to see if it's a dynamic variable.  This check will pick
 623          * up both thread-local variables and any global dynamically-allocated
 624          * variables.
 625          */
 626         if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
 627             vstate->dtvs_dynvars.dtds_size)) {
 628                 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
 629                 uintptr_t base = (uintptr_t)dstate->dtds_base +
 630                     (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
 631                 uintptr_t chunkoffs;
 632 
 633                 /*
 634                  * Before we assume that we can store here, we need to make
 635                  * sure that it isn't in our metadata -- storing to our
 636                  * dynamic variable metadata would corrupt our state.  For
 637                  * the range to not include any dynamic variable metadata,
 638                  * it must:
 639                  *
 640                  *      (1) Start above the hash table that is at the base of
 641                  *      the dynamic variable space
 642                  *
 643                  *      (2) Have a starting chunk offset that is beyond the
 644                  *      dtrace_dynvar_t that is at the base of every chunk
 645                  *
 646                  *      (3) Not span a chunk boundary
 647                  *
 648                  */
 649                 if (addr < base)
 650                         return (0);
 651 
 652                 chunkoffs = (addr - base) % dstate->dtds_chunksize;
 653 
 654                 if (chunkoffs < sizeof (dtrace_dynvar_t))
 655                         return (0);
 656 
 657                 if (chunkoffs + sz > dstate->dtds_chunksize)
 658                         return (0);
 659 
 660                 return (1);
 661         }
 662 
 663         /*
 664          * Finally, check the static local and global variables.  These checks
 665          * take the longest, so we perform them last.
 666          */
 667         if (dtrace_canstore_statvar(addr, sz,
 668             vstate->dtvs_locals, vstate->dtvs_nlocals))
 669                 return (1);
 670 
 671         if (dtrace_canstore_statvar(addr, sz,
 672             vstate->dtvs_globals, vstate->dtvs_nglobals))
 673                 return (1);
 674 
 675         return (0);
 676 }
 677 
 678 
 679 /*
 680  * Convenience routine to check to see if the address is within a memory
 681  * region in which a load may be issued given the user's privilege level;
 682  * if not, it sets the appropriate error flags and loads 'addr' into the
 683  * illegal value slot.
 684  *
 685  * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
 686  * appropriate memory access protection.
 687  */
 688 static int
 689 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 690     dtrace_vstate_t *vstate)
 691 {
 692         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
 693         file_t *fp;
 694 
 695         /*
 696          * If we hold the privilege to read from kernel memory, then
 697          * everything is readable.
 698          */
 699         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 700                 return (1);
 701 
 702         /*
 703          * You can obviously read that which you can store.
 704          */
 705         if (dtrace_canstore(addr, sz, mstate, vstate))
 706                 return (1);
 707 
 708         /*
 709          * We're allowed to read from our own string table.
 710          */
 711         if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
 712             mstate->dtms_difo->dtdo_strlen))
 713                 return (1);
 714 
 715         if (vstate->dtvs_state != NULL &&
 716             dtrace_priv_proc(vstate->dtvs_state, mstate)) {
 717                 proc_t *p;
 718 
 719                 /*
 720                  * When we have privileges to the current process, there are
 721                  * several context-related kernel structures that are safe to
 722                  * read, even absent the privilege to read from kernel memory.
 723                  * These reads are safe because these structures contain only
 724                  * state that (1) we're permitted to read, (2) is harmless or
 725                  * (3) contains pointers to additional kernel state that we're
 726                  * not permitted to read (and as such, do not present an
 727                  * opportunity for privilege escalation).  Finally (and
 728                  * critically), because of the nature of their relation with
 729                  * the current thread context, the memory associated with these
 730                  * structures cannot change over the duration of probe context,
 731                  * and it is therefore impossible for this memory to be
 732                  * deallocated and reallocated as something else while it's
 733                  * being operated upon.
 734                  */
 735                 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t)))
 736                         return (1);
 737 
 738                 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
 739                     sz, curthread->t_procp, sizeof (proc_t))) {
 740                         return (1);
 741                 }
 742 
 743                 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
 744                     curthread->t_cred, sizeof (cred_t))) {
 745                         return (1);
 746                 }
 747 
 748                 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
 749                     &(p->p_pidp->pid_id), sizeof (pid_t))) {
 750                         return (1);
 751                 }
 752 
 753                 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
 754                     curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
 755                         return (1);
 756                 }
 757         }
 758 
 759         if ((fp = mstate->dtms_getf) != NULL) {
 760                 uintptr_t psz = sizeof (void *);
 761                 vnode_t *vp;
 762                 vnodeops_t *op;
 763 
 764                 /*
 765                  * When getf() returns a file_t, the enabling is implicitly
 766                  * granted the (transient) right to read the returned file_t
 767                  * as well as the v_path and v_op->vnop_name of the underlying
 768                  * vnode.  These accesses are allowed after a successful
 769                  * getf() because the members that they refer to cannot change
 770                  * once set -- and the barrier logic in the kernel's closef()
 771                  * path assures that the file_t and its referenced vode_t
 772                  * cannot themselves be stale (that is, it impossible for
 773                  * either dtms_getf itself or its f_vnode member to reference
 774                  * freed memory).
 775                  */
 776                 if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t)))
 777                         return (1);
 778 
 779                 if ((vp = fp->f_vnode) != NULL) {
 780                         if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz))
 781                                 return (1);
 782 
 783                         if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz,
 784                             vp->v_path, strlen(vp->v_path) + 1)) {
 785                                 return (1);
 786                         }
 787 
 788                         if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz))
 789                                 return (1);
 790 
 791                         if ((op = vp->v_op) != NULL &&
 792                             DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) {
 793                                 return (1);
 794                         }
 795 
 796                         if (op != NULL && op->vnop_name != NULL &&
 797                             DTRACE_INRANGE(addr, sz, op->vnop_name,
 798                             strlen(op->vnop_name) + 1)) {
 799                                 return (1);
 800                         }
 801                 }
 802         }
 803 
 804         DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
 805         *illval = addr;
 806         return (0);
 807 }
 808 
 809 /*
 810  * Convenience routine to check to see if a given string is within a memory
 811  * region in which a load may be issued given the user's privilege level;
 812  * this exists so that we don't need to issue unnecessary dtrace_strlen()
 813  * calls in the event that the user has all privileges.
 814  */
 815 static int
 816 dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 817     dtrace_vstate_t *vstate)
 818 {
 819         size_t strsz;
 820 
 821         /*
 822          * If we hold the privilege to read from kernel memory, then
 823          * everything is readable.
 824          */
 825         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 826                 return (1);
 827 
 828         strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
 829         if (dtrace_canload(addr, strsz, mstate, vstate))
 830                 return (1);
 831 
 832         return (0);
 833 }
 834 
 835 /*
 836  * Convenience routine to check to see if a given variable is within a memory
 837  * region in which a load may be issued given the user's privilege level.
 838  */
 839 static int
 840 dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
 841     dtrace_vstate_t *vstate)
 842 {
 843         size_t sz;
 844         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
 845 
 846         /*
 847          * If we hold the privilege to read from kernel memory, then
 848          * everything is readable.
 849          */
 850         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 851                 return (1);
 852 
 853         if (type->dtdt_kind == DIF_TYPE_STRING)
 854                 sz = dtrace_strlen(src,
 855                     vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
 856         else
 857                 sz = type->dtdt_size;
 858 
 859         return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
 860 }
 861 
 862 /*
 863  * Convert a string to a signed integer using safe loads.
 864  *
 865  * NOTE: This function uses various macros from strtolctype.h to manipulate
 866  * digit values, etc -- these have all been checked to ensure they make
 867  * no additional function calls.
 868  */
 869 static int64_t
 870 dtrace_strtoll(char *input, int base, size_t limit)
 871 {
 872         uintptr_t pos = (uintptr_t)input;
 873         int64_t val = 0;
 874         int x;
 875         boolean_t neg = B_FALSE;
 876         char c, cc, ccc;
 877         uintptr_t end = pos + limit;
 878 
 879         /*
 880          * Consume any whitespace preceding digits.
 881          */
 882         while ((c = dtrace_load8(pos)) == ' ' || c == '\t')
 883                 pos++;
 884 
 885         /*
 886          * Handle an explicit sign if one is present.
 887          */
 888         if (c == '-' || c == '+') {
 889                 if (c == '-')
 890                         neg = B_TRUE;
 891                 c = dtrace_load8(++pos);
 892         }
 893 
 894         /*
 895          * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it
 896          * if present.
 897          */
 898         if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' ||
 899             cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) {
 900                 pos += 2;
 901                 c = ccc;
 902         }
 903 
 904         /*
 905          * Read in contiguous digits until the first non-digit character.
 906          */
 907         for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base;
 908             c = dtrace_load8(++pos))
 909                 val = val * base + x;
 910 
 911         return (neg ? -val : val);
 912 }
 913 
 914 /*
 915  * Compare two strings using safe loads.
 916  */
 917 static int
 918 dtrace_strncmp(char *s1, char *s2, size_t limit)
 919 {
 920         uint8_t c1, c2;
 921         volatile uint16_t *flags;
 922 
 923         if (s1 == s2 || limit == 0)
 924                 return (0);
 925 
 926         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 927 
 928         do {
 929                 if (s1 == NULL) {
 930                         c1 = '\0';
 931                 } else {
 932                         c1 = dtrace_load8((uintptr_t)s1++);
 933                 }
 934 
 935                 if (s2 == NULL) {
 936                         c2 = '\0';
 937                 } else {
 938                         c2 = dtrace_load8((uintptr_t)s2++);
 939                 }
 940 
 941                 if (c1 != c2)
 942                         return (c1 - c2);
 943         } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
 944 
 945         return (0);
 946 }
 947 
 948 /*
 949  * Compute strlen(s) for a string using safe memory accesses.  The additional
 950  * len parameter is used to specify a maximum length to ensure completion.
 951  */
 952 static size_t
 953 dtrace_strlen(const char *s, size_t lim)
 954 {
 955         uint_t len;
 956 
 957         for (len = 0; len != lim; len++) {
 958                 if (dtrace_load8((uintptr_t)s++) == '\0')
 959                         break;
 960         }
 961 
 962         return (len);
 963 }
 964 
 965 /*
 966  * Check if an address falls within a toxic region.
 967  */
 968 static int
 969 dtrace_istoxic(uintptr_t kaddr, size_t size)
 970 {
 971         uintptr_t taddr, tsize;
 972         int i;
 973 
 974         for (i = 0; i < dtrace_toxranges; i++) {
 975                 taddr = dtrace_toxrange[i].dtt_base;
 976                 tsize = dtrace_toxrange[i].dtt_limit - taddr;
 977 
 978                 if (kaddr - taddr < tsize) {
 979                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 980                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
 981                         return (1);
 982                 }
 983 
 984                 if (taddr - kaddr < size) {
 985                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 986                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
 987                         return (1);
 988                 }
 989         }
 990 
 991         return (0);
 992 }
 993 
 994 /*
 995  * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
 996  * memory specified by the DIF program.  The dst is assumed to be safe memory
 997  * that we can store to directly because it is managed by DTrace.  As with
 998  * standard bcopy, overlapping copies are handled properly.
 999  */
1000 static void
1001 dtrace_bcopy(const void *src, void *dst, size_t len)
1002 {
1003         if (len != 0) {
1004                 uint8_t *s1 = dst;
1005                 const uint8_t *s2 = src;
1006 
1007                 if (s1 <= s2) {
1008                         do {
1009                                 *s1++ = dtrace_load8((uintptr_t)s2++);
1010                         } while (--len != 0);
1011                 } else {
1012                         s2 += len;
1013                         s1 += len;
1014 
1015                         do {
1016                                 *--s1 = dtrace_load8((uintptr_t)--s2);
1017                         } while (--len != 0);
1018                 }
1019         }
1020 }
1021 
1022 /*
1023  * Copy src to dst using safe memory accesses, up to either the specified
1024  * length, or the point that a nul byte is encountered.  The src is assumed to
1025  * be unsafe memory specified by the DIF program.  The dst is assumed to be
1026  * safe memory that we can store to directly because it is managed by DTrace.
1027  * Unlike dtrace_bcopy(), overlapping regions are not handled.
1028  */
1029 static void
1030 dtrace_strcpy(const void *src, void *dst, size_t len)
1031 {
1032         if (len != 0) {
1033                 uint8_t *s1 = dst, c;
1034                 const uint8_t *s2 = src;
1035 
1036                 do {
1037                         *s1++ = c = dtrace_load8((uintptr_t)s2++);
1038                 } while (--len != 0 && c != '\0');
1039         }
1040 }
1041 
1042 /*
1043  * Copy src to dst, deriving the size and type from the specified (BYREF)
1044  * variable type.  The src is assumed to be unsafe memory specified by the DIF
1045  * program.  The dst is assumed to be DTrace variable memory that is of the
1046  * specified type; we assume that we can store to directly.
1047  */
1048 static void
1049 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
1050 {
1051         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
1052 
1053         if (type->dtdt_kind == DIF_TYPE_STRING) {
1054                 dtrace_strcpy(src, dst, type->dtdt_size);
1055         } else {
1056                 dtrace_bcopy(src, dst, type->dtdt_size);
1057         }
1058 }
1059 
1060 /*
1061  * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
1062  * unsafe memory specified by the DIF program.  The s2 data is assumed to be
1063  * safe memory that we can access directly because it is managed by DTrace.
1064  */
1065 static int
1066 dtrace_bcmp(const void *s1, const void *s2, size_t len)
1067 {
1068         volatile uint16_t *flags;
1069 
1070         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
1071 
1072         if (s1 == s2)
1073                 return (0);
1074 
1075         if (s1 == NULL || s2 == NULL)
1076                 return (1);
1077 
1078         if (s1 != s2 && len != 0) {
1079                 const uint8_t *ps1 = s1;
1080                 const uint8_t *ps2 = s2;
1081 
1082                 do {
1083                         if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
1084                                 return (1);
1085                 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
1086         }
1087         return (0);
1088 }
1089 
1090 /*
1091  * Zero the specified region using a simple byte-by-byte loop.  Note that this
1092  * is for safe DTrace-managed memory only.
1093  */
1094 static void
1095 dtrace_bzero(void *dst, size_t len)
1096 {
1097         uchar_t *cp;
1098 
1099         for (cp = dst; len != 0; len--)
1100                 *cp++ = 0;
1101 }
1102 
1103 static void
1104 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
1105 {
1106         uint64_t result[2];
1107 
1108         result[0] = addend1[0] + addend2[0];
1109         result[1] = addend1[1] + addend2[1] +
1110             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
1111 
1112         sum[0] = result[0];
1113         sum[1] = result[1];
1114 }
1115 
1116 /*
1117  * Shift the 128-bit value in a by b. If b is positive, shift left.
1118  * If b is negative, shift right.
1119  */
1120 static void
1121 dtrace_shift_128(uint64_t *a, int b)
1122 {
1123         uint64_t mask;
1124 
1125         if (b == 0)
1126                 return;
1127 
1128         if (b < 0) {
1129                 b = -b;
1130                 if (b >= 64) {
1131                         a[0] = a[1] >> (b - 64);
1132                         a[1] = 0;
1133                 } else {
1134                         a[0] >>= b;
1135                         mask = 1LL << (64 - b);
1136                         mask -= 1;
1137                         a[0] |= ((a[1] & mask) << (64 - b));
1138                         a[1] >>= b;
1139                 }
1140         } else {
1141                 if (b >= 64) {
1142                         a[1] = a[0] << (b - 64);
1143                         a[0] = 0;
1144                 } else {
1145                         a[1] <<= b;
1146                         mask = a[0] >> (64 - b);
1147                         a[1] |= mask;
1148                         a[0] <<= b;
1149                 }
1150         }
1151 }
1152 
1153 /*
1154  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1155  * use native multiplication on those, and then re-combine into the
1156  * resulting 128-bit value.
1157  *
1158  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1159  *     hi1 * hi2 << 64 +
1160  *     hi1 * lo2 << 32 +
1161  *     hi2 * lo1 << 32 +
1162  *     lo1 * lo2
1163  */
1164 static void
1165 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
1166 {
1167         uint64_t hi1, hi2, lo1, lo2;
1168         uint64_t tmp[2];
1169 
1170         hi1 = factor1 >> 32;
1171         hi2 = factor2 >> 32;
1172 
1173         lo1 = factor1 & DT_MASK_LO;
1174         lo2 = factor2 & DT_MASK_LO;
1175 
1176         product[0] = lo1 * lo2;
1177         product[1] = hi1 * hi2;
1178 
1179         tmp[0] = hi1 * lo2;
1180         tmp[1] = 0;
1181         dtrace_shift_128(tmp, 32);
1182         dtrace_add_128(product, tmp, product);
1183 
1184         tmp[0] = hi2 * lo1;
1185         tmp[1] = 0;
1186         dtrace_shift_128(tmp, 32);
1187         dtrace_add_128(product, tmp, product);
1188 }
1189 
1190 /*
1191  * This privilege check should be used by actions and subroutines to
1192  * verify that the user credentials of the process that enabled the
1193  * invoking ECB match the target credentials
1194  */
1195 static int
1196 dtrace_priv_proc_common_user(dtrace_state_t *state)
1197 {
1198         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1199 
1200         /*
1201          * We should always have a non-NULL state cred here, since if cred
1202          * is null (anonymous tracing), we fast-path bypass this routine.
1203          */
1204         ASSERT(s_cr != NULL);
1205 
1206         if ((cr = CRED()) != NULL &&
1207             s_cr->cr_uid == cr->cr_uid &&
1208             s_cr->cr_uid == cr->cr_ruid &&
1209             s_cr->cr_uid == cr->cr_suid &&
1210             s_cr->cr_gid == cr->cr_gid &&
1211             s_cr->cr_gid == cr->cr_rgid &&
1212             s_cr->cr_gid == cr->cr_sgid)
1213                 return (1);
1214 
1215         return (0);
1216 }
1217 
1218 /*
1219  * This privilege check should be used by actions and subroutines to
1220  * verify that the zone of the process that enabled the invoking ECB
1221  * matches the target credentials
1222  */
1223 static int
1224 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1225 {
1226         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1227 
1228         /*
1229          * We should always have a non-NULL state cred here, since if cred
1230          * is null (anonymous tracing), we fast-path bypass this routine.
1231          */
1232         ASSERT(s_cr != NULL);
1233 
1234         if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone)
1235                 return (1);
1236 
1237         return (0);
1238 }
1239 
1240 /*
1241  * This privilege check should be used by actions and subroutines to
1242  * verify that the process has not setuid or changed credentials.
1243  */
1244 static int
1245 dtrace_priv_proc_common_nocd()
1246 {
1247         proc_t *proc;
1248 
1249         if ((proc = ttoproc(curthread)) != NULL &&
1250             !(proc->p_flag & SNOCD))
1251                 return (1);
1252 
1253         return (0);
1254 }
1255 
1256 static int
1257 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate)
1258 {
1259         int action = state->dts_cred.dcr_action;
1260 
1261         if (!(mstate->dtms_access & DTRACE_ACCESS_PROC))
1262                 goto bad;
1263 
1264         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
1265             dtrace_priv_proc_common_zone(state) == 0)
1266                 goto bad;
1267 
1268         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
1269             dtrace_priv_proc_common_user(state) == 0)
1270                 goto bad;
1271 
1272         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
1273             dtrace_priv_proc_common_nocd() == 0)
1274                 goto bad;
1275 
1276         return (1);
1277 
1278 bad:
1279         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1280 
1281         return (0);
1282 }
1283 
1284 static int
1285 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate)
1286 {
1287         if (mstate->dtms_access & DTRACE_ACCESS_PROC) {
1288                 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1289                         return (1);
1290 
1291                 if (dtrace_priv_proc_common_zone(state) &&
1292                     dtrace_priv_proc_common_user(state) &&
1293                     dtrace_priv_proc_common_nocd())
1294                         return (1);
1295         }
1296 
1297         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1298 
1299         return (0);
1300 }
1301 
1302 static int
1303 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate)
1304 {
1305         if ((mstate->dtms_access & DTRACE_ACCESS_PROC) &&
1306             (state->dts_cred.dcr_action & DTRACE_CRA_PROC))
1307                 return (1);
1308 
1309         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1310 
1311         return (0);
1312 }
1313 
1314 static int
1315 dtrace_priv_kernel(dtrace_state_t *state)
1316 {
1317         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1318                 return (1);
1319 
1320         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1321 
1322         return (0);
1323 }
1324 
1325 static int
1326 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1327 {
1328         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1329                 return (1);
1330 
1331         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1332 
1333         return (0);
1334 }
1335 
1336 /*
1337  * Determine if the dte_cond of the specified ECB allows for processing of
1338  * the current probe to continue.  Note that this routine may allow continued
1339  * processing, but with access(es) stripped from the mstate's dtms_access
1340  * field.
1341  */
1342 static int
1343 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
1344     dtrace_ecb_t *ecb)
1345 {
1346         dtrace_probe_t *probe = ecb->dte_probe;
1347         dtrace_provider_t *prov = probe->dtpr_provider;
1348         dtrace_pops_t *pops = &prov->dtpv_pops;
1349         int mode = DTRACE_MODE_NOPRIV_DROP;
1350 
1351         ASSERT(ecb->dte_cond);
1352 
1353         if (pops->dtps_mode != NULL) {
1354                 mode = pops->dtps_mode(prov->dtpv_arg,
1355                     probe->dtpr_id, probe->dtpr_arg);
1356 
1357                 ASSERT(mode & (DTRACE_MODE_USER | DTRACE_MODE_KERNEL));
1358                 ASSERT(mode & (DTRACE_MODE_NOPRIV_RESTRICT |
1359                     DTRACE_MODE_NOPRIV_DROP));
1360         }
1361 
1362         /*
1363          * If the dte_cond bits indicate that this consumer is only allowed to
1364          * see user-mode firings of this probe, check that the probe was fired
1365          * while in a user context.  If that's not the case, use the policy
1366          * specified by the provider to determine if we drop the probe or
1367          * merely restrict operation.
1368          */
1369         if (ecb->dte_cond & DTRACE_COND_USERMODE) {
1370                 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
1371 
1372                 if (!(mode & DTRACE_MODE_USER)) {
1373                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1374                                 return (0);
1375 
1376                         mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1377                 }
1378         }
1379 
1380         /*
1381          * This is more subtle than it looks. We have to be absolutely certain
1382          * that CRED() isn't going to change out from under us so it's only
1383          * legit to examine that structure if we're in constrained situations.
1384          * Currently, the only times we'll this check is if a non-super-user
1385          * has enabled the profile or syscall providers -- providers that
1386          * allow visibility of all processes. For the profile case, the check
1387          * above will ensure that we're examining a user context.
1388          */
1389         if (ecb->dte_cond & DTRACE_COND_OWNER) {
1390                 cred_t *cr;
1391                 cred_t *s_cr = state->dts_cred.dcr_cred;
1392                 proc_t *proc;
1393 
1394                 ASSERT(s_cr != NULL);
1395 
1396                 if ((cr = CRED()) == NULL ||
1397                     s_cr->cr_uid != cr->cr_uid ||
1398                     s_cr->cr_uid != cr->cr_ruid ||
1399                     s_cr->cr_uid != cr->cr_suid ||
1400                     s_cr->cr_gid != cr->cr_gid ||
1401                     s_cr->cr_gid != cr->cr_rgid ||
1402                     s_cr->cr_gid != cr->cr_sgid ||
1403                     (proc = ttoproc(curthread)) == NULL ||
1404                     (proc->p_flag & SNOCD)) {
1405                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1406                                 return (0);
1407 
1408                         mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
1409                 }
1410         }
1411 
1412         /*
1413          * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1414          * in our zone, check to see if our mode policy is to restrict rather
1415          * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1416          * and DTRACE_ACCESS_ARGS
1417          */
1418         if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
1419                 cred_t *cr;
1420                 cred_t *s_cr = state->dts_cred.dcr_cred;
1421 
1422                 ASSERT(s_cr != NULL);
1423 
1424                 if ((cr = CRED()) == NULL ||
1425                     s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
1426                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1427                                 return (0);
1428 
1429                         mstate->dtms_access &=
1430                             ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
1431                 }
1432         }
1433 
1434         /*
1435          * By merits of being in this code path at all, we have limited
1436          * privileges.  If the provider has indicated that limited privileges
1437          * are to denote restricted operation, strip off the ability to access
1438          * arguments.
1439          */
1440         if (mode & DTRACE_MODE_LIMITEDPRIV_RESTRICT)
1441                 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1442 
1443         return (1);
1444 }
1445 
1446 /*
1447  * Note:  not called from probe context.  This function is called
1448  * asynchronously (and at a regular interval) from outside of probe context to
1449  * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
1450  * cleaning is explained in detail in <sys/dtrace_impl.h>.
1451  */
1452 void
1453 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
1454 {
1455         dtrace_dynvar_t *dirty;
1456         dtrace_dstate_percpu_t *dcpu;
1457         dtrace_dynvar_t **rinsep;
1458         int i, j, work = 0;
1459 
1460         for (i = 0; i < NCPU; i++) {
1461                 dcpu = &dstate->dtds_percpu[i];
1462                 rinsep = &dcpu->dtdsc_rinsing;
1463 
1464                 /*
1465                  * If the dirty list is NULL, there is no dirty work to do.
1466                  */
1467                 if (dcpu->dtdsc_dirty == NULL)
1468                         continue;
1469 
1470                 if (dcpu->dtdsc_rinsing != NULL) {
1471                         /*
1472                          * If the rinsing list is non-NULL, then it is because
1473                          * this CPU was selected to accept another CPU's
1474                          * dirty list -- and since that time, dirty buffers
1475                          * have accumulated.  This is a highly unlikely
1476                          * condition, but we choose to ignore the dirty
1477                          * buffers -- they'll be picked up a future cleanse.
1478                          */
1479                         continue;
1480                 }
1481 
1482                 if (dcpu->dtdsc_clean != NULL) {
1483                         /*
1484                          * If the clean list is non-NULL, then we're in a
1485                          * situation where a CPU has done deallocations (we
1486                          * have a non-NULL dirty list) but no allocations (we
1487                          * also have a non-NULL clean list).  We can't simply
1488                          * move the dirty list into the clean list on this
1489                          * CPU, yet we also don't want to allow this condition
1490                          * to persist, lest a short clean list prevent a
1491                          * massive dirty list from being cleaned (which in
1492                          * turn could lead to otherwise avoidable dynamic
1493                          * drops).  To deal with this, we look for some CPU
1494                          * with a NULL clean list, NULL dirty list, and NULL
1495                          * rinsing list -- and then we borrow this CPU to
1496                          * rinse our dirty list.
1497                          */
1498                         for (j = 0; j < NCPU; j++) {
1499                                 dtrace_dstate_percpu_t *rinser;
1500 
1501                                 rinser = &dstate->dtds_percpu[j];
1502 
1503                                 if (rinser->dtdsc_rinsing != NULL)
1504                                         continue;
1505 
1506                                 if (rinser->dtdsc_dirty != NULL)
1507                                         continue;
1508 
1509                                 if (rinser->dtdsc_clean != NULL)
1510                                         continue;
1511 
1512                                 rinsep = &rinser->dtdsc_rinsing;
1513                                 break;
1514                         }
1515 
1516                         if (j == NCPU) {
1517                                 /*
1518                                  * We were unable to find another CPU that
1519                                  * could accept this dirty list -- we are
1520                                  * therefore unable to clean it now.
1521                                  */
1522                                 dtrace_dynvar_failclean++;
1523                                 continue;
1524                         }
1525                 }
1526 
1527                 work = 1;
1528 
1529                 /*
1530                  * Atomically move the dirty list aside.
1531                  */
1532                 do {
1533                         dirty = dcpu->dtdsc_dirty;
1534 
1535                         /*
1536                          * Before we zap the dirty list, set the rinsing list.
1537                          * (This allows for a potential assertion in
1538                          * dtrace_dynvar():  if a free dynamic variable appears
1539                          * on a hash chain, either the dirty list or the
1540                          * rinsing list for some CPU must be non-NULL.)
1541                          */
1542                         *rinsep = dirty;
1543                         dtrace_membar_producer();
1544                 } while (dtrace_casptr(&dcpu->dtdsc_dirty,
1545                     dirty, NULL) != dirty);
1546         }
1547 
1548         if (!work) {
1549                 /*
1550                  * We have no work to do; we can simply return.
1551                  */
1552                 return;
1553         }
1554 
1555         dtrace_sync();
1556 
1557         for (i = 0; i < NCPU; i++) {
1558                 dcpu = &dstate->dtds_percpu[i];
1559 
1560                 if (dcpu->dtdsc_rinsing == NULL)
1561                         continue;
1562 
1563                 /*
1564                  * We are now guaranteed that no hash chain contains a pointer
1565                  * into this dirty list; we can make it clean.
1566                  */
1567                 ASSERT(dcpu->dtdsc_clean == NULL);
1568                 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
1569                 dcpu->dtdsc_rinsing = NULL;
1570         }
1571 
1572         /*
1573          * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1574          * sure that all CPUs have seen all of the dtdsc_clean pointers.
1575          * This prevents a race whereby a CPU incorrectly decides that
1576          * the state should be something other than DTRACE_DSTATE_CLEAN
1577          * after dtrace_dynvar_clean() has completed.
1578          */
1579         dtrace_sync();
1580 
1581         dstate->dtds_state = DTRACE_DSTATE_CLEAN;
1582 }
1583 
1584 /*
1585  * Depending on the value of the op parameter, this function looks-up,
1586  * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
1587  * allocation is requested, this function will return a pointer to a
1588  * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1589  * variable can be allocated.  If NULL is returned, the appropriate counter
1590  * will be incremented.
1591  */
1592 dtrace_dynvar_t *
1593 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
1594     dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
1595     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
1596 {
1597         uint64_t hashval = DTRACE_DYNHASH_VALID;
1598         dtrace_dynhash_t *hash = dstate->dtds_hash;
1599         dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
1600         processorid_t me = CPU->cpu_id, cpu = me;
1601         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
1602         size_t bucket, ksize;
1603         size_t chunksize = dstate->dtds_chunksize;
1604         uintptr_t kdata, lock, nstate;
1605         uint_t i;
1606 
1607         ASSERT(nkeys != 0);
1608 
1609         /*
1610          * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
1611          * algorithm.  For the by-value portions, we perform the algorithm in
1612          * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
1613          * bit, and seems to have only a minute effect on distribution.  For
1614          * the by-reference data, we perform "One-at-a-time" iterating (safely)
1615          * over each referenced byte.  It's painful to do this, but it's much
1616          * better than pathological hash distribution.  The efficacy of the
1617          * hashing algorithm (and a comparison with other algorithms) may be
1618          * found by running the ::dtrace_dynstat MDB dcmd.
1619          */
1620         for (i = 0; i < nkeys; i++) {
1621                 if (key[i].dttk_size == 0) {
1622                         uint64_t val = key[i].dttk_value;
1623 
1624                         hashval += (val >> 48) & 0xffff;
1625                         hashval += (hashval << 10);
1626                         hashval ^= (hashval >> 6);
1627 
1628                         hashval += (val >> 32) & 0xffff;
1629                         hashval += (hashval << 10);
1630                         hashval ^= (hashval >> 6);
1631 
1632                         hashval += (val >> 16) & 0xffff;
1633                         hashval += (hashval << 10);
1634                         hashval ^= (hashval >> 6);
1635 
1636                         hashval += val & 0xffff;
1637                         hashval += (hashval << 10);
1638                         hashval ^= (hashval >> 6);
1639                 } else {
1640                         /*
1641                          * This is incredibly painful, but it beats the hell
1642                          * out of the alternative.
1643                          */
1644                         uint64_t j, size = key[i].dttk_size;
1645                         uintptr_t base = (uintptr_t)key[i].dttk_value;
1646 
1647                         if (!dtrace_canload(base, size, mstate, vstate))
1648                                 break;
1649 
1650                         for (j = 0; j < size; j++) {
1651                                 hashval += dtrace_load8(base + j);
1652                                 hashval += (hashval << 10);
1653                                 hashval ^= (hashval >> 6);
1654                         }
1655                 }
1656         }
1657 
1658         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
1659                 return (NULL);
1660 
1661         hashval += (hashval << 3);
1662         hashval ^= (hashval >> 11);
1663         hashval += (hashval << 15);
1664 
1665         /*
1666          * There is a remote chance (ideally, 1 in 2^31) that our hashval
1667          * comes out to be one of our two sentinel hash values.  If this
1668          * actually happens, we set the hashval to be a value known to be a
1669          * non-sentinel value.
1670          */
1671         if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
1672                 hashval = DTRACE_DYNHASH_VALID;
1673 
1674         /*
1675          * Yes, it's painful to do a divide here.  If the cycle count becomes
1676          * important here, tricks can be pulled to reduce it.  (However, it's
1677          * critical that hash collisions be kept to an absolute minimum;
1678          * they're much more painful than a divide.)  It's better to have a
1679          * solution that generates few collisions and still keeps things
1680          * relatively simple.
1681          */
1682         bucket = hashval % dstate->dtds_hashsize;
1683 
1684         if (op == DTRACE_DYNVAR_DEALLOC) {
1685                 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1686 
1687                 for (;;) {
1688                         while ((lock = *lockp) & 1)
1689                                 continue;
1690 
1691                         if (dtrace_casptr((void *)lockp,
1692                             (void *)lock, (void *)(lock + 1)) == (void *)lock)
1693                                 break;
1694                 }
1695 
1696                 dtrace_membar_producer();
1697         }
1698 
1699 top:
1700         prev = NULL;
1701         lock = hash[bucket].dtdh_lock;
1702 
1703         dtrace_membar_consumer();
1704 
1705         start = hash[bucket].dtdh_chain;
1706         ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
1707             start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
1708             op != DTRACE_DYNVAR_DEALLOC));
1709 
1710         for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1711                 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1712                 dtrace_key_t *dkey = &dtuple->dtt_key[0];
1713 
1714                 if (dvar->dtdv_hashval != hashval) {
1715                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
1716                                 /*
1717                                  * We've reached the sink, and therefore the
1718                                  * end of the hash chain; we can kick out of
1719                                  * the loop knowing that we have seen a valid
1720                                  * snapshot of state.
1721                                  */
1722                                 ASSERT(dvar->dtdv_next == NULL);
1723                                 ASSERT(dvar == &dtrace_dynhash_sink);
1724                                 break;
1725                         }
1726 
1727                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
1728                                 /*
1729                                  * We've gone off the rails:  somewhere along
1730                                  * the line, one of the members of this hash
1731                                  * chain was deleted.  Note that we could also
1732                                  * detect this by simply letting this loop run
1733                                  * to completion, as we would eventually hit
1734                                  * the end of the dirty list.  However, we
1735                                  * want to avoid running the length of the
1736                                  * dirty list unnecessarily (it might be quite
1737                                  * long), so we catch this as early as
1738                                  * possible by detecting the hash marker.  In
1739                                  * this case, we simply set dvar to NULL and
1740                                  * break; the conditional after the loop will
1741                                  * send us back to top.
1742                                  */
1743                                 dvar = NULL;
1744                                 break;
1745                         }
1746 
1747                         goto next;
1748                 }
1749 
1750                 if (dtuple->dtt_nkeys != nkeys)
1751                         goto next;
1752 
1753                 for (i = 0; i < nkeys; i++, dkey++) {
1754                         if (dkey->dttk_size != key[i].dttk_size)
1755                                 goto next; /* size or type mismatch */
1756 
1757                         if (dkey->dttk_size != 0) {
1758                                 if (dtrace_bcmp(
1759                                     (void *)(uintptr_t)key[i].dttk_value,
1760                                     (void *)(uintptr_t)dkey->dttk_value,
1761                                     dkey->dttk_size))
1762                                         goto next;
1763                         } else {
1764                                 if (dkey->dttk_value != key[i].dttk_value)
1765                                         goto next;
1766                         }
1767                 }
1768 
1769                 if (op != DTRACE_DYNVAR_DEALLOC)
1770                         return (dvar);
1771 
1772                 ASSERT(dvar->dtdv_next == NULL ||
1773                     dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
1774 
1775                 if (prev != NULL) {
1776                         ASSERT(hash[bucket].dtdh_chain != dvar);
1777                         ASSERT(start != dvar);
1778                         ASSERT(prev->dtdv_next == dvar);
1779                         prev->dtdv_next = dvar->dtdv_next;
1780                 } else {
1781                         if (dtrace_casptr(&hash[bucket].dtdh_chain,
1782                             start, dvar->dtdv_next) != start) {
1783                                 /*
1784                                  * We have failed to atomically swing the
1785                                  * hash table head pointer, presumably because
1786                                  * of a conflicting allocation on another CPU.
1787                                  * We need to reread the hash chain and try
1788                                  * again.
1789                                  */
1790                                 goto top;
1791                         }
1792                 }
1793 
1794                 dtrace_membar_producer();
1795 
1796                 /*
1797                  * Now set the hash value to indicate that it's free.
1798                  */
1799                 ASSERT(hash[bucket].dtdh_chain != dvar);
1800                 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1801 
1802                 dtrace_membar_producer();
1803 
1804                 /*
1805                  * Set the next pointer to point at the dirty list, and
1806                  * atomically swing the dirty pointer to the newly freed dvar.
1807                  */
1808                 do {
1809                         next = dcpu->dtdsc_dirty;
1810                         dvar->dtdv_next = next;
1811                 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1812 
1813                 /*
1814                  * Finally, unlock this hash bucket.
1815                  */
1816                 ASSERT(hash[bucket].dtdh_lock == lock);
1817                 ASSERT(lock & 1);
1818                 hash[bucket].dtdh_lock++;
1819 
1820                 return (NULL);
1821 next:
1822                 prev = dvar;
1823                 continue;
1824         }
1825 
1826         if (dvar == NULL) {
1827                 /*
1828                  * If dvar is NULL, it is because we went off the rails:
1829                  * one of the elements that we traversed in the hash chain
1830                  * was deleted while we were traversing it.  In this case,
1831                  * we assert that we aren't doing a dealloc (deallocs lock
1832                  * the hash bucket to prevent themselves from racing with
1833                  * one another), and retry the hash chain traversal.
1834                  */
1835                 ASSERT(op != DTRACE_DYNVAR_DEALLOC);
1836                 goto top;
1837         }
1838 
1839         if (op != DTRACE_DYNVAR_ALLOC) {
1840                 /*
1841                  * If we are not to allocate a new variable, we want to
1842                  * return NULL now.  Before we return, check that the value
1843                  * of the lock word hasn't changed.  If it has, we may have
1844                  * seen an inconsistent snapshot.
1845                  */
1846                 if (op == DTRACE_DYNVAR_NOALLOC) {
1847                         if (hash[bucket].dtdh_lock != lock)
1848                                 goto top;
1849                 } else {
1850                         ASSERT(op == DTRACE_DYNVAR_DEALLOC);
1851                         ASSERT(hash[bucket].dtdh_lock == lock);
1852                         ASSERT(lock & 1);
1853                         hash[bucket].dtdh_lock++;
1854                 }
1855 
1856                 return (NULL);
1857         }
1858 
1859         /*
1860          * We need to allocate a new dynamic variable.  The size we need is the
1861          * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
1862          * size of any auxiliary key data (rounded up to 8-byte alignment) plus
1863          * the size of any referred-to data (dsize).  We then round the final
1864          * size up to the chunksize for allocation.
1865          */
1866         for (ksize = 0, i = 0; i < nkeys; i++)
1867                 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
1868 
1869         /*
1870          * This should be pretty much impossible, but could happen if, say,
1871          * strange DIF specified the tuple.  Ideally, this should be an
1872          * assertion and not an error condition -- but that requires that the
1873          * chunksize calculation in dtrace_difo_chunksize() be absolutely
1874          * bullet-proof.  (That is, it must not be able to be fooled by
1875          * malicious DIF.)  Given the lack of backwards branches in DIF,
1876          * solving this would presumably not amount to solving the Halting
1877          * Problem -- but it still seems awfully hard.
1878          */
1879         if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
1880             ksize + dsize > chunksize) {
1881                 dcpu->dtdsc_drops++;
1882                 return (NULL);
1883         }
1884 
1885         nstate = DTRACE_DSTATE_EMPTY;
1886 
1887         do {
1888 retry:
1889                 free = dcpu->dtdsc_free;
1890 
1891                 if (free == NULL) {
1892                         dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
1893                         void *rval;
1894 
1895                         if (clean == NULL) {
1896                                 /*
1897                                  * We're out of dynamic variable space on
1898                                  * this CPU.  Unless we have tried all CPUs,
1899                                  * we'll try to allocate from a different
1900                                  * CPU.
1901                                  */
1902                                 switch (dstate->dtds_state) {
1903                                 case DTRACE_DSTATE_CLEAN: {
1904                                         void *sp = &dstate->dtds_state;
1905 
1906                                         if (++cpu >= NCPU)
1907                                                 cpu = 0;
1908 
1909                                         if (dcpu->dtdsc_dirty != NULL &&
1910                                             nstate == DTRACE_DSTATE_EMPTY)
1911                                                 nstate = DTRACE_DSTATE_DIRTY;
1912 
1913                                         if (dcpu->dtdsc_rinsing != NULL)
1914                                                 nstate = DTRACE_DSTATE_RINSING;
1915 
1916                                         dcpu = &dstate->dtds_percpu[cpu];
1917 
1918                                         if (cpu != me)
1919                                                 goto retry;
1920 
1921                                         (void) dtrace_cas32(sp,
1922                                             DTRACE_DSTATE_CLEAN, nstate);
1923 
1924                                         /*
1925                                          * To increment the correct bean
1926                                          * counter, take another lap.
1927                                          */
1928                                         goto retry;
1929                                 }
1930 
1931                                 case DTRACE_DSTATE_DIRTY:
1932                                         dcpu->dtdsc_dirty_drops++;
1933                                         break;
1934 
1935                                 case DTRACE_DSTATE_RINSING:
1936                                         dcpu->dtdsc_rinsing_drops++;
1937                                         break;
1938 
1939                                 case DTRACE_DSTATE_EMPTY:
1940                                         dcpu->dtdsc_drops++;
1941                                         break;
1942                                 }
1943 
1944                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
1945                                 return (NULL);
1946                         }
1947 
1948                         /*
1949                          * The clean list appears to be non-empty.  We want to
1950                          * move the clean list to the free list; we start by
1951                          * moving the clean pointer aside.
1952                          */
1953                         if (dtrace_casptr(&dcpu->dtdsc_clean,
1954                             clean, NULL) != clean) {
1955                                 /*
1956                                  * We are in one of two situations:
1957                                  *
1958                                  *  (a) The clean list was switched to the
1959                                  *      free list by another CPU.
1960                                  *
1961                                  *  (b) The clean list was added to by the
1962                                  *      cleansing cyclic.
1963                                  *
1964                                  * In either of these situations, we can
1965                                  * just reattempt the free list allocation.
1966                                  */
1967                                 goto retry;
1968                         }
1969 
1970                         ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
1971 
1972                         /*
1973                          * Now we'll move the clean list to our free list.
1974                          * It's impossible for this to fail:  the only way
1975                          * the free list can be updated is through this
1976                          * code path, and only one CPU can own the clean list.
1977                          * Thus, it would only be possible for this to fail if
1978                          * this code were racing with dtrace_dynvar_clean().
1979                          * (That is, if dtrace_dynvar_clean() updated the clean
1980                          * list, and we ended up racing to update the free
1981                          * list.)  This race is prevented by the dtrace_sync()
1982                          * in dtrace_dynvar_clean() -- which flushes the
1983                          * owners of the clean lists out before resetting
1984                          * the clean lists.
1985                          */
1986                         dcpu = &dstate->dtds_percpu[me];
1987                         rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
1988                         ASSERT(rval == NULL);
1989                         goto retry;
1990                 }
1991 
1992                 dvar = free;
1993                 new_free = dvar->dtdv_next;
1994         } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
1995 
1996         /*
1997          * We have now allocated a new chunk.  We copy the tuple keys into the
1998          * tuple array and copy any referenced key data into the data space
1999          * following the tuple array.  As we do this, we relocate dttk_value
2000          * in the final tuple to point to the key data address in the chunk.
2001          */
2002         kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
2003         dvar->dtdv_data = (void *)(kdata + ksize);
2004         dvar->dtdv_tuple.dtt_nkeys = nkeys;
2005 
2006         for (i = 0; i < nkeys; i++) {
2007                 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
2008                 size_t kesize = key[i].dttk_size;
2009 
2010                 if (kesize != 0) {
2011                         dtrace_bcopy(
2012                             (const void *)(uintptr_t)key[i].dttk_value,
2013                             (void *)kdata, kesize);
2014                         dkey->dttk_value = kdata;
2015                         kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
2016                 } else {
2017                         dkey->dttk_value = key[i].dttk_value;
2018                 }
2019 
2020                 dkey->dttk_size = kesize;
2021         }
2022 
2023         ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
2024         dvar->dtdv_hashval = hashval;
2025         dvar->dtdv_next = start;
2026 
2027         if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
2028                 return (dvar);
2029 
2030         /*
2031          * The cas has failed.  Either another CPU is adding an element to
2032          * this hash chain, or another CPU is deleting an element from this
2033          * hash chain.  The simplest way to deal with both of these cases
2034          * (though not necessarily the most efficient) is to free our
2035          * allocated block and tail-call ourselves.  Note that the free is
2036          * to the dirty list and _not_ to the free list.  This is to prevent
2037          * races with allocators, above.
2038          */
2039         dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
2040 
2041         dtrace_membar_producer();
2042 
2043         do {
2044                 free = dcpu->dtdsc_dirty;
2045                 dvar->dtdv_next = free;
2046         } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
2047 
2048         return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
2049 }
2050 
2051 /*ARGSUSED*/
2052 static void
2053 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
2054 {
2055         if ((int64_t)nval < (int64_t)*oval)
2056                 *oval = nval;
2057 }
2058 
2059 /*ARGSUSED*/
2060 static void
2061 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
2062 {
2063         if ((int64_t)nval > (int64_t)*oval)
2064                 *oval = nval;
2065 }
2066 
2067 static void
2068 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
2069 {
2070         int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
2071         int64_t val = (int64_t)nval;
2072 
2073         if (val < 0) {
2074                 for (i = 0; i < zero; i++) {
2075                         if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
2076                                 quanta[i] += incr;
2077                                 return;
2078                         }
2079                 }
2080         } else {
2081                 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
2082                         if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
2083                                 quanta[i - 1] += incr;
2084                                 return;
2085                         }
2086                 }
2087 
2088                 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
2089                 return;
2090         }
2091 
2092         ASSERT(0);
2093 }
2094 
2095 static void
2096 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
2097 {
2098         uint64_t arg = *lquanta++;
2099         int32_t base = DTRACE_LQUANTIZE_BASE(arg);
2100         uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
2101         uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
2102         int32_t val = (int32_t)nval, level;
2103 
2104         ASSERT(step != 0);
2105         ASSERT(levels != 0);
2106 
2107         if (val < base) {
2108                 /*
2109                  * This is an underflow.
2110                  */
2111                 lquanta[0] += incr;
2112                 return;
2113         }
2114 
2115         level = (val - base) / step;
2116 
2117         if (level < levels) {
2118                 lquanta[level + 1] += incr;
2119                 return;
2120         }
2121 
2122         /*
2123          * This is an overflow.
2124          */
2125         lquanta[levels + 1] += incr;
2126 }
2127 
2128 static int
2129 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
2130     uint16_t high, uint16_t nsteps, int64_t value)
2131 {
2132         int64_t this = 1, last, next;
2133         int base = 1, order;
2134 
2135         ASSERT(factor <= nsteps);
2136         ASSERT(nsteps % factor == 0);
2137 
2138         for (order = 0; order < low; order++)
2139                 this *= factor;
2140 
2141         /*
2142          * If our value is less than our factor taken to the power of the
2143          * low order of magnitude, it goes into the zeroth bucket.
2144          */
2145         if (value < (last = this))
2146                 return (0);
2147 
2148         for (this *= factor; order <= high; order++) {
2149                 int nbuckets = this > nsteps ? nsteps : this;
2150 
2151                 if ((next = this * factor) < this) {
2152                         /*
2153                          * We should not generally get log/linear quantizations
2154                          * with a high magnitude that allows 64-bits to
2155                          * overflow, but we nonetheless protect against this
2156                          * by explicitly checking for overflow, and clamping
2157                          * our value accordingly.
2158                          */
2159                         value = this - 1;
2160                 }
2161 
2162                 if (value < this) {
2163                         /*
2164                          * If our value lies within this order of magnitude,
2165                          * determine its position by taking the offset within
2166                          * the order of magnitude, dividing by the bucket
2167                          * width, and adding to our (accumulated) base.
2168                          */
2169                         return (base + (value - last) / (this / nbuckets));
2170                 }
2171 
2172                 base += nbuckets - (nbuckets / factor);
2173                 last = this;
2174                 this = next;
2175         }
2176 
2177         /*
2178          * Our value is greater than or equal to our factor taken to the
2179          * power of one plus the high magnitude -- return the top bucket.
2180          */
2181         return (base);
2182 }
2183 
2184 static void
2185 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
2186 {
2187         uint64_t arg = *llquanta++;
2188         uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2189         uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2190         uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2191         uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2192 
2193         llquanta[dtrace_aggregate_llquantize_bucket(factor,
2194             low, high, nsteps, nval)] += incr;
2195 }
2196 
2197 /*ARGSUSED*/
2198 static void
2199 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
2200 {
2201         data[0]++;
2202         data[1] += nval;
2203 }
2204 
2205 /*ARGSUSED*/
2206 static void
2207 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
2208 {
2209         int64_t snval = (int64_t)nval;
2210         uint64_t tmp[2];
2211 
2212         data[0]++;
2213         data[1] += nval;
2214 
2215         /*
2216          * What we want to say here is:
2217          *
2218          * data[2] += nval * nval;
2219          *
2220          * But given that nval is 64-bit, we could easily overflow, so
2221          * we do this as 128-bit arithmetic.
2222          */
2223         if (snval < 0)
2224                 snval = -snval;
2225 
2226         dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
2227         dtrace_add_128(data + 2, tmp, data + 2);
2228 }
2229 
2230 /*ARGSUSED*/
2231 static void
2232 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
2233 {
2234         *oval = *oval + 1;
2235 }
2236 
2237 /*ARGSUSED*/
2238 static void
2239 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
2240 {
2241         *oval += nval;
2242 }
2243 
2244 /*
2245  * Aggregate given the tuple in the principal data buffer, and the aggregating
2246  * action denoted by the specified dtrace_aggregation_t.  The aggregation
2247  * buffer is specified as the buf parameter.  This routine does not return
2248  * failure; if there is no space in the aggregation buffer, the data will be
2249  * dropped, and a corresponding counter incremented.
2250  */
2251 static void
2252 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
2253     intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
2254 {
2255         dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
2256         uint32_t i, ndx, size, fsize;
2257         uint32_t align = sizeof (uint64_t) - 1;
2258         dtrace_aggbuffer_t *agb;
2259         dtrace_aggkey_t *key;
2260         uint32_t hashval = 0, limit, isstr;
2261         caddr_t tomax, data, kdata;
2262         dtrace_actkind_t action;
2263         dtrace_action_t *act;
2264         uintptr_t offs;
2265 
2266         if (buf == NULL)
2267                 return;
2268 
2269         if (!agg->dtag_hasarg) {
2270                 /*
2271                  * Currently, only quantize() and lquantize() take additional
2272                  * arguments, and they have the same semantics:  an increment
2273                  * value that defaults to 1 when not present.  If additional
2274                  * aggregating actions take arguments, the setting of the
2275                  * default argument value will presumably have to become more
2276                  * sophisticated...
2277                  */
2278                 arg = 1;
2279         }
2280 
2281         action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
2282         size = rec->dtrd_offset - agg->dtag_base;
2283         fsize = size + rec->dtrd_size;
2284 
2285         ASSERT(dbuf->dtb_tomax != NULL);
2286         data = dbuf->dtb_tomax + offset + agg->dtag_base;
2287 
2288         if ((tomax = buf->dtb_tomax) == NULL) {
2289                 dtrace_buffer_drop(buf);
2290                 return;
2291         }
2292 
2293         /*
2294          * The metastructure is always at the bottom of the buffer.
2295          */
2296         agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
2297             sizeof (dtrace_aggbuffer_t));
2298 
2299         if (buf->dtb_offset == 0) {
2300                 /*
2301                  * We just kludge up approximately 1/8th of the size to be
2302                  * buckets.  If this guess ends up being routinely
2303                  * off-the-mark, we may need to dynamically readjust this
2304                  * based on past performance.
2305                  */
2306                 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
2307 
2308                 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
2309                     (uintptr_t)tomax || hashsize == 0) {
2310                         /*
2311                          * We've been given a ludicrously small buffer;
2312                          * increment our drop count and leave.
2313                          */
2314                         dtrace_buffer_drop(buf);
2315                         return;
2316                 }
2317 
2318                 /*
2319                  * And now, a pathetic attempt to try to get a an odd (or
2320                  * perchance, a prime) hash size for better hash distribution.
2321                  */
2322                 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
2323                         hashsize -= DTRACE_AGGHASHSIZE_SLEW;
2324 
2325                 agb->dtagb_hashsize = hashsize;
2326                 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
2327                     agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
2328                 agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
2329 
2330                 for (i = 0; i < agb->dtagb_hashsize; i++)
2331                         agb->dtagb_hash[i] = NULL;
2332         }
2333 
2334         ASSERT(agg->dtag_first != NULL);
2335         ASSERT(agg->dtag_first->dta_intuple);
2336 
2337         /*
2338          * Calculate the hash value based on the key.  Note that we _don't_
2339          * include the aggid in the hashing (but we will store it as part of
2340          * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
2341          * algorithm: a simple, quick algorithm that has no known funnels, and
2342          * gets good distribution in practice.  The efficacy of the hashing
2343          * algorithm (and a comparison with other algorithms) may be found by
2344          * running the ::dtrace_aggstat MDB dcmd.
2345          */
2346         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2347                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2348                 limit = i + act->dta_rec.dtrd_size;
2349                 ASSERT(limit <= size);
2350                 isstr = DTRACEACT_ISSTRING(act);
2351 
2352                 for (; i < limit; i++) {
2353                         hashval += data[i];
2354                         hashval += (hashval << 10);
2355                         hashval ^= (hashval >> 6);
2356 
2357                         if (isstr && data[i] == '\0')
2358                                 break;
2359                 }
2360         }
2361 
2362         hashval += (hashval << 3);
2363         hashval ^= (hashval >> 11);
2364         hashval += (hashval << 15);
2365 
2366         /*
2367          * Yes, the divide here is expensive -- but it's generally the least
2368          * of the performance issues given the amount of data that we iterate
2369          * over to compute hash values, compare data, etc.
2370          */
2371         ndx = hashval % agb->dtagb_hashsize;
2372 
2373         for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
2374                 ASSERT((caddr_t)key >= tomax);
2375                 ASSERT((caddr_t)key < tomax + buf->dtb_size);
2376 
2377                 if (hashval != key->dtak_hashval || key->dtak_size != size)
2378                         continue;
2379 
2380                 kdata = key->dtak_data;
2381                 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
2382 
2383                 for (act = agg->dtag_first; act->dta_intuple;
2384                     act = act->dta_next) {
2385                         i = act->dta_rec.dtrd_offset - agg->dtag_base;
2386                         limit = i + act->dta_rec.dtrd_size;
2387                         ASSERT(limit <= size);
2388                         isstr = DTRACEACT_ISSTRING(act);
2389 
2390                         for (; i < limit; i++) {
2391                                 if (kdata[i] != data[i])
2392                                         goto next;
2393 
2394                                 if (isstr && data[i] == '\0')
2395                                         break;
2396                         }
2397                 }
2398 
2399                 if (action != key->dtak_action) {
2400                         /*
2401                          * We are aggregating on the same value in the same
2402                          * aggregation with two different aggregating actions.
2403                          * (This should have been picked up in the compiler,
2404                          * so we may be dealing with errant or devious DIF.)
2405                          * This is an error condition; we indicate as much,
2406                          * and return.
2407                          */
2408                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2409                         return;
2410                 }
2411 
2412                 /*
2413                  * This is a hit:  we need to apply the aggregator to
2414                  * the value at this key.
2415                  */
2416                 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
2417                 return;
2418 next:
2419                 continue;
2420         }
2421 
2422         /*
2423          * We didn't find it.  We need to allocate some zero-filled space,
2424          * link it into the hash table appropriately, and apply the aggregator
2425          * to the (zero-filled) value.
2426          */
2427         offs = buf->dtb_offset;
2428         while (offs & (align - 1))
2429                 offs += sizeof (uint32_t);
2430 
2431         /*
2432          * If we don't have enough room to both allocate a new key _and_
2433          * its associated data, increment the drop count and return.
2434          */
2435         if ((uintptr_t)tomax + offs + fsize >
2436             agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
2437                 dtrace_buffer_drop(buf);
2438                 return;
2439         }
2440 
2441         /*CONSTCOND*/
2442         ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
2443         key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
2444         agb->dtagb_free -= sizeof (dtrace_aggkey_t);
2445 
2446         key->dtak_data = kdata = tomax + offs;
2447         buf->dtb_offset = offs + fsize;
2448 
2449         /*
2450          * Now copy the data across.
2451          */
2452         *((dtrace_aggid_t *)kdata) = agg->dtag_id;
2453 
2454         for (i = sizeof (dtrace_aggid_t); i < size; i++)
2455                 kdata[i] = data[i];
2456 
2457         /*
2458          * Because strings are not zeroed out by default, we need to iterate
2459          * looking for actions that store strings, and we need to explicitly
2460          * pad these strings out with zeroes.
2461          */
2462         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2463                 int nul;
2464 
2465                 if (!DTRACEACT_ISSTRING(act))
2466                         continue;
2467 
2468                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2469                 limit = i + act->dta_rec.dtrd_size;
2470                 ASSERT(limit <= size);
2471 
2472                 for (nul = 0; i < limit; i++) {
2473                         if (nul) {
2474                                 kdata[i] = '\0';
2475                                 continue;
2476                         }
2477 
2478                         if (data[i] != '\0')
2479                                 continue;
2480 
2481                         nul = 1;
2482                 }
2483         }
2484 
2485         for (i = size; i < fsize; i++)
2486                 kdata[i] = 0;
2487 
2488         key->dtak_hashval = hashval;
2489         key->dtak_size = size;
2490         key->dtak_action = action;
2491         key->dtak_next = agb->dtagb_hash[ndx];
2492         agb->dtagb_hash[ndx] = key;
2493 
2494         /*
2495          * Finally, apply the aggregator.
2496          */
2497         *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
2498         agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
2499 }
2500 
2501 /*
2502  * Given consumer state, this routine finds a speculation in the INACTIVE
2503  * state and transitions it into the ACTIVE state.  If there is no speculation
2504  * in the INACTIVE state, 0 is returned.  In this case, no error counter is
2505  * incremented -- it is up to the caller to take appropriate action.
2506  */
2507 static int
2508 dtrace_speculation(dtrace_state_t *state)
2509 {
2510         int i = 0;
2511         dtrace_speculation_state_t current;
2512         uint32_t *stat = &state->dts_speculations_unavail, count;
2513 
2514         while (i < state->dts_nspeculations) {
2515                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2516 
2517                 current = spec->dtsp_state;
2518 
2519                 if (current != DTRACESPEC_INACTIVE) {
2520                         if (current == DTRACESPEC_COMMITTINGMANY ||
2521                             current == DTRACESPEC_COMMITTING ||
2522                             current == DTRACESPEC_DISCARDING)
2523                                 stat = &state->dts_speculations_busy;
2524                         i++;
2525                         continue;
2526                 }
2527 
2528                 if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2529                     current, DTRACESPEC_ACTIVE) == current)
2530                         return (i + 1);
2531         }
2532 
2533         /*
2534          * We couldn't find a speculation.  If we found as much as a single
2535          * busy speculation buffer, we'll attribute this failure as "busy"
2536          * instead of "unavail".
2537          */
2538         do {
2539                 count = *stat;
2540         } while (dtrace_cas32(stat, count, count + 1) != count);
2541 
2542         return (0);
2543 }
2544 
2545 /*
2546  * This routine commits an active speculation.  If the specified speculation
2547  * is not in a valid state to perform a commit(), this routine will silently do
2548  * nothing.  The state of the specified speculation is transitioned according
2549  * to the state transition diagram outlined in <sys/dtrace_impl.h>
2550  */
2551 static void
2552 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2553     dtrace_specid_t which)
2554 {
2555         dtrace_speculation_t *spec;
2556         dtrace_buffer_t *src, *dest;
2557         uintptr_t daddr, saddr, dlimit, slimit;
2558         dtrace_speculation_state_t current, new;
2559         intptr_t offs;
2560         uint64_t timestamp;
2561 
2562         if (which == 0)
2563                 return;
2564 
2565         if (which > state->dts_nspeculations) {
2566                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2567                 return;
2568         }
2569 
2570         spec = &state->dts_speculations[which - 1];
2571         src = &spec->dtsp_buffer[cpu];
2572         dest = &state->dts_buffer[cpu];
2573 
2574         do {
2575                 current = spec->dtsp_state;
2576 
2577                 if (current == DTRACESPEC_COMMITTINGMANY)
2578                         break;
2579 
2580                 switch (current) {
2581                 case DTRACESPEC_INACTIVE:
2582                 case DTRACESPEC_DISCARDING:
2583                         return;
2584 
2585                 case DTRACESPEC_COMMITTING:
2586                         /*
2587                          * This is only possible if we are (a) commit()'ing
2588                          * without having done a prior speculate() on this CPU
2589                          * and (b) racing with another commit() on a different
2590                          * CPU.  There's nothing to do -- we just assert that
2591                          * our offset is 0.
2592                          */
2593                         ASSERT(src->dtb_offset == 0);
2594                         return;
2595 
2596                 case DTRACESPEC_ACTIVE:
2597                         new = DTRACESPEC_COMMITTING;
2598                         break;
2599 
2600                 case DTRACESPEC_ACTIVEONE:
2601                         /*
2602                          * This speculation is active on one CPU.  If our
2603                          * buffer offset is non-zero, we know that the one CPU
2604                          * must be us.  Otherwise, we are committing on a
2605                          * different CPU from the speculate(), and we must
2606                          * rely on being asynchronously cleaned.
2607                          */
2608                         if (src->dtb_offset != 0) {
2609                                 new = DTRACESPEC_COMMITTING;
2610                                 break;
2611                         }
2612                         /*FALLTHROUGH*/
2613 
2614                 case DTRACESPEC_ACTIVEMANY:
2615                         new = DTRACESPEC_COMMITTINGMANY;
2616                         break;
2617 
2618                 default:
2619                         ASSERT(0);
2620                 }
2621         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2622             current, new) != current);
2623 
2624         /*
2625          * We have set the state to indicate that we are committing this
2626          * speculation.  Now reserve the necessary space in the destination
2627          * buffer.
2628          */
2629         if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
2630             sizeof (uint64_t), state, NULL)) < 0) {
2631                 dtrace_buffer_drop(dest);
2632                 goto out;
2633         }
2634 
2635         /*
2636          * We have sufficient space to copy the speculative buffer into the
2637          * primary buffer.  First, modify the speculative buffer, filling
2638          * in the timestamp of all entries with the current time.  The data
2639          * must have the commit() time rather than the time it was traced,
2640          * so that all entries in the primary buffer are in timestamp order.
2641          */
2642         timestamp = dtrace_gethrtime();
2643         saddr = (uintptr_t)src->dtb_tomax;
2644         slimit = saddr + src->dtb_offset;
2645         while (saddr < slimit) {
2646                 size_t size;
2647                 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
2648 
2649                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2650                         saddr += sizeof (dtrace_epid_t);
2651                         continue;
2652                 }
2653                 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
2654                 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
2655 
2656                 ASSERT3U(saddr + size, <=, slimit);
2657                 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
2658                 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
2659 
2660                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
2661 
2662                 saddr += size;
2663         }
2664 
2665         /*
2666          * Copy the buffer across.  (Note that this is a
2667          * highly subobtimal bcopy(); in the unlikely event that this becomes
2668          * a serious performance issue, a high-performance DTrace-specific
2669          * bcopy() should obviously be invented.)
2670          */
2671         daddr = (uintptr_t)dest->dtb_tomax + offs;
2672         dlimit = daddr + src->dtb_offset;
2673         saddr = (uintptr_t)src->dtb_tomax;
2674 
2675         /*
2676          * First, the aligned portion.
2677          */
2678         while (dlimit - daddr >= sizeof (uint64_t)) {
2679                 *((uint64_t *)daddr) = *((uint64_t *)saddr);
2680 
2681                 daddr += sizeof (uint64_t);
2682                 saddr += sizeof (uint64_t);
2683         }
2684 
2685         /*
2686          * Now any left-over bit...
2687          */
2688         while (dlimit - daddr)
2689                 *((uint8_t *)daddr++) = *((uint8_t *)saddr++);
2690 
2691         /*
2692          * Finally, commit the reserved space in the destination buffer.
2693          */
2694         dest->dtb_offset = offs + src->dtb_offset;
2695 
2696 out:
2697         /*
2698          * If we're lucky enough to be the only active CPU on this speculation
2699          * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2700          */
2701         if (current == DTRACESPEC_ACTIVE ||
2702             (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
2703                 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
2704                     DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
2705 
2706                 ASSERT(rval == DTRACESPEC_COMMITTING);
2707         }
2708 
2709         src->dtb_offset = 0;
2710         src->dtb_xamot_drops += src->dtb_drops;
2711         src->dtb_drops = 0;
2712 }
2713 
2714 /*
2715  * This routine discards an active speculation.  If the specified speculation
2716  * is not in a valid state to perform a discard(), this routine will silently
2717  * do nothing.  The state of the specified speculation is transitioned
2718  * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2719  */
2720 static void
2721 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2722     dtrace_specid_t which)
2723 {
2724         dtrace_speculation_t *spec;
2725         dtrace_speculation_state_t current, new;
2726         dtrace_buffer_t *buf;
2727 
2728         if (which == 0)
2729                 return;
2730 
2731         if (which > state->dts_nspeculations) {
2732                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2733                 return;
2734         }
2735 
2736         spec = &state->dts_speculations[which - 1];
2737         buf = &spec->dtsp_buffer[cpu];
2738 
2739         do {
2740                 current = spec->dtsp_state;
2741 
2742                 switch (current) {
2743                 case DTRACESPEC_INACTIVE:
2744                 case DTRACESPEC_COMMITTINGMANY:
2745                 case DTRACESPEC_COMMITTING:
2746                 case DTRACESPEC_DISCARDING:
2747                         return;
2748 
2749                 case DTRACESPEC_ACTIVE:
2750                 case DTRACESPEC_ACTIVEMANY:
2751                         new = DTRACESPEC_DISCARDING;
2752                         break;
2753 
2754                 case DTRACESPEC_ACTIVEONE:
2755                         if (buf->dtb_offset != 0) {
2756                                 new = DTRACESPEC_INACTIVE;
2757                         } else {
2758                                 new = DTRACESPEC_DISCARDING;
2759                         }
2760                         break;
2761 
2762                 default:
2763                         ASSERT(0);
2764                 }
2765         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2766             current, new) != current);
2767 
2768         buf->dtb_offset = 0;
2769         buf->dtb_drops = 0;
2770 }
2771 
2772 /*
2773  * Note:  not called from probe context.  This function is called
2774  * asynchronously from cross call context to clean any speculations that are
2775  * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
2776  * transitioned back to the INACTIVE state until all CPUs have cleaned the
2777  * speculation.
2778  */
2779 static void
2780 dtrace_speculation_clean_here(dtrace_state_t *state)
2781 {
2782         dtrace_icookie_t cookie;
2783         processorid_t cpu = CPU->cpu_id;
2784         dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2785         dtrace_specid_t i;
2786 
2787         cookie = dtrace_interrupt_disable();
2788 
2789         if (dest->dtb_tomax == NULL) {
2790                 dtrace_interrupt_enable(cookie);
2791                 return;
2792         }
2793 
2794         for (i = 0; i < state->dts_nspeculations; i++) {
2795                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2796                 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
2797 
2798                 if (src->dtb_tomax == NULL)
2799                         continue;
2800 
2801                 if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
2802                         src->dtb_offset = 0;
2803                         continue;
2804                 }
2805 
2806                 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2807                         continue;
2808 
2809                 if (src->dtb_offset == 0)
2810                         continue;
2811 
2812                 dtrace_speculation_commit(state, cpu, i + 1);
2813         }
2814 
2815         dtrace_interrupt_enable(cookie);
2816 }
2817 
2818 /*
2819  * Note:  not called from probe context.  This function is called
2820  * asynchronously (and at a regular interval) to clean any speculations that
2821  * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
2822  * is work to be done, it cross calls all CPUs to perform that work;
2823  * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2824  * INACTIVE state until they have been cleaned by all CPUs.
2825  */
2826 static void
2827 dtrace_speculation_clean(dtrace_state_t *state)
2828 {
2829         int work = 0, rv;
2830         dtrace_specid_t i;
2831 
2832         for (i = 0; i < state->dts_nspeculations; i++) {
2833                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2834 
2835                 ASSERT(!spec->dtsp_cleaning);
2836 
2837                 if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
2838                     spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2839                         continue;
2840 
2841                 work++;
2842                 spec->dtsp_cleaning = 1;
2843         }
2844 
2845         if (!work)
2846                 return;
2847 
2848         dtrace_xcall(DTRACE_CPUALL,
2849             (dtrace_xcall_t)dtrace_speculation_clean_here, state);
2850 
2851         /*
2852          * We now know that all CPUs have committed or discarded their
2853          * speculation buffers, as appropriate.  We can now set the state
2854          * to inactive.
2855          */
2856         for (i = 0; i < state->dts_nspeculations; i++) {
2857                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2858                 dtrace_speculation_state_t current, new;
2859 
2860                 if (!spec->dtsp_cleaning)
2861                         continue;
2862 
2863                 current = spec->dtsp_state;
2864                 ASSERT(current == DTRACESPEC_DISCARDING ||
2865                     current == DTRACESPEC_COMMITTINGMANY);
2866 
2867                 new = DTRACESPEC_INACTIVE;
2868 
2869                 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
2870                 ASSERT(rv == current);
2871                 spec->dtsp_cleaning = 0;
2872         }
2873 }
2874 
2875 /*
2876  * Called as part of a speculate() to get the speculative buffer associated
2877  * with a given speculation.  Returns NULL if the specified speculation is not
2878  * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
2879  * the active CPU is not the specified CPU -- the speculation will be
2880  * atomically transitioned into the ACTIVEMANY state.
2881  */
2882 static dtrace_buffer_t *
2883 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2884     dtrace_specid_t which)
2885 {
2886         dtrace_speculation_t *spec;
2887         dtrace_speculation_state_t current, new;
2888         dtrace_buffer_t *buf;
2889 
2890         if (which == 0)
2891                 return (NULL);
2892 
2893         if (which > state->dts_nspeculations) {
2894                 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2895                 return (NULL);
2896         }
2897 
2898         spec = &state->dts_speculations[which - 1];
2899         buf = &spec->dtsp_buffer[cpuid];
2900 
2901         do {
2902                 current = spec->dtsp_state;
2903 
2904                 switch (current) {
2905                 case DTRACESPEC_INACTIVE:
2906                 case DTRACESPEC_COMMITTINGMANY:
2907                 case DTRACESPEC_DISCARDING:
2908                         return (NULL);
2909 
2910                 case DTRACESPEC_COMMITTING:
2911                         ASSERT(buf->dtb_offset == 0);
2912                         return (NULL);
2913 
2914                 case DTRACESPEC_ACTIVEONE:
2915                         /*
2916                          * This speculation is currently active on one CPU.
2917                          * Check the offset in the buffer; if it's non-zero,
2918                          * that CPU must be us (and we leave the state alone).
2919                          * If it's zero, assume that we're starting on a new
2920                          * CPU -- and change the state to indicate that the
2921                          * speculation is active on more than one CPU.
2922                          */
2923                         if (buf->dtb_offset != 0)
2924                                 return (buf);
2925 
2926                         new = DTRACESPEC_ACTIVEMANY;
2927                         break;
2928 
2929                 case DTRACESPEC_ACTIVEMANY:
2930                         return (buf);
2931 
2932                 case DTRACESPEC_ACTIVE:
2933                         new = DTRACESPEC_ACTIVEONE;
2934                         break;
2935 
2936                 default:
2937                         ASSERT(0);
2938                 }
2939         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2940             current, new) != current);
2941 
2942         ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
2943         return (buf);
2944 }
2945 
2946 /*
2947  * Return a string.  In the event that the user lacks the privilege to access
2948  * arbitrary kernel memory, we copy the string out to scratch memory so that we
2949  * don't fail access checking.
2950  *
2951  * dtrace_dif_variable() uses this routine as a helper for various
2952  * builtin values such as 'execname' and 'probefunc.'
2953  */
2954 uintptr_t
2955 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
2956     dtrace_mstate_t *mstate)
2957 {
2958         uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2959         uintptr_t ret;
2960         size_t strsz;
2961 
2962         /*
2963          * The easy case: this probe is allowed to read all of memory, so
2964          * we can just return this as a vanilla pointer.
2965          */
2966         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
2967                 return (addr);
2968 
2969         /*
2970          * This is the tougher case: we copy the string in question from
2971          * kernel memory into scratch memory and return it that way: this
2972          * ensures that we won't trip up when access checking tests the
2973          * BYREF return value.
2974          */
2975         strsz = dtrace_strlen((char *)addr, size) + 1;
2976 
2977         if (mstate->dtms_scratch_ptr + strsz >
2978             mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2979                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2980                 return (NULL);
2981         }
2982 
2983         dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
2984             strsz);
2985         ret = mstate->dtms_scratch_ptr;
2986         mstate->dtms_scratch_ptr += strsz;
2987         return (ret);
2988 }
2989 
2990 /*
2991  * This function implements the DIF emulator's variable lookups.  The emulator
2992  * passes a reserved variable identifier and optional built-in array index.
2993  */
2994 static uint64_t
2995 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
2996     uint64_t ndx)
2997 {
2998         /*
2999          * If we're accessing one of the uncached arguments, we'll turn this
3000          * into a reference in the args array.
3001          */
3002         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
3003                 ndx = v - DIF_VAR_ARG0;
3004                 v = DIF_VAR_ARGS;
3005         }
3006 
3007         switch (v) {
3008         case DIF_VAR_ARGS:
3009                 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) {
3010                         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |=
3011                             CPU_DTRACE_KPRIV;
3012                         return (0);
3013                 }
3014 
3015                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
3016                 if (ndx >= sizeof (mstate->dtms_arg) /
3017                     sizeof (mstate->dtms_arg[0])) {
3018                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3019                         dtrace_provider_t *pv;
3020                         uint64_t val;
3021 
3022                         pv = mstate->dtms_probe->dtpr_provider;
3023                         if (pv->dtpv_pops.dtps_getargval != NULL)
3024                                 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
3025                                     mstate->dtms_probe->dtpr_id,
3026                                     mstate->dtms_probe->dtpr_arg, ndx, aframes);
3027                         else
3028                                 val = dtrace_getarg(ndx, aframes);
3029 
3030                         /*
3031                          * This is regrettably required to keep the compiler
3032                          * from tail-optimizing the call to dtrace_getarg().
3033                          * The condition always evaluates to true, but the
3034                          * compiler has no way of figuring that out a priori.
3035                          * (None of this would be necessary if the compiler
3036                          * could be relied upon to _always_ tail-optimize
3037                          * the call to dtrace_getarg() -- but it can't.)
3038                          */
3039                         if (mstate->dtms_probe != NULL)
3040                                 return (val);
3041 
3042                         ASSERT(0);
3043                 }
3044 
3045                 return (mstate->dtms_arg[ndx]);
3046 
3047         case DIF_VAR_UREGS: {
3048                 klwp_t *lwp;
3049 
3050                 if (!dtrace_priv_proc(state, mstate))
3051                         return (0);
3052 
3053                 if ((lwp = curthread->t_lwp) == NULL) {
3054                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
3055                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
3056                         return (0);
3057                 }
3058 
3059                 return (dtrace_getreg(lwp->lwp_regs, ndx));
3060         }
3061 
3062         case DIF_VAR_VMREGS: {
3063                 uint64_t rval;
3064 
3065                 if (!dtrace_priv_kernel(state))
3066                         return (0);
3067 
3068                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3069 
3070                 rval = dtrace_getvmreg(ndx,
3071                     &cpu_core[CPU->cpu_id].cpuc_dtrace_flags);
3072 
3073                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3074 
3075                 return (rval);
3076         }
3077 
3078         case DIF_VAR_CURTHREAD:
3079                 if (!dtrace_priv_proc(state, mstate))
3080                         return (0);
3081                 return ((uint64_t)(uintptr_t)curthread);
3082 
3083         case DIF_VAR_TIMESTAMP:
3084                 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
3085                         mstate->dtms_timestamp = dtrace_gethrtime();
3086                         mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
3087                 }
3088                 return (mstate->dtms_timestamp);
3089 
3090         case DIF_VAR_VTIMESTAMP:
3091                 ASSERT(dtrace_vtime_references != 0);
3092                 return (curthread->t_dtrace_vtime);
3093 
3094         case DIF_VAR_WALLTIMESTAMP:
3095                 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
3096                         mstate->dtms_walltimestamp = dtrace_gethrestime();
3097                         mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
3098                 }
3099                 return (mstate->dtms_walltimestamp);
3100 
3101         case DIF_VAR_IPL:
3102                 if (!dtrace_priv_kernel(state))
3103                         return (0);
3104                 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
3105                         mstate->dtms_ipl = dtrace_getipl();
3106                         mstate->dtms_present |= DTRACE_MSTATE_IPL;
3107                 }
3108                 return (mstate->dtms_ipl);
3109 
3110         case DIF_VAR_EPID:
3111                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
3112                 return (mstate->dtms_epid);
3113 
3114         case DIF_VAR_ID:
3115                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3116                 return (mstate->dtms_probe->dtpr_id);
3117 
3118         case DIF_VAR_STACKDEPTH:
3119                 if (!dtrace_priv_kernel(state))
3120                         return (0);
3121                 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
3122                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3123 
3124                         mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
3125                         mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
3126                 }
3127                 return (mstate->dtms_stackdepth);
3128 
3129         case DIF_VAR_USTACKDEPTH:
3130                 if (!dtrace_priv_proc(state, mstate))
3131                         return (0);
3132                 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
3133                         /*
3134                          * See comment in DIF_VAR_PID.
3135                          */
3136                         if (DTRACE_ANCHORED(mstate->dtms_probe) &&
3137                             CPU_ON_INTR(CPU)) {
3138                                 mstate->dtms_ustackdepth = 0;
3139                         } else {
3140                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3141                                 mstate->dtms_ustackdepth =
3142                                     dtrace_getustackdepth();
3143                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3144                         }
3145                         mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
3146                 }
3147                 return (mstate->dtms_ustackdepth);
3148 
3149         case DIF_VAR_CALLER:
3150                 if (!dtrace_priv_kernel(state))
3151                         return (0);
3152                 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
3153                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3154 
3155                         if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
3156                                 /*
3157                                  * If this is an unanchored probe, we are
3158                                  * required to go through the slow path:
3159                                  * dtrace_caller() only guarantees correct
3160                                  * results for anchored probes.
3161                                  */
3162                                 pc_t caller[2];
3163 
3164                                 dtrace_getpcstack(caller, 2, aframes,
3165                                     (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
3166                                 mstate->dtms_caller = caller[1];
3167                         } else if ((mstate->dtms_caller =
3168                             dtrace_caller(aframes)) == -1) {
3169                                 /*
3170                                  * We have failed to do this the quick way;
3171                                  * we must resort to the slower approach of
3172                                  * calling dtrace_getpcstack().
3173                                  */
3174                                 pc_t caller;
3175 
3176                                 dtrace_getpcstack(&caller, 1, aframes, NULL);
3177                                 mstate->dtms_caller = caller;
3178                         }
3179 
3180                         mstate->dtms_present |= DTRACE_MSTATE_CALLER;
3181                 }
3182                 return (mstate->dtms_caller);
3183 
3184         case DIF_VAR_UCALLER:
3185                 if (!dtrace_priv_proc(state, mstate))
3186                         return (0);
3187 
3188                 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
3189                         uint64_t ustack[3];
3190 
3191                         /*
3192                          * dtrace_getupcstack() fills in the first uint64_t
3193                          * with the current PID.  The second uint64_t will
3194                          * be the program counter at user-level.  The third
3195                          * uint64_t will contain the caller, which is what
3196                          * we're after.
3197                          */
3198                         ustack[2] = NULL;
3199                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3200                         dtrace_getupcstack(ustack, 3);
3201                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3202                         mstate->dtms_ucaller = ustack[2];
3203                         mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
3204                 }
3205 
3206                 return (mstate->dtms_ucaller);
3207 
3208         case DIF_VAR_PROBEPROV:
3209                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3210                 return (dtrace_dif_varstr(
3211                     (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
3212                     state, mstate));
3213 
3214         case DIF_VAR_PROBEMOD:
3215                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3216                 return (dtrace_dif_varstr(
3217                     (uintptr_t)mstate->dtms_probe->dtpr_mod,
3218                     state, mstate));
3219 
3220         case DIF_VAR_PROBEFUNC:
3221                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3222                 return (dtrace_dif_varstr(
3223                     (uintptr_t)mstate->dtms_probe->dtpr_func,
3224                     state, mstate));
3225 
3226         case DIF_VAR_PROBENAME:
3227                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3228                 return (dtrace_dif_varstr(
3229                     (uintptr_t)mstate->dtms_probe->dtpr_name,
3230                     state, mstate));
3231 
3232         case DIF_VAR_PID:
3233                 if (!dtrace_priv_proc(state, mstate))
3234                         return (0);
3235 
3236                 /*
3237                  * Note that we are assuming that an unanchored probe is
3238                  * always due to a high-level interrupt.  (And we're assuming
3239                  * that there is only a single high level interrupt.)
3240                  */
3241                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3242                         return (pid0.pid_id);
3243 
3244                 /*
3245                  * It is always safe to dereference one's own t_procp pointer:
3246                  * it always points to a valid, allocated proc structure.
3247                  * Further, it is always safe to dereference the p_pidp member
3248                  * of one's own proc structure.  (These are truisms becuase
3249                  * threads and processes don't clean up their own state --
3250                  * they leave that task to whomever reaps them.)
3251                  */
3252                 return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
3253 
3254         case DIF_VAR_PPID:
3255                 if (!dtrace_priv_proc(state, mstate))
3256                         return (0);
3257 
3258                 /*
3259                  * See comment in DIF_VAR_PID.
3260                  */
3261                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3262                         return (pid0.pid_id);
3263 
3264                 /*
3265                  * It is always safe to dereference one's own t_procp pointer:
3266                  * it always points to a valid, allocated proc structure.
3267                  * (This is true because threads don't clean up their own
3268                  * state -- they leave that task to whomever reaps them.)
3269                  */
3270                 return ((uint64_t)curthread->t_procp->p_ppid);
3271 
3272         case DIF_VAR_TID:
3273                 /*
3274                  * See comment in DIF_VAR_PID.
3275                  */
3276                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3277                         return (0);
3278 
3279                 return ((uint64_t)curthread->t_tid);
3280 
3281         case DIF_VAR_EXECNAME:
3282                 if (!dtrace_priv_proc(state, mstate))
3283                         return (0);
3284 
3285                 /*
3286                  * See comment in DIF_VAR_PID.
3287                  */
3288                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3289                         return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
3290 
3291                 /*
3292                  * It is always safe to dereference one's own t_procp pointer:
3293                  * it always points to a valid, allocated proc structure.
3294                  * (This is true because threads don't clean up their own
3295                  * state -- they leave that task to whomever reaps them.)
3296                  */
3297                 return (dtrace_dif_varstr(
3298                     (uintptr_t)curthread->t_procp->p_user.u_comm,
3299                     state, mstate));
3300 
3301         case DIF_VAR_ZONENAME:
3302                 if (!dtrace_priv_proc(state, mstate))
3303                         return (0);
3304 
3305                 /*
3306                  * See comment in DIF_VAR_PID.
3307                  */
3308                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3309                         return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
3310 
3311                 /*
3312                  * It is always safe to dereference one's own t_procp pointer:
3313                  * it always points to a valid, allocated proc structure.
3314                  * (This is true because threads don't clean up their own
3315                  * state -- they leave that task to whomever reaps them.)
3316                  */
3317                 return (dtrace_dif_varstr(
3318                     (uintptr_t)curthread->t_procp->p_zone->zone_name,
3319                     state, mstate));
3320 
3321         case DIF_VAR_UID:
3322                 if (!dtrace_priv_proc(state, mstate))
3323                         return (0);
3324 
3325                 /*
3326                  * See comment in DIF_VAR_PID.
3327                  */
3328                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3329                         return ((uint64_t)p0.p_cred->cr_uid);
3330 
3331                 /*
3332                  * It is always safe to dereference one's own t_procp pointer:
3333                  * it always points to a valid, allocated proc structure.
3334                  * (This is true because threads don't clean up their own
3335                  * state -- they leave that task to whomever reaps them.)
3336                  *
3337                  * Additionally, it is safe to dereference one's own process
3338                  * credential, since this is never NULL after process birth.
3339                  */
3340                 return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
3341 
3342         case DIF_VAR_GID:
3343                 if (!dtrace_priv_proc(state, mstate))
3344                         return (0);
3345 
3346                 /*
3347                  * See comment in DIF_VAR_PID.
3348                  */
3349                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3350                         return ((uint64_t)p0.p_cred->cr_gid);
3351 
3352                 /*
3353                  * It is always safe to dereference one's own t_procp pointer:
3354                  * it always points to a valid, allocated proc structure.
3355                  * (This is true because threads don't clean up their own
3356                  * state -- they leave that task to whomever reaps them.)
3357                  *
3358                  * Additionally, it is safe to dereference one's own process
3359                  * credential, since this is never NULL after process birth.
3360                  */
3361                 return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
3362 
3363         case DIF_VAR_ERRNO: {
3364                 klwp_t *lwp;
3365                 if (!dtrace_priv_proc(state, mstate))
3366                         return (0);
3367 
3368                 /*
3369                  * See comment in DIF_VAR_PID.
3370                  */
3371                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3372                         return (0);
3373 
3374                 /*
3375                  * It is always safe to dereference one's own t_lwp pointer in
3376                  * the event that this pointer is non-NULL.  (This is true
3377                  * because threads and lwps don't clean up their own state --
3378                  * they leave that task to whomever reaps them.)
3379                  */
3380                 if ((lwp = curthread->t_lwp) == NULL)
3381                         return (0);
3382 
3383                 return ((uint64_t)lwp->lwp_errno);
3384         }
3385         default:
3386                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3387                 return (0);
3388         }
3389 }
3390 
3391 
3392 typedef enum dtrace_json_state {
3393         DTRACE_JSON_REST = 1,
3394         DTRACE_JSON_OBJECT,
3395         DTRACE_JSON_STRING,
3396         DTRACE_JSON_STRING_ESCAPE,
3397         DTRACE_JSON_STRING_ESCAPE_UNICODE,
3398         DTRACE_JSON_COLON,
3399         DTRACE_JSON_COMMA,
3400         DTRACE_JSON_VALUE,
3401         DTRACE_JSON_IDENTIFIER,
3402         DTRACE_JSON_NUMBER,
3403         DTRACE_JSON_NUMBER_FRAC,
3404         DTRACE_JSON_NUMBER_EXP,
3405         DTRACE_JSON_COLLECT_OBJECT
3406 } dtrace_json_state_t;
3407 
3408 /*
3409  * This function possesses just enough knowledge about JSON to extract a single
3410  * value from a JSON string and store it in the scratch buffer.  It is able
3411  * to extract nested object values, and members of arrays by index.
3412  *
3413  * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to
3414  * be looked up as we descend into the object tree.  e.g.
3415  *
3416  *    foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL
3417  *       with nelems = 5.
3418  *
3419  * The run time of this function must be bounded above by strsize to limit the
3420  * amount of work done in probe context.  As such, it is implemented as a
3421  * simple state machine, reading one character at a time using safe loads
3422  * until we find the requested element, hit a parsing error or run off the
3423  * end of the object or string.
3424  *
3425  * As there is no way for a subroutine to return an error without interrupting
3426  * clause execution, we simply return NULL in the event of a missing key or any
3427  * other error condition.  Each NULL return in this function is commented with
3428  * the error condition it represents -- parsing or otherwise.
3429  *
3430  * The set of states for the state machine closely matches the JSON
3431  * specification (http://json.org/).  Briefly:
3432  *
3433  *   DTRACE_JSON_REST:
3434  *     Skip whitespace until we find either a top-level Object, moving
3435  *     to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE.
3436  *
3437  *   DTRACE_JSON_OBJECT:
3438  *     Locate the next key String in an Object.  Sets a flag to denote
3439  *     the next String as a key string and moves to DTRACE_JSON_STRING.
3440  *
3441  *   DTRACE_JSON_COLON:
3442  *     Skip whitespace until we find the colon that separates key Strings
3443  *     from their values.  Once found, move to DTRACE_JSON_VALUE.
3444  *
3445  *   DTRACE_JSON_VALUE:
3446  *     Detects the type of the next value (String, Number, Identifier, Object
3447  *     or Array) and routes to the states that process that type.  Here we also
3448  *     deal with the element selector list if we are requested to traverse down
3449  *     into the object tree.
3450  *
3451  *   DTRACE_JSON_COMMA:
3452  *     Skip whitespace until we find the comma that separates key-value pairs
3453  *     in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays
3454  *     (similarly DTRACE_JSON_VALUE).  All following literal value processing
3455  *     states return to this state at the end of their value, unless otherwise
3456  *     noted.
3457  *
3458  *   DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP:
3459  *     Processes a Number literal from the JSON, including any exponent
3460  *     component that may be present.  Numbers are returned as strings, which
3461  *     may be passed to strtoll() if an integer is required.
3462  *
3463  *   DTRACE_JSON_IDENTIFIER:
3464  *     Processes a "true", "false" or "null" literal in the JSON.
3465  *
3466  *   DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE,
3467  *   DTRACE_JSON_STRING_ESCAPE_UNICODE:
3468  *     Processes a String literal from the JSON, whether the String denotes
3469  *     a key, a value or part of a larger Object.  Handles all escape sequences
3470  *     present in the specification, including four-digit unicode characters,
3471  *     but merely includes the escape sequence without converting it to the
3472  *     actual escaped character.  If the String is flagged as a key, we
3473  *     move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA.
3474  *
3475  *   DTRACE_JSON_COLLECT_OBJECT:
3476  *     This state collects an entire Object (or Array), correctly handling
3477  *     embedded strings.  If the full element selector list matches this nested
3478  *     object, we return the Object in full as a string.  If not, we use this
3479  *     state to skip to the next value at this level and continue processing.
3480  *
3481  * NOTE: This function uses various macros from strtolctype.h to manipulate
3482  * digit values, etc -- these have all been checked to ensure they make
3483  * no additional function calls.
3484  */
3485 static char *
3486 dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems,
3487     char *dest)
3488 {
3489         dtrace_json_state_t state = DTRACE_JSON_REST;
3490         int64_t array_elem = INT64_MIN;
3491         int64_t array_pos = 0;
3492         uint8_t escape_unicount = 0;
3493         boolean_t string_is_key = B_FALSE;
3494         boolean_t collect_object = B_FALSE;
3495         boolean_t found_key = B_FALSE;
3496         boolean_t in_array = B_FALSE;
3497         uint32_t braces = 0, brackets = 0;
3498         char *elem = elemlist;
3499         char *dd = dest;
3500         uintptr_t cur;
3501 
3502         for (cur = json; cur < json + size; cur++) {
3503                 char cc = dtrace_load8(cur);
3504                 if (cc == '\0')
3505                         return (NULL);
3506 
3507                 switch (state) {
3508                 case DTRACE_JSON_REST:
3509                         if (isspace(cc))
3510                                 break;
3511 
3512                         if (cc == '{') {
3513                                 state = DTRACE_JSON_OBJECT;
3514                                 break;
3515                         }
3516 
3517                         if (cc == '[') {
3518                                 in_array = B_TRUE;
3519                                 array_pos = 0;
3520                                 array_elem = dtrace_strtoll(elem, 10, size);
3521                                 found_key = array_elem == 0 ? B_TRUE : B_FALSE;
3522                                 state = DTRACE_JSON_VALUE;
3523                                 break;
3524                         }
3525 
3526                         /*
3527                          * ERROR: expected to find a top-level object or array.
3528                          */
3529                         return (NULL);
3530                 case DTRACE_JSON_OBJECT:
3531                         if (isspace(cc))
3532                                 break;
3533 
3534                         if (cc == '"') {
3535                                 state = DTRACE_JSON_STRING;
3536                                 string_is_key = B_TRUE;
3537                                 break;
3538                         }
3539 
3540                         /*
3541                          * ERROR: either the object did not start with a key
3542                          * string, or we've run off the end of the object
3543                          * without finding the requested key.
3544                          */
3545                         return (NULL);
3546                 case DTRACE_JSON_STRING:
3547                         if (cc == '\\') {
3548                                 *dd++ = '\\';
3549                                 state = DTRACE_JSON_STRING_ESCAPE;
3550                                 break;
3551                         }
3552 
3553                         if (cc == '"') {
3554                                 if (collect_object) {
3555                                         /*
3556                                          * We don't reset the dest here, as
3557                                          * the string is part of a larger
3558                                          * object being collected.
3559                                          */
3560                                         *dd++ = cc;
3561                                         collect_object = B_FALSE;
3562                                         state = DTRACE_JSON_COLLECT_OBJECT;
3563                                         break;
3564                                 }
3565                                 *dd = '\0';
3566                                 dd = dest; /* reset string buffer */
3567                                 if (string_is_key) {
3568                                         if (dtrace_strncmp(dest, elem,
3569                                             size) == 0)
3570                                                 found_key = B_TRUE;
3571                                 } else if (found_key) {
3572                                         if (nelems > 1) {
3573                                                 /*
3574                                                  * We expected an object, not
3575                                                  * this string.
3576                                                  */
3577                                                 return (NULL);
3578                                         }
3579                                         return (dest);
3580                                 }
3581                                 state = string_is_key ? DTRACE_JSON_COLON :
3582                                     DTRACE_JSON_COMMA;
3583                                 string_is_key = B_FALSE;
3584                                 break;
3585                         }
3586 
3587                         *dd++ = cc;
3588                         break;
3589                 case DTRACE_JSON_STRING_ESCAPE:
3590                         *dd++ = cc;
3591                         if (cc == 'u') {
3592                                 escape_unicount = 0;
3593                                 state = DTRACE_JSON_STRING_ESCAPE_UNICODE;
3594                         } else {
3595                                 state = DTRACE_JSON_STRING;
3596                         }
3597                         break;
3598                 case DTRACE_JSON_STRING_ESCAPE_UNICODE:
3599                         if (!isxdigit(cc)) {
3600                                 /*
3601                                  * ERROR: invalid unicode escape, expected
3602                                  * four valid hexidecimal digits.
3603                                  */
3604                                 return (NULL);
3605                         }
3606 
3607                         *dd++ = cc;
3608                         if (++escape_unicount == 4)
3609                                 state = DTRACE_JSON_STRING;
3610                         break;
3611                 case DTRACE_JSON_COLON:
3612                         if (isspace(cc))
3613                                 break;
3614 
3615                         if (cc == ':') {
3616                                 state = DTRACE_JSON_VALUE;
3617                                 break;
3618                         }
3619 
3620                         /*
3621                          * ERROR: expected a colon.
3622                          */
3623                         return (NULL);
3624                 case DTRACE_JSON_COMMA:
3625                         if (isspace(cc))
3626                                 break;
3627 
3628                         if (cc == ',') {
3629                                 if (in_array) {
3630                                         state = DTRACE_JSON_VALUE;
3631                                         if (++array_pos == array_elem)
3632                                                 found_key = B_TRUE;
3633                                 } else {
3634                                         state = DTRACE_JSON_OBJECT;
3635                                 }
3636                                 break;
3637                         }
3638 
3639                         /*
3640                          * ERROR: either we hit an unexpected character, or
3641                          * we reached the end of the object or array without
3642                          * finding the requested key.
3643                          */
3644                         return (NULL);
3645                 case DTRACE_JSON_IDENTIFIER:
3646                         if (islower(cc)) {
3647                                 *dd++ = cc;
3648                                 break;
3649                         }
3650 
3651                         *dd = '\0';
3652                         dd = dest; /* reset string buffer */
3653 
3654                         if (dtrace_strncmp(dest, "true", 5) == 0 ||
3655                             dtrace_strncmp(dest, "false", 6) == 0 ||
3656                             dtrace_strncmp(dest, "null", 5) == 0) {
3657                                 if (found_key) {
3658                                         if (nelems > 1) {
3659                                                 /*
3660                                                  * ERROR: We expected an object,
3661                                                  * not this identifier.
3662                                                  */
3663                                                 return (NULL);
3664                                         }
3665                                         return (dest);
3666                                 } else {
3667                                         cur--;
3668                                         state = DTRACE_JSON_COMMA;
3669                                         break;
3670                                 }
3671                         }
3672 
3673                         /*
3674                          * ERROR: we did not recognise the identifier as one
3675                          * of those in the JSON specification.
3676                          */
3677                         return (NULL);
3678                 case DTRACE_JSON_NUMBER:
3679                         if (cc == '.') {
3680                                 *dd++ = cc;
3681                                 state = DTRACE_JSON_NUMBER_FRAC;
3682                                 break;
3683                         }
3684 
3685                         if (cc == 'x' || cc == 'X') {
3686                                 /*
3687                                  * ERROR: specification explicitly excludes
3688                                  * hexidecimal or octal numbers.
3689                                  */
3690                                 return (NULL);
3691                         }
3692 
3693                         /* FALLTHRU */
3694                 case DTRACE_JSON_NUMBER_FRAC:
3695                         if (cc == 'e' || cc == 'E') {
3696                                 *dd++ = cc;
3697                                 state = DTRACE_JSON_NUMBER_EXP;
3698                                 break;
3699                         }
3700 
3701                         if (cc == '+' || cc == '-') {
3702                                 /*
3703                                  * ERROR: expect sign as part of exponent only.
3704                                  */
3705                                 return (NULL);
3706                         }
3707                         /* FALLTHRU */
3708                 case DTRACE_JSON_NUMBER_EXP:
3709                         if (isdigit(cc) || cc == '+' || cc == '-') {
3710                                 *dd++ = cc;
3711                                 break;
3712                         }
3713 
3714                         *dd = '\0';
3715                         dd = dest; /* reset string buffer */
3716                         if (found_key) {
3717                                 if (nelems > 1) {
3718                                         /*
3719                                          * ERROR: We expected an object, not
3720                                          * this number.
3721                                          */
3722                                         return (NULL);
3723                                 }
3724                                 return (dest);
3725                         }
3726 
3727                         cur--;
3728                         state = DTRACE_JSON_COMMA;
3729                         break;
3730                 case DTRACE_JSON_VALUE:
3731                         if (isspace(cc))
3732                                 break;
3733 
3734                         if (cc == '{' || cc == '[') {
3735                                 if (nelems > 1 && found_key) {
3736                                         in_array = cc == '[' ? B_TRUE : B_FALSE;
3737                                         /*
3738                                          * If our element selector directs us
3739                                          * to descend into this nested object,
3740                                          * then move to the next selector
3741                                          * element in the list and restart the
3742                                          * state machine.
3743                                          */
3744                                         while (*elem != '\0')
3745                                                 elem++;
3746                                         elem++; /* skip the inter-element NUL */
3747                                         nelems--;
3748                                         dd = dest;
3749                                         if (in_array) {
3750                                                 state = DTRACE_JSON_VALUE;
3751                                                 array_pos = 0;
3752                                                 array_elem = dtrace_strtoll(
3753                                                     elem, 10, size);
3754                                                 found_key = array_elem == 0 ?
3755                                                     B_TRUE : B_FALSE;
3756                                         } else {
3757                                                 found_key = B_FALSE;
3758                                                 state = DTRACE_JSON_OBJECT;
3759                                         }
3760                                         break;
3761                                 }
3762 
3763                                 /*
3764                                  * Otherwise, we wish to either skip this
3765                                  * nested object or return it in full.
3766                                  */
3767                                 if (cc == '[')
3768                                         brackets = 1;
3769                                 else
3770                                         braces = 1;
3771                                 *dd++ = cc;
3772                                 state = DTRACE_JSON_COLLECT_OBJECT;
3773                                 break;
3774                         }
3775 
3776                         if (cc == '"') {
3777                                 state = DTRACE_JSON_STRING;
3778                                 break;
3779                         }
3780 
3781                         if (islower(cc)) {
3782                                 /*
3783                                  * Here we deal with true, false and null.
3784                                  */
3785                                 *dd++ = cc;
3786                                 state = DTRACE_JSON_IDENTIFIER;
3787                                 break;
3788                         }
3789 
3790                         if (cc == '-' || isdigit(cc)) {
3791                                 *dd++ = cc;
3792                                 state = DTRACE_JSON_NUMBER;
3793                                 break;
3794                         }
3795 
3796                         /*
3797                          * ERROR: unexpected character at start of value.
3798                          */
3799                         return (NULL);
3800                 case DTRACE_JSON_COLLECT_OBJECT:
3801                         if (cc == '\0')
3802                                 /*
3803                                  * ERROR: unexpected end of input.
3804                                  */
3805                                 return (NULL);
3806 
3807                         *dd++ = cc;
3808                         if (cc == '"') {
3809                                 collect_object = B_TRUE;
3810                                 state = DTRACE_JSON_STRING;
3811                                 break;
3812                         }
3813 
3814                         if (cc == ']') {
3815                                 if (brackets-- == 0) {
3816                                         /*
3817                                          * ERROR: unbalanced brackets.
3818                                          */
3819                                         return (NULL);
3820                                 }
3821                         } else if (cc == '}') {
3822                                 if (braces-- == 0) {
3823                                         /*
3824                                          * ERROR: unbalanced braces.
3825                                          */
3826                                         return (NULL);
3827                                 }
3828                         } else if (cc == '{') {
3829                                 braces++;
3830                         } else if (cc == '[') {
3831                                 brackets++;
3832                         }
3833 
3834                         if (brackets == 0 && braces == 0) {
3835                                 if (found_key) {
3836                                         *dd = '\0';
3837                                         return (dest);
3838                                 }
3839                                 dd = dest; /* reset string buffer */
3840                                 state = DTRACE_JSON_COMMA;
3841                         }
3842                         break;
3843                 }
3844         }
3845         return (NULL);
3846 }
3847 
3848 /*
3849  * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
3850  * Notice that we don't bother validating the proper number of arguments or
3851  * their types in the tuple stack.  This isn't needed because all argument
3852  * interpretation is safe because of our load safety -- the worst that can
3853  * happen is that a bogus program can obtain bogus results.
3854  */
3855 static void
3856 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
3857     dtrace_key_t *tupregs, int nargs,
3858     dtrace_mstate_t *mstate, dtrace_state_t *state)
3859 {
3860         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
3861         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
3862         dtrace_vstate_t *vstate = &state->dts_vstate;
3863 
3864         union {
3865                 mutex_impl_t mi;
3866                 uint64_t mx;
3867         } m;
3868 
3869         union {
3870                 krwlock_t ri;
3871                 uintptr_t rw;
3872         } r;
3873 
3874         switch (subr) {
3875         case DIF_SUBR_RAND:
3876                 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
3877                 break;
3878 
3879         case DIF_SUBR_MUTEX_OWNED:
3880                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3881                     mstate, vstate)) {
3882                         regs[rd] = NULL;
3883                         break;
3884                 }
3885 
3886                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3887                 if (MUTEX_TYPE_ADAPTIVE(&m.mi))
3888                         regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
3889                 else
3890                         regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
3891                 break;
3892 
3893         case DIF_SUBR_MUTEX_OWNER:
3894                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3895                     mstate, vstate)) {
3896                         regs[rd] = NULL;
3897                         break;
3898                 }
3899 
3900                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3901                 if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
3902                     MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
3903                         regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
3904                 else
3905                         regs[rd] = 0;
3906                 break;
3907 
3908         case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
3909                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3910                     mstate, vstate)) {
3911                         regs[rd] = NULL;
3912                         break;
3913                 }
3914 
3915                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3916                 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
3917                 break;
3918 
3919         case DIF_SUBR_MUTEX_TYPE_SPIN:
3920                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3921                     mstate, vstate)) {
3922                         regs[rd] = NULL;
3923                         break;
3924                 }
3925 
3926                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3927                 regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
3928                 break;
3929 
3930         case DIF_SUBR_RW_READ_HELD: {
3931                 uintptr_t tmp;
3932 
3933                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
3934                     mstate, vstate)) {
3935                         regs[rd] = NULL;
3936                         break;
3937                 }
3938 
3939                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3940                 regs[rd] = _RW_READ_HELD(&r.ri, tmp);
3941                 break;
3942         }
3943 
3944         case DIF_SUBR_RW_WRITE_HELD:
3945                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3946                     mstate, vstate)) {
3947                         regs[rd] = NULL;
3948                         break;
3949                 }
3950 
3951                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3952                 regs[rd] = _RW_WRITE_HELD(&r.ri);
3953                 break;
3954 
3955         case DIF_SUBR_RW_ISWRITER:
3956                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3957                     mstate, vstate)) {
3958                         regs[rd] = NULL;
3959                         break;
3960                 }
3961 
3962                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3963                 regs[rd] = _RW_ISWRITER(&r.ri);
3964                 break;
3965 
3966         case DIF_SUBR_BCOPY: {
3967                 /*
3968                  * We need to be sure that the destination is in the scratch
3969                  * region -- no other region is allowed.
3970                  */
3971                 uintptr_t src = tupregs[0].dttk_value;
3972                 uintptr_t dest = tupregs[1].dttk_value;
3973                 size_t size = tupregs[2].dttk_value;
3974 
3975                 if (!dtrace_inscratch(dest, size, mstate)) {
3976                         *flags |= CPU_DTRACE_BADADDR;
3977                         *illval = regs[rd];
3978                         break;
3979                 }
3980 
3981                 if (!dtrace_canload(src, size, mstate, vstate)) {
3982                         regs[rd] = NULL;
3983                         break;
3984                 }
3985 
3986                 dtrace_bcopy((void *)src, (void *)dest, size);
3987                 break;
3988         }
3989 
3990         case DIF_SUBR_ALLOCA:
3991         case DIF_SUBR_COPYIN: {
3992                 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
3993                 uint64_t size =
3994                     tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
3995                 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
3996 
3997                 /*
3998                  * This action doesn't require any credential checks since
3999                  * probes will not activate in user contexts to which the
4000                  * enabling user does not have permissions.
4001                  */
4002 
4003                 /*
4004                  * Rounding up the user allocation size could have overflowed
4005                  * a large, bogus allocation (like -1ULL) to 0.
4006                  */
4007                 if (scratch_size < size ||
4008                     !DTRACE_INSCRATCH(mstate, scratch_size)) {
4009                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4010                         regs[rd] = NULL;
4011                         break;
4012                 }
4013 
4014                 if (subr == DIF_SUBR_COPYIN) {
4015                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4016                         dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4017                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4018                 }
4019 
4020                 mstate->dtms_scratch_ptr += scratch_size;
4021                 regs[rd] = dest;
4022                 break;
4023         }
4024 
4025         case DIF_SUBR_COPYINTO: {
4026                 uint64_t size = tupregs[1].dttk_value;
4027                 uintptr_t dest = tupregs[2].dttk_value;
4028 
4029                 /*
4030                  * This action doesn't require any credential checks since
4031                  * probes will not activate in user contexts to which the
4032                  * enabling user does not have permissions.
4033                  */
4034                 if (!dtrace_inscratch(dest, size, mstate)) {
4035                         *flags |= CPU_DTRACE_BADADDR;
4036                         *illval = regs[rd];
4037                         break;
4038                 }
4039 
4040                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4041                 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4042                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4043                 break;
4044         }
4045 
4046         case DIF_SUBR_COPYINSTR: {
4047                 uintptr_t dest = mstate->dtms_scratch_ptr;
4048                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4049 
4050                 if (nargs > 1 && tupregs[1].dttk_value < size)
4051                         size = tupregs[1].dttk_value + 1;
4052 
4053                 /*
4054                  * This action doesn't require any credential checks since
4055                  * probes will not activate in user contexts to which the
4056                  * enabling user does not have permissions.
4057                  */
4058                 if (!DTRACE_INSCRATCH(mstate, size)) {
4059                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4060                         regs[rd] = NULL;
4061                         break;
4062                 }
4063 
4064                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4065                 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
4066                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4067 
4068                 ((char *)dest)[size - 1] = '\0';
4069                 mstate->dtms_scratch_ptr += size;
4070                 regs[rd] = dest;
4071                 break;
4072         }
4073 
4074         case DIF_SUBR_MSGSIZE:
4075         case DIF_SUBR_MSGDSIZE: {
4076                 uintptr_t baddr = tupregs[0].dttk_value, daddr;
4077                 uintptr_t wptr, rptr;
4078                 size_t count = 0;
4079                 int cont = 0;
4080 
4081                 while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4082 
4083                         if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
4084                             vstate)) {
4085                                 regs[rd] = NULL;
4086                                 break;
4087                         }
4088 
4089                         wptr = dtrace_loadptr(baddr +
4090                             offsetof(mblk_t, b_wptr));
4091 
4092                         rptr = dtrace_loadptr(baddr +
4093                             offsetof(mblk_t, b_rptr));
4094 
4095                         if (wptr < rptr) {
4096                                 *flags |= CPU_DTRACE_BADADDR;
4097                                 *illval = tupregs[0].dttk_value;
4098                                 break;
4099                         }
4100 
4101                         daddr = dtrace_loadptr(baddr +
4102                             offsetof(mblk_t, b_datap));
4103 
4104                         baddr = dtrace_loadptr(baddr +
4105                             offsetof(mblk_t, b_cont));
4106 
4107                         /*
4108                          * We want to prevent against denial-of-service here,
4109                          * so we're only going to search the list for
4110                          * dtrace_msgdsize_max mblks.
4111                          */
4112                         if (cont++ > dtrace_msgdsize_max) {
4113                                 *flags |= CPU_DTRACE_ILLOP;
4114                                 break;
4115                         }
4116 
4117                         if (subr == DIF_SUBR_MSGDSIZE) {
4118                                 if (dtrace_load8(daddr +
4119                                     offsetof(dblk_t, db_type)) != M_DATA)
4120                                         continue;
4121                         }
4122 
4123                         count += wptr - rptr;
4124                 }
4125 
4126                 if (!(*flags & CPU_DTRACE_FAULT))
4127                         regs[rd] = count;
4128 
4129                 break;
4130         }
4131 
4132         case DIF_SUBR_PROGENYOF: {
4133                 pid_t pid = tupregs[0].dttk_value;
4134                 proc_t *p;
4135                 int rval = 0;
4136 
4137                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4138 
4139                 for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
4140                         if (p->p_pidp->pid_id == pid) {
4141                                 rval = 1;
4142                                 break;
4143                         }
4144                 }
4145 
4146                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4147 
4148                 regs[rd] = rval;
4149                 break;
4150         }
4151 
4152         case DIF_SUBR_SPECULATION:
4153                 regs[rd] = dtrace_speculation(state);
4154                 break;
4155 
4156         case DIF_SUBR_COPYOUT: {
4157                 uintptr_t kaddr = tupregs[0].dttk_value;
4158                 uintptr_t uaddr = tupregs[1].dttk_value;
4159                 uint64_t size = tupregs[2].dttk_value;
4160 
4161                 if (!dtrace_destructive_disallow &&
4162                     dtrace_priv_proc_control(state, mstate) &&
4163                     !dtrace_istoxic(kaddr, size)) {
4164                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4165                         dtrace_copyout(kaddr, uaddr, size, flags);
4166                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4167                 }
4168                 break;
4169         }
4170 
4171         case DIF_SUBR_COPYOUTSTR: {
4172                 uintptr_t kaddr = tupregs[0].dttk_value;
4173                 uintptr_t uaddr = tupregs[1].dttk_value;
4174                 uint64_t size = tupregs[2].dttk_value;
4175 
4176                 if (!dtrace_destructive_disallow &&
4177                     dtrace_priv_proc_control(state, mstate) &&
4178                     !dtrace_istoxic(kaddr, size)) {
4179                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4180                         dtrace_copyoutstr(kaddr, uaddr, size, flags);
4181                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4182                 }
4183                 break;
4184         }
4185 
4186         case DIF_SUBR_STRLEN: {
4187                 size_t sz;
4188                 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
4189                 sz = dtrace_strlen((char *)addr,
4190                     state->dts_options[DTRACEOPT_STRSIZE]);
4191 
4192                 if (!dtrace_canload(addr, sz + 1, mstate, vstate)) {
4193                         regs[rd] = NULL;
4194                         break;
4195                 }
4196 
4197                 regs[rd] = sz;
4198 
4199                 break;
4200         }
4201 
4202         case DIF_SUBR_STRCHR:
4203         case DIF_SUBR_STRRCHR: {
4204                 /*
4205                  * We're going to iterate over the string looking for the
4206                  * specified character.  We will iterate until we have reached
4207                  * the string length or we have found the character.  If this
4208                  * is DIF_SUBR_STRRCHR, we will look for the last occurrence
4209                  * of the specified character instead of the first.
4210                  */
4211                 uintptr_t saddr = tupregs[0].dttk_value;
4212                 uintptr_t addr = tupregs[0].dttk_value;
4213                 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
4214                 char c, target = (char)tupregs[1].dttk_value;
4215 
4216                 for (regs[rd] = NULL; addr < limit; addr++) {
4217                         if ((c = dtrace_load8(addr)) == target) {
4218                                 regs[rd] = addr;
4219 
4220                                 if (subr == DIF_SUBR_STRCHR)
4221                                         break;
4222                         }
4223 
4224                         if (c == '\0')
4225                                 break;
4226                 }
4227 
4228                 if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) {
4229                         regs[rd] = NULL;
4230                         break;
4231                 }
4232 
4233                 break;
4234         }
4235 
4236         case DIF_SUBR_STRSTR:
4237         case DIF_SUBR_INDEX:
4238         case DIF_SUBR_RINDEX: {
4239                 /*
4240                  * We're going to iterate over the string looking for the
4241                  * specified string.  We will iterate until we have reached
4242                  * the string length or we have found the string.  (Yes, this
4243                  * is done in the most naive way possible -- but considering
4244                  * that the string we're searching for is likely to be
4245                  * relatively short, the complexity of Rabin-Karp or similar
4246                  * hardly seems merited.)
4247                  */
4248                 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
4249                 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
4250                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4251                 size_t len = dtrace_strlen(addr, size);
4252                 size_t sublen = dtrace_strlen(substr, size);
4253                 char *limit = addr + len, *orig = addr;
4254                 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
4255                 int inc = 1;
4256 
4257                 regs[rd] = notfound;
4258 
4259                 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
4260                         regs[rd] = NULL;
4261                         break;
4262                 }
4263 
4264                 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
4265                     vstate)) {
4266                         regs[rd] = NULL;
4267                         break;
4268                 }
4269 
4270                 /*
4271                  * strstr() and index()/rindex() have similar semantics if
4272                  * both strings are the empty string: strstr() returns a
4273                  * pointer to the (empty) string, and index() and rindex()
4274                  * both return index 0 (regardless of any position argument).
4275                  */
4276                 if (sublen == 0 && len == 0) {
4277                         if (subr == DIF_SUBR_STRSTR)
4278                                 regs[rd] = (uintptr_t)addr;
4279                         else
4280                                 regs[rd] = 0;
4281                         break;
4282                 }
4283 
4284                 if (subr != DIF_SUBR_STRSTR) {
4285                         if (subr == DIF_SUBR_RINDEX) {
4286                                 limit = orig - 1;
4287                                 addr += len;
4288                                 inc = -1;
4289                         }
4290 
4291                         /*
4292                          * Both index() and rindex() take an optional position
4293                          * argument that denotes the starting position.
4294                          */
4295                         if (nargs == 3) {
4296                                 int64_t pos = (int64_t)tupregs[2].dttk_value;
4297 
4298                                 /*
4299                                  * If the position argument to index() is
4300                                  * negative, Perl implicitly clamps it at
4301                                  * zero.  This semantic is a little surprising
4302                                  * given the special meaning of negative
4303                                  * positions to similar Perl functions like
4304                                  * substr(), but it appears to reflect a
4305                                  * notion that index() can start from a
4306                                  * negative index and increment its way up to
4307                                  * the string.  Given this notion, Perl's
4308                                  * rindex() is at least self-consistent in
4309                                  * that it implicitly clamps positions greater
4310                                  * than the string length to be the string
4311                                  * length.  Where Perl completely loses
4312                                  * coherence, however, is when the specified
4313                                  * substring is the empty string ("").  In
4314                                  * this case, even if the position is
4315                                  * negative, rindex() returns 0 -- and even if
4316                                  * the position is greater than the length,
4317                                  * index() returns the string length.  These
4318                                  * semantics violate the notion that index()
4319                                  * should never return a value less than the
4320                                  * specified position and that rindex() should
4321                                  * never return a value greater than the
4322                                  * specified position.  (One assumes that
4323                                  * these semantics are artifacts of Perl's
4324                                  * implementation and not the results of
4325                                  * deliberate design -- it beggars belief that
4326                                  * even Larry Wall could desire such oddness.)
4327                                  * While in the abstract one would wish for
4328                                  * consistent position semantics across
4329                                  * substr(), index() and rindex() -- or at the
4330                                  * very least self-consistent position
4331                                  * semantics for index() and rindex() -- we
4332                                  * instead opt to keep with the extant Perl
4333                                  * semantics, in all their broken glory.  (Do
4334                                  * we have more desire to maintain Perl's
4335                                  * semantics than Perl does?  Probably.)
4336                                  */
4337                                 if (subr == DIF_SUBR_RINDEX) {
4338                                         if (pos < 0) {
4339                                                 if (sublen == 0)
4340                                                         regs[rd] = 0;
4341                                                 break;
4342                                         }
4343 
4344                                         if (pos > len)
4345                                                 pos = len;
4346                                 } else {
4347                                         if (pos < 0)
4348                                                 pos = 0;
4349 
4350                                         if (pos >= len) {
4351                                                 if (sublen == 0)
4352                                                         regs[rd] = len;
4353                                                 break;
4354                                         }
4355                                 }
4356 
4357                                 addr = orig + pos;
4358                         }
4359                 }
4360 
4361                 for (regs[rd] = notfound; addr != limit; addr += inc) {
4362                         if (dtrace_strncmp(addr, substr, sublen) == 0) {
4363                                 if (subr != DIF_SUBR_STRSTR) {
4364                                         /*
4365                                          * As D index() and rindex() are
4366                                          * modeled on Perl (and not on awk),
4367                                          * we return a zero-based (and not a
4368                                          * one-based) index.  (For you Perl
4369                                          * weenies: no, we're not going to add
4370                                          * $[ -- and shouldn't you be at a con
4371                                          * or something?)
4372                                          */
4373                                         regs[rd] = (uintptr_t)(addr - orig);
4374                                         break;
4375                                 }
4376 
4377                                 ASSERT(subr == DIF_SUBR_STRSTR);
4378                                 regs[rd] = (uintptr_t)addr;
4379                                 break;
4380                         }
4381                 }
4382 
4383                 break;
4384         }
4385 
4386         case DIF_SUBR_STRTOK: {
4387                 uintptr_t addr = tupregs[0].dttk_value;
4388                 uintptr_t tokaddr = tupregs[1].dttk_value;
4389                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4390                 uintptr_t limit, toklimit = tokaddr + size;
4391                 uint8_t c, tokmap[32];   /* 256 / 8 */
4392                 char *dest = (char *)mstate->dtms_scratch_ptr;
4393                 int i;
4394 
4395                 /*
4396                  * Check both the token buffer and (later) the input buffer,
4397                  * since both could be non-scratch addresses.
4398                  */
4399                 if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) {
4400                         regs[rd] = NULL;
4401                         break;
4402                 }
4403 
4404                 if (!DTRACE_INSCRATCH(mstate, size)) {
4405                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4406                         regs[rd] = NULL;
4407                         break;
4408                 }
4409 
4410                 if (addr == NULL) {
4411                         /*
4412                          * If the address specified is NULL, we use our saved
4413                          * strtok pointer from the mstate.  Note that this
4414                          * means that the saved strtok pointer is _only_
4415                          * valid within multiple enablings of the same probe --
4416                          * it behaves like an implicit clause-local variable.
4417                          */
4418                         addr = mstate->dtms_strtok;
4419                 } else {
4420                         /*
4421                          * If the user-specified address is non-NULL we must
4422                          * access check it.  This is the only time we have
4423                          * a chance to do so, since this address may reside
4424                          * in the string table of this clause-- future calls
4425                          * (when we fetch addr from mstate->dtms_strtok)
4426                          * would fail this access check.
4427                          */
4428                         if (!dtrace_strcanload(addr, size, mstate, vstate)) {
4429                                 regs[rd] = NULL;
4430                                 break;
4431                         }
4432                 }
4433 
4434                 /*
4435                  * First, zero the token map, and then process the token
4436                  * string -- setting a bit in the map for every character
4437                  * found in the token string.
4438                  */
4439                 for (i = 0; i < sizeof (tokmap); i++)
4440                         tokmap[i] = 0;
4441 
4442                 for (; tokaddr < toklimit; tokaddr++) {
4443                         if ((c = dtrace_load8(tokaddr)) == '\0')
4444                                 break;
4445 
4446                         ASSERT((c >> 3) < sizeof (tokmap));
4447                         tokmap[c >> 3] |= (1 << (c & 0x7));
4448                 }
4449 
4450                 for (limit = addr + size; addr < limit; addr++) {
4451                         /*
4452                          * We're looking for a character that is _not_ contained
4453                          * in the token string.
4454                          */
4455                         if ((c = dtrace_load8(addr)) == '\0')
4456                                 break;
4457 
4458                         if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
4459                                 break;
4460                 }
4461 
4462                 if (c == '\0') {
4463                         /*
4464                          * We reached the end of the string without finding
4465                          * any character that was not in the token string.
4466                          * We return NULL in this case, and we set the saved
4467                          * address to NULL as well.
4468                          */
4469                         regs[rd] = NULL;
4470                         mstate->dtms_strtok = NULL;
4471                         break;
4472                 }
4473 
4474                 /*
4475                  * From here on, we're copying into the destination string.
4476                  */
4477                 for (i = 0; addr < limit && i < size - 1; addr++) {
4478                         if ((c = dtrace_load8(addr)) == '\0')
4479                                 break;
4480 
4481                         if (tokmap[c >> 3] & (1 << (c & 0x7)))
4482                                 break;
4483 
4484                         ASSERT(i < size);
4485                         dest[i++] = c;
4486                 }
4487 
4488                 ASSERT(i < size);
4489                 dest[i] = '\0';
4490                 regs[rd] = (uintptr_t)dest;
4491                 mstate->dtms_scratch_ptr += size;
4492                 mstate->dtms_strtok = addr;
4493                 break;
4494         }
4495 
4496         case DIF_SUBR_SUBSTR: {
4497                 uintptr_t s = tupregs[0].dttk_value;
4498                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4499                 char *d = (char *)mstate->dtms_scratch_ptr;
4500                 int64_t index = (int64_t)tupregs[1].dttk_value;
4501                 int64_t remaining = (int64_t)tupregs[2].dttk_value;
4502                 size_t len = dtrace_strlen((char *)s, size);
4503                 int64_t i;
4504 
4505                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4506                         regs[rd] = NULL;
4507                         break;
4508                 }
4509 
4510                 if (!DTRACE_INSCRATCH(mstate, size)) {
4511                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4512                         regs[rd] = NULL;
4513                         break;
4514                 }
4515 
4516                 if (nargs <= 2)
4517                         remaining = (int64_t)size;
4518 
4519                 if (index < 0) {
4520                         index += len;
4521 
4522                         if (index < 0 && index + remaining > 0) {
4523                                 remaining += index;
4524                                 index = 0;
4525                         }
4526                 }
4527 
4528                 if (index >= len || index < 0) {
4529                         remaining = 0;
4530                 } else if (remaining < 0) {
4531                         remaining += len - index;
4532                 } else if (index + remaining > size) {
4533                         remaining = size - index;
4534                 }
4535 
4536                 for (i = 0; i < remaining; i++) {
4537                         if ((d[i] = dtrace_load8(s + index + i)) == '\0')
4538                                 break;
4539                 }
4540 
4541                 d[i] = '\0';
4542 
4543                 mstate->dtms_scratch_ptr += size;
4544                 regs[rd] = (uintptr_t)d;
4545                 break;
4546         }
4547 
4548         case DIF_SUBR_JSON: {
4549                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4550                 uintptr_t json = tupregs[0].dttk_value;
4551                 size_t jsonlen = dtrace_strlen((char *)json, size);
4552                 uintptr_t elem = tupregs[1].dttk_value;
4553                 size_t elemlen = dtrace_strlen((char *)elem, size);
4554 
4555                 char *dest = (char *)mstate->dtms_scratch_ptr;
4556                 char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1;
4557                 char *ee = elemlist;
4558                 int nelems = 1;
4559                 uintptr_t cur;
4560 
4561                 if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) ||
4562                     !dtrace_canload(elem, elemlen + 1, mstate, vstate)) {
4563                         regs[rd] = NULL;
4564                         break;
4565                 }
4566 
4567                 if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) {
4568                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4569                         regs[rd] = NULL;
4570                         break;
4571                 }
4572 
4573                 /*
4574                  * Read the element selector and split it up into a packed list
4575                  * of strings.
4576                  */
4577                 for (cur = elem; cur < elem + elemlen; cur++) {
4578                         char cc = dtrace_load8(cur);
4579 
4580                         if (cur == elem && cc == '[') {
4581                                 /*
4582                                  * If the first element selector key is
4583                                  * actually an array index then ignore the
4584                                  * bracket.
4585                                  */
4586                                 continue;
4587                         }
4588 
4589                         if (cc == ']')
4590                                 continue;
4591 
4592                         if (cc == '.' || cc == '[') {
4593                                 nelems++;
4594                                 cc = '\0';
4595                         }
4596 
4597                         *ee++ = cc;
4598                 }
4599                 *ee++ = '\0';
4600 
4601                 if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist,
4602                     nelems, dest)) != NULL)
4603                         mstate->dtms_scratch_ptr += jsonlen + 1;
4604                 break;
4605         }
4606 
4607         case DIF_SUBR_TOUPPER:
4608         case DIF_SUBR_TOLOWER: {
4609                 uintptr_t s = tupregs[0].dttk_value;
4610                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4611                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
4612                 size_t len = dtrace_strlen((char *)s, size);
4613                 char lower, upper, convert;
4614                 int64_t i;
4615 
4616                 if (subr == DIF_SUBR_TOUPPER) {
4617                         lower = 'a';
4618                         upper = 'z';
4619                         convert = 'A';
4620                 } else {
4621                         lower = 'A';
4622                         upper = 'Z';
4623                         convert = 'a';
4624                 }
4625 
4626                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4627                         regs[rd] = NULL;
4628                         break;
4629                 }
4630 
4631                 if (!DTRACE_INSCRATCH(mstate, size)) {
4632                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4633                         regs[rd] = NULL;
4634                         break;
4635                 }
4636 
4637                 for (i = 0; i < size - 1; i++) {
4638                         if ((c = dtrace_load8(s + i)) == '\0')
4639                                 break;
4640 
4641                         if (c >= lower && c <= upper)
4642                                 c = convert + (c - lower);
4643 
4644                         dest[i] = c;
4645                 }
4646 
4647                 ASSERT(i < size);
4648                 dest[i] = '\0';
4649                 regs[rd] = (uintptr_t)dest;
4650                 mstate->dtms_scratch_ptr += size;
4651                 break;
4652         }
4653 
4654 case DIF_SUBR_GETMAJOR:
4655 #ifdef _LP64
4656                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
4657 #else
4658                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
4659 #endif
4660                 break;
4661 
4662         case DIF_SUBR_GETMINOR:
4663 #ifdef _LP64
4664                 regs[rd] = tupregs[0].dttk_value & MAXMIN64;
4665 #else
4666                 regs[rd] = tupregs[0].dttk_value & MAXMIN;
4667 #endif
4668                 break;
4669 
4670         case DIF_SUBR_DDI_PATHNAME: {
4671                 /*
4672                  * This one is a galactic mess.  We are going to roughly
4673                  * emulate ddi_pathname(), but it's made more complicated
4674                  * by the fact that we (a) want to include the minor name and
4675                  * (b) must proceed iteratively instead of recursively.
4676                  */
4677                 uintptr_t dest = mstate->dtms_scratch_ptr;
4678                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4679                 char *start = (char *)dest, *end = start + size - 1;
4680                 uintptr_t daddr = tupregs[0].dttk_value;
4681                 int64_t minor = (int64_t)tupregs[1].dttk_value;
4682                 char *s;
4683                 int i, len, depth = 0;
4684 
4685                 /*
4686                  * Due to all the pointer jumping we do and context we must
4687                  * rely upon, we just mandate that the user must have kernel
4688                  * read privileges to use this routine.
4689                  */
4690                 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
4691                         *flags |= CPU_DTRACE_KPRIV;
4692                         *illval = daddr;
4693                         regs[rd] = NULL;
4694                 }
4695 
4696                 if (!DTRACE_INSCRATCH(mstate, size)) {
4697                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4698                         regs[rd] = NULL;
4699                         break;
4700                 }
4701 
4702                 *end = '\0';
4703 
4704                 /*
4705                  * We want to have a name for the minor.  In order to do this,
4706                  * we need to walk the minor list from the devinfo.  We want
4707                  * to be sure that we don't infinitely walk a circular list,
4708                  * so we check for circularity by sending a scout pointer
4709                  * ahead two elements for every element that we iterate over;
4710                  * if the list is circular, these will ultimately point to the
4711                  * same element.  You may recognize this little trick as the
4712                  * answer to a stupid interview question -- one that always
4713                  * seems to be asked by those who had to have it laboriously
4714                  * explained to them, and who can't even concisely describe
4715                  * the conditions under which one would be forced to resort to
4716                  * this technique.  Needless to say, those conditions are
4717                  * found here -- and probably only here.  Is this the only use
4718                  * of this infamous trick in shipping, production code?  If it
4719                  * isn't, it probably should be...
4720                  */
4721                 if (minor != -1) {
4722                         uintptr_t maddr = dtrace_loadptr(daddr +
4723                             offsetof(struct dev_info, devi_minor));
4724 
4725                         uintptr_t next = offsetof(struct ddi_minor_data, next);
4726                         uintptr_t name = offsetof(struct ddi_minor_data,
4727                             d_minor) + offsetof(struct ddi_minor, name);
4728                         uintptr_t dev = offsetof(struct ddi_minor_data,
4729                             d_minor) + offsetof(struct ddi_minor, dev);
4730                         uintptr_t scout;
4731 
4732                         if (maddr != NULL)
4733                                 scout = dtrace_loadptr(maddr + next);
4734 
4735                         while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4736                                 uint64_t m;
4737 #ifdef _LP64
4738                                 m = dtrace_load64(maddr + dev) & MAXMIN64;
4739 #else
4740                                 m = dtrace_load32(maddr + dev) & MAXMIN;
4741 #endif
4742                                 if (m != minor) {
4743                                         maddr = dtrace_loadptr(maddr + next);
4744 
4745                                         if (scout == NULL)
4746                                                 continue;
4747 
4748                                         scout = dtrace_loadptr(scout + next);
4749 
4750                                         if (scout == NULL)
4751                                                 continue;
4752 
4753                                         scout = dtrace_loadptr(scout + next);
4754 
4755                                         if (scout == NULL)
4756                                                 continue;
4757 
4758                                         if (scout == maddr) {
4759                                                 *flags |= CPU_DTRACE_ILLOP;
4760                                                 break;
4761                                         }
4762 
4763                                         continue;
4764                                 }
4765 
4766                                 /*
4767                                  * We have the minor data.  Now we need to
4768                                  * copy the minor's name into the end of the
4769                                  * pathname.
4770                                  */
4771                                 s = (char *)dtrace_loadptr(maddr + name);
4772                                 len = dtrace_strlen(s, size);
4773 
4774                                 if (*flags & CPU_DTRACE_FAULT)
4775                                         break;
4776 
4777                                 if (len != 0) {
4778                                         if ((end -= (len + 1)) < start)
4779                                                 break;
4780 
4781                                         *end = ':';
4782                                 }
4783 
4784                                 for (i = 1; i <= len; i++)
4785                                         end[i] = dtrace_load8((uintptr_t)s++);
4786                                 break;
4787                         }
4788                 }
4789 
4790                 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4791                         ddi_node_state_t devi_state;
4792 
4793                         devi_state = dtrace_load32(daddr +
4794                             offsetof(struct dev_info, devi_node_state));
4795 
4796                         if (*flags & CPU_DTRACE_FAULT)
4797                                 break;
4798 
4799                         if (devi_state >= DS_INITIALIZED) {
4800                                 s = (char *)dtrace_loadptr(daddr +
4801                                     offsetof(struct dev_info, devi_addr));
4802                                 len = dtrace_strlen(s, size);
4803 
4804                                 if (*flags & CPU_DTRACE_FAULT)
4805                                         break;
4806 
4807                                 if (len != 0) {
4808                                         if ((end -= (len + 1)) < start)
4809                                                 break;
4810 
4811                                         *end = '@';
4812                                 }
4813 
4814                                 for (i = 1; i <= len; i++)
4815                                         end[i] = dtrace_load8((uintptr_t)s++);
4816                         }
4817 
4818                         /*
4819                          * Now for the node name...
4820                          */
4821                         s = (char *)dtrace_loadptr(daddr +
4822                             offsetof(struct dev_info, devi_node_name));
4823 
4824                         daddr = dtrace_loadptr(daddr +
4825                             offsetof(struct dev_info, devi_parent));
4826 
4827                         /*
4828                          * If our parent is NULL (that is, if we're the root
4829                          * node), we're going to use the special path
4830                          * "devices".
4831                          */
4832                         if (daddr == NULL)
4833                                 s = "devices";
4834 
4835                         len = dtrace_strlen(s, size);
4836                         if (*flags & CPU_DTRACE_FAULT)
4837                                 break;
4838 
4839                         if ((end -= (len + 1)) < start)
4840                                 break;
4841 
4842                         for (i = 1; i <= len; i++)
4843                                 end[i] = dtrace_load8((uintptr_t)s++);
4844                         *end = '/';
4845 
4846                         if (depth++ > dtrace_devdepth_max) {
4847                                 *flags |= CPU_DTRACE_ILLOP;
4848                                 break;
4849                         }
4850                 }
4851 
4852                 if (end < start)
4853                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4854 
4855                 if (daddr == NULL) {
4856                         regs[rd] = (uintptr_t)end;
4857                         mstate->dtms_scratch_ptr += size;
4858                 }
4859 
4860                 break;
4861         }
4862 
4863         case DIF_SUBR_STRJOIN: {
4864                 char *d = (char *)mstate->dtms_scratch_ptr;
4865                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4866                 uintptr_t s1 = tupregs[0].dttk_value;
4867                 uintptr_t s2 = tupregs[1].dttk_value;
4868                 int i = 0;
4869 
4870                 if (!dtrace_strcanload(s1, size, mstate, vstate) ||
4871                     !dtrace_strcanload(s2, size, mstate, vstate)) {
4872                         regs[rd] = NULL;
4873                         break;
4874                 }
4875 
4876                 if (!DTRACE_INSCRATCH(mstate, size)) {
4877                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4878                         regs[rd] = NULL;
4879                         break;
4880                 }
4881 
4882                 for (;;) {
4883                         if (i >= size) {
4884                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4885                                 regs[rd] = NULL;
4886                                 break;
4887                         }
4888 
4889                         if ((d[i++] = dtrace_load8(s1++)) == '\0') {
4890                                 i--;
4891                                 break;
4892                         }
4893                 }
4894 
4895                 for (;;) {
4896                         if (i >= size) {
4897                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4898                                 regs[rd] = NULL;
4899                                 break;
4900                         }
4901 
4902                         if ((d[i++] = dtrace_load8(s2++)) == '\0')
4903                                 break;
4904                 }
4905 
4906                 if (i < size) {
4907                         mstate->dtms_scratch_ptr += i;
4908                         regs[rd] = (uintptr_t)d;
4909                 }
4910 
4911                 break;
4912         }
4913 
4914         case DIF_SUBR_STRTOLL: {
4915                 uintptr_t s = tupregs[0].dttk_value;
4916                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4917                 int base = 10;
4918 
4919                 if (nargs > 1) {
4920                         if ((base = tupregs[1].dttk_value) <= 1 ||
4921                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4922                                 *flags |= CPU_DTRACE_ILLOP;
4923                                 break;
4924                         }
4925                 }
4926 
4927                 if (!dtrace_strcanload(s, size, mstate, vstate)) {
4928                         regs[rd] = INT64_MIN;
4929                         break;
4930                 }
4931 
4932                 regs[rd] = dtrace_strtoll((char *)s, base, size);
4933                 break;
4934         }
4935 
4936         case DIF_SUBR_LLTOSTR: {
4937                 int64_t i = (int64_t)tupregs[0].dttk_value;
4938                 uint64_t val, digit;
4939                 uint64_t size = 65;     /* enough room for 2^64 in binary */
4940                 char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
4941                 int base = 10;
4942 
4943                 if (nargs > 1) {
4944                         if ((base = tupregs[1].dttk_value) <= 1 ||
4945                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4946                                 *flags |= CPU_DTRACE_ILLOP;
4947                                 break;
4948                         }
4949                 }
4950 
4951                 val = (base == 10 && i < 0) ? i * -1 : i;
4952 
4953                 if (!DTRACE_INSCRATCH(mstate, size)) {
4954                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4955                         regs[rd] = NULL;
4956                         break;
4957                 }
4958 
4959                 for (*end-- = '\0'; val; val /= base) {
4960                         if ((digit = val % base) <= '9' - '0') {
4961                                 *end-- = '0' + digit;
4962                         } else {
4963                                 *end-- = 'a' + (digit - ('9' - '0') - 1);
4964                         }
4965                 }
4966 
4967                 if (i == 0 && base == 16)
4968                         *end-- = '0';
4969 
4970                 if (base == 16)
4971                         *end-- = 'x';
4972 
4973                 if (i == 0 || base == 8 || base == 16)
4974                         *end-- = '0';
4975 
4976                 if (i < 0 && base == 10)
4977                         *end-- = '-';
4978 
4979                 regs[rd] = (uintptr_t)end + 1;
4980                 mstate->dtms_scratch_ptr += size;
4981                 break;
4982         }
4983 
4984         case DIF_SUBR_HTONS:
4985         case DIF_SUBR_NTOHS:
4986 #ifdef _BIG_ENDIAN
4987                 regs[rd] = (uint16_t)tupregs[0].dttk_value;
4988 #else
4989                 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
4990 #endif
4991                 break;
4992 
4993 
4994         case DIF_SUBR_HTONL:
4995         case DIF_SUBR_NTOHL:
4996 #ifdef _BIG_ENDIAN
4997                 regs[rd] = (uint32_t)tupregs[0].dttk_value;
4998 #else
4999                 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
5000 #endif
5001                 break;
5002 
5003 
5004         case DIF_SUBR_HTONLL:
5005         case DIF_SUBR_NTOHLL:
5006 #ifdef _BIG_ENDIAN
5007                 regs[rd] = (uint64_t)tupregs[0].dttk_value;
5008 #else
5009                 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
5010 #endif
5011                 break;
5012 
5013 
5014         case DIF_SUBR_DIRNAME:
5015         case DIF_SUBR_BASENAME: {
5016                 char *dest = (char *)mstate->dtms_scratch_ptr;
5017                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5018                 uintptr_t src = tupregs[0].dttk_value;
5019                 int i, j, len = dtrace_strlen((char *)src, size);
5020                 int lastbase = -1, firstbase = -1, lastdir = -1;
5021                 int start, end;
5022 
5023                 if (!dtrace_canload(src, len + 1, mstate, vstate)) {
5024                         regs[rd] = NULL;
5025                         break;
5026                 }
5027 
5028                 if (!DTRACE_INSCRATCH(mstate, size)) {
5029                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5030                         regs[rd] = NULL;
5031                         break;
5032                 }
5033 
5034                 /*
5035                  * The basename and dirname for a zero-length string is
5036                  * defined to be "."
5037                  */
5038                 if (len == 0) {
5039                         len = 1;
5040                         src = (uintptr_t)".";
5041                 }
5042 
5043                 /*
5044                  * Start from the back of the string, moving back toward the
5045                  * front until we see a character that isn't a slash.  That
5046                  * character is the last character in the basename.
5047                  */
5048                 for (i = len - 1; i >= 0; i--) {
5049                         if (dtrace_load8(src + i) != '/')
5050                                 break;
5051                 }
5052 
5053                 if (i >= 0)
5054                         lastbase = i;
5055 
5056                 /*
5057                  * Starting from the last character in the basename, move
5058                  * towards the front until we find a slash.  The character
5059                  * that we processed immediately before that is the first
5060                  * character in the basename.
5061                  */
5062                 for (; i >= 0; i--) {
5063                         if (dtrace_load8(src + i) == '/')
5064                                 break;
5065                 }
5066 
5067                 if (i >= 0)
5068                         firstbase = i + 1;
5069 
5070                 /*
5071                  * Now keep going until we find a non-slash character.  That
5072                  * character is the last character in the dirname.
5073                  */
5074                 for (; i >= 0; i--) {
5075                         if (dtrace_load8(src + i) != '/')
5076                                 break;
5077                 }
5078 
5079                 if (i >= 0)
5080                         lastdir = i;
5081 
5082                 ASSERT(!(lastbase == -1 && firstbase != -1));
5083                 ASSERT(!(firstbase == -1 && lastdir != -1));
5084 
5085                 if (lastbase == -1) {
5086                         /*
5087                          * We didn't find a non-slash character.  We know that
5088                          * the length is non-zero, so the whole string must be
5089                          * slashes.  In either the dirname or the basename
5090                          * case, we return '/'.
5091                          */
5092                         ASSERT(firstbase == -1);
5093                         firstbase = lastbase = lastdir = 0;
5094                 }
5095 
5096                 if (firstbase == -1) {
5097                         /*
5098                          * The entire string consists only of a basename
5099                          * component.  If we're looking for dirname, we need
5100                          * to change our string to be just "."; if we're
5101                          * looking for a basename, we'll just set the first
5102                          * character of the basename to be 0.
5103                          */
5104                         if (subr == DIF_SUBR_DIRNAME) {
5105                                 ASSERT(lastdir == -1);
5106                                 src = (uintptr_t)".";
5107                                 lastdir = 0;
5108                         } else {
5109                                 firstbase = 0;
5110                         }
5111                 }
5112 
5113                 if (subr == DIF_SUBR_DIRNAME) {
5114                         if (lastdir == -1) {
5115                                 /*
5116                                  * We know that we have a slash in the name --
5117                                  * or lastdir would be set to 0, above.  And
5118                                  * because lastdir is -1, we know that this
5119                                  * slash must be the first character.  (That
5120                                  * is, the full string must be of the form
5121                                  * "/basename".)  In this case, the last
5122                                  * character of the directory name is 0.
5123                                  */
5124                                 lastdir = 0;
5125                         }
5126 
5127                         start = 0;
5128                         end = lastdir;
5129                 } else {
5130                         ASSERT(subr == DIF_SUBR_BASENAME);
5131                         ASSERT(firstbase != -1 && lastbase != -1);
5132                         start = firstbase;
5133                         end = lastbase;
5134                 }
5135 
5136                 for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
5137                         dest[j] = dtrace_load8(src + i);
5138 
5139                 dest[j] = '\0';
5140                 regs[rd] = (uintptr_t)dest;
5141                 mstate->dtms_scratch_ptr += size;
5142                 break;
5143         }
5144 
5145         case DIF_SUBR_GETF: {
5146                 uintptr_t fd = tupregs[0].dttk_value;
5147                 uf_info_t *finfo = &curthread->t_procp->p_user.u_finfo;
5148                 file_t *fp;
5149 
5150                 if (!dtrace_priv_proc(state, mstate)) {
5151                         regs[rd] = NULL;
5152                         break;
5153                 }
5154 
5155                 /*
5156                  * This is safe because fi_nfiles only increases, and the
5157                  * fi_list array is not freed when the array size doubles.
5158                  * (See the comment in flist_grow() for details on the
5159                  * management of the u_finfo structure.)
5160                  */
5161                 fp = fd < finfo->fi_nfiles ? finfo->fi_list[fd].uf_file : NULL;
5162 
5163                 mstate->dtms_getf = fp;
5164                 regs[rd] = (uintptr_t)fp;
5165                 break;
5166         }
5167 
5168         case DIF_SUBR_CLEANPATH: {
5169                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
5170                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5171                 uintptr_t src = tupregs[0].dttk_value;
5172                 int i = 0, j = 0;
5173                 zone_t *z;
5174 
5175                 if (!dtrace_strcanload(src, size, mstate, vstate)) {
5176                         regs[rd] = NULL;
5177                         break;
5178                 }
5179 
5180                 if (!DTRACE_INSCRATCH(mstate, size)) {
5181                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5182                         regs[rd] = NULL;
5183                         break;
5184                 }
5185 
5186                 /*
5187                  * Move forward, loading each character.
5188                  */
5189                 do {
5190                         c = dtrace_load8(src + i++);
5191 next:
5192                         if (j + 5 >= size)   /* 5 = strlen("/..c\0") */
5193                                 break;
5194 
5195                         if (c != '/') {
5196                                 dest[j++] = c;
5197                                 continue;
5198                         }
5199 
5200                         c = dtrace_load8(src + i++);
5201 
5202                         if (c == '/') {
5203                                 /*
5204                                  * We have two slashes -- we can just advance
5205                                  * to the next character.
5206                                  */
5207                                 goto next;
5208                         }
5209 
5210                         if (c != '.') {
5211                                 /*
5212                                  * This is not "." and it's not ".." -- we can
5213                                  * just store the "/" and this character and
5214                                  * drive on.
5215                                  */
5216                                 dest[j++] = '/';
5217                                 dest[j++] = c;
5218                                 continue;
5219                         }
5220 
5221                         c = dtrace_load8(src + i++);
5222 
5223                         if (c == '/') {
5224                                 /*
5225                                  * This is a "/./" component.  We're not going
5226                                  * to store anything in the destination buffer;
5227                                  * we're just going to go to the next component.
5228                                  */
5229                                 goto next;
5230                         }
5231 
5232                         if (c != '.') {
5233                                 /*
5234                                  * This is not ".." -- we can just store the
5235                                  * "/." and this character and continue
5236                                  * processing.
5237                                  */
5238                                 dest[j++] = '/';
5239                                 dest[j++] = '.';
5240                                 dest[j++] = c;
5241                                 continue;
5242                         }
5243 
5244                         c = dtrace_load8(src + i++);
5245 
5246                         if (c != '/' && c != '\0') {
5247                                 /*
5248                                  * This is not ".." -- it's "..[mumble]".
5249                                  * We'll store the "/.." and this character
5250                                  * and continue processing.
5251                                  */
5252                                 dest[j++] = '/';
5253                                 dest[j++] = '.';
5254                                 dest[j++] = '.';
5255                                 dest[j++] = c;
5256                                 continue;
5257                         }
5258 
5259                         /*
5260                          * This is "/../" or "/..\0".  We need to back up
5261                          * our destination pointer until we find a "/".
5262                          */
5263                         i--;
5264                         while (j != 0 && dest[--j] != '/')
5265                                 continue;
5266 
5267                         if (c == '\0')
5268                                 dest[++j] = '/';
5269                 } while (c != '\0');
5270 
5271                 dest[j] = '\0';
5272 
5273                 if (mstate->dtms_getf != NULL &&
5274                     !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) &&
5275                     (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) {
5276                         /*
5277                          * If we've done a getf() as a part of this ECB and we
5278                          * don't have kernel access (and we're not in the global
5279                          * zone), check if the path we cleaned up begins with
5280                          * the zone's root path, and trim it off if so.  Note
5281                          * that this is an output cleanliness issue, not a
5282                          * security issue: knowing one's zone root path does
5283                          * not enable privilege escalation.
5284                          */
5285                         if (strstr(dest, z->zone_rootpath) == dest)
5286                                 dest += strlen(z->zone_rootpath) - 1;
5287                 }
5288 
5289                 regs[rd] = (uintptr_t)dest;
5290                 mstate->dtms_scratch_ptr += size;
5291                 break;
5292         }
5293 
5294         case DIF_SUBR_INET_NTOA:
5295         case DIF_SUBR_INET_NTOA6:
5296         case DIF_SUBR_INET_NTOP: {
5297                 size_t size;
5298                 int af, argi, i;
5299                 char *base, *end;
5300 
5301                 if (subr == DIF_SUBR_INET_NTOP) {
5302                         af = (int)tupregs[0].dttk_value;
5303                         argi = 1;
5304                 } else {
5305                         af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
5306                         argi = 0;
5307                 }
5308 
5309                 if (af == AF_INET) {
5310                         ipaddr_t ip4;
5311                         uint8_t *ptr8, val;
5312 
5313                         /*
5314                          * Safely load the IPv4 address.
5315                          */
5316                         ip4 = dtrace_load32(tupregs[argi].dttk_value);
5317 
5318                         /*
5319                          * Check an IPv4 string will fit in scratch.
5320                          */
5321                         size = INET_ADDRSTRLEN;
5322                         if (!DTRACE_INSCRATCH(mstate, size)) {
5323                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5324                                 regs[rd] = NULL;
5325                                 break;
5326                         }
5327                         base = (char *)mstate->dtms_scratch_ptr;
5328                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5329 
5330                         /*
5331                          * Stringify as a dotted decimal quad.
5332                          */
5333                         *end-- = '\0';
5334                         ptr8 = (uint8_t *)&ip4;
5335                         for (i = 3; i >= 0; i--) {
5336                                 val = ptr8[i];
5337 
5338                                 if (val == 0) {
5339                                         *end-- = '0';
5340                                 } else {
5341                                         for (; val; val /= 10) {
5342                                                 *end-- = '0' + (val % 10);
5343                                         }
5344                                 }
5345 
5346                                 if (i > 0)
5347                                         *end-- = '.';
5348                         }
5349                         ASSERT(end + 1 >= base);
5350 
5351                 } else if (af == AF_INET6) {
5352                         struct in6_addr ip6;
5353                         int firstzero, tryzero, numzero, v6end;
5354                         uint16_t val;
5355                         const char digits[] = "0123456789abcdef";
5356 
5357                         /*
5358                          * Stringify using RFC 1884 convention 2 - 16 bit
5359                          * hexadecimal values with a zero-run compression.
5360                          * Lower case hexadecimal digits are used.
5361                          *      eg, fe80::214:4fff:fe0b:76c8.
5362                          * The IPv4 embedded form is returned for inet_ntop,
5363                          * just the IPv4 string is returned for inet_ntoa6.
5364                          */
5365 
5366                         /*
5367                          * Safely load the IPv6 address.
5368                          */
5369                         dtrace_bcopy(
5370                             (void *)(uintptr_t)tupregs[argi].dttk_value,
5371                             (void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
5372 
5373                         /*
5374                          * Check an IPv6 string will fit in scratch.
5375                          */
5376                         size = INET6_ADDRSTRLEN;
5377                         if (!DTRACE_INSCRATCH(mstate, size)) {
5378                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5379                                 regs[rd] = NULL;
5380                                 break;
5381                         }
5382                         base = (char *)mstate->dtms_scratch_ptr;
5383                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5384                         *end-- = '\0';
5385 
5386                         /*
5387                          * Find the longest run of 16 bit zero values
5388                          * for the single allowed zero compression - "::".
5389                          */
5390                         firstzero = -1;
5391                         tryzero = -1;
5392                         numzero = 1;
5393                         for (i = 0; i < sizeof (struct in6_addr); i++) {
5394                                 if (ip6._S6_un._S6_u8[i] == 0 &&
5395                                     tryzero == -1 && i % 2 == 0) {
5396                                         tryzero = i;
5397                                         continue;
5398                                 }
5399 
5400                                 if (tryzero != -1 &&
5401                                     (ip6._S6_un._S6_u8[i] != 0 ||
5402                                     i == sizeof (struct in6_addr) - 1)) {
5403 
5404                                         if (i - tryzero <= numzero) {
5405                                                 tryzero = -1;
5406                                                 continue;
5407                                         }
5408 
5409                                         firstzero = tryzero;
5410                                         numzero = i - i % 2 - tryzero;
5411                                         tryzero = -1;
5412 
5413                                         if (ip6._S6_un._S6_u8[i] == 0 &&
5414                                             i == sizeof (struct in6_addr) - 1)
5415                                                 numzero += 2;
5416                                 }
5417                         }
5418                         ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
5419 
5420                         /*
5421                          * Check for an IPv4 embedded address.
5422                          */
5423                         v6end = sizeof (struct in6_addr) - 2;
5424                         if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
5425                             IN6_IS_ADDR_V4COMPAT(&ip6)) {
5426                                 for (i = sizeof (struct in6_addr) - 1;
5427                                     i >= DTRACE_V4MAPPED_OFFSET; i--) {
5428                                         ASSERT(end >= base);
5429 
5430                                         val = ip6._S6_un._S6_u8[i];
5431 
5432                                         if (val == 0) {
5433                                                 *end-- = '0';
5434                                         } else {
5435                                                 for (; val; val /= 10) {
5436                                                         *end-- = '0' + val % 10;
5437                                                 }
5438                                         }
5439 
5440                                         if (i > DTRACE_V4MAPPED_OFFSET)
5441                                                 *end-- = '.';
5442                                 }
5443 
5444                                 if (subr == DIF_SUBR_INET_NTOA6)
5445                                         goto inetout;
5446 
5447                                 /*
5448                                  * Set v6end to skip the IPv4 address that
5449                                  * we have already stringified.
5450                                  */
5451                                 v6end = 10;
5452                         }
5453 
5454                         /*
5455                          * Build the IPv6 string by working through the
5456                          * address in reverse.
5457                          */
5458                         for (i = v6end; i >= 0; i -= 2) {
5459                                 ASSERT(end >= base);
5460 
5461                                 if (i == firstzero + numzero - 2) {
5462                                         *end-- = ':';
5463                                         *end-- = ':';
5464                                         i -= numzero - 2;
5465                                         continue;
5466                                 }
5467 
5468                                 if (i < 14 && i != firstzero - 2)
5469                                         *end-- = ':';
5470 
5471                                 val = (ip6._S6_un._S6_u8[i] << 8) +
5472                                     ip6._S6_un._S6_u8[i + 1];
5473 
5474                                 if (val == 0) {
5475                                         *end-- = '0';
5476                                 } else {
5477                                         for (; val; val /= 16) {
5478                                                 *end-- = digits[val % 16];
5479                                         }
5480                                 }
5481                         }
5482                         ASSERT(end + 1 >= base);
5483 
5484                 } else {
5485                         /*
5486                          * The user didn't use AH_INET or AH_INET6.
5487                          */
5488                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
5489                         regs[rd] = NULL;
5490                         break;
5491                 }
5492 
5493 inetout:        regs[rd] = (uintptr_t)end + 1;
5494                 mstate->dtms_scratch_ptr += size;
5495                 break;
5496         }
5497 
5498         }
5499 }
5500 
5501 /*
5502  * Emulate the execution of DTrace IR instructions specified by the given
5503  * DIF object.  This function is deliberately void of assertions as all of
5504  * the necessary checks are handled by a call to dtrace_difo_validate().
5505  */
5506 static uint64_t
5507 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
5508     dtrace_vstate_t *vstate, dtrace_state_t *state)
5509 {
5510         const dif_instr_t *text = difo->dtdo_buf;
5511         const uint_t textlen = difo->dtdo_len;
5512         const char *strtab = difo->dtdo_strtab;
5513         const uint64_t *inttab = difo->dtdo_inttab;
5514 
5515         uint64_t rval = 0;
5516         dtrace_statvar_t *svar;
5517         dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
5518         dtrace_difv_t *v;
5519         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5520         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
5521 
5522         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
5523         uint64_t regs[DIF_DIR_NREGS];
5524         uint64_t *tmp;
5525 
5526         uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
5527         int64_t cc_r;
5528         uint_t pc = 0, id, opc;
5529         uint8_t ttop = 0;
5530         dif_instr_t instr;
5531         uint_t r1, r2, rd;
5532 
5533         /*
5534          * We stash the current DIF object into the machine state: we need it
5535          * for subsequent access checking.
5536          */
5537         mstate->dtms_difo = difo;
5538 
5539         regs[DIF_REG_R0] = 0;           /* %r0 is fixed at zero */
5540 
5541         while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
5542                 opc = pc;
5543 
5544                 instr = text[pc++];
5545                 r1 = DIF_INSTR_R1(instr);
5546                 r2 = DIF_INSTR_R2(instr);
5547                 rd = DIF_INSTR_RD(instr);
5548 
5549                 switch (DIF_INSTR_OP(instr)) {
5550                 case DIF_OP_OR:
5551                         regs[rd] = regs[r1] | regs[r2];
5552                         break;
5553                 case DIF_OP_XOR:
5554                         regs[rd] = regs[r1] ^ regs[r2];
5555                         break;
5556                 case DIF_OP_AND:
5557                         regs[rd] = regs[r1] & regs[r2];
5558                         break;
5559                 case DIF_OP_SLL:
5560                         regs[rd] = regs[r1] << regs[r2];
5561                         break;
5562                 case DIF_OP_SRL:
5563                         regs[rd] = regs[r1] >> regs[r2];
5564                         break;
5565                 case DIF_OP_SUB:
5566                         regs[rd] = regs[r1] - regs[r2];
5567                         break;
5568                 case DIF_OP_ADD:
5569                         regs[rd] = regs[r1] + regs[r2];
5570                         break;
5571                 case DIF_OP_MUL:
5572                         regs[rd] = regs[r1] * regs[r2];
5573                         break;
5574                 case DIF_OP_SDIV:
5575                         if (regs[r2] == 0) {
5576                                 regs[rd] = 0;
5577                                 *flags |= CPU_DTRACE_DIVZERO;
5578                         } else {
5579                                 regs[rd] = (int64_t)regs[r1] /
5580                                     (int64_t)regs[r2];
5581                         }
5582                         break;
5583 
5584                 case DIF_OP_UDIV:
5585                         if (regs[r2] == 0) {
5586                                 regs[rd] = 0;
5587                                 *flags |= CPU_DTRACE_DIVZERO;
5588                         } else {
5589                                 regs[rd] = regs[r1] / regs[r2];
5590                         }
5591                         break;
5592 
5593                 case DIF_OP_SREM:
5594                         if (regs[r2] == 0) {
5595                                 regs[rd] = 0;
5596                                 *flags |= CPU_DTRACE_DIVZERO;
5597                         } else {
5598                                 regs[rd] = (int64_t)regs[r1] %
5599                                     (int64_t)regs[r2];
5600                         }
5601                         break;
5602 
5603                 case DIF_OP_UREM:
5604                         if (regs[r2] == 0) {
5605                                 regs[rd] = 0;
5606                                 *flags |= CPU_DTRACE_DIVZERO;
5607                         } else {
5608                                 regs[rd] = regs[r1] % regs[r2];
5609                         }
5610                         break;
5611 
5612                 case DIF_OP_NOT:
5613                         regs[rd] = ~regs[r1];
5614                         break;
5615                 case DIF_OP_MOV:
5616                         regs[rd] = regs[r1];
5617                         break;
5618                 case DIF_OP_CMP:
5619                         cc_r = regs[r1] - regs[r2];
5620                         cc_n = cc_r < 0;
5621                         cc_z = cc_r == 0;
5622                         cc_v = 0;
5623                         cc_c = regs[r1] < regs[r2];
5624                         break;
5625                 case DIF_OP_TST:
5626                         cc_n = cc_v = cc_c = 0;
5627                         cc_z = regs[r1] == 0;
5628                         break;
5629                 case DIF_OP_BA:
5630                         pc = DIF_INSTR_LABEL(instr);
5631                         break;
5632                 case DIF_OP_BE:
5633                         if (cc_z)
5634                                 pc = DIF_INSTR_LABEL(instr);
5635                         break;
5636                 case DIF_OP_BNE:
5637                         if (cc_z == 0)
5638                                 pc = DIF_INSTR_LABEL(instr);
5639                         break;
5640                 case DIF_OP_BG:
5641                         if ((cc_z | (cc_n ^ cc_v)) == 0)
5642                                 pc = DIF_INSTR_LABEL(instr);
5643                         break;
5644                 case DIF_OP_BGU:
5645                         if ((cc_c | cc_z) == 0)
5646                                 pc = DIF_INSTR_LABEL(instr);
5647                         break;
5648                 case DIF_OP_BGE:
5649                         if ((cc_n ^ cc_v) == 0)
5650                                 pc = DIF_INSTR_LABEL(instr);
5651                         break;
5652                 case DIF_OP_BGEU:
5653                         if (cc_c == 0)
5654                                 pc = DIF_INSTR_LABEL(instr);
5655                         break;
5656                 case DIF_OP_BL:
5657                         if (cc_n ^ cc_v)
5658                                 pc = DIF_INSTR_LABEL(instr);
5659                         break;
5660                 case DIF_OP_BLU:
5661                         if (cc_c)
5662                                 pc = DIF_INSTR_LABEL(instr);
5663                         break;
5664                 case DIF_OP_BLE:
5665                         if (cc_z | (cc_n ^ cc_v))
5666                                 pc = DIF_INSTR_LABEL(instr);
5667                         break;
5668                 case DIF_OP_BLEU:
5669                         if (cc_c | cc_z)
5670                                 pc = DIF_INSTR_LABEL(instr);
5671                         break;
5672                 case DIF_OP_RLDSB:
5673                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5674                                 break;
5675                         /*FALLTHROUGH*/
5676                 case DIF_OP_LDSB:
5677                         regs[rd] = (int8_t)dtrace_load8(regs[r1]);
5678                         break;
5679                 case DIF_OP_RLDSH:
5680                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5681                                 break;
5682                         /*FALLTHROUGH*/
5683                 case DIF_OP_LDSH:
5684                         regs[rd] = (int16_t)dtrace_load16(regs[r1]);
5685                         break;
5686                 case DIF_OP_RLDSW:
5687                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5688                                 break;
5689                         /*FALLTHROUGH*/
5690                 case DIF_OP_LDSW:
5691                         regs[rd] = (int32_t)dtrace_load32(regs[r1]);
5692                         break;
5693                 case DIF_OP_RLDUB:
5694                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5695                                 break;
5696                         /*FALLTHROUGH*/
5697                 case DIF_OP_LDUB:
5698                         regs[rd] = dtrace_load8(regs[r1]);
5699                         break;
5700                 case DIF_OP_RLDUH:
5701                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5702                                 break;
5703                         /*FALLTHROUGH*/
5704                 case DIF_OP_LDUH:
5705                         regs[rd] = dtrace_load16(regs[r1]);
5706                         break;
5707                 case DIF_OP_RLDUW:
5708                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5709                                 break;
5710                         /*FALLTHROUGH*/
5711                 case DIF_OP_LDUW:
5712                         regs[rd] = dtrace_load32(regs[r1]);
5713                         break;
5714                 case DIF_OP_RLDX:
5715                         if (!dtrace_canload(regs[r1], 8, mstate, vstate))
5716                                 break;
5717                         /*FALLTHROUGH*/
5718                 case DIF_OP_LDX:
5719                         regs[rd] = dtrace_load64(regs[r1]);
5720                         break;
5721                 case DIF_OP_ULDSB:
5722                         regs[rd] = (int8_t)
5723                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5724                         break;
5725                 case DIF_OP_ULDSH:
5726                         regs[rd] = (int16_t)
5727                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5728                         break;
5729                 case DIF_OP_ULDSW:
5730                         regs[rd] = (int32_t)
5731                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5732                         break;
5733                 case DIF_OP_ULDUB:
5734                         regs[rd] =
5735                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5736                         break;
5737                 case DIF_OP_ULDUH:
5738                         regs[rd] =
5739                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5740                         break;
5741                 case DIF_OP_ULDUW:
5742                         regs[rd] =
5743                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5744                         break;
5745                 case DIF_OP_ULDX:
5746                         regs[rd] =
5747                             dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5748                         break;
5749                 case DIF_OP_RET:
5750                         rval = regs[rd];
5751                         pc = textlen;
5752                         break;
5753                 case DIF_OP_NOP:
5754                         break;
5755                 case DIF_OP_SETX:
5756                         regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5757                         break;
5758                 case DIF_OP_SETS:
5759                         regs[rd] = (uint64_t)(uintptr_t)
5760                             (strtab + DIF_INSTR_STRING(instr));
5761                         break;
5762                 case DIF_OP_SCMP: {
5763                         size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5764                         uintptr_t s1 = regs[r1];
5765                         uintptr_t s2 = regs[r2];
5766 
5767                         if (s1 != NULL &&
5768                             !dtrace_strcanload(s1, sz, mstate, vstate))
5769                                 break;
5770                         if (s2 != NULL &&
5771                             !dtrace_strcanload(s2, sz, mstate, vstate))
5772                                 break;
5773 
5774                         cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz);
5775 
5776                         cc_n = cc_r < 0;
5777                         cc_z = cc_r == 0;
5778                         cc_v = cc_c = 0;
5779                         break;
5780                 }
5781                 case DIF_OP_LDGA:
5782                         regs[rd] = dtrace_dif_variable(mstate, state,
5783                             r1, regs[r2]);
5784                         break;
5785                 case DIF_OP_LDGS:
5786                         id = DIF_INSTR_VAR(instr);
5787 
5788                         if (id >= DIF_VAR_OTHER_UBASE) {
5789                                 uintptr_t a;
5790 
5791                                 id -= DIF_VAR_OTHER_UBASE;
5792                                 svar = vstate->dtvs_globals[id];
5793                                 ASSERT(svar != NULL);
5794                                 v = &svar->dtsv_var;
5795 
5796                                 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
5797                                         regs[rd] = svar->dtsv_data;
5798                                         break;
5799                                 }
5800 
5801                                 a = (uintptr_t)svar->dtsv_data;
5802 
5803                                 if (*(uint8_t *)a == UINT8_MAX) {
5804                                         /*
5805                                          * If the 0th byte is set to UINT8_MAX
5806                                          * then this is to be treated as a
5807                                          * reference to a NULL variable.
5808                                          */
5809                                         regs[rd] = NULL;
5810                                 } else {
5811                                         regs[rd] = a + sizeof (uint64_t);
5812                                 }
5813 
5814                                 break;
5815                         }
5816 
5817                         regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
5818                         break;
5819 
5820                 case DIF_OP_STGS:
5821                         id = DIF_INSTR_VAR(instr);
5822 
5823                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5824                         id -= DIF_VAR_OTHER_UBASE;
5825 
5826                         svar = vstate->dtvs_globals[id];
5827                         ASSERT(svar != NULL);
5828                         v = &svar->dtsv_var;
5829 
5830                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5831                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5832 
5833                                 ASSERT(a != NULL);
5834                                 ASSERT(svar->dtsv_size != 0);
5835 
5836                                 if (regs[rd] == NULL) {
5837                                         *(uint8_t *)a = UINT8_MAX;
5838                                         break;
5839                                 } else {
5840                                         *(uint8_t *)a = 0;
5841                                         a += sizeof (uint64_t);
5842                                 }
5843                                 if (!dtrace_vcanload(
5844                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5845                                     mstate, vstate))
5846                                         break;
5847 
5848                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5849                                     (void *)a, &v->dtdv_type);
5850                                 break;
5851                         }
5852 
5853                         svar->dtsv_data = regs[rd];
5854                         break;
5855 
5856                 case DIF_OP_LDTA:
5857                         /*
5858                          * There are no DTrace built-in thread-local arrays at
5859                          * present.  This opcode is saved for future work.
5860                          */
5861                         *flags |= CPU_DTRACE_ILLOP;
5862                         regs[rd] = 0;
5863                         break;
5864 
5865                 case DIF_OP_LDLS:
5866                         id = DIF_INSTR_VAR(instr);
5867 
5868                         if (id < DIF_VAR_OTHER_UBASE) {
5869                                 /*
5870                                  * For now, this has no meaning.
5871                                  */
5872                                 regs[rd] = 0;
5873                                 break;
5874                         }
5875 
5876                         id -= DIF_VAR_OTHER_UBASE;
5877 
5878                         ASSERT(id < vstate->dtvs_nlocals);
5879                         ASSERT(vstate->dtvs_locals != NULL);
5880 
5881                         svar = vstate->dtvs_locals[id];
5882                         ASSERT(svar != NULL);
5883                         v = &svar->dtsv_var;
5884 
5885                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5886                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5887                                 size_t sz = v->dtdv_type.dtdt_size;
5888 
5889                                 sz += sizeof (uint64_t);
5890                                 ASSERT(svar->dtsv_size == NCPU * sz);
5891                                 a += CPU->cpu_id * sz;
5892 
5893                                 if (*(uint8_t *)a == UINT8_MAX) {
5894                                         /*
5895                                          * If the 0th byte is set to UINT8_MAX
5896                                          * then this is to be treated as a
5897                                          * reference to a NULL variable.
5898                                          */
5899                                         regs[rd] = NULL;
5900                                 } else {
5901                                         regs[rd] = a + sizeof (uint64_t);
5902                                 }
5903 
5904                                 break;
5905                         }
5906 
5907                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5908                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5909                         regs[rd] = tmp[CPU->cpu_id];
5910                         break;
5911 
5912                 case DIF_OP_STLS:
5913                         id = DIF_INSTR_VAR(instr);
5914 
5915                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5916                         id -= DIF_VAR_OTHER_UBASE;
5917                         ASSERT(id < vstate->dtvs_nlocals);
5918 
5919                         ASSERT(vstate->dtvs_locals != NULL);
5920                         svar = vstate->dtvs_locals[id];
5921                         ASSERT(svar != NULL);
5922                         v = &svar->dtsv_var;
5923 
5924                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5925                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5926                                 size_t sz = v->dtdv_type.dtdt_size;
5927 
5928                                 sz += sizeof (uint64_t);
5929                                 ASSERT(svar->dtsv_size == NCPU * sz);
5930                                 a += CPU->cpu_id * sz;
5931 
5932                                 if (regs[rd] == NULL) {
5933                                         *(uint8_t *)a = UINT8_MAX;
5934                                         break;
5935                                 } else {
5936                                         *(uint8_t *)a = 0;
5937                                         a += sizeof (uint64_t);
5938                                 }
5939 
5940                                 if (!dtrace_vcanload(
5941                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5942                                     mstate, vstate))
5943                                         break;
5944 
5945                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5946                                     (void *)a, &v->dtdv_type);
5947                                 break;
5948                         }
5949 
5950                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5951                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5952                         tmp[CPU->cpu_id] = regs[rd];
5953                         break;
5954 
5955                 case DIF_OP_LDTS: {
5956                         dtrace_dynvar_t *dvar;
5957                         dtrace_key_t *key;
5958 
5959                         id = DIF_INSTR_VAR(instr);
5960                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5961                         id -= DIF_VAR_OTHER_UBASE;
5962                         v = &vstate->dtvs_tlocals[id];
5963 
5964                         key = &tupregs[DIF_DTR_NREGS];
5965                         key[0].dttk_value = (uint64_t)id;
5966                         key[0].dttk_size = 0;
5967                         DTRACE_TLS_THRKEY(key[1].dttk_value);
5968                         key[1].dttk_size = 0;
5969 
5970                         dvar = dtrace_dynvar(dstate, 2, key,
5971                             sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
5972                             mstate, vstate);
5973 
5974                         if (dvar == NULL) {
5975                                 regs[rd] = 0;
5976                                 break;
5977                         }
5978 
5979                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5980                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5981                         } else {
5982                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5983                         }
5984 
5985                         break;
5986                 }
5987 
5988                 case DIF_OP_STTS: {
5989                         dtrace_dynvar_t *dvar;
5990                         dtrace_key_t *key;
5991 
5992                         id = DIF_INSTR_VAR(instr);
5993                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5994                         id -= DIF_VAR_OTHER_UBASE;
5995 
5996                         key = &tupregs[DIF_DTR_NREGS];
5997                         key[0].dttk_value = (uint64_t)id;
5998                         key[0].dttk_size = 0;
5999                         DTRACE_TLS_THRKEY(key[1].dttk_value);
6000                         key[1].dttk_size = 0;
6001                         v = &vstate->dtvs_tlocals[id];
6002 
6003                         dvar = dtrace_dynvar(dstate, 2, key,
6004                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6005                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6006                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6007                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6008 
6009                         /*
6010                          * Given that we're storing to thread-local data,
6011                          * we need to flush our predicate cache.
6012                          */
6013                         curthread->t_predcache = NULL;
6014 
6015                         if (dvar == NULL)
6016                                 break;
6017 
6018                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6019                                 if (!dtrace_vcanload(
6020                                     (void *)(uintptr_t)regs[rd],
6021                                     &v->dtdv_type, mstate, vstate))
6022                                         break;
6023 
6024                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6025                                     dvar->dtdv_data, &v->dtdv_type);
6026                         } else {
6027                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6028                         }
6029 
6030                         break;
6031                 }
6032 
6033                 case DIF_OP_SRA:
6034                         regs[rd] = (int64_t)regs[r1] >> regs[r2];
6035                         break;
6036 
6037                 case DIF_OP_CALL:
6038                         dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
6039                             regs, tupregs, ttop, mstate, state);
6040                         break;
6041 
6042                 case DIF_OP_PUSHTR:
6043                         if (ttop == DIF_DTR_NREGS) {
6044                                 *flags |= CPU_DTRACE_TUPOFLOW;
6045                                 break;
6046                         }
6047 
6048                         if (r1 == DIF_TYPE_STRING) {
6049                                 /*
6050                                  * If this is a string type and the size is 0,
6051                                  * we'll use the system-wide default string
6052                                  * size.  Note that we are _not_ looking at
6053                                  * the value of the DTRACEOPT_STRSIZE option;
6054                                  * had this been set, we would expect to have
6055                                  * a non-zero size value in the "pushtr".
6056                                  */
6057                                 tupregs[ttop].dttk_size =
6058                                     dtrace_strlen((char *)(uintptr_t)regs[rd],
6059                                     regs[r2] ? regs[r2] :
6060                                     dtrace_strsize_default) + 1;
6061                         } else {
6062                                 tupregs[ttop].dttk_size = regs[r2];
6063                         }
6064 
6065                         tupregs[ttop++].dttk_value = regs[rd];
6066                         break;
6067 
6068                 case DIF_OP_PUSHTV:
6069                         if (ttop == DIF_DTR_NREGS) {
6070                                 *flags |= CPU_DTRACE_TUPOFLOW;
6071                                 break;
6072                         }
6073 
6074                         tupregs[ttop].dttk_value = regs[rd];
6075                         tupregs[ttop++].dttk_size = 0;
6076                         break;
6077 
6078                 case DIF_OP_POPTS:
6079                         if (ttop != 0)
6080                                 ttop--;
6081                         break;
6082 
6083                 case DIF_OP_FLUSHTS:
6084                         ttop = 0;
6085                         break;
6086 
6087                 case DIF_OP_LDGAA:
6088                 case DIF_OP_LDTAA: {
6089                         dtrace_dynvar_t *dvar;
6090                         dtrace_key_t *key = tupregs;
6091                         uint_t nkeys = ttop;
6092 
6093                         id = DIF_INSTR_VAR(instr);
6094                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6095                         id -= DIF_VAR_OTHER_UBASE;
6096 
6097                         key[nkeys].dttk_value = (uint64_t)id;
6098                         key[nkeys++].dttk_size = 0;
6099 
6100                         if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
6101                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6102                                 key[nkeys++].dttk_size = 0;
6103                                 v = &vstate->dtvs_tlocals[id];
6104                         } else {
6105                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6106                         }
6107 
6108                         dvar = dtrace_dynvar(dstate, nkeys, key,
6109                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6110                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6111                             DTRACE_DYNVAR_NOALLOC, mstate, vstate);
6112 
6113                         if (dvar == NULL) {
6114                                 regs[rd] = 0;
6115                                 break;
6116                         }
6117 
6118                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6119                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
6120                         } else {
6121                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
6122                         }
6123 
6124                         break;
6125                 }
6126 
6127                 case DIF_OP_STGAA:
6128                 case DIF_OP_STTAA: {
6129                         dtrace_dynvar_t *dvar;
6130                         dtrace_key_t *key = tupregs;
6131                         uint_t nkeys = ttop;
6132 
6133                         id = DIF_INSTR_VAR(instr);
6134                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6135                         id -= DIF_VAR_OTHER_UBASE;
6136 
6137                         key[nkeys].dttk_value = (uint64_t)id;
6138                         key[nkeys++].dttk_size = 0;
6139 
6140                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
6141                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6142                                 key[nkeys++].dttk_size = 0;
6143                                 v = &vstate->dtvs_tlocals[id];
6144                         } else {
6145                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6146                         }
6147 
6148                         dvar = dtrace_dynvar(dstate, nkeys, key,
6149                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6150                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6151                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6152                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6153 
6154                         if (dvar == NULL)
6155                                 break;
6156 
6157                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6158                                 if (!dtrace_vcanload(
6159                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
6160                                     mstate, vstate))
6161                                         break;
6162 
6163                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6164                                     dvar->dtdv_data, &v->dtdv_type);
6165                         } else {
6166                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6167                         }
6168 
6169                         break;
6170                 }
6171 
6172                 case DIF_OP_ALLOCS: {
6173                         uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6174                         size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
6175 
6176                         /*
6177                          * Rounding up the user allocation size could have
6178                          * overflowed large, bogus allocations (like -1ULL) to
6179                          * 0.
6180                          */
6181                         if (size < regs[r1] ||
6182                             !DTRACE_INSCRATCH(mstate, size)) {
6183                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6184                                 regs[rd] = NULL;
6185                                 break;
6186                         }
6187 
6188                         dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
6189                         mstate->dtms_scratch_ptr += size;
6190                         regs[rd] = ptr;
6191                         break;
6192                 }
6193 
6194                 case DIF_OP_COPYS:
6195                         if (!dtrace_canstore(regs[rd], regs[r2],
6196                             mstate, vstate)) {
6197                                 *flags |= CPU_DTRACE_BADADDR;
6198                                 *illval = regs[rd];
6199                                 break;
6200                         }
6201 
6202                         if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
6203                                 break;
6204 
6205                         dtrace_bcopy((void *)(uintptr_t)regs[r1],
6206                             (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
6207                         break;
6208 
6209                 case DIF_OP_STB:
6210                         if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
6211                                 *flags |= CPU_DTRACE_BADADDR;
6212                                 *illval = regs[rd];
6213                                 break;
6214                         }
6215                         *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
6216                         break;
6217 
6218                 case DIF_OP_STH:
6219                         if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
6220                                 *flags |= CPU_DTRACE_BADADDR;
6221                                 *illval = regs[rd];
6222                                 break;
6223                         }
6224                         if (regs[rd] & 1) {
6225                                 *flags |= CPU_DTRACE_BADALIGN;
6226                                 *illval = regs[rd];
6227                                 break;
6228                         }
6229                         *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
6230                         break;
6231 
6232                 case DIF_OP_STW:
6233                         if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
6234                                 *flags |= CPU_DTRACE_BADADDR;
6235                                 *illval = regs[rd];
6236                                 break;
6237                         }
6238                         if (regs[rd] & 3) {
6239                                 *flags |= CPU_DTRACE_BADALIGN;
6240                                 *illval = regs[rd];
6241                                 break;
6242                         }
6243                         *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
6244                         break;
6245 
6246                 case DIF_OP_STX:
6247                         if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
6248                                 *flags |= CPU_DTRACE_BADADDR;
6249                                 *illval = regs[rd];
6250                                 break;
6251                         }
6252                         if (regs[rd] & 7) {
6253                                 *flags |= CPU_DTRACE_BADALIGN;
6254                                 *illval = regs[rd];
6255                                 break;
6256                         }
6257                         *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
6258                         break;
6259                 }
6260         }
6261 
6262         if (!(*flags & CPU_DTRACE_FAULT))
6263                 return (rval);
6264 
6265         mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
6266         mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
6267 
6268         return (0);
6269 }
6270 
6271 static void
6272 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
6273 {
6274         dtrace_probe_t *probe = ecb->dte_probe;
6275         dtrace_provider_t *prov = probe->dtpr_provider;
6276         char c[DTRACE_FULLNAMELEN + 80], *str;
6277         char *msg = "dtrace: breakpoint action at probe ";
6278         char *ecbmsg = " (ecb ";
6279         uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
6280         uintptr_t val = (uintptr_t)ecb;
6281         int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
6282 
6283         if (dtrace_destructive_disallow)
6284                 return;
6285 
6286         /*
6287          * It's impossible to be taking action on the NULL probe.
6288          */
6289         ASSERT(probe != NULL);
6290 
6291         /*
6292          * This is a poor man's (destitute man's?) sprintf():  we want to
6293          * print the provider name, module name, function name and name of
6294          * the probe, along with the hex address of the ECB with the breakpoint
6295          * action -- all of which we must place in the character buffer by
6296          * hand.
6297          */
6298         while (*msg != '\0')
6299                 c[i++] = *msg++;
6300 
6301         for (str = prov->dtpv_name; *str != '\0'; str++)
6302                 c[i++] = *str;
6303         c[i++] = ':';
6304 
6305         for (str = probe->dtpr_mod; *str != '\0'; str++)
6306                 c[i++] = *str;
6307         c[i++] = ':';
6308 
6309         for (str = probe->dtpr_func; *str != '\0'; str++)
6310                 c[i++] = *str;
6311         c[i++] = ':';
6312 
6313         for (str = probe->dtpr_name; *str != '\0'; str++)
6314                 c[i++] = *str;
6315 
6316         while (*ecbmsg != '\0')
6317                 c[i++] = *ecbmsg++;
6318 
6319         while (shift >= 0) {
6320                 mask = (uintptr_t)0xf << shift;
6321 
6322                 if (val >= ((uintptr_t)1 << shift))
6323                         c[i++] = "0123456789abcdef"[(val & mask) >> shift];
6324                 shift -= 4;
6325         }
6326 
6327         c[i++] = ')';
6328         c[i] = '\0';
6329 
6330         debug_enter(c);
6331 }
6332 
6333 static void
6334 dtrace_action_panic(dtrace_ecb_t *ecb)
6335 {
6336         dtrace_probe_t *probe = ecb->dte_probe;
6337 
6338         /*
6339          * It's impossible to be taking action on the NULL probe.
6340          */
6341         ASSERT(probe != NULL);
6342 
6343         if (dtrace_destructive_disallow)
6344                 return;
6345 
6346         if (dtrace_panicked != NULL)
6347                 return;
6348 
6349         if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
6350                 return;
6351 
6352         /*
6353          * We won the right to panic.  (We want to be sure that only one
6354          * thread calls panic() from dtrace_probe(), and that panic() is
6355          * called exactly once.)
6356          */
6357         dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
6358             probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
6359             probe->dtpr_func, probe->dtpr_name, (void *)ecb);
6360 }
6361 
6362 static void
6363 dtrace_action_raise(uint64_t sig)
6364 {
6365         if (dtrace_destructive_disallow)
6366                 return;
6367 
6368         if (sig >= NSIG) {
6369                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
6370                 return;
6371         }
6372 
6373         /*
6374          * raise() has a queue depth of 1 -- we ignore all subsequent
6375          * invocations of the raise() action.
6376          */
6377         if (curthread->t_dtrace_sig == 0)
6378                 curthread->t_dtrace_sig = (uint8_t)sig;
6379 
6380         curthread->t_sig_check = 1;
6381         aston(curthread);
6382 }
6383 
6384 static void
6385 dtrace_action_stop(void)
6386 {
6387         if (dtrace_destructive_disallow)
6388                 return;
6389 
6390         if (!curthread->t_dtrace_stop) {
6391                 curthread->t_dtrace_stop = 1;
6392                 curthread->t_sig_check = 1;
6393                 aston(curthread);
6394         }
6395 }
6396 
6397 static void
6398 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
6399 {
6400         hrtime_t now;
6401         volatile uint16_t *flags;
6402         cpu_t *cpu = CPU;
6403 
6404         if (dtrace_destructive_disallow)
6405                 return;
6406 
6407         flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
6408 
6409         now = dtrace_gethrtime();
6410 
6411         if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
6412                 /*
6413                  * We need to advance the mark to the current time.
6414                  */
6415                 cpu->cpu_dtrace_chillmark = now;
6416                 cpu->cpu_dtrace_chilled = 0;
6417         }
6418 
6419         /*
6420          * Now check to see if the requested chill time would take us over
6421          * the maximum amount of time allowed in the chill interval.  (Or
6422          * worse, if the calculation itself induces overflow.)
6423          */
6424         if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
6425             cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
6426                 *flags |= CPU_DTRACE_ILLOP;
6427                 return;
6428         }
6429 
6430         while (dtrace_gethrtime() - now < val)
6431                 continue;
6432 
6433         /*
6434          * Normally, we assure that the value of the variable "timestamp" does
6435          * not change within an ECB.  The presence of chill() represents an
6436          * exception to this rule, however.
6437          */
6438         mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
6439         cpu->cpu_dtrace_chilled += val;
6440 }
6441 
6442 static void
6443 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
6444     uint64_t *buf, uint64_t arg)
6445 {
6446         int nframes = DTRACE_USTACK_NFRAMES(arg);
6447         int strsize = DTRACE_USTACK_STRSIZE(arg);
6448         uint64_t *pcs = &buf[1], *fps;
6449         char *str = (char *)&pcs[nframes];
6450         int size, offs = 0, i, j;
6451         uintptr_t old = mstate->dtms_scratch_ptr, saved;
6452         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6453         char *sym;
6454 
6455         /*
6456          * Should be taking a faster path if string space has not been
6457          * allocated.
6458          */
6459         ASSERT(strsize != 0);
6460 
6461         /*
6462          * We will first allocate some temporary space for the frame pointers.
6463          */
6464         fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6465         size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
6466             (nframes * sizeof (uint64_t));
6467 
6468         if (!DTRACE_INSCRATCH(mstate, size)) {
6469                 /*
6470                  * Not enough room for our frame pointers -- need to indicate
6471                  * that we ran out of scratch space.
6472                  */
6473                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6474                 return;
6475         }
6476 
6477         mstate->dtms_scratch_ptr += size;
6478         saved = mstate->dtms_scratch_ptr;
6479 
6480         /*
6481          * Now get a stack with both program counters and frame pointers.
6482          */
6483         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6484         dtrace_getufpstack(buf, fps, nframes + 1);
6485         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6486 
6487         /*
6488          * If that faulted, we're cooked.
6489          */
6490         if (*flags & CPU_DTRACE_FAULT)
6491                 goto out;
6492 
6493         /*
6494          * Now we want to walk up the stack, calling the USTACK helper.  For
6495          * each iteration, we restore the scratch pointer.
6496          */
6497         for (i = 0; i < nframes; i++) {
6498                 mstate->dtms_scratch_ptr = saved;
6499 
6500                 if (offs >= strsize)
6501                         break;
6502 
6503                 sym = (char *)(uintptr_t)dtrace_helper(
6504                     DTRACE_HELPER_ACTION_USTACK,
6505                     mstate, state, pcs[i], fps[i]);
6506 
6507                 /*
6508                  * If we faulted while running the helper, we're going to
6509                  * clear the fault and null out the corresponding string.
6510                  */
6511                 if (*flags & CPU_DTRACE_FAULT) {
6512                         *flags &= ~CPU_DTRACE_FAULT;
6513                         str[offs++] = '\0';
6514                         continue;
6515                 }
6516 
6517                 if (sym == NULL) {
6518                         str[offs++] = '\0';
6519                         continue;
6520                 }
6521 
6522                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6523 
6524                 /*
6525                  * Now copy in the string that the helper returned to us.
6526                  */
6527                 for (j = 0; offs + j < strsize; j++) {
6528                         if ((str[offs + j] = sym[j]) == '\0')
6529                                 break;
6530                 }
6531 
6532                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6533 
6534                 offs += j + 1;
6535         }
6536 
6537         if (offs >= strsize) {
6538                 /*
6539                  * If we didn't have room for all of the strings, we don't
6540                  * abort processing -- this needn't be a fatal error -- but we
6541                  * still want to increment a counter (dts_stkstroverflows) to
6542                  * allow this condition to be warned about.  (If this is from
6543                  * a jstack() action, it is easily tuned via jstackstrsize.)
6544                  */
6545                 dtrace_error(&state->dts_stkstroverflows);
6546         }
6547 
6548         while (offs < strsize)
6549                 str[offs++] = '\0';
6550 
6551 out:
6552         mstate->dtms_scratch_ptr = old;
6553 }
6554 
6555 /*
6556  * If you're looking for the epicenter of DTrace, you just found it.  This
6557  * is the function called by the provider to fire a probe -- from which all
6558  * subsequent probe-context DTrace activity emanates.
6559  */
6560 void
6561 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
6562     uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
6563 {
6564         processorid_t cpuid;
6565         dtrace_icookie_t cookie;
6566         dtrace_probe_t *probe;
6567         dtrace_mstate_t mstate;
6568         dtrace_ecb_t *ecb;
6569         dtrace_action_t *act;
6570         intptr_t offs;
6571         size_t size;
6572         int vtime, onintr;
6573         volatile uint16_t *flags;
6574         hrtime_t now, end;
6575 
6576         /*
6577          * Kick out immediately if this CPU is still being born (in which case
6578          * curthread will be set to -1) or the current thread can't allow
6579          * probes in its current context.
6580          */
6581         if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
6582                 return;
6583 
6584         cookie = dtrace_interrupt_disable();
6585         probe = dtrace_probes[id - 1];
6586         cpuid = CPU->cpu_id;
6587         onintr = CPU_ON_INTR(CPU);
6588 
6589         CPU->cpu_dtrace_probes++;
6590 
6591         if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
6592             probe->dtpr_predcache == curthread->t_predcache) {
6593                 /*
6594                  * We have hit in the predicate cache; we know that
6595                  * this predicate would evaluate to be false.
6596                  */
6597                 dtrace_interrupt_enable(cookie);
6598                 return;
6599         }
6600 
6601         if (panic_quiesce) {
6602                 /*
6603                  * We don't trace anything if we're panicking.
6604                  */
6605                 dtrace_interrupt_enable(cookie);
6606                 return;
6607         }
6608 
6609         now = dtrace_gethrtime();
6610         vtime = dtrace_vtime_references != 0;
6611 
6612         if (vtime && curthread->t_dtrace_start)
6613                 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
6614 
6615         mstate.dtms_difo = NULL;
6616         mstate.dtms_probe = probe;
6617         mstate.dtms_strtok = NULL;
6618         mstate.dtms_arg[0] = arg0;
6619         mstate.dtms_arg[1] = arg1;
6620         mstate.dtms_arg[2] = arg2;
6621         mstate.dtms_arg[3] = arg3;
6622         mstate.dtms_arg[4] = arg4;
6623 
6624         flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
6625 
6626         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
6627                 dtrace_predicate_t *pred = ecb->dte_predicate;
6628                 dtrace_state_t *state = ecb->dte_state;
6629                 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
6630                 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
6631                 dtrace_vstate_t *vstate = &state->dts_vstate;
6632                 dtrace_provider_t *prov = probe->dtpr_provider;
6633                 uint64_t tracememsize = 0;
6634                 int committed = 0;
6635                 caddr_t tomax;
6636 
6637                 /*
6638                  * A little subtlety with the following (seemingly innocuous)
6639                  * declaration of the automatic 'val':  by looking at the
6640                  * code, you might think that it could be declared in the
6641                  * action processing loop, below.  (That is, it's only used in
6642                  * the action processing loop.)  However, it must be declared
6643                  * out of that scope because in the case of DIF expression
6644                  * arguments to aggregating actions, one iteration of the
6645                  * action loop will use the last iteration's value.
6646                  */
6647 #ifdef lint
6648                 uint64_t val = 0;
6649 #else
6650                 uint64_t val;
6651 #endif
6652 
6653                 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
6654                 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC;
6655                 mstate.dtms_getf = NULL;
6656 
6657                 *flags &= ~CPU_DTRACE_ERROR;
6658 
6659                 if (prov == dtrace_provider) {
6660                         /*
6661                          * If dtrace itself is the provider of this probe,
6662                          * we're only going to continue processing the ECB if
6663                          * arg0 (the dtrace_state_t) is equal to the ECB's
6664                          * creating state.  (This prevents disjoint consumers
6665                          * from seeing one another's metaprobes.)
6666                          */
6667                         if (arg0 != (uint64_t)(uintptr_t)state)
6668                                 continue;
6669                 }
6670 
6671                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
6672                         /*
6673                          * We're not currently active.  If our provider isn't
6674                          * the dtrace pseudo provider, we're not interested.
6675                          */
6676                         if (prov != dtrace_provider)
6677                                 continue;
6678 
6679                         /*
6680                          * Now we must further check if we are in the BEGIN
6681                          * probe.  If we are, we will only continue processing
6682                          * if we're still in WARMUP -- if one BEGIN enabling
6683                          * has invoked the exit() action, we don't want to
6684                          * evaluate subsequent BEGIN enablings.
6685                          */
6686                         if (probe->dtpr_id == dtrace_probeid_begin &&
6687                             state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
6688                                 ASSERT(state->dts_activity ==
6689                                     DTRACE_ACTIVITY_DRAINING);
6690                                 continue;
6691                         }
6692                 }
6693 
6694                 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb))
6695                         continue;
6696 
6697                 if (now - state->dts_alive > dtrace_deadman_timeout) {
6698                         /*
6699                          * We seem to be dead.  Unless we (a) have kernel
6700                          * destructive permissions (b) have explicitly enabled
6701                          * destructive actions and (c) destructive actions have
6702                          * not been disabled, we're going to transition into
6703                          * the KILLED state, from which no further processing
6704                          * on this state will be performed.
6705                          */
6706                         if (!dtrace_priv_kernel_destructive(state) ||
6707                             !state->dts_cred.dcr_destructive ||
6708                             dtrace_destructive_disallow) {
6709                                 void *activity = &state->dts_activity;
6710                                 dtrace_activity_t current;
6711 
6712                                 do {
6713                                         current = state->dts_activity;
6714                                 } while (dtrace_cas32(activity, current,
6715                                     DTRACE_ACTIVITY_KILLED) != current);
6716 
6717                                 continue;
6718                         }
6719                 }
6720 
6721                 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
6722                     ecb->dte_alignment, state, &mstate)) < 0)
6723                         continue;
6724 
6725                 tomax = buf->dtb_tomax;
6726                 ASSERT(tomax != NULL);
6727 
6728                 if (ecb->dte_size != 0) {
6729                         dtrace_rechdr_t dtrh;
6730                         if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
6731                                 mstate.dtms_timestamp = dtrace_gethrtime();
6732                                 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
6733                         }
6734                         ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
6735                         dtrh.dtrh_epid = ecb->dte_epid;
6736                         DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
6737                             mstate.dtms_timestamp);
6738                         *((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
6739                 }
6740 
6741                 mstate.dtms_epid = ecb->dte_epid;
6742                 mstate.dtms_present |= DTRACE_MSTATE_EPID;
6743 
6744                 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
6745                         mstate.dtms_access |= DTRACE_ACCESS_KERNEL;
6746 
6747                 if (pred != NULL) {
6748                         dtrace_difo_t *dp = pred->dtp_difo;
6749                         int rval;
6750 
6751                         rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
6752 
6753                         if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
6754                                 dtrace_cacheid_t cid = probe->dtpr_predcache;
6755 
6756                                 if (cid != DTRACE_CACHEIDNONE && !onintr) {
6757                                         /*
6758                                          * Update the predicate cache...
6759                                          */
6760                                         ASSERT(cid == pred->dtp_cacheid);
6761                                         curthread->t_predcache = cid;
6762                                 }
6763 
6764                                 continue;
6765                         }
6766                 }
6767 
6768                 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
6769                     act != NULL; act = act->dta_next) {
6770                         size_t valoffs;
6771                         dtrace_difo_t *dp;
6772                         dtrace_recdesc_t *rec = &act->dta_rec;
6773 
6774                         size = rec->dtrd_size;
6775                         valoffs = offs + rec->dtrd_offset;
6776 
6777                         if (DTRACEACT_ISAGG(act->dta_kind)) {
6778                                 uint64_t v = 0xbad;
6779                                 dtrace_aggregation_t *agg;
6780 
6781                                 agg = (dtrace_aggregation_t *)act;
6782 
6783                                 if ((dp = act->dta_difo) != NULL)
6784                                         v = dtrace_dif_emulate(dp,
6785                                             &mstate, vstate, state);
6786 
6787                                 if (*flags & CPU_DTRACE_ERROR)
6788                                         continue;
6789 
6790                                 /*
6791                                  * Note that we always pass the expression
6792                                  * value from the previous iteration of the
6793                                  * action loop.  This value will only be used
6794                                  * if there is an expression argument to the
6795                                  * aggregating action, denoted by the
6796                                  * dtag_hasarg field.
6797                                  */
6798                                 dtrace_aggregate(agg, buf,
6799                                     offs, aggbuf, v, val);
6800                                 continue;
6801                         }
6802 
6803                         switch (act->dta_kind) {
6804                         case DTRACEACT_STOP:
6805                                 if (dtrace_priv_proc_destructive(state,
6806                                     &mstate))
6807                                         dtrace_action_stop();
6808                                 continue;
6809 
6810                         case DTRACEACT_BREAKPOINT:
6811                                 if (dtrace_priv_kernel_destructive(state))
6812                                         dtrace_action_breakpoint(ecb);
6813                                 continue;
6814 
6815                         case DTRACEACT_PANIC:
6816                                 if (dtrace_priv_kernel_destructive(state))
6817                                         dtrace_action_panic(ecb);
6818                                 continue;
6819 
6820                         case DTRACEACT_STACK:
6821                                 if (!dtrace_priv_kernel(state))
6822                                         continue;
6823 
6824                                 dtrace_getpcstack((pc_t *)(tomax + valoffs),
6825                                     size / sizeof (pc_t), probe->dtpr_aframes,
6826                                     DTRACE_ANCHORED(probe) ? NULL :
6827                                     (uint32_t *)arg0);
6828 
6829                                 continue;
6830 
6831                         case DTRACEACT_JSTACK:
6832                         case DTRACEACT_USTACK:
6833                                 if (!dtrace_priv_proc(state, &mstate))
6834                                         continue;
6835 
6836                                 /*
6837                                  * See comment in DIF_VAR_PID.
6838                                  */
6839                                 if (DTRACE_ANCHORED(mstate.dtms_probe) &&
6840                                     CPU_ON_INTR(CPU)) {
6841                                         int depth = DTRACE_USTACK_NFRAMES(
6842                                             rec->dtrd_arg) + 1;
6843 
6844                                         dtrace_bzero((void *)(tomax + valoffs),
6845                                             DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
6846                                             + depth * sizeof (uint64_t));
6847 
6848                                         continue;
6849                                 }
6850 
6851                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
6852                                     curproc->p_dtrace_helpers != NULL) {
6853                                         /*
6854                                          * This is the slow path -- we have
6855                                          * allocated string space, and we're
6856                                          * getting the stack of a process that
6857                                          * has helpers.  Call into a separate
6858                                          * routine to perform this processing.
6859                                          */
6860                                         dtrace_action_ustack(&mstate, state,
6861                                             (uint64_t *)(tomax + valoffs),
6862                                             rec->dtrd_arg);
6863                                         continue;
6864                                 }
6865 
6866                                 /*
6867                                  * Clear the string space, since there's no
6868                                  * helper to do it for us.
6869                                  */
6870                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) {
6871                                         int depth = DTRACE_USTACK_NFRAMES(
6872                                             rec->dtrd_arg);
6873                                         size_t strsize = DTRACE_USTACK_STRSIZE(
6874                                             rec->dtrd_arg);
6875                                         uint64_t *buf = (uint64_t *)(tomax +
6876                                             valoffs);
6877                                         void *strspace = &buf[depth + 1];
6878 
6879                                         dtrace_bzero(strspace,
6880                                             MIN(depth, strsize));
6881                                 }
6882 
6883                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6884                                 dtrace_getupcstack((uint64_t *)
6885                                     (tomax + valoffs),
6886                                     DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
6887                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6888                                 continue;
6889 
6890                         default:
6891                                 break;
6892                         }
6893 
6894                         dp = act->dta_difo;
6895                         ASSERT(dp != NULL);
6896 
6897                         val = dtrace_dif_emulate(dp, &mstate, vstate, state);
6898 
6899                         if (*flags & CPU_DTRACE_ERROR)
6900                                 continue;
6901 
6902                         switch (act->dta_kind) {
6903                         case DTRACEACT_SPECULATE: {
6904                                 dtrace_rechdr_t *dtrh;
6905 
6906                                 ASSERT(buf == &state->dts_buffer[cpuid]);
6907                                 buf = dtrace_speculation_buffer(state,
6908                                     cpuid, val);
6909 
6910                                 if (buf == NULL) {
6911                                         *flags |= CPU_DTRACE_DROP;
6912                                         continue;
6913                                 }
6914 
6915                                 offs = dtrace_buffer_reserve(buf,
6916                                     ecb->dte_needed, ecb->dte_alignment,
6917                                     state, NULL);
6918 
6919                                 if (offs < 0) {
6920                                         *flags |= CPU_DTRACE_DROP;
6921                                         continue;
6922                                 }
6923 
6924                                 tomax = buf->dtb_tomax;
6925                                 ASSERT(tomax != NULL);
6926 
6927                                 if (ecb->dte_size == 0)
6928                                         continue;
6929 
6930                                 ASSERT3U(ecb->dte_size, >=,
6931                                     sizeof (dtrace_rechdr_t));
6932                                 dtrh = ((void *)(tomax + offs));
6933                                 dtrh->dtrh_epid = ecb->dte_epid;
6934                                 /*
6935                                  * When the speculation is committed, all of
6936                                  * the records in the speculative buffer will
6937                                  * have their timestamps set to the commit
6938                                  * time.  Until then, it is set to a sentinel
6939                                  * value, for debugability.
6940                                  */
6941                                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
6942                                 continue;
6943                         }
6944 
6945                         case DTRACEACT_CHILL:
6946                                 if (dtrace_priv_kernel_destructive(state))
6947                                         dtrace_action_chill(&mstate, val);
6948                                 continue;
6949 
6950                         case DTRACEACT_RAISE:
6951                                 if (dtrace_priv_proc_destructive(state,
6952                                     &mstate))
6953                                         dtrace_action_raise(val);
6954                                 continue;
6955 
6956                         case DTRACEACT_COMMIT:
6957                                 ASSERT(!committed);
6958 
6959                                 /*
6960                                  * We need to commit our buffer state.
6961                                  */
6962                                 if (ecb->dte_size)
6963                                         buf->dtb_offset = offs + ecb->dte_size;
6964                                 buf = &state->dts_buffer[cpuid];
6965                                 dtrace_speculation_commit(state, cpuid, val);
6966                                 committed = 1;
6967                                 continue;
6968 
6969                         case DTRACEACT_DISCARD:
6970                                 dtrace_speculation_discard(state, cpuid, val);
6971                                 continue;
6972 
6973                         case DTRACEACT_DIFEXPR:
6974                         case DTRACEACT_LIBACT:
6975                         case DTRACEACT_PRINTF:
6976                         case DTRACEACT_PRINTA:
6977                         case DTRACEACT_SYSTEM:
6978                         case DTRACEACT_FREOPEN:
6979                         case DTRACEACT_TRACEMEM:
6980                                 break;
6981 
6982                         case DTRACEACT_TRACEMEM_DYNSIZE:
6983                                 tracememsize = val;
6984                                 break;
6985 
6986                         case DTRACEACT_SYM:
6987                         case DTRACEACT_MOD:
6988                                 if (!dtrace_priv_kernel(state))
6989                                         continue;
6990                                 break;
6991 
6992                         case DTRACEACT_USYM:
6993                         case DTRACEACT_UMOD:
6994                         case DTRACEACT_UADDR: {
6995                                 struct pid *pid = curthread->t_procp->p_pidp;
6996 
6997                                 if (!dtrace_priv_proc(state, &mstate))
6998                                         continue;
6999 
7000                                 DTRACE_STORE(uint64_t, tomax,
7001                                     valoffs, (uint64_t)pid->pid_id);
7002                                 DTRACE_STORE(uint64_t, tomax,
7003                                     valoffs + sizeof (uint64_t), val);
7004 
7005                                 continue;
7006                         }
7007 
7008                         case DTRACEACT_EXIT: {
7009                                 /*
7010                                  * For the exit action, we are going to attempt
7011                                  * to atomically set our activity to be
7012                                  * draining.  If this fails (either because
7013                                  * another CPU has beat us to the exit action,
7014                                  * or because our current activity is something
7015                                  * other than ACTIVE or WARMUP), we will
7016                                  * continue.  This assures that the exit action
7017                                  * can be successfully recorded at most once
7018                                  * when we're in the ACTIVE state.  If we're
7019                                  * encountering the exit() action while in
7020                                  * COOLDOWN, however, we want to honor the new
7021                                  * status code.  (We know that we're the only
7022                                  * thread in COOLDOWN, so there is no race.)
7023                                  */
7024                                 void *activity = &state->dts_activity;
7025                                 dtrace_activity_t current = state->dts_activity;
7026 
7027                                 if (current == DTRACE_ACTIVITY_COOLDOWN)
7028                                         break;
7029 
7030                                 if (current != DTRACE_ACTIVITY_WARMUP)
7031                                         current = DTRACE_ACTIVITY_ACTIVE;
7032 
7033                                 if (dtrace_cas32(activity, current,
7034                                     DTRACE_ACTIVITY_DRAINING) != current) {
7035                                         *flags |= CPU_DTRACE_DROP;
7036                                         continue;
7037                                 }
7038 
7039                                 break;
7040                         }
7041 
7042                         default:
7043                                 ASSERT(0);
7044                         }
7045 
7046                         if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
7047                                 uintptr_t end = valoffs + size;
7048 
7049                                 if (tracememsize != 0 &&
7050                                     valoffs + tracememsize < end) {
7051                                         end = valoffs + tracememsize;
7052                                         tracememsize = 0;
7053                                 }
7054 
7055                                 if (!dtrace_vcanload((void *)(uintptr_t)val,
7056                                     &dp->dtdo_rtype, &mstate, vstate))
7057                                         continue;
7058 
7059                                 /*
7060                                  * If this is a string, we're going to only
7061                                  * load until we find the zero byte -- after
7062                                  * which we'll store zero bytes.
7063                                  */
7064                                 if (dp->dtdo_rtype.dtdt_kind ==
7065                                     DIF_TYPE_STRING) {
7066                                         char c = '\0' + 1;
7067                                         int intuple = act->dta_intuple;
7068                                         size_t s;
7069 
7070                                         for (s = 0; s < size; s++) {
7071                                                 if (c != '\0')
7072                                                         c = dtrace_load8(val++);
7073 
7074                                                 DTRACE_STORE(uint8_t, tomax,
7075                                                     valoffs++, c);
7076 
7077                                                 if (c == '\0' && intuple)
7078                                                         break;
7079                                         }
7080 
7081                                         continue;
7082                                 }
7083 
7084                                 while (valoffs < end) {
7085                                         DTRACE_STORE(uint8_t, tomax, valoffs++,
7086                                             dtrace_load8(val++));
7087                                 }
7088 
7089                                 continue;
7090                         }
7091 
7092                         switch (size) {
7093                         case 0:
7094                                 break;
7095 
7096                         case sizeof (uint8_t):
7097                                 DTRACE_STORE(uint8_t, tomax, valoffs, val);
7098                                 break;
7099                         case sizeof (uint16_t):
7100                                 DTRACE_STORE(uint16_t, tomax, valoffs, val);
7101                                 break;
7102                         case sizeof (uint32_t):
7103                                 DTRACE_STORE(uint32_t, tomax, valoffs, val);
7104                                 break;
7105                         case sizeof (uint64_t):
7106                                 DTRACE_STORE(uint64_t, tomax, valoffs, val);
7107                                 break;
7108                         default:
7109                                 /*
7110                                  * Any other size should have been returned by
7111                                  * reference, not by value.
7112                                  */
7113                                 ASSERT(0);
7114                                 break;
7115                         }
7116                 }
7117 
7118                 if (*flags & CPU_DTRACE_DROP)
7119                         continue;
7120 
7121                 if (*flags & CPU_DTRACE_FAULT) {
7122                         int ndx;
7123                         dtrace_action_t *err;
7124 
7125                         buf->dtb_errors++;
7126 
7127                         if (probe->dtpr_id == dtrace_probeid_error) {
7128                                 /*
7129                                  * There's nothing we can do -- we had an
7130                                  * error on the error probe.  We bump an
7131                                  * error counter to at least indicate that
7132                                  * this condition happened.
7133                                  */
7134                                 dtrace_error(&state->dts_dblerrors);
7135                                 continue;
7136                         }
7137 
7138                         if (vtime) {
7139                                 /*
7140                                  * Before recursing on dtrace_probe(), we
7141                                  * need to explicitly clear out our start
7142                                  * time to prevent it from being accumulated
7143                                  * into t_dtrace_vtime.
7144                                  */
7145                                 curthread->t_dtrace_start = 0;
7146                         }
7147 
7148                         /*
7149                          * Iterate over the actions to figure out which action
7150                          * we were processing when we experienced the error.
7151                          * Note that act points _past_ the faulting action; if
7152                          * act is ecb->dte_action, the fault was in the
7153                          * predicate, if it's ecb->dte_action->dta_next it's
7154                          * in action #1, and so on.
7155                          */
7156                         for (err = ecb->dte_action, ndx = 0;
7157                             err != act; err = err->dta_next, ndx++)
7158                                 continue;
7159 
7160                         dtrace_probe_error(state, ecb->dte_epid, ndx,
7161                             (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
7162                             mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
7163                             cpu_core[cpuid].cpuc_dtrace_illval);
7164 
7165                         continue;
7166                 }
7167 
7168                 if (!committed)
7169                         buf->dtb_offset = offs + ecb->dte_size;
7170         }
7171 
7172         end = dtrace_gethrtime();
7173         if (vtime)
7174                 curthread->t_dtrace_start = end;
7175 
7176         CPU->cpu_dtrace_nsec += end - now;
7177 
7178         dtrace_interrupt_enable(cookie);
7179 }
7180 
7181 /*
7182  * DTrace Probe Hashing Functions
7183  *
7184  * The functions in this section (and indeed, the functions in remaining
7185  * sections) are not _called_ from probe context.  (Any exceptions to this are
7186  * marked with a "Note:".)  Rather, they are called from elsewhere in the
7187  * DTrace framework to look-up probes in, add probes to and remove probes from
7188  * the DTrace probe hashes.  (Each probe is hashed by each element of the
7189  * probe tuple -- allowing for fast lookups, regardless of what was
7190  * specified.)
7191  */
7192 static uint_t
7193 dtrace_hash_str(char *p)
7194 {
7195         unsigned int g;
7196         uint_t hval = 0;
7197 
7198         while (*p) {
7199                 hval = (hval << 4) + *p++;
7200                 if ((g = (hval & 0xf0000000)) != 0)
7201                         hval ^= g >> 24;
7202                 hval &= ~g;
7203         }
7204         return (hval);
7205 }
7206 
7207 static dtrace_hash_t *
7208 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
7209 {
7210         dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
7211 
7212         hash->dth_stroffs = stroffs;
7213         hash->dth_nextoffs = nextoffs;
7214         hash->dth_prevoffs = prevoffs;
7215 
7216         hash->dth_size = 1;
7217         hash->dth_mask = hash->dth_size - 1;
7218 
7219         hash->dth_tab = kmem_zalloc(hash->dth_size *
7220             sizeof (dtrace_hashbucket_t *), KM_SLEEP);
7221 
7222         return (hash);
7223 }
7224 
7225 static void
7226 dtrace_hash_destroy(dtrace_hash_t *hash)
7227 {
7228 #ifdef DEBUG
7229         int i;
7230 
7231         for (i = 0; i < hash->dth_size; i++)
7232                 ASSERT(hash->dth_tab[i] == NULL);
7233 #endif
7234 
7235         kmem_free(hash->dth_tab,
7236             hash->dth_size * sizeof (dtrace_hashbucket_t *));
7237         kmem_free(hash, sizeof (dtrace_hash_t));
7238 }
7239 
7240 static void
7241 dtrace_hash_resize(dtrace_hash_t *hash)
7242 {
7243         int size = hash->dth_size, i, ndx;
7244         int new_size = hash->dth_size << 1;
7245         int new_mask = new_size - 1;
7246         dtrace_hashbucket_t **new_tab, *bucket, *next;
7247 
7248         ASSERT((new_size & new_mask) == 0);
7249 
7250         new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
7251 
7252         for (i = 0; i < size; i++) {
7253                 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
7254                         dtrace_probe_t *probe = bucket->dthb_chain;
7255 
7256                         ASSERT(probe != NULL);
7257                         ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
7258 
7259                         next = bucket->dthb_next;
7260                         bucket->dthb_next = new_tab[ndx];
7261                         new_tab[ndx] = bucket;
7262                 }
7263         }
7264 
7265         kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
7266         hash->dth_tab = new_tab;
7267         hash->dth_size = new_size;
7268         hash->dth_mask = new_mask;
7269 }
7270 
7271 static void
7272 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
7273 {
7274         int hashval = DTRACE_HASHSTR(hash, new);
7275         int ndx = hashval & hash->dth_mask;
7276         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7277         dtrace_probe_t **nextp, **prevp;
7278 
7279         for (; bucket != NULL; bucket = bucket->dthb_next) {
7280                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
7281                         goto add;
7282         }
7283 
7284         if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
7285                 dtrace_hash_resize(hash);
7286                 dtrace_hash_add(hash, new);
7287                 return;
7288         }
7289 
7290         bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
7291         bucket->dthb_next = hash->dth_tab[ndx];
7292         hash->dth_tab[ndx] = bucket;
7293         hash->dth_nbuckets++;
7294 
7295 add:
7296         nextp = DTRACE_HASHNEXT(hash, new);
7297         ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
7298         *nextp = bucket->dthb_chain;
7299 
7300         if (bucket->dthb_chain != NULL) {
7301                 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
7302                 ASSERT(*prevp == NULL);
7303                 *prevp = new;
7304         }
7305 
7306         bucket->dthb_chain = new;
7307         bucket->dthb_len++;
7308 }
7309 
7310 static dtrace_probe_t *
7311 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
7312 {
7313         int hashval = DTRACE_HASHSTR(hash, template);
7314         int ndx = hashval & hash->dth_mask;
7315         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7316 
7317         for (; bucket != NULL; bucket = bucket->dthb_next) {
7318                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7319                         return (bucket->dthb_chain);
7320         }
7321 
7322         return (NULL);
7323 }
7324 
7325 static int
7326 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
7327 {
7328         int hashval = DTRACE_HASHSTR(hash, template);
7329         int ndx = hashval & hash->dth_mask;
7330         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7331 
7332         for (; bucket != NULL; bucket = bucket->dthb_next) {
7333                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7334                         return (bucket->dthb_len);
7335         }
7336 
7337         return (NULL);
7338 }
7339 
7340 static void
7341 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
7342 {
7343         int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
7344         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7345 
7346         dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
7347         dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
7348 
7349         /*
7350          * Find the bucket that we're removing this probe from.
7351          */
7352         for (; bucket != NULL; bucket = bucket->dthb_next) {
7353                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
7354                         break;
7355         }
7356 
7357         ASSERT(bucket != NULL);
7358 
7359         if (*prevp == NULL) {
7360                 if (*nextp == NULL) {
7361                         /*
7362                          * The removed probe was the only probe on this
7363                          * bucket; we need to remove the bucket.
7364                          */
7365                         dtrace_hashbucket_t *b = hash->dth_tab[ndx];
7366 
7367                         ASSERT(bucket->dthb_chain == probe);
7368                         ASSERT(b != NULL);
7369 
7370                         if (b == bucket) {
7371                                 hash->dth_tab[ndx] = bucket->dthb_next;
7372                         } else {
7373                                 while (b->dthb_next != bucket)
7374                                         b = b->dthb_next;
7375                                 b->dthb_next = bucket->dthb_next;
7376                         }
7377 
7378                         ASSERT(hash->dth_nbuckets > 0);
7379                         hash->dth_nbuckets--;
7380                         kmem_free(bucket, sizeof (dtrace_hashbucket_t));
7381                         return;
7382                 }
7383 
7384                 bucket->dthb_chain = *nextp;
7385         } else {
7386                 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
7387         }
7388 
7389         if (*nextp != NULL)
7390                 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
7391 }
7392 
7393 /*
7394  * DTrace Utility Functions
7395  *
7396  * These are random utility functions that are _not_ called from probe context.
7397  */
7398 static int
7399 dtrace_badattr(const dtrace_attribute_t *a)
7400 {
7401         return (a->dtat_name > DTRACE_STABILITY_MAX ||
7402             a->dtat_data > DTRACE_STABILITY_MAX ||
7403             a->dtat_class > DTRACE_CLASS_MAX);
7404 }
7405 
7406 /*
7407  * Return a duplicate copy of a string.  If the specified string is NULL,
7408  * this function returns a zero-length string.
7409  */
7410 static char *
7411 dtrace_strdup(const char *str)
7412 {
7413         char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
7414 
7415         if (str != NULL)
7416                 (void) strcpy(new, str);
7417 
7418         return (new);
7419 }
7420 
7421 #define DTRACE_ISALPHA(c)       \
7422         (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
7423 
7424 static int
7425 dtrace_badname(const char *s)
7426 {
7427         char c;
7428 
7429         if (s == NULL || (c = *s++) == '\0')
7430                 return (0);
7431 
7432         if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
7433                 return (1);
7434 
7435         while ((c = *s++) != '\0') {
7436                 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
7437                     c != '-' && c != '_' && c != '.' && c != '`')
7438                         return (1);
7439         }
7440 
7441         return (0);
7442 }
7443 
7444 static void
7445 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
7446 {
7447         uint32_t priv;
7448 
7449         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
7450                 /*
7451                  * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
7452                  */
7453                 priv = DTRACE_PRIV_ALL;
7454         } else {
7455                 *uidp = crgetuid(cr);
7456                 *zoneidp = crgetzoneid(cr);
7457 
7458                 priv = 0;
7459                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
7460                         priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
7461                 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
7462                         priv |= DTRACE_PRIV_USER;
7463                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
7464                         priv |= DTRACE_PRIV_PROC;
7465                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
7466                         priv |= DTRACE_PRIV_OWNER;
7467                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
7468                         priv |= DTRACE_PRIV_ZONEOWNER;
7469         }
7470 
7471         *privp = priv;
7472 }
7473 
7474 #ifdef DTRACE_ERRDEBUG
7475 static void
7476 dtrace_errdebug(const char *str)
7477 {
7478         int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
7479         int occupied = 0;
7480 
7481         mutex_enter(&dtrace_errlock);
7482         dtrace_errlast = str;
7483         dtrace_errthread = curthread;
7484 
7485         while (occupied++ < DTRACE_ERRHASHSZ) {
7486                 if (dtrace_errhash[hval].dter_msg == str) {
7487                         dtrace_errhash[hval].dter_count++;
7488                         goto out;
7489                 }
7490 
7491                 if (dtrace_errhash[hval].dter_msg != NULL) {
7492                         hval = (hval + 1) % DTRACE_ERRHASHSZ;
7493                         continue;
7494                 }
7495 
7496                 dtrace_errhash[hval].dter_msg = str;
7497                 dtrace_errhash[hval].dter_count = 1;
7498                 goto out;
7499         }
7500 
7501         panic("dtrace: undersized error hash");
7502 out:
7503         mutex_exit(&dtrace_errlock);
7504 }
7505 #endif
7506 
7507 /*
7508  * DTrace Matching Functions
7509  *
7510  * These functions are used to match groups of probes, given some elements of
7511  * a probe tuple, or some globbed expressions for elements of a probe tuple.
7512  */
7513 static int
7514 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
7515     zoneid_t zoneid)
7516 {
7517         if (priv != DTRACE_PRIV_ALL) {
7518                 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
7519                 uint32_t match = priv & ppriv;
7520 
7521                 /*
7522                  * No PRIV_DTRACE_* privileges...
7523                  */
7524                 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
7525                     DTRACE_PRIV_KERNEL)) == 0)
7526                         return (0);
7527 
7528                 /*
7529                  * No matching bits, but there were bits to match...
7530                  */
7531                 if (match == 0 && ppriv != 0)
7532                         return (0);
7533 
7534                 /*
7535                  * Need to have permissions to the process, but don't...
7536                  */
7537                 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
7538                     uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
7539                         return (0);
7540                 }
7541 
7542                 /*
7543                  * Need to be in the same zone unless we possess the
7544                  * privilege to examine all zones.
7545                  */
7546                 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
7547                     zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
7548                         return (0);
7549                 }
7550         }
7551 
7552         return (1);
7553 }
7554 
7555 /*
7556  * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
7557  * consists of input pattern strings and an ops-vector to evaluate them.
7558  * This function returns >0 for match, 0 for no match, and <0 for error.
7559  */
7560 static int
7561 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
7562     uint32_t priv, uid_t uid, zoneid_t zoneid)
7563 {
7564         dtrace_provider_t *pvp = prp->dtpr_provider;
7565         int rv;
7566 
7567         if (pvp->dtpv_defunct)
7568                 return (0);
7569 
7570         if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
7571                 return (rv);
7572 
7573         if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
7574                 return (rv);
7575 
7576         if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
7577                 return (rv);
7578 
7579         if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
7580                 return (rv);
7581 
7582         if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
7583                 return (0);
7584 
7585         return (rv);
7586 }
7587 
7588 /*
7589  * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
7590  * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
7591  * libc's version, the kernel version only applies to 8-bit ASCII strings.
7592  * In addition, all of the recursion cases except for '*' matching have been
7593  * unwound.  For '*', we still implement recursive evaluation, but a depth
7594  * counter is maintained and matching is aborted if we recurse too deep.
7595  * The function returns 0 if no match, >0 if match, and <0 if recursion error.
7596  */
7597 static int
7598 dtrace_match_glob(const char *s, const char *p, int depth)
7599 {
7600         const char *olds;
7601         char s1, c;
7602         int gs;
7603 
7604         if (depth > DTRACE_PROBEKEY_MAXDEPTH)
7605                 return (-1);
7606 
7607         if (s == NULL)
7608                 s = ""; /* treat NULL as empty string */
7609 
7610 top:
7611         olds = s;
7612         s1 = *s++;
7613 
7614         if (p == NULL)
7615                 return (0);
7616 
7617         if ((c = *p++) == '\0')
7618                 return (s1 == '\0');
7619 
7620         switch (c) {
7621         case '[': {
7622                 int ok = 0, notflag = 0;
7623                 char lc = '\0';
7624 
7625                 if (s1 == '\0')
7626                         return (0);
7627 
7628                 if (*p == '!') {
7629                         notflag = 1;
7630                         p++;
7631                 }
7632 
7633                 if ((c = *p++) == '\0')
7634                         return (0);
7635 
7636                 do {
7637                         if (c == '-' && lc != '\0' && *p != ']') {
7638                                 if ((c = *p++) == '\0')
7639                                         return (0);
7640                                 if (c == '\\' && (c = *p++) == '\0')
7641                                         return (0);
7642 
7643                                 if (notflag) {
7644                                         if (s1 < lc || s1 > c)
7645                                                 ok++;
7646                                         else
7647                                                 return (0);
7648                                 } else if (lc <= s1 && s1 <= c)
7649                                         ok++;
7650 
7651                         } else if (c == '\\' && (c = *p++) == '\0')
7652                                 return (0);
7653 
7654                         lc = c; /* save left-hand 'c' for next iteration */
7655 
7656                         if (notflag) {
7657                                 if (s1 != c)
7658                                         ok++;
7659                                 else
7660                                         return (0);
7661                         } else if (s1 == c)
7662                                 ok++;
7663 
7664                         if ((c = *p++) == '\0')
7665                                 return (0);
7666 
7667                 } while (c != ']');
7668 
7669                 if (ok)
7670                         goto top;
7671 
7672                 return (0);
7673         }
7674 
7675         case '\\':
7676                 if ((c = *p++) == '\0')
7677                         return (0);
7678                 /*FALLTHRU*/
7679 
7680         default:
7681                 if (c != s1)
7682                         return (0);
7683                 /*FALLTHRU*/
7684 
7685         case '?':
7686                 if (s1 != '\0')
7687                         goto top;
7688                 return (0);
7689 
7690         case '*':
7691                 while (*p == '*')
7692                         p++; /* consecutive *'s are identical to a single one */
7693 
7694                 if (*p == '\0')
7695                         return (1);
7696 
7697                 for (s = olds; *s != '\0'; s++) {
7698                         if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
7699                                 return (gs);
7700                 }
7701 
7702                 return (0);
7703         }
7704 }
7705 
7706 /*ARGSUSED*/
7707 static int
7708 dtrace_match_string(const char *s, const char *p, int depth)
7709 {
7710         return (s != NULL && strcmp(s, p) == 0);
7711 }
7712 
7713 /*ARGSUSED*/
7714 static int
7715 dtrace_match_nul(const char *s, const char *p, int depth)
7716 {
7717         return (1); /* always match the empty pattern */
7718 }
7719 
7720 /*ARGSUSED*/
7721 static int
7722 dtrace_match_nonzero(const char *s, const char *p, int depth)
7723 {
7724         return (s != NULL && s[0] != '\0');
7725 }
7726 
7727 static int
7728 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
7729     zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
7730 {
7731         dtrace_probe_t template, *probe;
7732         dtrace_hash_t *hash = NULL;
7733         int len, rc, best = INT_MAX, nmatched = 0;
7734         dtrace_id_t i;
7735 
7736         ASSERT(MUTEX_HELD(&dtrace_lock));
7737 
7738         /*
7739          * If the probe ID is specified in the key, just lookup by ID and
7740          * invoke the match callback once if a matching probe is found.
7741          */
7742         if (pkp->dtpk_id != DTRACE_IDNONE) {
7743                 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
7744                     dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
7745                         if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
7746                                 return (DTRACE_MATCH_FAIL);
7747                         nmatched++;
7748                 }
7749                 return (nmatched);
7750         }
7751 
7752         template.dtpr_mod = (char *)pkp->dtpk_mod;
7753         template.dtpr_func = (char *)pkp->dtpk_func;
7754         template.dtpr_name = (char *)pkp->dtpk_name;
7755 
7756         /*
7757          * We want to find the most distinct of the module name, function
7758          * name, and name.  So for each one that is not a glob pattern or
7759          * empty string, we perform a lookup in the corresponding hash and
7760          * use the hash table with the fewest collisions to do our search.
7761          */
7762         if (pkp->dtpk_mmatch == &dtrace_match_string &&
7763             (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
7764                 best = len;
7765                 hash = dtrace_bymod;
7766         }
7767 
7768         if (pkp->dtpk_fmatch == &dtrace_match_string &&
7769             (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
7770                 best = len;
7771                 hash = dtrace_byfunc;
7772         }
7773 
7774         if (pkp->dtpk_nmatch == &dtrace_match_string &&
7775             (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
7776                 best = len;
7777                 hash = dtrace_byname;
7778         }
7779 
7780         /*
7781          * If we did not select a hash table, iterate over every probe and
7782          * invoke our callback for each one that matches our input probe key.
7783          */
7784         if (hash == NULL) {
7785                 for (i = 0; i < dtrace_nprobes; i++) {
7786                         if ((probe = dtrace_probes[i]) == NULL ||
7787                             dtrace_match_probe(probe, pkp, priv, uid,
7788                             zoneid) <= 0)
7789                                 continue;
7790 
7791                         nmatched++;
7792 
7793                         if ((rc = (*matched)(probe, arg)) !=
7794                             DTRACE_MATCH_NEXT) {
7795                                 if (rc == DTRACE_MATCH_FAIL)
7796                                         return (DTRACE_MATCH_FAIL);
7797                                 break;
7798                         }
7799                 }
7800 
7801                 return (nmatched);
7802         }
7803 
7804         /*
7805          * If we selected a hash table, iterate over each probe of the same key
7806          * name and invoke the callback for every probe that matches the other
7807          * attributes of our input probe key.
7808          */
7809         for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
7810             probe = *(DTRACE_HASHNEXT(hash, probe))) {
7811 
7812                 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
7813                         continue;
7814 
7815                 nmatched++;
7816 
7817                 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
7818                         if (rc == DTRACE_MATCH_FAIL)
7819                                 return (DTRACE_MATCH_FAIL);
7820                         break;
7821                 }
7822         }
7823 
7824         return (nmatched);
7825 }
7826 
7827 /*
7828  * Return the function pointer dtrace_probecmp() should use to compare the
7829  * specified pattern with a string.  For NULL or empty patterns, we select
7830  * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
7831  * For non-empty non-glob strings, we use dtrace_match_string().
7832  */
7833 static dtrace_probekey_f *
7834 dtrace_probekey_func(const char *p)
7835 {
7836         char c;
7837 
7838         if (p == NULL || *p == '\0')
7839                 return (&dtrace_match_nul);
7840 
7841         while ((c = *p++) != '\0') {
7842                 if (c == '[' || c == '?' || c == '*' || c == '\\')
7843                         return (&dtrace_match_glob);
7844         }
7845 
7846         return (&dtrace_match_string);
7847 }
7848 
7849 /*
7850  * Build a probe comparison key for use with dtrace_match_probe() from the
7851  * given probe description.  By convention, a null key only matches anchored
7852  * probes: if each field is the empty string, reset dtpk_fmatch to
7853  * dtrace_match_nonzero().
7854  */
7855 static void
7856 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
7857 {
7858         pkp->dtpk_prov = pdp->dtpd_provider;
7859         pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
7860 
7861         pkp->dtpk_mod = pdp->dtpd_mod;
7862         pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
7863 
7864         pkp->dtpk_func = pdp->dtpd_func;
7865         pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
7866 
7867         pkp->dtpk_name = pdp->dtpd_name;
7868         pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
7869 
7870         pkp->dtpk_id = pdp->dtpd_id;
7871 
7872         if (pkp->dtpk_id == DTRACE_IDNONE &&
7873             pkp->dtpk_pmatch == &dtrace_match_nul &&
7874             pkp->dtpk_mmatch == &dtrace_match_nul &&
7875             pkp->dtpk_fmatch == &dtrace_match_nul &&
7876             pkp->dtpk_nmatch == &dtrace_match_nul)
7877                 pkp->dtpk_fmatch = &dtrace_match_nonzero;
7878 }
7879 
7880 /*
7881  * DTrace Provider-to-Framework API Functions
7882  *
7883  * These functions implement much of the Provider-to-Framework API, as
7884  * described in <sys/dtrace.h>.  The parts of the API not in this section are
7885  * the functions in the API for probe management (found below), and
7886  * dtrace_probe() itself (found above).
7887  */
7888 
7889 /*
7890  * Register the calling provider with the DTrace framework.  This should
7891  * generally be called by DTrace providers in their attach(9E) entry point.
7892  */
7893 int
7894 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
7895     cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
7896 {
7897         dtrace_provider_t *provider;
7898 
7899         if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
7900                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7901                     "arguments", name ? name : "<NULL>");
7902                 return (EINVAL);
7903         }
7904 
7905         if (name[0] == '\0' || dtrace_badname(name)) {
7906                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7907                     "provider name", name);
7908                 return (EINVAL);
7909         }
7910 
7911         if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
7912             pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
7913             pops->dtps_destroy == NULL ||
7914             ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
7915                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7916                     "provider ops", name);
7917                 return (EINVAL);
7918         }
7919 
7920         if (dtrace_badattr(&pap->dtpa_provider) ||
7921             dtrace_badattr(&pap->dtpa_mod) ||
7922             dtrace_badattr(&pap->dtpa_func) ||
7923             dtrace_badattr(&pap->dtpa_name) ||
7924             dtrace_badattr(&pap->dtpa_args)) {
7925                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7926                     "provider attributes", name);
7927                 return (EINVAL);
7928         }
7929 
7930         if (priv & ~DTRACE_PRIV_ALL) {
7931                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7932                     "privilege attributes", name);
7933                 return (EINVAL);
7934         }
7935 
7936         if ((priv & DTRACE_PRIV_KERNEL) &&
7937             (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
7938             pops->dtps_mode == NULL) {
7939                 cmn_err(CE_WARN, "failed to register provider '%s': need "
7940                     "dtps_mode() op for given privilege attributes", name);
7941                 return (EINVAL);
7942         }
7943 
7944         provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
7945         provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
7946         (void) strcpy(provider->dtpv_name, name);
7947 
7948         provider->dtpv_attr = *pap;
7949         provider->dtpv_priv.dtpp_flags = priv;
7950         if (cr != NULL) {
7951                 provider->dtpv_priv.dtpp_uid = crgetuid(cr);
7952                 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
7953         }
7954         provider->dtpv_pops = *pops;
7955 
7956         if (pops->dtps_provide == NULL) {
7957                 ASSERT(pops->dtps_provide_module != NULL);
7958                 provider->dtpv_pops.dtps_provide =
7959                     (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
7960         }
7961 
7962         if (pops->dtps_provide_module == NULL) {
7963                 ASSERT(pops->dtps_provide != NULL);
7964                 provider->dtpv_pops.dtps_provide_module =
7965                     (void (*)(void *, struct modctl *))dtrace_nullop;
7966         }
7967 
7968         if (pops->dtps_suspend == NULL) {
7969                 ASSERT(pops->dtps_resume == NULL);
7970                 provider->dtpv_pops.dtps_suspend =
7971                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7972                 provider->dtpv_pops.dtps_resume =
7973                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7974         }
7975 
7976         provider->dtpv_arg = arg;
7977         *idp = (dtrace_provider_id_t)provider;
7978 
7979         if (pops == &dtrace_provider_ops) {
7980                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7981                 ASSERT(MUTEX_HELD(&dtrace_lock));
7982                 ASSERT(dtrace_anon.dta_enabling == NULL);
7983 
7984                 /*
7985                  * We make sure that the DTrace provider is at the head of
7986                  * the provider chain.
7987                  */
7988                 provider->dtpv_next = dtrace_provider;
7989                 dtrace_provider = provider;
7990                 return (0);
7991         }
7992 
7993         mutex_enter(&dtrace_provider_lock);
7994         mutex_enter(&dtrace_lock);
7995 
7996         /*
7997          * If there is at least one provider registered, we'll add this
7998          * provider after the first provider.
7999          */
8000         if (dtrace_provider != NULL) {
8001                 provider->dtpv_next = dtrace_provider->dtpv_next;
8002                 dtrace_provider->dtpv_next = provider;
8003         } else {
8004                 dtrace_provider = provider;
8005         }
8006 
8007         if (dtrace_retained != NULL) {
8008                 dtrace_enabling_provide(provider);
8009 
8010                 /*
8011                  * Now we need to call dtrace_enabling_matchall() -- which
8012                  * will acquire cpu_lock and dtrace_lock.  We therefore need
8013                  * to drop all of our locks before calling into it...
8014                  */
8015                 mutex_exit(&dtrace_lock);
8016                 mutex_exit(&dtrace_provider_lock);
8017                 dtrace_enabling_matchall();
8018 
8019                 return (0);
8020         }
8021 
8022         mutex_exit(&dtrace_lock);
8023         mutex_exit(&dtrace_provider_lock);
8024 
8025         return (0);
8026 }
8027 
8028 /*
8029  * Unregister the specified provider from the DTrace framework.  This should
8030  * generally be called by DTrace providers in their detach(9E) entry point.
8031  */
8032 int
8033 dtrace_unregister(dtrace_provider_id_t id)
8034 {
8035         dtrace_provider_t *old = (dtrace_provider_t *)id;
8036         dtrace_provider_t *prev = NULL;
8037         int i, self = 0, noreap = 0;
8038         dtrace_probe_t *probe, *first = NULL;
8039 
8040         if (old->dtpv_pops.dtps_enable ==
8041             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) {
8042                 /*
8043                  * If DTrace itself is the provider, we're called with locks
8044                  * already held.
8045                  */
8046                 ASSERT(old == dtrace_provider);
8047                 ASSERT(dtrace_devi != NULL);
8048                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8049                 ASSERT(MUTEX_HELD(&dtrace_lock));
8050                 self = 1;
8051 
8052                 if (dtrace_provider->dtpv_next != NULL) {
8053                         /*
8054                          * There's another provider here; return failure.
8055                          */
8056                         return (EBUSY);
8057                 }
8058         } else {
8059                 mutex_enter(&dtrace_provider_lock);
8060                 mutex_enter(&mod_lock);
8061                 mutex_enter(&dtrace_lock);
8062         }
8063 
8064         /*
8065          * If anyone has /dev/dtrace open, or if there are anonymous enabled
8066          * probes, we refuse to let providers slither away, unless this
8067          * provider has already been explicitly invalidated.
8068          */
8069         if (!old->dtpv_defunct &&
8070             (dtrace_opens || (dtrace_anon.dta_state != NULL &&
8071             dtrace_anon.dta_state->dts_necbs > 0))) {
8072                 if (!self) {
8073                         mutex_exit(&dtrace_lock);
8074                         mutex_exit(&mod_lock);
8075                         mutex_exit(&dtrace_provider_lock);
8076                 }
8077                 return (EBUSY);
8078         }
8079 
8080         /*
8081          * Attempt to destroy the probes associated with this provider.
8082          */
8083         for (i = 0; i < dtrace_nprobes; i++) {
8084                 if ((probe = dtrace_probes[i]) == NULL)
8085                         continue;
8086 
8087                 if (probe->dtpr_provider != old)
8088                         continue;
8089 
8090                 if (probe->dtpr_ecb == NULL)
8091                         continue;
8092 
8093                 /*
8094                  * If we are trying to unregister a defunct provider, and the
8095                  * provider was made defunct within the interval dictated by
8096                  * dtrace_unregister_defunct_reap, we'll (asynchronously)
8097                  * attempt to reap our enablings.  To denote that the provider
8098                  * should reattempt to unregister itself at some point in the
8099                  * future, we will return a differentiable error code (EAGAIN
8100                  * instead of EBUSY) in this case.
8101                  */
8102                 if (dtrace_gethrtime() - old->dtpv_defunct >
8103                     dtrace_unregister_defunct_reap)
8104                         noreap = 1;
8105 
8106                 if (!self) {
8107                         mutex_exit(&dtrace_lock);
8108                         mutex_exit(&mod_lock);
8109                         mutex_exit(&dtrace_provider_lock);
8110                 }
8111 
8112                 if (noreap)
8113                         return (EBUSY);
8114 
8115                 (void) taskq_dispatch(dtrace_taskq,
8116                     (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
8117 
8118                 return (EAGAIN);
8119         }
8120 
8121         /*
8122          * All of the probes for this provider are disabled; we can safely
8123          * remove all of them from their hash chains and from the probe array.
8124          */
8125         for (i = 0; i < dtrace_nprobes; i++) {
8126                 if ((probe = dtrace_probes[i]) == NULL)
8127                         continue;
8128 
8129                 if (probe->dtpr_provider != old)
8130                         continue;
8131 
8132                 dtrace_probes[i] = NULL;
8133 
8134                 dtrace_hash_remove(dtrace_bymod, probe);
8135                 dtrace_hash_remove(dtrace_byfunc, probe);
8136                 dtrace_hash_remove(dtrace_byname, probe);
8137 
8138                 if (first == NULL) {
8139                         first = probe;
8140                         probe->dtpr_nextmod = NULL;
8141                 } else {
8142                         probe->dtpr_nextmod = first;
8143                         first = probe;
8144                 }
8145         }
8146 
8147         /*
8148          * The provider's probes have been removed from the hash chains and
8149          * from the probe array.  Now issue a dtrace_sync() to be sure that
8150          * everyone has cleared out from any probe array processing.
8151          */
8152         dtrace_sync();
8153 
8154         for (probe = first; probe != NULL; probe = first) {
8155                 first = probe->dtpr_nextmod;
8156 
8157                 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
8158                     probe->dtpr_arg);
8159                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8160                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8161                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8162                 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
8163                 kmem_free(probe, sizeof (dtrace_probe_t));
8164         }
8165 
8166         if ((prev = dtrace_provider) == old) {
8167                 ASSERT(self || dtrace_devi == NULL);
8168                 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
8169                 dtrace_provider = old->dtpv_next;
8170         } else {
8171                 while (prev != NULL && prev->dtpv_next != old)
8172                         prev = prev->dtpv_next;
8173 
8174                 if (prev == NULL) {
8175                         panic("attempt to unregister non-existent "
8176                             "dtrace provider %p\n", (void *)id);
8177                 }
8178 
8179                 prev->dtpv_next = old->dtpv_next;
8180         }
8181 
8182         if (!self) {
8183                 mutex_exit(&dtrace_lock);
8184                 mutex_exit(&mod_lock);
8185                 mutex_exit(&dtrace_provider_lock);
8186         }
8187 
8188         kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
8189         kmem_free(old, sizeof (dtrace_provider_t));
8190 
8191         return (0);
8192 }
8193 
8194 /*
8195  * Invalidate the specified provider.  All subsequent probe lookups for the
8196  * specified provider will fail, but its probes will not be removed.
8197  */
8198 void
8199 dtrace_invalidate(dtrace_provider_id_t id)
8200 {
8201         dtrace_provider_t *pvp = (dtrace_provider_t *)id;
8202 
8203         ASSERT(pvp->dtpv_pops.dtps_enable !=
8204             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
8205 
8206         mutex_enter(&dtrace_provider_lock);
8207         mutex_enter(&dtrace_lock);
8208 
8209         pvp->dtpv_defunct = dtrace_gethrtime();
8210 
8211         mutex_exit(&dtrace_lock);
8212         mutex_exit(&dtrace_provider_lock);
8213 }
8214 
8215 /*
8216  * Indicate whether or not DTrace has attached.
8217  */
8218 int
8219 dtrace_attached(void)
8220 {
8221         /*
8222          * dtrace_provider will be non-NULL iff the DTrace driver has
8223          * attached.  (It's non-NULL because DTrace is always itself a
8224          * provider.)
8225          */
8226         return (dtrace_provider != NULL);
8227 }
8228 
8229 /*
8230  * Remove all the unenabled probes for the given provider.  This function is
8231  * not unlike dtrace_unregister(), except that it doesn't remove the provider
8232  * -- just as many of its associated probes as it can.
8233  */
8234 int
8235 dtrace_condense(dtrace_provider_id_t id)
8236 {
8237         dtrace_provider_t *prov = (dtrace_provider_t *)id;
8238         int i;
8239         dtrace_probe_t *probe;
8240 
8241         /*
8242          * Make sure this isn't the dtrace provider itself.
8243          */
8244         ASSERT(prov->dtpv_pops.dtps_enable !=
8245             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
8246 
8247         mutex_enter(&dtrace_provider_lock);
8248         mutex_enter(&dtrace_lock);
8249 
8250         /*
8251          * Attempt to destroy the probes associated with this provider.
8252          */
8253         for (i = 0; i < dtrace_nprobes; i++) {
8254                 if ((probe = dtrace_probes[i]) == NULL)
8255                         continue;
8256 
8257                 if (probe->dtpr_provider != prov)
8258                         continue;
8259 
8260                 if (probe->dtpr_ecb != NULL)
8261                         continue;
8262 
8263                 dtrace_probes[i] = NULL;
8264 
8265                 dtrace_hash_remove(dtrace_bymod, probe);
8266                 dtrace_hash_remove(dtrace_byfunc, probe);
8267                 dtrace_hash_remove(dtrace_byname, probe);
8268 
8269                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
8270                     probe->dtpr_arg);
8271                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8272                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8273                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8274                 kmem_free(probe, sizeof (dtrace_probe_t));
8275                 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
8276         }
8277 
8278         mutex_exit(&dtrace_lock);
8279         mutex_exit(&dtrace_provider_lock);
8280 
8281         return (0);
8282 }
8283 
8284 /*
8285  * DTrace Probe Management Functions
8286  *
8287  * The functions in this section perform the DTrace probe management,
8288  * including functions to create probes, look-up probes, and call into the
8289  * providers to request that probes be provided.  Some of these functions are
8290  * in the Provider-to-Framework API; these functions can be identified by the
8291  * fact that they are not declared "static".
8292  */
8293 
8294 /*
8295  * Create a probe with the specified module name, function name, and name.
8296  */
8297 dtrace_id_t
8298 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
8299     const char *func, const char *name, int aframes, void *arg)
8300 {
8301         dtrace_probe_t *probe, **probes;
8302         dtrace_provider_t *provider = (dtrace_provider_t *)prov;
8303         dtrace_id_t id;
8304 
8305         if (provider == dtrace_provider) {
8306                 ASSERT(MUTEX_HELD(&dtrace_lock));
8307         } else {
8308                 mutex_enter(&dtrace_lock);
8309         }
8310 
8311         id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
8312             VM_BESTFIT | VM_SLEEP);
8313         probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
8314 
8315         probe->dtpr_id = id;
8316         probe->dtpr_gen = dtrace_probegen++;
8317         probe->dtpr_mod = dtrace_strdup(mod);
8318         probe->dtpr_func = dtrace_strdup(func);
8319         probe->dtpr_name = dtrace_strdup(name);
8320         probe->dtpr_arg = arg;
8321         probe->dtpr_aframes = aframes;
8322         probe->dtpr_provider = provider;
8323 
8324         dtrace_hash_add(dtrace_bymod, probe);
8325         dtrace_hash_add(dtrace_byfunc, probe);
8326         dtrace_hash_add(dtrace_byname, probe);
8327 
8328         if (id - 1 >= dtrace_nprobes) {
8329                 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
8330                 size_t nsize = osize << 1;
8331 
8332                 if (nsize == 0) {
8333                         ASSERT(osize == 0);
8334                         ASSERT(dtrace_probes == NULL);
8335                         nsize = sizeof (dtrace_probe_t *);
8336                 }
8337 
8338                 probes = kmem_zalloc(nsize, KM_SLEEP);
8339 
8340                 if (dtrace_probes == NULL) {
8341                         ASSERT(osize == 0);
8342                         dtrace_probes = probes;
8343                         dtrace_nprobes = 1;
8344                 } else {
8345                         dtrace_probe_t **oprobes = dtrace_probes;
8346 
8347                         bcopy(oprobes, probes, osize);
8348                         dtrace_membar_producer();
8349                         dtrace_probes = probes;
8350 
8351                         dtrace_sync();
8352 
8353                         /*
8354                          * All CPUs are now seeing the new probes array; we can
8355                          * safely free the old array.
8356                          */
8357                         kmem_free(oprobes, osize);
8358                         dtrace_nprobes <<= 1;
8359                 }
8360 
8361                 ASSERT(id - 1 < dtrace_nprobes);
8362         }
8363 
8364         ASSERT(dtrace_probes[id - 1] == NULL);
8365         dtrace_probes[id - 1] = probe;
8366 
8367         if (provider != dtrace_provider)
8368                 mutex_exit(&dtrace_lock);
8369 
8370         return (id);
8371 }
8372 
8373 static dtrace_probe_t *
8374 dtrace_probe_lookup_id(dtrace_id_t id)
8375 {
8376         ASSERT(MUTEX_HELD(&dtrace_lock));
8377 
8378         if (id == 0 || id > dtrace_nprobes)
8379                 return (NULL);
8380 
8381         return (dtrace_probes[id - 1]);
8382 }
8383 
8384 static int
8385 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
8386 {
8387         *((dtrace_id_t *)arg) = probe->dtpr_id;
8388 
8389         return (DTRACE_MATCH_DONE);
8390 }
8391 
8392 /*
8393  * Look up a probe based on provider and one or more of module name, function
8394  * name and probe name.
8395  */
8396 dtrace_id_t
8397 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
8398     const char *func, const char *name)
8399 {
8400         dtrace_probekey_t pkey;
8401         dtrace_id_t id;
8402         int match;
8403 
8404         pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
8405         pkey.dtpk_pmatch = &dtrace_match_string;
8406         pkey.dtpk_mod = mod;
8407         pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
8408         pkey.dtpk_func = func;
8409         pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
8410         pkey.dtpk_name = name;
8411         pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
8412         pkey.dtpk_id = DTRACE_IDNONE;
8413 
8414         mutex_enter(&dtrace_lock);
8415         match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
8416             dtrace_probe_lookup_match, &id);
8417         mutex_exit(&dtrace_lock);
8418 
8419         ASSERT(match == 1 || match == 0);
8420         return (match ? id : 0);
8421 }
8422 
8423 /*
8424  * Returns the probe argument associated with the specified probe.
8425  */
8426 void *
8427 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
8428 {
8429         dtrace_probe_t *probe;
8430         void *rval = NULL;
8431 
8432         mutex_enter(&dtrace_lock);
8433 
8434         if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
8435             probe->dtpr_provider == (dtrace_provider_t *)id)
8436                 rval = probe->dtpr_arg;
8437 
8438         mutex_exit(&dtrace_lock);
8439 
8440         return (rval);
8441 }
8442 
8443 /*
8444  * Copy a probe into a probe description.
8445  */
8446 static void
8447 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
8448 {
8449         bzero(pdp, sizeof (dtrace_probedesc_t));
8450         pdp->dtpd_id = prp->dtpr_id;
8451 
8452         (void) strncpy(pdp->dtpd_provider,
8453             prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
8454 
8455         (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
8456         (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
8457         (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
8458 }
8459 
8460 /*
8461  * Called to indicate that a probe -- or probes -- should be provided by a
8462  * specfied provider.  If the specified description is NULL, the provider will
8463  * be told to provide all of its probes.  (This is done whenever a new
8464  * consumer comes along, or whenever a retained enabling is to be matched.) If
8465  * the specified description is non-NULL, the provider is given the
8466  * opportunity to dynamically provide the specified probe, allowing providers
8467  * to support the creation of probes on-the-fly.  (So-called _autocreated_
8468  * probes.)  If the provider is NULL, the operations will be applied to all
8469  * providers; if the provider is non-NULL the operations will only be applied
8470  * to the specified provider.  The dtrace_provider_lock must be held, and the
8471  * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
8472  * will need to grab the dtrace_lock when it reenters the framework through
8473  * dtrace_probe_lookup(), dtrace_probe_create(), etc.
8474  */
8475 static void
8476 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
8477 {
8478         struct modctl *ctl;
8479         int all = 0;
8480 
8481         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8482 
8483         if (prv == NULL) {
8484                 all = 1;
8485                 prv = dtrace_provider;
8486         }
8487 
8488         do {
8489                 /*
8490                  * First, call the blanket provide operation.
8491                  */
8492                 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
8493 
8494                 /*
8495                  * Now call the per-module provide operation.  We will grab
8496                  * mod_lock to prevent the list from being modified.  Note
8497                  * that this also prevents the mod_busy bits from changing.
8498                  * (mod_busy can only be changed with mod_lock held.)
8499                  */
8500                 mutex_enter(&mod_lock);
8501 
8502                 ctl = &modules;
8503                 do {
8504                         if (ctl->mod_busy || ctl->mod_mp == NULL)
8505                                 continue;
8506 
8507                         prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
8508 
8509                 } while ((ctl = ctl->mod_next) != &modules);
8510 
8511                 mutex_exit(&mod_lock);
8512         } while (all && (prv = prv->dtpv_next) != NULL);
8513 }
8514 
8515 /*
8516  * Iterate over each probe, and call the Framework-to-Provider API function
8517  * denoted by offs.
8518  */
8519 static void
8520 dtrace_probe_foreach(uintptr_t offs)
8521 {
8522         dtrace_provider_t *prov;
8523         void (*func)(void *, dtrace_id_t, void *);
8524         dtrace_probe_t *probe;
8525         dtrace_icookie_t cookie;
8526         int i;
8527 
8528         /*
8529          * We disable interrupts to walk through the probe array.  This is
8530          * safe -- the dtrace_sync() in dtrace_unregister() assures that we
8531          * won't see stale data.
8532          */
8533         cookie = dtrace_interrupt_disable();
8534 
8535         for (i = 0; i < dtrace_nprobes; i++) {
8536                 if ((probe = dtrace_probes[i]) == NULL)
8537                         continue;
8538 
8539                 if (probe->dtpr_ecb == NULL) {
8540                         /*
8541                          * This probe isn't enabled -- don't call the function.
8542                          */
8543                         continue;
8544                 }
8545 
8546                 prov = probe->dtpr_provider;
8547                 func = *((void(**)(void *, dtrace_id_t, void *))
8548                     ((uintptr_t)&prov->dtpv_pops + offs));
8549 
8550                 func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
8551         }
8552 
8553         dtrace_interrupt_enable(cookie);
8554 }
8555 
8556 static int
8557 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
8558 {
8559         dtrace_probekey_t pkey;
8560         uint32_t priv;
8561         uid_t uid;
8562         zoneid_t zoneid;
8563 
8564         ASSERT(MUTEX_HELD(&dtrace_lock));
8565         dtrace_ecb_create_cache = NULL;
8566 
8567         if (desc == NULL) {
8568                 /*
8569                  * If we're passed a NULL description, we're being asked to
8570                  * create an ECB with a NULL probe.
8571                  */
8572                 (void) dtrace_ecb_create_enable(NULL, enab);
8573                 return (0);
8574         }
8575 
8576         dtrace_probekey(desc, &pkey);
8577         dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
8578             &priv, &uid, &zoneid);
8579 
8580         return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
8581             enab));
8582 }
8583 
8584 /*
8585  * DTrace Helper Provider Functions
8586  */
8587 static void
8588 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
8589 {
8590         attr->dtat_name = DOF_ATTR_NAME(dofattr);
8591         attr->dtat_data = DOF_ATTR_DATA(dofattr);
8592         attr->dtat_class = DOF_ATTR_CLASS(dofattr);
8593 }
8594 
8595 static void
8596 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
8597     const dof_provider_t *dofprov, char *strtab)
8598 {
8599         hprov->dthpv_provname = strtab + dofprov->dofpv_name;
8600         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
8601             dofprov->dofpv_provattr);
8602         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
8603             dofprov->dofpv_modattr);
8604         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
8605             dofprov->dofpv_funcattr);
8606         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
8607             dofprov->dofpv_nameattr);
8608         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
8609             dofprov->dofpv_argsattr);
8610 }
8611 
8612 static void
8613 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8614 {
8615         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8616         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8617         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
8618         dof_provider_t *provider;
8619         dof_probe_t *probe;
8620         uint32_t *off, *enoff;
8621         uint8_t *arg;
8622         char *strtab;
8623         uint_t i, nprobes;
8624         dtrace_helper_provdesc_t dhpv;
8625         dtrace_helper_probedesc_t dhpb;
8626         dtrace_meta_t *meta = dtrace_meta_pid;
8627         dtrace_mops_t *mops = &meta->dtm_mops;
8628         void *parg;
8629 
8630         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8631         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8632             provider->dofpv_strtab * dof->dofh_secsize);
8633         prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8634             provider->dofpv_probes * dof->dofh_secsize);
8635         arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8636             provider->dofpv_prargs * dof->dofh_secsize);
8637         off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8638             provider->dofpv_proffs * dof->dofh_secsize);
8639 
8640         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8641         off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
8642         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
8643         enoff = NULL;
8644 
8645         /*
8646          * See dtrace_helper_provider_validate().
8647          */
8648         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
8649             provider->dofpv_prenoffs != DOF_SECT_NONE) {
8650                 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8651                     provider->dofpv_prenoffs * dof->dofh_secsize);
8652                 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
8653         }
8654 
8655         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
8656 
8657         /*
8658          * Create the provider.
8659          */
8660         dtrace_dofprov2hprov(&dhpv, provider, strtab);
8661 
8662         if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
8663                 return;
8664 
8665         meta->dtm_count++;
8666 
8667         /*
8668          * Create the probes.
8669          */
8670         for (i = 0; i < nprobes; i++) {
8671                 probe = (dof_probe_t *)(uintptr_t)(daddr +
8672                     prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
8673 
8674                 dhpb.dthpb_mod = dhp->dofhp_mod;
8675                 dhpb.dthpb_func = strtab + probe->dofpr_func;
8676                 dhpb.dthpb_name = strtab + probe->dofpr_name;
8677                 dhpb.dthpb_base = probe->dofpr_addr;
8678                 dhpb.dthpb_offs = off + probe->dofpr_offidx;
8679                 dhpb.dthpb_noffs = probe->dofpr_noffs;
8680                 if (enoff != NULL) {
8681                         dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
8682                         dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
8683                 } else {
8684                         dhpb.dthpb_enoffs = NULL;
8685                         dhpb.dthpb_nenoffs = 0;
8686                 }
8687                 dhpb.dthpb_args = arg + probe->dofpr_argidx;
8688                 dhpb.dthpb_nargc = probe->dofpr_nargc;
8689                 dhpb.dthpb_xargc = probe->dofpr_xargc;
8690                 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
8691                 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
8692 
8693                 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
8694         }
8695 }
8696 
8697 static void
8698 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
8699 {
8700         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8701         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8702         int i;
8703 
8704         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8705 
8706         for (i = 0; i < dof->dofh_secnum; i++) {
8707                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8708                     dof->dofh_secoff + i * dof->dofh_secsize);
8709 
8710                 if (sec->dofs_type != DOF_SECT_PROVIDER)
8711                         continue;
8712 
8713                 dtrace_helper_provide_one(dhp, sec, pid);
8714         }
8715 
8716         /*
8717          * We may have just created probes, so we must now rematch against
8718          * any retained enablings.  Note that this call will acquire both
8719          * cpu_lock and dtrace_lock; the fact that we are holding
8720          * dtrace_meta_lock now is what defines the ordering with respect to
8721          * these three locks.
8722          */
8723         dtrace_enabling_matchall();
8724 }
8725 
8726 static void
8727 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8728 {
8729         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8730         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8731         dof_sec_t *str_sec;
8732         dof_provider_t *provider;
8733         char *strtab;
8734         dtrace_helper_provdesc_t dhpv;
8735         dtrace_meta_t *meta = dtrace_meta_pid;
8736         dtrace_mops_t *mops = &meta->dtm_mops;
8737 
8738         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8739         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8740             provider->dofpv_strtab * dof->dofh_secsize);
8741 
8742         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8743 
8744         /*
8745          * Create the provider.
8746          */
8747         dtrace_dofprov2hprov(&dhpv, provider, strtab);
8748 
8749         mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
8750 
8751         meta->dtm_count--;
8752 }
8753 
8754 static void
8755 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
8756 {
8757         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8758         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8759         int i;
8760 
8761         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8762 
8763         for (i = 0; i < dof->dofh_secnum; i++) {
8764                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8765                     dof->dofh_secoff + i * dof->dofh_secsize);
8766 
8767                 if (sec->dofs_type != DOF_SECT_PROVIDER)
8768                         continue;
8769 
8770                 dtrace_helper_provider_remove_one(dhp, sec, pid);
8771         }
8772 }
8773 
8774 /*
8775  * DTrace Meta Provider-to-Framework API Functions
8776  *
8777  * These functions implement the Meta Provider-to-Framework API, as described
8778  * in <sys/dtrace.h>.
8779  */
8780 int
8781 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
8782     dtrace_meta_provider_id_t *idp)
8783 {
8784         dtrace_meta_t *meta;
8785         dtrace_helpers_t *help, *next;
8786         int i;
8787 
8788         *idp = DTRACE_METAPROVNONE;
8789 
8790         /*
8791          * We strictly don't need the name, but we hold onto it for
8792          * debuggability. All hail error queues!
8793          */
8794         if (name == NULL) {
8795                 cmn_err(CE_WARN, "failed to register meta-provider: "
8796                     "invalid name");
8797                 return (EINVAL);
8798         }
8799 
8800         if (mops == NULL ||
8801             mops->dtms_create_probe == NULL ||
8802             mops->dtms_provide_pid == NULL ||
8803             mops->dtms_remove_pid == NULL) {
8804                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8805                     "invalid ops", name);
8806                 return (EINVAL);
8807         }
8808 
8809         meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
8810         meta->dtm_mops = *mops;
8811         meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
8812         (void) strcpy(meta->dtm_name, name);
8813         meta->dtm_arg = arg;
8814 
8815         mutex_enter(&dtrace_meta_lock);
8816         mutex_enter(&dtrace_lock);
8817 
8818         if (dtrace_meta_pid != NULL) {
8819                 mutex_exit(&dtrace_lock);
8820                 mutex_exit(&dtrace_meta_lock);
8821                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8822                     "user-land meta-provider exists", name);
8823                 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
8824                 kmem_free(meta, sizeof (dtrace_meta_t));
8825                 return (EINVAL);
8826         }
8827 
8828         dtrace_meta_pid = meta;
8829         *idp = (dtrace_meta_provider_id_t)meta;
8830 
8831         /*
8832          * If there are providers and probes ready to go, pass them
8833          * off to the new meta provider now.
8834          */
8835 
8836         help = dtrace_deferred_pid;
8837         dtrace_deferred_pid = NULL;
8838 
8839         mutex_exit(&dtrace_lock);
8840 
8841         while (help != NULL) {
8842                 for (i = 0; i < help->dthps_nprovs; i++) {
8843                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
8844                             help->dthps_pid);
8845                 }
8846 
8847                 next = help->dthps_next;
8848                 help->dthps_next = NULL;
8849                 help->dthps_prev = NULL;
8850                 help->dthps_deferred = 0;
8851                 help = next;
8852         }
8853 
8854         mutex_exit(&dtrace_meta_lock);
8855 
8856         return (0);
8857 }
8858 
8859 int
8860 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
8861 {
8862         dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
8863 
8864         mutex_enter(&dtrace_meta_lock);
8865         mutex_enter(&dtrace_lock);
8866 
8867         if (old == dtrace_meta_pid) {
8868                 pp = &dtrace_meta_pid;
8869         } else {
8870                 panic("attempt to unregister non-existent "
8871                     "dtrace meta-provider %p\n", (void *)old);
8872         }
8873 
8874         if (old->dtm_count != 0) {
8875                 mutex_exit(&dtrace_lock);
8876                 mutex_exit(&dtrace_meta_lock);
8877                 return (EBUSY);
8878         }
8879 
8880         *pp = NULL;
8881 
8882         mutex_exit(&dtrace_lock);
8883         mutex_exit(&dtrace_meta_lock);
8884 
8885         kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
8886         kmem_free(old, sizeof (dtrace_meta_t));
8887 
8888         return (0);
8889 }
8890 
8891 
8892 /*
8893  * DTrace DIF Object Functions
8894  */
8895 static int
8896 dtrace_difo_err(uint_t pc, const char *format, ...)
8897 {
8898         if (dtrace_err_verbose) {
8899                 va_list alist;
8900 
8901                 (void) uprintf("dtrace DIF object error: [%u]: ", pc);
8902                 va_start(alist, format);
8903                 (void) vuprintf(format, alist);
8904                 va_end(alist);
8905         }
8906 
8907 #ifdef DTRACE_ERRDEBUG
8908         dtrace_errdebug(format);
8909 #endif
8910         return (1);
8911 }
8912 
8913 /*
8914  * Validate a DTrace DIF object by checking the IR instructions.  The following
8915  * rules are currently enforced by dtrace_difo_validate():
8916  *
8917  * 1. Each instruction must have a valid opcode
8918  * 2. Each register, string, variable, or subroutine reference must be valid
8919  * 3. No instruction can modify register %r0 (must be zero)
8920  * 4. All instruction reserved bits must be set to zero
8921  * 5. The last instruction must be a "ret" instruction
8922  * 6. All branch targets must reference a valid instruction _after_ the branch
8923  */
8924 static int
8925 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
8926     cred_t *cr)
8927 {
8928         int err = 0, i;
8929         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8930         int kcheckload;
8931         uint_t pc;
8932 
8933         kcheckload = cr == NULL ||
8934             (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
8935 
8936         dp->dtdo_destructive = 0;
8937 
8938         for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
8939                 dif_instr_t instr = dp->dtdo_buf[pc];
8940 
8941                 uint_t r1 = DIF_INSTR_R1(instr);
8942                 uint_t r2 = DIF_INSTR_R2(instr);
8943                 uint_t rd = DIF_INSTR_RD(instr);
8944                 uint_t rs = DIF_INSTR_RS(instr);
8945                 uint_t label = DIF_INSTR_LABEL(instr);
8946                 uint_t v = DIF_INSTR_VAR(instr);
8947                 uint_t subr = DIF_INSTR_SUBR(instr);
8948                 uint_t type = DIF_INSTR_TYPE(instr);
8949                 uint_t op = DIF_INSTR_OP(instr);
8950 
8951                 switch (op) {
8952                 case DIF_OP_OR:
8953                 case DIF_OP_XOR:
8954                 case DIF_OP_AND:
8955                 case DIF_OP_SLL:
8956                 case DIF_OP_SRL:
8957                 case DIF_OP_SRA:
8958                 case DIF_OP_SUB:
8959                 case DIF_OP_ADD:
8960                 case DIF_OP_MUL:
8961                 case DIF_OP_SDIV:
8962                 case DIF_OP_UDIV:
8963                 case DIF_OP_SREM:
8964                 case DIF_OP_UREM:
8965                 case DIF_OP_COPYS:
8966                         if (r1 >= nregs)
8967                                 err += efunc(pc, "invalid register %u\n", r1);
8968                         if (r2 >= nregs)
8969                                 err += efunc(pc, "invalid register %u\n", r2);
8970                         if (rd >= nregs)
8971                                 err += efunc(pc, "invalid register %u\n", rd);
8972                         if (rd == 0)
8973                                 err += efunc(pc, "cannot write to %r0\n");
8974                         break;
8975                 case DIF_OP_NOT:
8976                 case DIF_OP_MOV:
8977                 case DIF_OP_ALLOCS:
8978                         if (r1 >= nregs)
8979                                 err += efunc(pc, "invalid register %u\n", r1);
8980                         if (r2 != 0)
8981                                 err += efunc(pc, "non-zero reserved bits\n");
8982                         if (rd >= nregs)
8983                                 err += efunc(pc, "invalid register %u\n", rd);
8984                         if (rd == 0)
8985                                 err += efunc(pc, "cannot write to %r0\n");
8986                         break;
8987                 case DIF_OP_LDSB:
8988                 case DIF_OP_LDSH:
8989                 case DIF_OP_LDSW:
8990                 case DIF_OP_LDUB:
8991                 case DIF_OP_LDUH:
8992                 case DIF_OP_LDUW:
8993                 case DIF_OP_LDX:
8994                         if (r1 >= nregs)
8995                                 err += efunc(pc, "invalid register %u\n", r1);
8996                         if (r2 != 0)
8997                                 err += efunc(pc, "non-zero reserved bits\n");
8998                         if (rd >= nregs)
8999                                 err += efunc(pc, "invalid register %u\n", rd);
9000                         if (rd == 0)
9001                                 err += efunc(pc, "cannot write to %r0\n");
9002                         if (kcheckload)
9003                                 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
9004                                     DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
9005                         break;
9006                 case DIF_OP_RLDSB:
9007                 case DIF_OP_RLDSH:
9008                 case DIF_OP_RLDSW:
9009                 case DIF_OP_RLDUB:
9010                 case DIF_OP_RLDUH:
9011                 case DIF_OP_RLDUW:
9012                 case DIF_OP_RLDX:
9013                         if (r1 >= nregs)
9014                                 err += efunc(pc, "invalid register %u\n", r1);
9015                         if (r2 != 0)
9016                                 err += efunc(pc, "non-zero reserved bits\n");
9017                         if (rd >= nregs)
9018                                 err += efunc(pc, "invalid register %u\n", rd);
9019                         if (rd == 0)
9020                                 err += efunc(pc, "cannot write to %r0\n");
9021                         break;
9022                 case DIF_OP_ULDSB:
9023                 case DIF_OP_ULDSH:
9024                 case DIF_OP_ULDSW:
9025                 case DIF_OP_ULDUB:
9026                 case DIF_OP_ULDUH:
9027                 case DIF_OP_ULDUW:
9028                 case DIF_OP_ULDX:
9029                         if (r1 >= nregs)
9030                                 err += efunc(pc, "invalid register %u\n", r1);
9031                         if (r2 != 0)
9032                                 err += efunc(pc, "non-zero reserved bits\n");
9033                         if (rd >= nregs)
9034                                 err += efunc(pc, "invalid register %u\n", rd);
9035                         if (rd == 0)
9036                                 err += efunc(pc, "cannot write to %r0\n");
9037                         break;
9038                 case DIF_OP_STB:
9039                 case DIF_OP_STH:
9040                 case DIF_OP_STW:
9041                 case DIF_OP_STX:
9042                         if (r1 >= nregs)
9043                                 err += efunc(pc, "invalid register %u\n", r1);
9044                         if (r2 != 0)
9045                                 err += efunc(pc, "non-zero reserved bits\n");
9046                         if (rd >= nregs)
9047                                 err += efunc(pc, "invalid register %u\n", rd);
9048                         if (rd == 0)
9049                                 err += efunc(pc, "cannot write to 0 address\n");
9050                         break;
9051                 case DIF_OP_CMP:
9052                 case DIF_OP_SCMP:
9053                         if (r1 >= nregs)
9054                                 err += efunc(pc, "invalid register %u\n", r1);
9055                         if (r2 >= nregs)
9056                                 err += efunc(pc, "invalid register %u\n", r2);
9057                         if (rd != 0)
9058                                 err += efunc(pc, "non-zero reserved bits\n");
9059                         break;
9060                 case DIF_OP_TST:
9061                         if (r1 >= nregs)
9062                                 err += efunc(pc, "invalid register %u\n", r1);
9063                         if (r2 != 0 || rd != 0)
9064                                 err += efunc(pc, "non-zero reserved bits\n");
9065                         break;
9066                 case DIF_OP_BA:
9067                 case DIF_OP_BE:
9068                 case DIF_OP_BNE:
9069                 case DIF_OP_BG:
9070                 case DIF_OP_BGU:
9071                 case DIF_OP_BGE:
9072                 case DIF_OP_BGEU:
9073                 case DIF_OP_BL:
9074                 case DIF_OP_BLU:
9075                 case DIF_OP_BLE:
9076                 case DIF_OP_BLEU:
9077                         if (label >= dp->dtdo_len) {
9078                                 err += efunc(pc, "invalid branch target %u\n",
9079                                     label);
9080                         }
9081                         if (label <= pc) {
9082                                 err += efunc(pc, "backward branch to %u\n",
9083                                     label);
9084                         }
9085                         break;
9086                 case DIF_OP_RET:
9087                         if (r1 != 0 || r2 != 0)
9088                                 err += efunc(pc, "non-zero reserved bits\n");
9089                         if (rd >= nregs)
9090                                 err += efunc(pc, "invalid register %u\n", rd);
9091                         break;
9092                 case DIF_OP_NOP:
9093                 case DIF_OP_POPTS:
9094                 case DIF_OP_FLUSHTS:
9095                         if (r1 != 0 || r2 != 0 || rd != 0)
9096                                 err += efunc(pc, "non-zero reserved bits\n");
9097                         break;
9098                 case DIF_OP_SETX:
9099                         if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
9100                                 err += efunc(pc, "invalid integer ref %u\n",
9101                                     DIF_INSTR_INTEGER(instr));
9102                         }
9103                         if (rd >= nregs)
9104                                 err += efunc(pc, "invalid register %u\n", rd);
9105                         if (rd == 0)
9106                                 err += efunc(pc, "cannot write to %r0\n");
9107                         break;
9108                 case DIF_OP_SETS:
9109                         if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
9110                                 err += efunc(pc, "invalid string ref %u\n",
9111                                     DIF_INSTR_STRING(instr));
9112                         }
9113                         if (rd >= nregs)
9114                                 err += efunc(pc, "invalid register %u\n", rd);
9115                         if (rd == 0)
9116                                 err += efunc(pc, "cannot write to %r0\n");
9117                         break;
9118                 case DIF_OP_LDGA:
9119                 case DIF_OP_LDTA:
9120                         if (r1 > DIF_VAR_ARRAY_MAX)
9121                                 err += efunc(pc, "invalid array %u\n", r1);
9122                         if (r2 >= nregs)
9123                                 err += efunc(pc, "invalid register %u\n", r2);
9124                         if (rd >= nregs)
9125                                 err += efunc(pc, "invalid register %u\n", rd);
9126                         if (rd == 0)
9127                                 err += efunc(pc, "cannot write to %r0\n");
9128                         break;
9129                 case DIF_OP_LDGS:
9130                 case DIF_OP_LDTS:
9131                 case DIF_OP_LDLS:
9132                 case DIF_OP_LDGAA:
9133                 case DIF_OP_LDTAA:
9134                         if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
9135                                 err += efunc(pc, "invalid variable %u\n", v);
9136                         if (rd >= nregs)
9137                                 err += efunc(pc, "invalid register %u\n", rd);
9138                         if (rd == 0)
9139                                 err += efunc(pc, "cannot write to %r0\n");
9140                         break;
9141                 case DIF_OP_STGS:
9142                 case DIF_OP_STTS:
9143                 case DIF_OP_STLS:
9144                 case DIF_OP_STGAA:
9145                 case DIF_OP_STTAA:
9146                         if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
9147                                 err += efunc(pc, "invalid variable %u\n", v);
9148                         if (rs >= nregs)
9149                                 err += efunc(pc, "invalid register %u\n", rd);
9150                         break;
9151                 case DIF_OP_CALL:
9152                         if (subr > DIF_SUBR_MAX)
9153                                 err += efunc(pc, "invalid subr %u\n", subr);
9154                         if (rd >= nregs)
9155                                 err += efunc(pc, "invalid register %u\n", rd);
9156                         if (rd == 0)
9157                                 err += efunc(pc, "cannot write to %r0\n");
9158 
9159                         if (subr == DIF_SUBR_COPYOUT ||
9160                             subr == DIF_SUBR_COPYOUTSTR) {
9161                                 dp->dtdo_destructive = 1;
9162                         }
9163 
9164                         if (subr == DIF_SUBR_GETF) {
9165                                 /*
9166                                  * If we have a getf() we need to record that
9167                                  * in our state.  Note that our state can be
9168                                  * NULL if this is a helper -- but in that
9169                                  * case, the call to getf() is itself illegal,
9170                                  * and will be caught (slightly later) when
9171                                  * the helper is validated.
9172                                  */
9173                                 if (vstate->dtvs_state != NULL)
9174                                         vstate->dtvs_state->dts_getf++;
9175                         }
9176 
9177                         break;
9178                 case DIF_OP_PUSHTR:
9179                         if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
9180                                 err += efunc(pc, "invalid ref type %u\n", type);
9181                         if (r2 >= nregs)
9182                                 err += efunc(pc, "invalid register %u\n", r2);
9183                         if (rs >= nregs)
9184                                 err += efunc(pc, "invalid register %u\n", rs);
9185                         break;
9186                 case DIF_OP_PUSHTV:
9187                         if (type != DIF_TYPE_CTF)
9188                                 err += efunc(pc, "invalid val type %u\n", type);
9189                         if (r2 >= nregs)
9190                                 err += efunc(pc, "invalid register %u\n", r2);
9191                         if (rs >= nregs)
9192                                 err += efunc(pc, "invalid register %u\n", rs);
9193                         break;
9194                 default:
9195                         err += efunc(pc, "invalid opcode %u\n",
9196                             DIF_INSTR_OP(instr));
9197                 }
9198         }
9199 
9200         if (dp->dtdo_len != 0 &&
9201             DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
9202                 err += efunc(dp->dtdo_len - 1,
9203                     "expected 'ret' as last DIF instruction\n");
9204         }
9205 
9206         if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
9207                 /*
9208                  * If we're not returning by reference, the size must be either
9209                  * 0 or the size of one of the base types.
9210                  */
9211                 switch (dp->dtdo_rtype.dtdt_size) {
9212                 case 0:
9213                 case sizeof (uint8_t):
9214                 case sizeof (uint16_t):
9215                 case sizeof (uint32_t):
9216                 case sizeof (uint64_t):
9217                         break;
9218 
9219                 default:
9220                         err += efunc(dp->dtdo_len - 1, "bad return size\n");
9221                 }
9222         }
9223 
9224         for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
9225                 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
9226                 dtrace_diftype_t *vt, *et;
9227                 uint_t id, ndx;
9228 
9229                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
9230                     v->dtdv_scope != DIFV_SCOPE_THREAD &&
9231                     v->dtdv_scope != DIFV_SCOPE_LOCAL) {
9232                         err += efunc(i, "unrecognized variable scope %d\n",
9233                             v->dtdv_scope);
9234                         break;
9235                 }
9236 
9237                 if (v->dtdv_kind != DIFV_KIND_ARRAY &&
9238                     v->dtdv_kind != DIFV_KIND_SCALAR) {
9239                         err += efunc(i, "unrecognized variable type %d\n",
9240                             v->dtdv_kind);
9241                         break;
9242                 }
9243 
9244                 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
9245                         err += efunc(i, "%d exceeds variable id limit\n", id);
9246                         break;
9247                 }
9248 
9249                 if (id < DIF_VAR_OTHER_UBASE)
9250                         continue;
9251 
9252                 /*
9253                  * For user-defined variables, we need to check that this
9254                  * definition is identical to any previous definition that we
9255                  * encountered.
9256                  */
9257                 ndx = id - DIF_VAR_OTHER_UBASE;
9258 
9259                 switch (v->dtdv_scope) {
9260                 case DIFV_SCOPE_GLOBAL:
9261                         if (ndx < vstate->dtvs_nglobals) {
9262                                 dtrace_statvar_t *svar;
9263 
9264                                 if ((svar = vstate->dtvs_globals[ndx]) != NULL)
9265                                         existing = &svar->dtsv_var;
9266                         }
9267 
9268                         break;
9269 
9270                 case DIFV_SCOPE_THREAD:
9271                         if (ndx < vstate->dtvs_ntlocals)
9272                                 existing = &vstate->dtvs_tlocals[ndx];
9273                         break;
9274 
9275                 case DIFV_SCOPE_LOCAL:
9276                         if (ndx < vstate->dtvs_nlocals) {
9277                                 dtrace_statvar_t *svar;
9278 
9279                                 if ((svar = vstate->dtvs_locals[ndx]) != NULL)
9280                                         existing = &svar->dtsv_var;
9281                         }
9282 
9283                         break;
9284                 }
9285 
9286                 vt = &v->dtdv_type;
9287 
9288                 if (vt->dtdt_flags & DIF_TF_BYREF) {
9289                         if (vt->dtdt_size == 0) {
9290                                 err += efunc(i, "zero-sized variable\n");
9291                                 break;
9292                         }
9293 
9294                         if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
9295                             vt->dtdt_size > dtrace_global_maxsize) {
9296                                 err += efunc(i, "oversized by-ref global\n");
9297                                 break;
9298                         }
9299                 }
9300 
9301                 if (existing == NULL || existing->dtdv_id == 0)
9302                         continue;
9303 
9304                 ASSERT(existing->dtdv_id == v->dtdv_id);
9305                 ASSERT(existing->dtdv_scope == v->dtdv_scope);
9306 
9307                 if (existing->dtdv_kind != v->dtdv_kind)
9308                         err += efunc(i, "%d changed variable kind\n", id);
9309 
9310                 et = &existing->dtdv_type;
9311 
9312                 if (vt->dtdt_flags != et->dtdt_flags) {
9313                         err += efunc(i, "%d changed variable type flags\n", id);
9314                         break;
9315                 }
9316 
9317                 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
9318                         err += efunc(i, "%d changed variable type size\n", id);
9319                         break;
9320                 }
9321         }
9322 
9323         return (err);
9324 }
9325 
9326 /*
9327  * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
9328  * are much more constrained than normal DIFOs.  Specifically, they may
9329  * not:
9330  *
9331  * 1. Make calls to subroutines other than copyin(), copyinstr() or
9332  *    miscellaneous string routines
9333  * 2. Access DTrace variables other than the args[] array, and the
9334  *    curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
9335  * 3. Have thread-local variables.
9336  * 4. Have dynamic variables.
9337  */
9338 static int
9339 dtrace_difo_validate_helper(dtrace_difo_t *dp)
9340 {
9341         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
9342         int err = 0;
9343         uint_t pc;
9344 
9345         for (pc = 0; pc < dp->dtdo_len; pc++) {
9346                 dif_instr_t instr = dp->dtdo_buf[pc];
9347 
9348                 uint_t v = DIF_INSTR_VAR(instr);
9349                 uint_t subr = DIF_INSTR_SUBR(instr);
9350                 uint_t op = DIF_INSTR_OP(instr);
9351 
9352                 switch (op) {
9353                 case DIF_OP_OR:
9354                 case DIF_OP_XOR:
9355                 case DIF_OP_AND:
9356                 case DIF_OP_SLL:
9357                 case DIF_OP_SRL:
9358                 case DIF_OP_SRA:
9359                 case DIF_OP_SUB:
9360                 case DIF_OP_ADD:
9361                 case DIF_OP_MUL:
9362                 case DIF_OP_SDIV:
9363                 case DIF_OP_UDIV:
9364                 case DIF_OP_SREM:
9365                 case DIF_OP_UREM:
9366                 case DIF_OP_COPYS:
9367                 case DIF_OP_NOT:
9368                 case DIF_OP_MOV:
9369                 case DIF_OP_RLDSB:
9370                 case DIF_OP_RLDSH:
9371                 case DIF_OP_RLDSW:
9372                 case DIF_OP_RLDUB:
9373                 case DIF_OP_RLDUH:
9374                 case DIF_OP_RLDUW:
9375                 case DIF_OP_RLDX:
9376                 case DIF_OP_ULDSB:
9377                 case DIF_OP_ULDSH:
9378                 case DIF_OP_ULDSW:
9379                 case DIF_OP_ULDUB:
9380                 case DIF_OP_ULDUH:
9381                 case DIF_OP_ULDUW:
9382                 case DIF_OP_ULDX:
9383                 case DIF_OP_STB:
9384                 case DIF_OP_STH:
9385                 case DIF_OP_STW:
9386                 case DIF_OP_STX:
9387                 case DIF_OP_ALLOCS:
9388                 case DIF_OP_CMP:
9389                 case DIF_OP_SCMP:
9390                 case DIF_OP_TST:
9391                 case DIF_OP_BA:
9392                 case DIF_OP_BE:
9393                 case DIF_OP_BNE:
9394                 case DIF_OP_BG:
9395                 case DIF_OP_BGU:
9396                 case DIF_OP_BGE:
9397                 case DIF_OP_BGEU:
9398                 case DIF_OP_BL:
9399                 case DIF_OP_BLU:
9400                 case DIF_OP_BLE:
9401                 case DIF_OP_BLEU:
9402                 case DIF_OP_RET:
9403                 case DIF_OP_NOP:
9404                 case DIF_OP_POPTS:
9405                 case DIF_OP_FLUSHTS:
9406                 case DIF_OP_SETX:
9407                 case DIF_OP_SETS:
9408                 case DIF_OP_LDGA:
9409                 case DIF_OP_LDLS:
9410                 case DIF_OP_STGS:
9411                 case DIF_OP_STLS:
9412                 case DIF_OP_PUSHTR:
9413                 case DIF_OP_PUSHTV:
9414                         break;
9415 
9416                 case DIF_OP_LDGS:
9417                         if (v >= DIF_VAR_OTHER_UBASE)
9418                                 break;
9419 
9420                         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
9421                                 break;
9422 
9423                         if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
9424                             v == DIF_VAR_PPID || v == DIF_VAR_TID ||
9425                             v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
9426                             v == DIF_VAR_UID || v == DIF_VAR_GID)
9427                                 break;
9428 
9429                         err += efunc(pc, "illegal variable %u\n", v);
9430                         break;
9431 
9432                 case DIF_OP_LDTA:
9433                 case DIF_OP_LDTS:
9434                 case DIF_OP_LDGAA:
9435                 case DIF_OP_LDTAA:
9436                         err += efunc(pc, "illegal dynamic variable load\n");
9437                         break;
9438 
9439                 case DIF_OP_STTS:
9440                 case DIF_OP_STGAA:
9441                 case DIF_OP_STTAA:
9442                         err += efunc(pc, "illegal dynamic variable store\n");
9443                         break;
9444 
9445                 case DIF_OP_CALL:
9446                         if (subr == DIF_SUBR_ALLOCA ||
9447                             subr == DIF_SUBR_BCOPY ||
9448                             subr == DIF_SUBR_COPYIN ||
9449                             subr == DIF_SUBR_COPYINTO ||
9450                             subr == DIF_SUBR_COPYINSTR ||
9451                             subr == DIF_SUBR_INDEX ||
9452                             subr == DIF_SUBR_INET_NTOA ||
9453                             subr == DIF_SUBR_INET_NTOA6 ||
9454                             subr == DIF_SUBR_INET_NTOP ||
9455                             subr == DIF_SUBR_JSON ||
9456                             subr == DIF_SUBR_LLTOSTR ||
9457                             subr == DIF_SUBR_STRTOLL ||
9458                             subr == DIF_SUBR_RINDEX ||
9459                             subr == DIF_SUBR_STRCHR ||
9460                             subr == DIF_SUBR_STRJOIN ||
9461                             subr == DIF_SUBR_STRRCHR ||
9462                             subr == DIF_SUBR_STRSTR ||
9463                             subr == DIF_SUBR_HTONS ||
9464                             subr == DIF_SUBR_HTONL ||
9465                             subr == DIF_SUBR_HTONLL ||
9466                             subr == DIF_SUBR_NTOHS ||
9467                             subr == DIF_SUBR_NTOHL ||
9468                             subr == DIF_SUBR_NTOHLL)
9469                                 break;
9470 
9471                         err += efunc(pc, "invalid subr %u\n", subr);
9472                         break;
9473 
9474                 default:
9475                         err += efunc(pc, "invalid opcode %u\n",
9476                             DIF_INSTR_OP(instr));
9477                 }
9478         }
9479 
9480         return (err);
9481 }
9482 
9483 /*
9484  * Returns 1 if the expression in the DIF object can be cached on a per-thread
9485  * basis; 0 if not.
9486  */
9487 static int
9488 dtrace_difo_cacheable(dtrace_difo_t *dp)
9489 {
9490         int i;
9491 
9492         if (dp == NULL)
9493                 return (0);
9494 
9495         for (i = 0; i < dp->dtdo_varlen; i++) {
9496                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9497 
9498                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
9499                         continue;
9500 
9501                 switch (v->dtdv_id) {
9502                 case DIF_VAR_CURTHREAD:
9503                 case DIF_VAR_PID:
9504                 case DIF_VAR_TID:
9505                 case DIF_VAR_EXECNAME:
9506                 case DIF_VAR_ZONENAME:
9507                         break;
9508 
9509                 default:
9510                         return (0);
9511                 }
9512         }
9513 
9514         /*
9515          * This DIF object may be cacheable.  Now we need to look for any
9516          * array loading instructions, any memory loading instructions, or
9517          * any stores to thread-local variables.
9518          */
9519         for (i = 0; i < dp->dtdo_len; i++) {
9520                 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
9521 
9522                 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
9523                     (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
9524                     (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
9525                     op == DIF_OP_LDGA || op == DIF_OP_STTS)
9526                         return (0);
9527         }
9528 
9529         return (1);
9530 }
9531 
9532 static void
9533 dtrace_difo_hold(dtrace_difo_t *dp)
9534 {
9535         int i;
9536 
9537         ASSERT(MUTEX_HELD(&dtrace_lock));
9538 
9539         dp->dtdo_refcnt++;
9540         ASSERT(dp->dtdo_refcnt != 0);
9541 
9542         /*
9543          * We need to check this DIF object for references to the variable
9544          * DIF_VAR_VTIMESTAMP.
9545          */
9546         for (i = 0; i < dp->dtdo_varlen; i++) {
9547                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9548 
9549                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9550                         continue;
9551 
9552                 if (dtrace_vtime_references++ == 0)
9553                         dtrace_vtime_enable();
9554         }
9555 }
9556 
9557 /*
9558  * This routine calculates the dynamic variable chunksize for a given DIF
9559  * object.  The calculation is not fool-proof, and can probably be tricked by
9560  * malicious DIF -- but it works for all compiler-generated DIF.  Because this
9561  * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
9562  * if a dynamic variable size exceeds the chunksize.
9563  */
9564 static void
9565 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9566 {
9567         uint64_t sval;
9568         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
9569         const dif_instr_t *text = dp->dtdo_buf;
9570         uint_t pc, srd = 0;
9571         uint_t ttop = 0;
9572         size_t size, ksize;
9573         uint_t id, i;
9574 
9575         for (pc = 0; pc < dp->dtdo_len; pc++) {
9576                 dif_instr_t instr = text[pc];
9577                 uint_t op = DIF_INSTR_OP(instr);
9578                 uint_t rd = DIF_INSTR_RD(instr);
9579                 uint_t r1 = DIF_INSTR_R1(instr);
9580                 uint_t nkeys = 0;
9581                 uchar_t scope;
9582 
9583                 dtrace_key_t *key = tupregs;
9584 
9585                 switch (op) {
9586                 case DIF_OP_SETX:
9587                         sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
9588                         srd = rd;
9589                         continue;
9590 
9591                 case DIF_OP_STTS:
9592                         key = &tupregs[DIF_DTR_NREGS];
9593                         key[0].dttk_size = 0;
9594                         key[1].dttk_size = 0;
9595                         nkeys = 2;
9596                         scope = DIFV_SCOPE_THREAD;
9597                         break;
9598 
9599                 case DIF_OP_STGAA:
9600                 case DIF_OP_STTAA:
9601                         nkeys = ttop;
9602 
9603                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
9604                                 key[nkeys++].dttk_size = 0;
9605 
9606                         key[nkeys++].dttk_size = 0;
9607 
9608                         if (op == DIF_OP_STTAA) {
9609                                 scope = DIFV_SCOPE_THREAD;
9610                         } else {
9611                                 scope = DIFV_SCOPE_GLOBAL;
9612                         }
9613 
9614                         break;
9615 
9616                 case DIF_OP_PUSHTR:
9617                         if (ttop == DIF_DTR_NREGS)
9618                                 return;
9619 
9620                         if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
9621                                 /*
9622                                  * If the register for the size of the "pushtr"
9623                                  * is %r0 (or the value is 0) and the type is
9624                                  * a string, we'll use the system-wide default
9625                                  * string size.
9626                                  */
9627                                 tupregs[ttop++].dttk_size =
9628                                     dtrace_strsize_default;
9629                         } else {
9630                                 if (srd == 0)
9631                                         return;
9632 
9633                                 tupregs[ttop++].dttk_size = sval;
9634                         }
9635 
9636                         break;
9637 
9638                 case DIF_OP_PUSHTV:
9639                         if (ttop == DIF_DTR_NREGS)
9640                                 return;
9641 
9642                         tupregs[ttop++].dttk_size = 0;
9643                         break;
9644 
9645                 case DIF_OP_FLUSHTS:
9646                         ttop = 0;
9647                         break;
9648 
9649                 case DIF_OP_POPTS:
9650                         if (ttop != 0)
9651                                 ttop--;
9652                         break;
9653                 }
9654 
9655                 sval = 0;
9656                 srd = 0;
9657 
9658                 if (nkeys == 0)
9659                         continue;
9660 
9661                 /*
9662                  * We have a dynamic variable allocation; calculate its size.
9663                  */
9664                 for (ksize = 0, i = 0; i < nkeys; i++)
9665                         ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
9666 
9667                 size = sizeof (dtrace_dynvar_t);
9668                 size += sizeof (dtrace_key_t) * (nkeys - 1);
9669                 size += ksize;
9670 
9671                 /*
9672                  * Now we need to determine the size of the stored data.
9673                  */
9674                 id = DIF_INSTR_VAR(instr);
9675 
9676                 for (i = 0; i < dp->dtdo_varlen; i++) {
9677                         dtrace_difv_t *v = &dp->dtdo_vartab[i];
9678 
9679                         if (v->dtdv_id == id && v->dtdv_scope == scope) {
9680                                 size += v->dtdv_type.dtdt_size;
9681                                 break;
9682                         }
9683                 }
9684 
9685                 if (i == dp->dtdo_varlen)
9686                         return;
9687 
9688                 /*
9689                  * We have the size.  If this is larger than the chunk size
9690                  * for our dynamic variable state, reset the chunk size.
9691                  */
9692                 size = P2ROUNDUP(size, sizeof (uint64_t));
9693 
9694                 if (size > vstate->dtvs_dynvars.dtds_chunksize)
9695                         vstate->dtvs_dynvars.dtds_chunksize = size;
9696         }
9697 }
9698 
9699 static void
9700 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9701 {
9702         int i, oldsvars, osz, nsz, otlocals, ntlocals;
9703         uint_t id;
9704 
9705         ASSERT(MUTEX_HELD(&dtrace_lock));
9706         ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
9707 
9708         for (i = 0; i < dp->dtdo_varlen; i++) {
9709                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9710                 dtrace_statvar_t *svar, ***svarp;
9711                 size_t dsize = 0;
9712                 uint8_t scope = v->dtdv_scope;
9713                 int *np;
9714 
9715                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9716                         continue;
9717 
9718                 id -= DIF_VAR_OTHER_UBASE;
9719 
9720                 switch (scope) {
9721                 case DIFV_SCOPE_THREAD:
9722                         while (id >= (otlocals = vstate->dtvs_ntlocals)) {
9723                                 dtrace_difv_t *tlocals;
9724 
9725                                 if ((ntlocals = (otlocals << 1)) == 0)
9726                                         ntlocals = 1;
9727 
9728                                 osz = otlocals * sizeof (dtrace_difv_t);
9729                                 nsz = ntlocals * sizeof (dtrace_difv_t);
9730 
9731                                 tlocals = kmem_zalloc(nsz, KM_SLEEP);
9732 
9733                                 if (osz != 0) {
9734                                         bcopy(vstate->dtvs_tlocals,
9735                                             tlocals, osz);
9736                                         kmem_free(vstate->dtvs_tlocals, osz);
9737                                 }
9738 
9739                                 vstate->dtvs_tlocals = tlocals;
9740                                 vstate->dtvs_ntlocals = ntlocals;
9741                         }
9742 
9743                         vstate->dtvs_tlocals[id] = *v;
9744                         continue;
9745 
9746                 case DIFV_SCOPE_LOCAL:
9747                         np = &vstate->dtvs_nlocals;
9748                         svarp = &vstate->dtvs_locals;
9749 
9750                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9751                                 dsize = NCPU * (v->dtdv_type.dtdt_size +
9752                                     sizeof (uint64_t));
9753                         else
9754                                 dsize = NCPU * sizeof (uint64_t);
9755 
9756                         break;
9757 
9758                 case DIFV_SCOPE_GLOBAL:
9759                         np = &vstate->dtvs_nglobals;
9760                         svarp = &vstate->dtvs_globals;
9761 
9762                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9763                                 dsize = v->dtdv_type.dtdt_size +
9764                                     sizeof (uint64_t);
9765 
9766                         break;
9767 
9768                 default:
9769                         ASSERT(0);
9770                 }
9771 
9772                 while (id >= (oldsvars = *np)) {
9773                         dtrace_statvar_t **statics;
9774                         int newsvars, oldsize, newsize;
9775 
9776                         if ((newsvars = (oldsvars << 1)) == 0)
9777                                 newsvars = 1;
9778 
9779                         oldsize = oldsvars * sizeof (dtrace_statvar_t *);
9780                         newsize = newsvars * sizeof (dtrace_statvar_t *);
9781 
9782                         statics = kmem_zalloc(newsize, KM_SLEEP);
9783 
9784                         if (oldsize != 0) {
9785                                 bcopy(*svarp, statics, oldsize);
9786                                 kmem_free(*svarp, oldsize);
9787                         }
9788 
9789                         *svarp = statics;
9790                         *np = newsvars;
9791                 }
9792 
9793                 if ((svar = (*svarp)[id]) == NULL) {
9794                         svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
9795                         svar->dtsv_var = *v;
9796 
9797                         if ((svar->dtsv_size = dsize) != 0) {
9798                                 svar->dtsv_data = (uint64_t)(uintptr_t)
9799                                     kmem_zalloc(dsize, KM_SLEEP);
9800                         }
9801 
9802                         (*svarp)[id] = svar;
9803                 }
9804 
9805                 svar->dtsv_refcnt++;
9806         }
9807 
9808         dtrace_difo_chunksize(dp, vstate);
9809         dtrace_difo_hold(dp);
9810 }
9811 
9812 static dtrace_difo_t *
9813 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9814 {
9815         dtrace_difo_t *new;
9816         size_t sz;
9817 
9818         ASSERT(dp->dtdo_buf != NULL);
9819         ASSERT(dp->dtdo_refcnt != 0);
9820 
9821         new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
9822 
9823         ASSERT(dp->dtdo_buf != NULL);
9824         sz = dp->dtdo_len * sizeof (dif_instr_t);
9825         new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
9826         bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
9827         new->dtdo_len = dp->dtdo_len;
9828 
9829         if (dp->dtdo_strtab != NULL) {
9830                 ASSERT(dp->dtdo_strlen != 0);
9831                 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
9832                 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
9833                 new->dtdo_strlen = dp->dtdo_strlen;
9834         }
9835 
9836         if (dp->dtdo_inttab != NULL) {
9837                 ASSERT(dp->dtdo_intlen != 0);
9838                 sz = dp->dtdo_intlen * sizeof (uint64_t);
9839                 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
9840                 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
9841                 new->dtdo_intlen = dp->dtdo_intlen;
9842         }
9843 
9844         if (dp->dtdo_vartab != NULL) {
9845                 ASSERT(dp->dtdo_varlen != 0);
9846                 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
9847                 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
9848                 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
9849                 new->dtdo_varlen = dp->dtdo_varlen;
9850         }
9851 
9852         dtrace_difo_init(new, vstate);
9853         return (new);
9854 }
9855 
9856 static void
9857 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9858 {
9859         int i;
9860 
9861         ASSERT(dp->dtdo_refcnt == 0);
9862 
9863         for (i = 0; i < dp->dtdo_varlen; i++) {
9864                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9865                 dtrace_statvar_t *svar, **svarp;
9866                 uint_t id;
9867                 uint8_t scope = v->dtdv_scope;
9868                 int *np;
9869 
9870                 switch (scope) {
9871                 case DIFV_SCOPE_THREAD:
9872                         continue;
9873 
9874                 case DIFV_SCOPE_LOCAL:
9875                         np = &vstate->dtvs_nlocals;
9876                         svarp = vstate->dtvs_locals;
9877                         break;
9878 
9879                 case DIFV_SCOPE_GLOBAL:
9880                         np = &vstate->dtvs_nglobals;
9881                         svarp = vstate->dtvs_globals;
9882                         break;
9883 
9884                 default:
9885                         ASSERT(0);
9886                 }
9887 
9888                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9889                         continue;
9890 
9891                 id -= DIF_VAR_OTHER_UBASE;
9892                 ASSERT(id < *np);
9893 
9894                 svar = svarp[id];
9895                 ASSERT(svar != NULL);
9896                 ASSERT(svar->dtsv_refcnt > 0);
9897 
9898                 if (--svar->dtsv_refcnt > 0)
9899                         continue;
9900 
9901                 if (svar->dtsv_size != 0) {
9902                         ASSERT(svar->dtsv_data != NULL);
9903                         kmem_free((void *)(uintptr_t)svar->dtsv_data,
9904                             svar->dtsv_size);
9905                 }
9906 
9907                 kmem_free(svar, sizeof (dtrace_statvar_t));
9908                 svarp[id] = NULL;
9909         }
9910 
9911         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
9912         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
9913         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
9914         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
9915 
9916         kmem_free(dp, sizeof (dtrace_difo_t));
9917 }
9918 
9919 static void
9920 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9921 {
9922         int i;
9923 
9924         ASSERT(MUTEX_HELD(&dtrace_lock));
9925         ASSERT(dp->dtdo_refcnt != 0);
9926 
9927         for (i = 0; i < dp->dtdo_varlen; i++) {
9928                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9929 
9930                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9931                         continue;
9932 
9933                 ASSERT(dtrace_vtime_references > 0);
9934                 if (--dtrace_vtime_references == 0)
9935                         dtrace_vtime_disable();
9936         }
9937 
9938         if (--dp->dtdo_refcnt == 0)
9939                 dtrace_difo_destroy(dp, vstate);
9940 }
9941 
9942 /*
9943  * DTrace Format Functions
9944  */
9945 static uint16_t
9946 dtrace_format_add(dtrace_state_t *state, char *str)
9947 {
9948         char *fmt, **new;
9949         uint16_t ndx, len = strlen(str) + 1;
9950 
9951         fmt = kmem_zalloc(len, KM_SLEEP);
9952         bcopy(str, fmt, len);
9953 
9954         for (ndx = 0; ndx < state->dts_nformats; ndx++) {
9955                 if (state->dts_formats[ndx] == NULL) {
9956                         state->dts_formats[ndx] = fmt;
9957                         return (ndx + 1);
9958                 }
9959         }
9960 
9961         if (state->dts_nformats == USHRT_MAX) {
9962                 /*
9963                  * This is only likely if a denial-of-service attack is being
9964                  * attempted.  As such, it's okay to fail silently here.
9965                  */
9966                 kmem_free(fmt, len);
9967                 return (0);
9968         }
9969 
9970         /*
9971          * For simplicity, we always resize the formats array to be exactly the
9972          * number of formats.
9973          */
9974         ndx = state->dts_nformats++;
9975         new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
9976 
9977         if (state->dts_formats != NULL) {
9978                 ASSERT(ndx != 0);
9979                 bcopy(state->dts_formats, new, ndx * sizeof (char *));
9980                 kmem_free(state->dts_formats, ndx * sizeof (char *));
9981         }
9982 
9983         state->dts_formats = new;
9984         state->dts_formats[ndx] = fmt;
9985 
9986         return (ndx + 1);
9987 }
9988 
9989 static void
9990 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
9991 {
9992         char *fmt;
9993 
9994         ASSERT(state->dts_formats != NULL);
9995         ASSERT(format <= state->dts_nformats);
9996         ASSERT(state->dts_formats[format - 1] != NULL);
9997 
9998         fmt = state->dts_formats[format - 1];
9999         kmem_free(fmt, strlen(fmt) + 1);
10000         state->dts_formats[format - 1] = NULL;
10001 }
10002 
10003 static void
10004 dtrace_format_destroy(dtrace_state_t *state)
10005 {
10006         int i;
10007 
10008         if (state->dts_nformats == 0) {
10009                 ASSERT(state->dts_formats == NULL);
10010                 return;
10011         }
10012 
10013         ASSERT(state->dts_formats != NULL);
10014 
10015         for (i = 0; i < state->dts_nformats; i++) {
10016                 char *fmt = state->dts_formats[i];
10017 
10018                 if (fmt == NULL)
10019                         continue;
10020 
10021                 kmem_free(fmt, strlen(fmt) + 1);
10022         }
10023 
10024         kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
10025         state->dts_nformats = 0;
10026         state->dts_formats = NULL;
10027 }
10028 
10029 /*
10030  * DTrace Predicate Functions
10031  */
10032 static dtrace_predicate_t *
10033 dtrace_predicate_create(dtrace_difo_t *dp)
10034 {
10035         dtrace_predicate_t *pred;
10036 
10037         ASSERT(MUTEX_HELD(&dtrace_lock));
10038         ASSERT(dp->dtdo_refcnt != 0);
10039 
10040         pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
10041         pred->dtp_difo = dp;
10042         pred->dtp_refcnt = 1;
10043 
10044         if (!dtrace_difo_cacheable(dp))
10045                 return (pred);
10046 
10047         if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
10048                 /*
10049                  * This is only theoretically possible -- we have had 2^32
10050                  * cacheable predicates on this machine.  We cannot allow any
10051                  * more predicates to become cacheable:  as unlikely as it is,
10052                  * there may be a thread caching a (now stale) predicate cache
10053                  * ID. (N.B.: the temptation is being successfully resisted to
10054                  * have this cmn_err() "Holy shit -- we executed this code!")
10055                  */
10056                 return (pred);
10057         }
10058 
10059         pred->dtp_cacheid = dtrace_predcache_id++;
10060 
10061         return (pred);
10062 }
10063 
10064 static void
10065 dtrace_predicate_hold(dtrace_predicate_t *pred)
10066 {
10067         ASSERT(MUTEX_HELD(&dtrace_lock));
10068         ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
10069         ASSERT(pred->dtp_refcnt > 0);
10070 
10071         pred->dtp_refcnt++;
10072 }
10073 
10074 static void
10075 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
10076 {
10077         dtrace_difo_t *dp = pred->dtp_difo;
10078 
10079         ASSERT(MUTEX_HELD(&dtrace_lock));
10080         ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
10081         ASSERT(pred->dtp_refcnt > 0);
10082 
10083         if (--pred->dtp_refcnt == 0) {
10084                 dtrace_difo_release(pred->dtp_difo, vstate);
10085                 kmem_free(pred, sizeof (dtrace_predicate_t));
10086         }
10087 }
10088 
10089 /*
10090  * DTrace Action Description Functions
10091  */
10092 static dtrace_actdesc_t *
10093 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
10094     uint64_t uarg, uint64_t arg)
10095 {
10096         dtrace_actdesc_t *act;
10097 
10098         ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
10099             arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
10100 
10101         act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
10102         act->dtad_kind = kind;
10103         act->dtad_ntuple = ntuple;
10104         act->dtad_uarg = uarg;
10105         act->dtad_arg = arg;
10106         act->dtad_refcnt = 1;
10107 
10108         return (act);
10109 }
10110 
10111 static void
10112 dtrace_actdesc_hold(dtrace_actdesc_t *act)
10113 {
10114         ASSERT(act->dtad_refcnt >= 1);
10115         act->dtad_refcnt++;
10116 }
10117 
10118 static void
10119 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
10120 {
10121         dtrace_actkind_t kind = act->dtad_kind;
10122         dtrace_difo_t *dp;
10123 
10124         ASSERT(act->dtad_refcnt >= 1);
10125 
10126         if (--act->dtad_refcnt != 0)
10127                 return;
10128 
10129         if ((dp = act->dtad_difo) != NULL)
10130                 dtrace_difo_release(dp, vstate);
10131 
10132         if (DTRACEACT_ISPRINTFLIKE(kind)) {
10133                 char *str = (char *)(uintptr_t)act->dtad_arg;
10134 
10135                 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
10136                     (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
10137 
10138                 if (str != NULL)
10139                         kmem_free(str, strlen(str) + 1);
10140         }
10141 
10142         kmem_free(act, sizeof (dtrace_actdesc_t));
10143 }
10144 
10145 /*
10146  * DTrace ECB Functions
10147  */
10148 static dtrace_ecb_t *
10149 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
10150 {
10151         dtrace_ecb_t *ecb;
10152         dtrace_epid_t epid;
10153 
10154         ASSERT(MUTEX_HELD(&dtrace_lock));
10155 
10156         ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
10157         ecb->dte_predicate = NULL;
10158         ecb->dte_probe = probe;
10159 
10160         /*
10161          * The default size is the size of the default action: recording
10162          * the header.
10163          */
10164         ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
10165         ecb->dte_alignment = sizeof (dtrace_epid_t);
10166 
10167         epid = state->dts_epid++;
10168 
10169         if (epid - 1 >= state->dts_necbs) {
10170                 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
10171                 int necbs = state->dts_necbs << 1;
10172 
10173                 ASSERT(epid == state->dts_necbs + 1);
10174 
10175                 if (necbs == 0) {
10176                         ASSERT(oecbs == NULL);
10177                         necbs = 1;
10178                 }
10179 
10180                 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
10181 
10182                 if (oecbs != NULL)
10183                         bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
10184 
10185                 dtrace_membar_producer();
10186                 state->dts_ecbs = ecbs;
10187 
10188                 if (oecbs != NULL) {
10189                         /*
10190                          * If this state is active, we must dtrace_sync()
10191                          * before we can free the old dts_ecbs array:  we're
10192                          * coming in hot, and there may be active ring
10193                          * buffer processing (which indexes into the dts_ecbs
10194                          * array) on another CPU.
10195                          */
10196                         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
10197                                 dtrace_sync();
10198 
10199                         kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
10200                 }
10201 
10202                 dtrace_membar_producer();
10203                 state->dts_necbs = necbs;
10204         }
10205 
10206         ecb->dte_state = state;
10207 
10208         ASSERT(state->dts_ecbs[epid - 1] == NULL);
10209         dtrace_membar_producer();
10210         state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
10211 
10212         return (ecb);
10213 }
10214 
10215 static int
10216 dtrace_ecb_enable(dtrace_ecb_t *ecb)
10217 {
10218         dtrace_probe_t *probe = ecb->dte_probe;
10219 
10220         ASSERT(MUTEX_HELD(&cpu_lock));
10221         ASSERT(MUTEX_HELD(&dtrace_lock));
10222         ASSERT(ecb->dte_next == NULL);
10223 
10224         if (probe == NULL) {
10225                 /*
10226                  * This is the NULL probe -- there's nothing to do.
10227                  */
10228                 return (0);
10229         }
10230 
10231         if (probe->dtpr_ecb == NULL) {
10232                 dtrace_provider_t *prov = probe->dtpr_provider;
10233 
10234                 /*
10235                  * We're the first ECB on this probe.
10236                  */
10237                 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
10238 
10239                 if (ecb->dte_predicate != NULL)
10240                         probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
10241 
10242                 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
10243                     probe->dtpr_id, probe->dtpr_arg));
10244         } else {
10245                 /*
10246                  * This probe is already active.  Swing the last pointer to
10247                  * point to the new ECB, and issue a dtrace_sync() to assure
10248                  * that all CPUs have seen the change.
10249                  */
10250                 ASSERT(probe->dtpr_ecb_last != NULL);
10251                 probe->dtpr_ecb_last->dte_next = ecb;
10252                 probe->dtpr_ecb_last = ecb;
10253                 probe->dtpr_predcache = 0;
10254 
10255                 dtrace_sync();
10256                 return (0);
10257         }
10258 }
10259 
10260 static void
10261 dtrace_ecb_resize(dtrace_ecb_t *ecb)
10262 {
10263         dtrace_action_t *act;
10264         uint32_t curneeded = UINT32_MAX;
10265         uint32_t aggbase = UINT32_MAX;
10266 
10267         /*
10268          * If we record anything, we always record the dtrace_rechdr_t.  (And
10269          * we always record it first.)
10270          */
10271         ecb->dte_size = sizeof (dtrace_rechdr_t);
10272         ecb->dte_alignment = sizeof (dtrace_epid_t);
10273 
10274         for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
10275                 dtrace_recdesc_t *rec = &act->dta_rec;
10276                 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
10277 
10278                 ecb->dte_alignment = MAX(ecb->dte_alignment,
10279                     rec->dtrd_alignment);
10280 
10281                 if (DTRACEACT_ISAGG(act->dta_kind)) {
10282                         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10283 
10284                         ASSERT(rec->dtrd_size != 0);
10285                         ASSERT(agg->dtag_first != NULL);
10286                         ASSERT(act->dta_prev->dta_intuple);
10287                         ASSERT(aggbase != UINT32_MAX);
10288                         ASSERT(curneeded != UINT32_MAX);
10289 
10290                         agg->dtag_base = aggbase;
10291 
10292                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10293                         rec->dtrd_offset = curneeded;
10294                         curneeded += rec->dtrd_size;
10295                         ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
10296 
10297                         aggbase = UINT32_MAX;
10298                         curneeded = UINT32_MAX;
10299                 } else if (act->dta_intuple) {
10300                         if (curneeded == UINT32_MAX) {
10301                                 /*
10302                                  * This is the first record in a tuple.  Align
10303                                  * curneeded to be at offset 4 in an 8-byte
10304                                  * aligned block.
10305                                  */
10306                                 ASSERT(act->dta_prev == NULL ||
10307                                     !act->dta_prev->dta_intuple);
10308                                 ASSERT3U(aggbase, ==, UINT32_MAX);
10309                                 curneeded = P2PHASEUP(ecb->dte_size,
10310                                     sizeof (uint64_t), sizeof (dtrace_aggid_t));
10311 
10312                                 aggbase = curneeded - sizeof (dtrace_aggid_t);
10313                                 ASSERT(IS_P2ALIGNED(aggbase,
10314                                     sizeof (uint64_t)));
10315                         }
10316                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10317                         rec->dtrd_offset = curneeded;
10318                         curneeded += rec->dtrd_size;
10319                 } else {
10320                         /* tuples must be followed by an aggregation */
10321                         ASSERT(act->dta_prev == NULL ||
10322                             !act->dta_prev->dta_intuple);
10323 
10324                         ecb->dte_size = P2ROUNDUP(ecb->dte_size,
10325                             rec->dtrd_alignment);
10326                         rec->dtrd_offset = ecb->dte_size;
10327                         ecb->dte_size += rec->dtrd_size;
10328                         ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
10329                 }
10330         }
10331 
10332         if ((act = ecb->dte_action) != NULL &&
10333             !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
10334             ecb->dte_size == sizeof (dtrace_rechdr_t)) {
10335                 /*
10336                  * If the size is still sizeof (dtrace_rechdr_t), then all
10337                  * actions store no data; set the size to 0.
10338                  */
10339                 ecb->dte_size = 0;
10340         }
10341 
10342         ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
10343         ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
10344         ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
10345             ecb->dte_needed);
10346 }
10347 
10348 static dtrace_action_t *
10349 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10350 {
10351         dtrace_aggregation_t *agg;
10352         size_t size = sizeof (uint64_t);
10353         int ntuple = desc->dtad_ntuple;
10354         dtrace_action_t *act;
10355         dtrace_recdesc_t *frec;
10356         dtrace_aggid_t aggid;
10357         dtrace_state_t *state = ecb->dte_state;
10358 
10359         agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
10360         agg->dtag_ecb = ecb;
10361 
10362         ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
10363 
10364         switch (desc->dtad_kind) {
10365         case DTRACEAGG_MIN:
10366                 agg->dtag_initial = INT64_MAX;
10367                 agg->dtag_aggregate = dtrace_aggregate_min;
10368                 break;
10369 
10370         case DTRACEAGG_MAX:
10371                 agg->dtag_initial = INT64_MIN;
10372                 agg->dtag_aggregate = dtrace_aggregate_max;
10373                 break;
10374 
10375         case DTRACEAGG_COUNT:
10376                 agg->dtag_aggregate = dtrace_aggregate_count;
10377                 break;
10378 
10379         case DTRACEAGG_QUANTIZE:
10380                 agg->dtag_aggregate = dtrace_aggregate_quantize;
10381                 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
10382                     sizeof (uint64_t);
10383                 break;
10384 
10385         case DTRACEAGG_LQUANTIZE: {
10386                 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
10387                 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
10388 
10389                 agg->dtag_initial = desc->dtad_arg;
10390                 agg->dtag_aggregate = dtrace_aggregate_lquantize;
10391 
10392                 if (step == 0 || levels == 0)
10393                         goto err;
10394 
10395                 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
10396                 break;
10397         }
10398 
10399         case DTRACEAGG_LLQUANTIZE: {
10400                 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
10401                 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
10402                 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
10403                 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
10404                 int64_t v;
10405 
10406                 agg->dtag_initial = desc->dtad_arg;
10407                 agg->dtag_aggregate = dtrace_aggregate_llquantize;
10408 
10409                 if (factor < 2 || low >= high || nsteps < factor)
10410                         goto err;
10411 
10412                 /*
10413                  * Now check that the number of steps evenly divides a power
10414                  * of the factor.  (This assures both integer bucket size and
10415                  * linearity within each magnitude.)
10416                  */
10417                 for (v = factor; v < nsteps; v *= factor)
10418                         continue;
10419 
10420                 if ((v % nsteps) || (nsteps % factor))
10421                         goto err;
10422 
10423                 size = (dtrace_aggregate_llquantize_bucket(factor,
10424                     low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
10425                 break;
10426         }
10427 
10428         case DTRACEAGG_AVG:
10429                 agg->dtag_aggregate = dtrace_aggregate_avg;
10430                 size = sizeof (uint64_t) * 2;
10431                 break;
10432 
10433         case DTRACEAGG_STDDEV:
10434                 agg->dtag_aggregate = dtrace_aggregate_stddev;
10435                 size = sizeof (uint64_t) * 4;
10436                 break;
10437 
10438         case DTRACEAGG_SUM:
10439                 agg->dtag_aggregate = dtrace_aggregate_sum;
10440                 break;
10441 
10442         default:
10443                 goto err;
10444         }
10445 
10446         agg->dtag_action.dta_rec.dtrd_size = size;
10447 
10448         if (ntuple == 0)
10449                 goto err;
10450 
10451         /*
10452          * We must make sure that we have enough actions for the n-tuple.
10453          */
10454         for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
10455                 if (DTRACEACT_ISAGG(act->dta_kind))
10456                         break;
10457 
10458                 if (--ntuple == 0) {
10459                         /*
10460                          * This is the action with which our n-tuple begins.
10461                          */
10462                         agg->dtag_first = act;
10463                         goto success;
10464                 }
10465         }
10466 
10467         /*
10468          * This n-tuple is short by ntuple elements.  Return failure.
10469          */
10470         ASSERT(ntuple != 0);
10471 err:
10472         kmem_free(agg, sizeof (dtrace_aggregation_t));
10473         return (NULL);
10474 
10475 success:
10476         /*
10477          * If the last action in the tuple has a size of zero, it's actually
10478          * an expression argument for the aggregating action.
10479          */
10480         ASSERT(ecb->dte_action_last != NULL);
10481         act = ecb->dte_action_last;
10482 
10483         if (act->dta_kind == DTRACEACT_DIFEXPR) {
10484                 ASSERT(act->dta_difo != NULL);
10485 
10486                 if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
10487                         agg->dtag_hasarg = 1;
10488         }
10489 
10490         /*
10491          * We need to allocate an id for this aggregation.
10492          */
10493         aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
10494             VM_BESTFIT | VM_SLEEP);
10495 
10496         if (aggid - 1 >= state->dts_naggregations) {
10497                 dtrace_aggregation_t **oaggs = state->dts_aggregations;
10498                 dtrace_aggregation_t **aggs;
10499                 int naggs = state->dts_naggregations << 1;
10500                 int onaggs = state->dts_naggregations;
10501 
10502                 ASSERT(aggid == state->dts_naggregations + 1);
10503 
10504                 if (naggs == 0) {
10505                         ASSERT(oaggs == NULL);
10506                         naggs = 1;
10507                 }
10508 
10509                 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
10510 
10511                 if (oaggs != NULL) {
10512                         bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
10513                         kmem_free(oaggs, onaggs * sizeof (*aggs));
10514                 }
10515 
10516                 state->dts_aggregations = aggs;
10517                 state->dts_naggregations = naggs;
10518         }
10519 
10520         ASSERT(state->dts_aggregations[aggid - 1] == NULL);
10521         state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
10522 
10523         frec = &agg->dtag_first->dta_rec;
10524         if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
10525                 frec->dtrd_alignment = sizeof (dtrace_aggid_t);
10526 
10527         for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
10528                 ASSERT(!act->dta_intuple);
10529                 act->dta_intuple = 1;
10530         }
10531 
10532         return (&agg->dtag_action);
10533 }
10534 
10535 static void
10536 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
10537 {
10538         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10539         dtrace_state_t *state = ecb->dte_state;
10540         dtrace_aggid_t aggid = agg->dtag_id;
10541 
10542         ASSERT(DTRACEACT_ISAGG(act->dta_kind));
10543         vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
10544 
10545         ASSERT(state->dts_aggregations[aggid - 1] == agg);
10546         state->dts_aggregations[aggid - 1] = NULL;
10547 
10548         kmem_free(agg, sizeof (dtrace_aggregation_t));
10549 }
10550 
10551 static int
10552 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10553 {
10554         dtrace_action_t *action, *last;
10555         dtrace_difo_t *dp = desc->dtad_difo;
10556         uint32_t size = 0, align = sizeof (uint8_t), mask;
10557         uint16_t format = 0;
10558         dtrace_recdesc_t *rec;
10559         dtrace_state_t *state = ecb->dte_state;
10560         dtrace_optval_t *opt = state->dts_options, nframes, strsize;
10561         uint64_t arg = desc->dtad_arg;
10562 
10563         ASSERT(MUTEX_HELD(&dtrace_lock));
10564         ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
10565 
10566         if (DTRACEACT_ISAGG(desc->dtad_kind)) {
10567                 /*
10568                  * If this is an aggregating action, there must be neither
10569                  * a speculate nor a commit on the action chain.
10570                  */
10571                 dtrace_action_t *act;
10572 
10573                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
10574                         if (act->dta_kind == DTRACEACT_COMMIT)
10575                                 return (EINVAL);
10576 
10577                         if (act->dta_kind == DTRACEACT_SPECULATE)
10578                                 return (EINVAL);
10579                 }
10580 
10581                 action = dtrace_ecb_aggregation_create(ecb, desc);
10582 
10583                 if (action == NULL)
10584                         return (EINVAL);
10585         } else {
10586                 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
10587                     (desc->dtad_kind == DTRACEACT_DIFEXPR &&
10588                     dp != NULL && dp->dtdo_destructive)) {
10589                         state->dts_destructive = 1;
10590                 }
10591 
10592                 switch (desc->dtad_kind) {
10593                 case DTRACEACT_PRINTF:
10594                 case DTRACEACT_PRINTA:
10595                 case DTRACEACT_SYSTEM:
10596                 case DTRACEACT_FREOPEN:
10597                 case DTRACEACT_DIFEXPR:
10598                         /*
10599                          * We know that our arg is a string -- turn it into a
10600                          * format.
10601                          */
10602                         if (arg == NULL) {
10603                                 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
10604                                     desc->dtad_kind == DTRACEACT_DIFEXPR);
10605                                 format = 0;
10606                         } else {
10607                                 ASSERT(arg != NULL);
10608                                 ASSERT(arg > KERNELBASE);
10609                                 format = dtrace_format_add(state,
10610                                     (char *)(uintptr_t)arg);
10611                         }
10612 
10613                         /*FALLTHROUGH*/
10614                 case DTRACEACT_LIBACT:
10615                 case DTRACEACT_TRACEMEM:
10616                 case DTRACEACT_TRACEMEM_DYNSIZE:
10617                         if (dp == NULL)
10618                                 return (EINVAL);
10619 
10620                         if ((size = dp->dtdo_rtype.dtdt_size) != 0)
10621                                 break;
10622 
10623                         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
10624                                 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10625                                         return (EINVAL);
10626 
10627                                 size = opt[DTRACEOPT_STRSIZE];
10628                         }
10629 
10630                         break;
10631 
10632                 case DTRACEACT_STACK:
10633                         if ((nframes = arg) == 0) {
10634                                 nframes = opt[DTRACEOPT_STACKFRAMES];
10635                                 ASSERT(nframes > 0);
10636                                 arg = nframes;
10637                         }
10638 
10639                         size = nframes * sizeof (pc_t);
10640                         break;
10641 
10642                 case DTRACEACT_JSTACK:
10643                         if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
10644                                 strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
10645 
10646                         if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
10647                                 nframes = opt[DTRACEOPT_JSTACKFRAMES];
10648 
10649                         arg = DTRACE_USTACK_ARG(nframes, strsize);
10650 
10651                         /*FALLTHROUGH*/
10652                 case DTRACEACT_USTACK:
10653                         if (desc->dtad_kind != DTRACEACT_JSTACK &&
10654                             (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
10655                                 strsize = DTRACE_USTACK_STRSIZE(arg);
10656                                 nframes = opt[DTRACEOPT_USTACKFRAMES];
10657                                 ASSERT(nframes > 0);
10658                                 arg = DTRACE_USTACK_ARG(nframes, strsize);
10659                         }
10660 
10661                         /*
10662                          * Save a slot for the pid.
10663                          */
10664                         size = (nframes + 1) * sizeof (uint64_t);
10665                         size += DTRACE_USTACK_STRSIZE(arg);
10666                         size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
10667 
10668                         break;
10669 
10670                 case DTRACEACT_SYM:
10671                 case DTRACEACT_MOD:
10672                         if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
10673                             sizeof (uint64_t)) ||
10674                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10675                                 return (EINVAL);
10676                         break;
10677 
10678                 case DTRACEACT_USYM:
10679                 case DTRACEACT_UMOD:
10680                 case DTRACEACT_UADDR:
10681                         if (dp == NULL ||
10682                             (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
10683                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10684                                 return (EINVAL);
10685 
10686                         /*
10687                          * We have a slot for the pid, plus a slot for the
10688                          * argument.  To keep things simple (aligned with
10689                          * bitness-neutral sizing), we store each as a 64-bit
10690                          * quantity.
10691                          */
10692                         size = 2 * sizeof (uint64_t);
10693                         break;
10694 
10695                 case DTRACEACT_STOP:
10696                 case DTRACEACT_BREAKPOINT:
10697                 case DTRACEACT_PANIC:
10698                         break;
10699 
10700                 case DTRACEACT_CHILL:
10701                 case DTRACEACT_DISCARD:
10702                 case DTRACEACT_RAISE:
10703                         if (dp == NULL)
10704                                 return (EINVAL);
10705                         break;
10706 
10707                 case DTRACEACT_EXIT:
10708                         if (dp == NULL ||
10709                             (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
10710                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10711                                 return (EINVAL);
10712                         break;
10713 
10714                 case DTRACEACT_SPECULATE:
10715                         if (ecb->dte_size > sizeof (dtrace_rechdr_t))
10716                                 return (EINVAL);
10717 
10718                         if (dp == NULL)
10719                                 return (EINVAL);
10720 
10721                         state->dts_speculates = 1;
10722                         break;
10723 
10724                 case DTRACEACT_COMMIT: {
10725                         dtrace_action_t *act = ecb->dte_action;
10726 
10727                         for (; act != NULL; act = act->dta_next) {
10728                                 if (act->dta_kind == DTRACEACT_COMMIT)
10729                                         return (EINVAL);
10730                         }
10731 
10732                         if (dp == NULL)
10733                                 return (EINVAL);
10734                         break;
10735                 }
10736 
10737                 default:
10738                         return (EINVAL);
10739                 }
10740 
10741                 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
10742                         /*
10743                          * If this is a data-storing action or a speculate,
10744                          * we must be sure that there isn't a commit on the
10745                          * action chain.
10746                          */
10747                         dtrace_action_t *act = ecb->dte_action;
10748 
10749                         for (; act != NULL; act = act->dta_next) {
10750                                 if (act->dta_kind == DTRACEACT_COMMIT)
10751                                         return (EINVAL);
10752                         }
10753                 }
10754 
10755                 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
10756                 action->dta_rec.dtrd_size = size;
10757         }
10758 
10759         action->dta_refcnt = 1;
10760         rec = &action->dta_rec;
10761         size = rec->dtrd_size;
10762 
10763         for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
10764                 if (!(size & mask)) {
10765                         align = mask + 1;
10766                         break;
10767                 }
10768         }
10769 
10770         action->dta_kind = desc->dtad_kind;
10771 
10772         if ((action->dta_difo = dp) != NULL)
10773                 dtrace_difo_hold(dp);
10774 
10775         rec->dtrd_action = action->dta_kind;
10776         rec->dtrd_arg = arg;
10777         rec->dtrd_uarg = desc->dtad_uarg;
10778         rec->dtrd_alignment = (uint16_t)align;
10779         rec->dtrd_format = format;
10780 
10781         if ((last = ecb->dte_action_last) != NULL) {
10782                 ASSERT(ecb->dte_action != NULL);
10783                 action->dta_prev = last;
10784                 last->dta_next = action;
10785         } else {
10786                 ASSERT(ecb->dte_action == NULL);
10787                 ecb->dte_action = action;
10788         }
10789 
10790         ecb->dte_action_last = action;
10791 
10792         return (0);
10793 }
10794 
10795 static void
10796 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
10797 {
10798         dtrace_action_t *act = ecb->dte_action, *next;
10799         dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
10800         dtrace_difo_t *dp;
10801         uint16_t format;
10802 
10803         if (act != NULL && act->dta_refcnt > 1) {
10804                 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
10805                 act->dta_refcnt--;
10806         } else {
10807                 for (; act != NULL; act = next) {
10808                         next = act->dta_next;
10809                         ASSERT(next != NULL || act == ecb->dte_action_last);
10810                         ASSERT(act->dta_refcnt == 1);
10811 
10812                         if ((format = act->dta_rec.dtrd_format) != 0)
10813                                 dtrace_format_remove(ecb->dte_state, format);
10814 
10815                         if ((dp = act->dta_difo) != NULL)
10816                                 dtrace_difo_release(dp, vstate);
10817 
10818                         if (DTRACEACT_ISAGG(act->dta_kind)) {
10819                                 dtrace_ecb_aggregation_destroy(ecb, act);
10820                         } else {
10821                                 kmem_free(act, sizeof (dtrace_action_t));
10822                         }
10823                 }
10824         }
10825 
10826         ecb->dte_action = NULL;
10827         ecb->dte_action_last = NULL;
10828         ecb->dte_size = 0;
10829 }
10830 
10831 static void
10832 dtrace_ecb_disable(dtrace_ecb_t *ecb)
10833 {
10834         /*
10835          * We disable the ECB by removing it from its probe.
10836          */
10837         dtrace_ecb_t *pecb, *prev = NULL;
10838         dtrace_probe_t *probe = ecb->dte_probe;
10839 
10840         ASSERT(MUTEX_HELD(&dtrace_lock));
10841 
10842         if (probe == NULL) {
10843                 /*
10844                  * This is the NULL probe; there is nothing to disable.
10845                  */
10846                 return;
10847         }
10848 
10849         for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
10850                 if (pecb == ecb)
10851                         break;
10852                 prev = pecb;
10853         }
10854 
10855         ASSERT(pecb != NULL);
10856 
10857         if (prev == NULL) {
10858                 probe->dtpr_ecb = ecb->dte_next;
10859         } else {
10860                 prev->dte_next = ecb->dte_next;
10861         }
10862 
10863         if (ecb == probe->dtpr_ecb_last) {
10864                 ASSERT(ecb->dte_next == NULL);
10865                 probe->dtpr_ecb_last = prev;
10866         }
10867 
10868         /*
10869          * The ECB has been disconnected from the probe; now sync to assure
10870          * that all CPUs have seen the change before returning.
10871          */
10872         dtrace_sync();
10873 
10874         if (probe->dtpr_ecb == NULL) {
10875                 /*
10876                  * That was the last ECB on the probe; clear the predicate
10877                  * cache ID for the probe, disable it and sync one more time
10878                  * to assure that we'll never hit it again.
10879                  */
10880                 dtrace_provider_t *prov = probe->dtpr_provider;
10881 
10882                 ASSERT(ecb->dte_next == NULL);
10883                 ASSERT(probe->dtpr_ecb_last == NULL);
10884                 probe->dtpr_predcache = DTRACE_CACHEIDNONE;
10885                 prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
10886                     probe->dtpr_id, probe->dtpr_arg);
10887                 dtrace_sync();
10888         } else {
10889                 /*
10890                  * There is at least one ECB remaining on the probe.  If there
10891                  * is _exactly_ one, set the probe's predicate cache ID to be
10892                  * the predicate cache ID of the remaining ECB.
10893                  */
10894                 ASSERT(probe->dtpr_ecb_last != NULL);
10895                 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
10896 
10897                 if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
10898                         dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
10899 
10900                         ASSERT(probe->dtpr_ecb->dte_next == NULL);
10901 
10902                         if (p != NULL)
10903                                 probe->dtpr_predcache = p->dtp_cacheid;
10904                 }
10905 
10906                 ecb->dte_next = NULL;
10907         }
10908 }
10909 
10910 static void
10911 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
10912 {
10913         dtrace_state_t *state = ecb->dte_state;
10914         dtrace_vstate_t *vstate = &state->dts_vstate;
10915         dtrace_predicate_t *pred;
10916         dtrace_epid_t epid = ecb->dte_epid;
10917 
10918         ASSERT(MUTEX_HELD(&dtrace_lock));
10919         ASSERT(ecb->dte_next == NULL);
10920         ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
10921 
10922         if ((pred = ecb->dte_predicate) != NULL)
10923                 dtrace_predicate_release(pred, vstate);
10924 
10925         dtrace_ecb_action_remove(ecb);
10926 
10927         ASSERT(state->dts_ecbs[epid - 1] == ecb);
10928         state->dts_ecbs[epid - 1] = NULL;
10929 
10930         kmem_free(ecb, sizeof (dtrace_ecb_t));
10931 }
10932 
10933 static dtrace_ecb_t *
10934 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
10935     dtrace_enabling_t *enab)
10936 {
10937         dtrace_ecb_t *ecb;
10938         dtrace_predicate_t *pred;
10939         dtrace_actdesc_t *act;
10940         dtrace_provider_t *prov;
10941         dtrace_ecbdesc_t *desc = enab->dten_current;
10942 
10943         ASSERT(MUTEX_HELD(&dtrace_lock));
10944         ASSERT(state != NULL);
10945 
10946         ecb = dtrace_ecb_add(state, probe);
10947         ecb->dte_uarg = desc->dted_uarg;
10948 
10949         if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
10950                 dtrace_predicate_hold(pred);
10951                 ecb->dte_predicate = pred;
10952         }
10953 
10954         if (probe != NULL) {
10955                 /*
10956                  * If the provider shows more leg than the consumer is old
10957                  * enough to see, we need to enable the appropriate implicit
10958                  * predicate bits to prevent the ecb from activating at
10959                  * revealing times.
10960                  *
10961                  * Providers specifying DTRACE_PRIV_USER at register time
10962                  * are stating that they need the /proc-style privilege
10963                  * model to be enforced, and this is what DTRACE_COND_OWNER
10964                  * and DTRACE_COND_ZONEOWNER will then do at probe time.
10965                  */
10966                 prov = probe->dtpr_provider;
10967                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
10968                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10969                         ecb->dte_cond |= DTRACE_COND_OWNER;
10970 
10971                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
10972                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10973                         ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
10974 
10975                 /*
10976                  * If the provider shows us kernel innards and the user
10977                  * is lacking sufficient privilege, enable the
10978                  * DTRACE_COND_USERMODE implicit predicate.
10979                  */
10980                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
10981                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
10982                         ecb->dte_cond |= DTRACE_COND_USERMODE;
10983         }
10984 
10985         if (dtrace_ecb_create_cache != NULL) {
10986                 /*
10987                  * If we have a cached ecb, we'll use its action list instead
10988                  * of creating our own (saving both time and space).
10989                  */
10990                 dtrace_ecb_t *cached = dtrace_ecb_create_cache;
10991                 dtrace_action_t *act = cached->dte_action;
10992 
10993                 if (act != NULL) {
10994                         ASSERT(act->dta_refcnt > 0);
10995                         act->dta_refcnt++;
10996                         ecb->dte_action = act;
10997                         ecb->dte_action_last = cached->dte_action_last;
10998                         ecb->dte_needed = cached->dte_needed;
10999                         ecb->dte_size = cached->dte_size;
11000                         ecb->dte_alignment = cached->dte_alignment;
11001                 }
11002 
11003                 return (ecb);
11004         }
11005 
11006         for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
11007                 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
11008                         dtrace_ecb_destroy(ecb);
11009                         return (NULL);
11010                 }
11011         }
11012 
11013         dtrace_ecb_resize(ecb);
11014 
11015         return (dtrace_ecb_create_cache = ecb);
11016 }
11017 
11018 static int
11019 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
11020 {
11021         dtrace_ecb_t *ecb;
11022         dtrace_enabling_t *enab = arg;
11023         dtrace_state_t *state = enab->dten_vstate->dtvs_state;
11024 
11025         ASSERT(state != NULL);
11026 
11027         if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
11028                 /*
11029                  * This probe was created in a generation for which this
11030                  * enabling has previously created ECBs; we don't want to
11031                  * enable it again, so just kick out.
11032                  */
11033                 return (DTRACE_MATCH_NEXT);
11034         }
11035 
11036         if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
11037                 return (DTRACE_MATCH_DONE);
11038 
11039         if (dtrace_ecb_enable(ecb) < 0)
11040                 return (DTRACE_MATCH_FAIL);
11041 
11042         return (DTRACE_MATCH_NEXT);
11043 }
11044 
11045 static dtrace_ecb_t *
11046 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
11047 {
11048         dtrace_ecb_t *ecb;
11049 
11050         ASSERT(MUTEX_HELD(&dtrace_lock));
11051 
11052         if (id == 0 || id > state->dts_necbs)
11053                 return (NULL);
11054 
11055         ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
11056         ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
11057 
11058         return (state->dts_ecbs[id - 1]);
11059 }
11060 
11061 static dtrace_aggregation_t *
11062 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
11063 {
11064         dtrace_aggregation_t *agg;
11065 
11066         ASSERT(MUTEX_HELD(&dtrace_lock));
11067 
11068         if (id == 0 || id > state->dts_naggregations)
11069                 return (NULL);
11070 
11071         ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
11072         ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
11073             agg->dtag_id == id);
11074 
11075         return (state->dts_aggregations[id - 1]);
11076 }
11077 
11078 /*
11079  * DTrace Buffer Functions
11080  *
11081  * The following functions manipulate DTrace buffers.  Most of these functions
11082  * are called in the context of establishing or processing consumer state;
11083  * exceptions are explicitly noted.
11084  */
11085 
11086 /*
11087  * Note:  called from cross call context.  This function switches the two
11088  * buffers on a given CPU.  The atomicity of this operation is assured by
11089  * disabling interrupts while the actual switch takes place; the disabling of
11090  * interrupts serializes the execution with any execution of dtrace_probe() on
11091  * the same CPU.
11092  */
11093 static void
11094 dtrace_buffer_switch(dtrace_buffer_t *buf)
11095 {
11096         caddr_t tomax = buf->dtb_tomax;
11097         caddr_t xamot = buf->dtb_xamot;
11098         dtrace_icookie_t cookie;
11099         hrtime_t now;
11100 
11101         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
11102         ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
11103 
11104         cookie = dtrace_interrupt_disable();
11105         now = dtrace_gethrtime();
11106         buf->dtb_tomax = xamot;
11107         buf->dtb_xamot = tomax;
11108         buf->dtb_xamot_drops = buf->dtb_drops;
11109         buf->dtb_xamot_offset = buf->dtb_offset;
11110         buf->dtb_xamot_errors = buf->dtb_errors;
11111         buf->dtb_xamot_flags = buf->dtb_flags;
11112         buf->dtb_offset = 0;
11113         buf->dtb_drops = 0;
11114         buf->dtb_errors = 0;
11115         buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
11116         buf->dtb_interval = now - buf->dtb_switched;
11117         buf->dtb_switched = now;
11118         dtrace_interrupt_enable(cookie);
11119 }
11120 
11121 /*
11122  * Note:  called from cross call context.  This function activates a buffer
11123  * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
11124  * is guaranteed by the disabling of interrupts.
11125  */
11126 static void
11127 dtrace_buffer_activate(dtrace_state_t *state)
11128 {
11129         dtrace_buffer_t *buf;
11130         dtrace_icookie_t cookie = dtrace_interrupt_disable();
11131 
11132         buf = &state->dts_buffer[CPU->cpu_id];
11133 
11134         if (buf->dtb_tomax != NULL) {
11135                 /*
11136                  * We might like to assert that the buffer is marked inactive,
11137                  * but this isn't necessarily true:  the buffer for the CPU
11138                  * that processes the BEGIN probe has its buffer activated
11139                  * manually.  In this case, we take the (harmless) action
11140                  * re-clearing the bit INACTIVE bit.
11141                  */
11142                 buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
11143         }
11144 
11145         dtrace_interrupt_enable(cookie);
11146 }
11147 
11148 static int
11149 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
11150     processorid_t cpu, int *factor)
11151 {
11152         cpu_t *cp;
11153         dtrace_buffer_t *buf;
11154         int allocated = 0, desired = 0;
11155 
11156         ASSERT(MUTEX_HELD(&cpu_lock));
11157         ASSERT(MUTEX_HELD(&dtrace_lock));
11158 
11159         *factor = 1;
11160 
11161         if (size > dtrace_nonroot_maxsize &&
11162             !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
11163                 return (EFBIG);
11164 
11165         cp = cpu_list;
11166 
11167         do {
11168                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11169                         continue;
11170 
11171                 buf = &bufs[cp->cpu_id];
11172 
11173                 /*
11174                  * If there is already a buffer allocated for this CPU, it
11175                  * is only possible that this is a DR event.  In this case,
11176                  * the buffer size must match our specified size.
11177                  */
11178                 if (buf->dtb_tomax != NULL) {
11179                         ASSERT(buf->dtb_size == size);
11180                         continue;
11181                 }
11182 
11183                 ASSERT(buf->dtb_xamot == NULL);
11184 
11185                 if ((buf->dtb_tomax = kmem_zalloc(size,
11186                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11187                         goto err;
11188 
11189                 buf->dtb_size = size;
11190                 buf->dtb_flags = flags;
11191                 buf->dtb_offset = 0;
11192                 buf->dtb_drops = 0;
11193 
11194                 if (flags & DTRACEBUF_NOSWITCH)
11195                         continue;
11196 
11197                 if ((buf->dtb_xamot = kmem_zalloc(size,
11198                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11199                         goto err;
11200         } while ((cp = cp->cpu_next) != cpu_list);
11201 
11202         return (0);
11203 
11204 err:
11205         cp = cpu_list;
11206 
11207         do {
11208                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11209                         continue;
11210 
11211                 buf = &bufs[cp->cpu_id];
11212                 desired += 2;
11213 
11214                 if (buf->dtb_xamot != NULL) {
11215                         ASSERT(buf->dtb_tomax != NULL);
11216                         ASSERT(buf->dtb_size == size);
11217                         kmem_free(buf->dtb_xamot, size);
11218                         allocated++;
11219                 }
11220 
11221                 if (buf->dtb_tomax != NULL) {
11222                         ASSERT(buf->dtb_size == size);
11223                         kmem_free(buf->dtb_tomax, size);
11224                         allocated++;
11225                 }
11226 
11227                 buf->dtb_tomax = NULL;
11228                 buf->dtb_xamot = NULL;
11229                 buf->dtb_size = 0;
11230         } while ((cp = cp->cpu_next) != cpu_list);
11231 
11232         *factor = desired / (allocated > 0 ? allocated : 1);
11233 
11234         return (ENOMEM);
11235 }
11236 
11237 /*
11238  * Note:  called from probe context.  This function just increments the drop
11239  * count on a buffer.  It has been made a function to allow for the
11240  * possibility of understanding the source of mysterious drop counts.  (A
11241  * problem for which one may be particularly disappointed that DTrace cannot
11242  * be used to understand DTrace.)
11243  */
11244 static void
11245 dtrace_buffer_drop(dtrace_buffer_t *buf)
11246 {
11247         buf->dtb_drops++;
11248 }
11249 
11250 /*
11251  * Note:  called from probe context.  This function is called to reserve space
11252  * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
11253  * mstate.  Returns the new offset in the buffer, or a negative value if an
11254  * error has occurred.
11255  */
11256 static intptr_t
11257 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
11258     dtrace_state_t *state, dtrace_mstate_t *mstate)
11259 {
11260         intptr_t offs = buf->dtb_offset, soffs;
11261         intptr_t woffs;
11262         caddr_t tomax;
11263         size_t total;
11264 
11265         if (buf->dtb_flags & DTRACEBUF_INACTIVE)
11266                 return (-1);
11267 
11268         if ((tomax = buf->dtb_tomax) == NULL) {
11269                 dtrace_buffer_drop(buf);
11270                 return (-1);
11271         }
11272 
11273         if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
11274                 while (offs & (align - 1)) {
11275                         /*
11276                          * Assert that our alignment is off by a number which
11277                          * is itself sizeof (uint32_t) aligned.
11278                          */
11279                         ASSERT(!((align - (offs & (align - 1))) &
11280                             (sizeof (uint32_t) - 1)));
11281                         DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11282                         offs += sizeof (uint32_t);
11283                 }
11284 
11285                 if ((soffs = offs + needed) > buf->dtb_size) {
11286                         dtrace_buffer_drop(buf);
11287                         return (-1);
11288                 }
11289 
11290                 if (mstate == NULL)
11291                         return (offs);
11292 
11293                 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
11294                 mstate->dtms_scratch_size = buf->dtb_size - soffs;
11295                 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11296 
11297                 return (offs);
11298         }
11299 
11300         if (buf->dtb_flags & DTRACEBUF_FILL) {
11301                 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
11302                     (buf->dtb_flags & DTRACEBUF_FULL))
11303                         return (-1);
11304                 goto out;
11305         }
11306 
11307         total = needed + (offs & (align - 1));
11308 
11309         /*
11310          * For a ring buffer, life is quite a bit more complicated.  Before
11311          * we can store any padding, we need to adjust our wrapping offset.
11312          * (If we've never before wrapped or we're not about to, no adjustment
11313          * is required.)
11314          */
11315         if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
11316             offs + total > buf->dtb_size) {
11317                 woffs = buf->dtb_xamot_offset;
11318 
11319                 if (offs + total > buf->dtb_size) {
11320                         /*
11321                          * We can't fit in the end of the buffer.  First, a
11322                          * sanity check that we can fit in the buffer at all.
11323                          */
11324                         if (total > buf->dtb_size) {
11325                                 dtrace_buffer_drop(buf);
11326                                 return (-1);
11327                         }
11328 
11329                         /*
11330                          * We're going to be storing at the top of the buffer,
11331                          * so now we need to deal with the wrapped offset.  We
11332                          * only reset our wrapped offset to 0 if it is
11333                          * currently greater than the current offset.  If it
11334                          * is less than the current offset, it is because a
11335                          * previous allocation induced a wrap -- but the
11336                          * allocation didn't subsequently take the space due
11337                          * to an error or false predicate evaluation.  In this
11338                          * case, we'll just leave the wrapped offset alone: if
11339                          * the wrapped offset hasn't been advanced far enough
11340                          * for this allocation, it will be adjusted in the
11341                          * lower loop.
11342                          */
11343                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
11344                                 if (woffs >= offs)
11345                                         woffs = 0;
11346                         } else {
11347                                 woffs = 0;
11348                         }
11349 
11350                         /*
11351                          * Now we know that we're going to be storing to the
11352                          * top of the buffer and that there is room for us
11353                          * there.  We need to clear the buffer from the current
11354                          * offset to the end (there may be old gunk there).
11355                          */
11356                         while (offs < buf->dtb_size)
11357                                 tomax[offs++] = 0;
11358 
11359                         /*
11360                          * We need to set our offset to zero.  And because we
11361                          * are wrapping, we need to set the bit indicating as
11362                          * much.  We can also adjust our needed space back
11363                          * down to the space required by the ECB -- we know
11364                          * that the top of the buffer is aligned.
11365                          */
11366                         offs = 0;
11367                         total = needed;
11368                         buf->dtb_flags |= DTRACEBUF_WRAPPED;
11369                 } else {
11370                         /*
11371                          * There is room for us in the buffer, so we simply
11372                          * need to check the wrapped offset.
11373                          */
11374                         if (woffs < offs) {
11375                                 /*
11376                                  * The wrapped offset is less than the offset.
11377                                  * This can happen if we allocated buffer space
11378                                  * that induced a wrap, but then we didn't
11379                                  * subsequently take the space due to an error
11380                                  * or false predicate evaluation.  This is
11381                                  * okay; we know that _this_ allocation isn't
11382                                  * going to induce a wrap.  We still can't
11383                                  * reset the wrapped offset to be zero,
11384                                  * however: the space may have been trashed in
11385                                  * the previous failed probe attempt.  But at
11386                                  * least the wrapped offset doesn't need to
11387                                  * be adjusted at all...
11388                                  */
11389                                 goto out;
11390                         }
11391                 }
11392 
11393                 while (offs + total > woffs) {
11394                         dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
11395                         size_t size;
11396 
11397                         if (epid == DTRACE_EPIDNONE) {
11398                                 size = sizeof (uint32_t);
11399                         } else {
11400                                 ASSERT3U(epid, <=, state->dts_necbs);
11401                                 ASSERT(state->dts_ecbs[epid - 1] != NULL);
11402 
11403                                 size = state->dts_ecbs[epid - 1]->dte_size;
11404                         }
11405 
11406                         ASSERT(woffs + size <= buf->dtb_size);
11407                         ASSERT(size != 0);
11408 
11409                         if (woffs + size == buf->dtb_size) {
11410                                 /*
11411                                  * We've reached the end of the buffer; we want
11412                                  * to set the wrapped offset to 0 and break
11413                                  * out.  However, if the offs is 0, then we're
11414                                  * in a strange edge-condition:  the amount of
11415                                  * space that we want to reserve plus the size
11416                                  * of the record that we're overwriting is
11417                                  * greater than the size of the buffer.  This
11418                                  * is problematic because if we reserve the
11419                                  * space but subsequently don't consume it (due
11420                                  * to a failed predicate or error) the wrapped
11421                                  * offset will be 0 -- yet the EPID at offset 0
11422                                  * will not be committed.  This situation is
11423                                  * relatively easy to deal with:  if we're in
11424                                  * this case, the buffer is indistinguishable
11425                                  * from one that hasn't wrapped; we need only
11426                                  * finish the job by clearing the wrapped bit,
11427                                  * explicitly setting the offset to be 0, and
11428                                  * zero'ing out the old data in the buffer.
11429                                  */
11430                                 if (offs == 0) {
11431                                         buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
11432                                         buf->dtb_offset = 0;
11433                                         woffs = total;
11434 
11435                                         while (woffs < buf->dtb_size)
11436                                                 tomax[woffs++] = 0;
11437                                 }
11438 
11439                                 woffs = 0;
11440                                 break;
11441                         }
11442 
11443                         woffs += size;
11444                 }
11445 
11446                 /*
11447                  * We have a wrapped offset.  It may be that the wrapped offset
11448                  * has become zero -- that's okay.
11449                  */
11450                 buf->dtb_xamot_offset = woffs;
11451         }
11452 
11453 out:
11454         /*
11455          * Now we can plow the buffer with any necessary padding.
11456          */
11457         while (offs & (align - 1)) {
11458                 /*
11459                  * Assert that our alignment is off by a number which
11460                  * is itself sizeof (uint32_t) aligned.
11461                  */
11462                 ASSERT(!((align - (offs & (align - 1))) &
11463                     (sizeof (uint32_t) - 1)));
11464                 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11465                 offs += sizeof (uint32_t);
11466         }
11467 
11468         if (buf->dtb_flags & DTRACEBUF_FILL) {
11469                 if (offs + needed > buf->dtb_size - state->dts_reserve) {
11470                         buf->dtb_flags |= DTRACEBUF_FULL;
11471                         return (-1);
11472                 }
11473         }
11474 
11475         if (mstate == NULL)
11476                 return (offs);
11477 
11478         /*
11479          * For ring buffers and fill buffers, the scratch space is always
11480          * the inactive buffer.
11481          */
11482         mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
11483         mstate->dtms_scratch_size = buf->dtb_size;
11484         mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11485 
11486         return (offs);
11487 }
11488 
11489 static void
11490 dtrace_buffer_polish(dtrace_buffer_t *buf)
11491 {
11492         ASSERT(buf->dtb_flags & DTRACEBUF_RING);
11493         ASSERT(MUTEX_HELD(&dtrace_lock));
11494 
11495         if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
11496                 return;
11497 
11498         /*
11499          * We need to polish the ring buffer.  There are three cases:
11500          *
11501          * - The first (and presumably most common) is that there is no gap
11502          *   between the buffer offset and the wrapped offset.  In this case,
11503          *   there is nothing in the buffer that isn't valid data; we can
11504          *   mark the buffer as polished and return.
11505          *
11506          * - The second (less common than the first but still more common
11507          *   than the third) is that there is a gap between the buffer offset
11508          *   and the wrapped offset, and the wrapped offset is larger than the
11509          *   buffer offset.  This can happen because of an alignment issue, or
11510          *   can happen because of a call to dtrace_buffer_reserve() that
11511          *   didn't subsequently consume the buffer space.  In this case,
11512          *   we need to zero the data from the buffer offset to the wrapped
11513          *   offset.
11514          *
11515          * - The third (and least common) is that there is a gap between the
11516          *   buffer offset and the wrapped offset, but the wrapped offset is
11517          *   _less_ than the buffer offset.  This can only happen because a
11518          *   call to dtrace_buffer_reserve() induced a wrap, but the space
11519          *   was not subsequently consumed.  In this case, we need to zero the
11520          *   space from the offset to the end of the buffer _and_ from the
11521          *   top of the buffer to the wrapped offset.
11522          */
11523         if (buf->dtb_offset < buf->dtb_xamot_offset) {
11524                 bzero(buf->dtb_tomax + buf->dtb_offset,
11525                     buf->dtb_xamot_offset - buf->dtb_offset);
11526         }
11527 
11528         if (buf->dtb_offset > buf->dtb_xamot_offset) {
11529                 bzero(buf->dtb_tomax + buf->dtb_offset,
11530                     buf->dtb_size - buf->dtb_offset);
11531                 bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
11532         }
11533 }
11534 
11535 /*
11536  * This routine determines if data generated at the specified time has likely
11537  * been entirely consumed at user-level.  This routine is called to determine
11538  * if an ECB on a defunct probe (but for an active enabling) can be safely
11539  * disabled and destroyed.
11540  */
11541 static int
11542 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
11543 {
11544         int i;
11545 
11546         for (i = 0; i < NCPU; i++) {
11547                 dtrace_buffer_t *buf = &bufs[i];
11548 
11549                 if (buf->dtb_size == 0)
11550                         continue;
11551 
11552                 if (buf->dtb_flags & DTRACEBUF_RING)
11553                         return (0);
11554 
11555                 if (!buf->dtb_switched && buf->dtb_offset != 0)
11556                         return (0);
11557 
11558                 if (buf->dtb_switched - buf->dtb_interval < when)
11559                         return (0);
11560         }
11561 
11562         return (1);
11563 }
11564 
11565 static void
11566 dtrace_buffer_free(dtrace_buffer_t *bufs)
11567 {
11568         int i;
11569 
11570         for (i = 0; i < NCPU; i++) {
11571                 dtrace_buffer_t *buf = &bufs[i];
11572 
11573                 if (buf->dtb_tomax == NULL) {
11574                         ASSERT(buf->dtb_xamot == NULL);
11575                         ASSERT(buf->dtb_size == 0);
11576                         continue;
11577                 }
11578 
11579                 if (buf->dtb_xamot != NULL) {
11580                         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
11581                         kmem_free(buf->dtb_xamot, buf->dtb_size);
11582                 }
11583 
11584                 kmem_free(buf->dtb_tomax, buf->dtb_size);
11585                 buf->dtb_size = 0;
11586                 buf->dtb_tomax = NULL;
11587                 buf->dtb_xamot = NULL;
11588         }
11589 }
11590 
11591 /*
11592  * DTrace Enabling Functions
11593  */
11594 static dtrace_enabling_t *
11595 dtrace_enabling_create(dtrace_vstate_t *vstate)
11596 {
11597         dtrace_enabling_t *enab;
11598 
11599         enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
11600         enab->dten_vstate = vstate;
11601 
11602         return (enab);
11603 }
11604 
11605 static void
11606 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
11607 {
11608         dtrace_ecbdesc_t **ndesc;
11609         size_t osize, nsize;
11610 
11611         /*
11612          * We can't add to enablings after we've enabled them, or after we've
11613          * retained them.
11614          */
11615         ASSERT(enab->dten_probegen == 0);
11616         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11617 
11618         if (enab->dten_ndesc < enab->dten_maxdesc) {
11619                 enab->dten_desc[enab->dten_ndesc++] = ecb;
11620                 return;
11621         }
11622 
11623         osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
11624 
11625         if (enab->dten_maxdesc == 0) {
11626                 enab->dten_maxdesc = 1;
11627         } else {
11628                 enab->dten_maxdesc <<= 1;
11629         }
11630 
11631         ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
11632 
11633         nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
11634         ndesc = kmem_zalloc(nsize, KM_SLEEP);
11635         bcopy(enab->dten_desc, ndesc, osize);
11636         kmem_free(enab->dten_desc, osize);
11637 
11638         enab->dten_desc = ndesc;
11639         enab->dten_desc[enab->dten_ndesc++] = ecb;
11640 }
11641 
11642 static void
11643 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
11644     dtrace_probedesc_t *pd)
11645 {
11646         dtrace_ecbdesc_t *new;
11647         dtrace_predicate_t *pred;
11648         dtrace_actdesc_t *act;
11649 
11650         /*
11651          * We're going to create a new ECB description that matches the
11652          * specified ECB in every way, but has the specified probe description.
11653          */
11654         new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
11655 
11656         if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
11657                 dtrace_predicate_hold(pred);
11658 
11659         for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
11660                 dtrace_actdesc_hold(act);
11661 
11662         new->dted_action = ecb->dted_action;
11663         new->dted_pred = ecb->dted_pred;
11664         new->dted_probe = *pd;
11665         new->dted_uarg = ecb->dted_uarg;
11666 
11667         dtrace_enabling_add(enab, new);
11668 }
11669 
11670 static void
11671 dtrace_enabling_dump(dtrace_enabling_t *enab)
11672 {
11673         int i;
11674 
11675         for (i = 0; i < enab->dten_ndesc; i++) {
11676                 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
11677 
11678                 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
11679                     desc->dtpd_provider, desc->dtpd_mod,
11680                     desc->dtpd_func, desc->dtpd_name);
11681         }
11682 }
11683 
11684 static void
11685 dtrace_enabling_destroy(dtrace_enabling_t *enab)
11686 {
11687         int i;
11688         dtrace_ecbdesc_t *ep;
11689         dtrace_vstate_t *vstate = enab->dten_vstate;
11690 
11691         ASSERT(MUTEX_HELD(&dtrace_lock));
11692 
11693         for (i = 0; i < enab->dten_ndesc; i++) {
11694                 dtrace_actdesc_t *act, *next;
11695                 dtrace_predicate_t *pred;
11696 
11697                 ep = enab->dten_desc[i];
11698 
11699                 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
11700                         dtrace_predicate_release(pred, vstate);
11701 
11702                 for (act = ep->dted_action; act != NULL; act = next) {
11703                         next = act->dtad_next;
11704                         dtrace_actdesc_release(act, vstate);
11705                 }
11706 
11707                 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
11708         }
11709 
11710         kmem_free(enab->dten_desc,
11711             enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
11712 
11713         /*
11714          * If this was a retained enabling, decrement the dts_nretained count
11715          * and take it off of the dtrace_retained list.
11716          */
11717         if (enab->dten_prev != NULL || enab->dten_next != NULL ||
11718             dtrace_retained == enab) {
11719                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11720                 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
11721                 enab->dten_vstate->dtvs_state->dts_nretained--;
11722                 dtrace_retained_gen++;
11723         }
11724 
11725         if (enab->dten_prev == NULL) {
11726                 if (dtrace_retained == enab) {
11727                         dtrace_retained = enab->dten_next;
11728 
11729                         if (dtrace_retained != NULL)
11730                                 dtrace_retained->dten_prev = NULL;
11731                 }
11732         } else {
11733                 ASSERT(enab != dtrace_retained);
11734                 ASSERT(dtrace_retained != NULL);
11735                 enab->dten_prev->dten_next = enab->dten_next;
11736         }
11737 
11738         if (enab->dten_next != NULL) {
11739                 ASSERT(dtrace_retained != NULL);
11740                 enab->dten_next->dten_prev = enab->dten_prev;
11741         }
11742 
11743         kmem_free(enab, sizeof (dtrace_enabling_t));
11744 }
11745 
11746 static int
11747 dtrace_enabling_retain(dtrace_enabling_t *enab)
11748 {
11749         dtrace_state_t *state;
11750 
11751         ASSERT(MUTEX_HELD(&dtrace_lock));
11752         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11753         ASSERT(enab->dten_vstate != NULL);
11754 
11755         state = enab->dten_vstate->dtvs_state;
11756         ASSERT(state != NULL);
11757 
11758         /*
11759          * We only allow each state to retain dtrace_retain_max enablings.
11760          */
11761         if (state->dts_nretained >= dtrace_retain_max)
11762                 return (ENOSPC);
11763 
11764         state->dts_nretained++;
11765         dtrace_retained_gen++;
11766 
11767         if (dtrace_retained == NULL) {
11768                 dtrace_retained = enab;
11769                 return (0);
11770         }
11771 
11772         enab->dten_next = dtrace_retained;
11773         dtrace_retained->dten_prev = enab;
11774         dtrace_retained = enab;
11775 
11776         return (0);
11777 }
11778 
11779 static int
11780 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
11781     dtrace_probedesc_t *create)
11782 {
11783         dtrace_enabling_t *new, *enab;
11784         int found = 0, err = ENOENT;
11785 
11786         ASSERT(MUTEX_HELD(&dtrace_lock));
11787         ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
11788         ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
11789         ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
11790         ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
11791 
11792         new = dtrace_enabling_create(&state->dts_vstate);
11793 
11794         /*
11795          * Iterate over all retained enablings, looking for enablings that
11796          * match the specified state.
11797          */
11798         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11799                 int i;
11800 
11801                 /*
11802                  * dtvs_state can only be NULL for helper enablings -- and
11803                  * helper enablings can't be retained.
11804                  */
11805                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11806 
11807                 if (enab->dten_vstate->dtvs_state != state)
11808                         continue;
11809 
11810                 /*
11811                  * Now iterate over each probe description; we're looking for
11812                  * an exact match to the specified probe description.
11813                  */
11814                 for (i = 0; i < enab->dten_ndesc; i++) {
11815                         dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11816                         dtrace_probedesc_t *pd = &ep->dted_probe;
11817 
11818                         if (strcmp(pd->dtpd_provider, match->dtpd_provider))
11819                                 continue;
11820 
11821                         if (strcmp(pd->dtpd_mod, match->dtpd_mod))
11822                                 continue;
11823 
11824                         if (strcmp(pd->dtpd_func, match->dtpd_func))
11825                                 continue;
11826 
11827                         if (strcmp(pd->dtpd_name, match->dtpd_name))
11828                                 continue;
11829 
11830                         /*
11831                          * We have a winning probe!  Add it to our growing
11832                          * enabling.
11833                          */
11834                         found = 1;
11835                         dtrace_enabling_addlike(new, ep, create);
11836                 }
11837         }
11838 
11839         if (!found || (err = dtrace_enabling_retain(new)) != 0) {
11840                 dtrace_enabling_destroy(new);
11841                 return (err);
11842         }
11843 
11844         return (0);
11845 }
11846 
11847 static void
11848 dtrace_enabling_retract(dtrace_state_t *state)
11849 {
11850         dtrace_enabling_t *enab, *next;
11851 
11852         ASSERT(MUTEX_HELD(&dtrace_lock));
11853 
11854         /*
11855          * Iterate over all retained enablings, destroy the enablings retained
11856          * for the specified state.
11857          */
11858         for (enab = dtrace_retained; enab != NULL; enab = next) {
11859                 next = enab->dten_next;
11860 
11861                 /*
11862                  * dtvs_state can only be NULL for helper enablings -- and
11863                  * helper enablings can't be retained.
11864                  */
11865                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11866 
11867                 if (enab->dten_vstate->dtvs_state == state) {
11868                         ASSERT(state->dts_nretained > 0);
11869                         dtrace_enabling_destroy(enab);
11870                 }
11871         }
11872 
11873         ASSERT(state->dts_nretained == 0);
11874 }
11875 
11876 static int
11877 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
11878 {
11879         int i = 0;
11880         int total_matched = 0, matched = 0;
11881 
11882         ASSERT(MUTEX_HELD(&cpu_lock));
11883         ASSERT(MUTEX_HELD(&dtrace_lock));
11884 
11885         for (i = 0; i < enab->dten_ndesc; i++) {
11886                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11887 
11888                 enab->dten_current = ep;
11889                 enab->dten_error = 0;
11890 
11891                 /*
11892                  * If a provider failed to enable a probe then get out and
11893                  * let the consumer know we failed.
11894                  */
11895                 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
11896                         return (EBUSY);
11897 
11898                 total_matched += matched;
11899 
11900                 if (enab->dten_error != 0) {
11901                         /*
11902                          * If we get an error half-way through enabling the
11903                          * probes, we kick out -- perhaps with some number of
11904                          * them enabled.  Leaving enabled probes enabled may
11905                          * be slightly confusing for user-level, but we expect
11906                          * that no one will attempt to actually drive on in
11907                          * the face of such errors.  If this is an anonymous
11908                          * enabling (indicated with a NULL nmatched pointer),
11909                          * we cmn_err() a message.  We aren't expecting to
11910                          * get such an error -- such as it can exist at all,
11911                          * it would be a result of corrupted DOF in the driver
11912                          * properties.
11913                          */
11914                         if (nmatched == NULL) {
11915                                 cmn_err(CE_WARN, "dtrace_enabling_match() "
11916                                     "error on %p: %d", (void *)ep,
11917                                     enab->dten_error);
11918                         }
11919 
11920                         return (enab->dten_error);
11921                 }
11922         }
11923 
11924         enab->dten_probegen = dtrace_probegen;
11925         if (nmatched != NULL)
11926                 *nmatched = total_matched;
11927 
11928         return (0);
11929 }
11930 
11931 static void
11932 dtrace_enabling_matchall(void)
11933 {
11934         dtrace_enabling_t *enab;
11935 
11936         mutex_enter(&cpu_lock);
11937         mutex_enter(&dtrace_lock);
11938 
11939         /*
11940          * Iterate over all retained enablings to see if any probes match
11941          * against them.  We only perform this operation on enablings for which
11942          * we have sufficient permissions by virtue of being in the global zone
11943          * or in the same zone as the DTrace client.  Because we can be called
11944          * after dtrace_detach() has been called, we cannot assert that there
11945          * are retained enablings.  We can safely load from dtrace_retained,
11946          * however:  the taskq_destroy() at the end of dtrace_detach() will
11947          * block pending our completion.
11948          */
11949         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11950                 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred;
11951                 cred_t *cr = dcr->dcr_cred;
11952                 zoneid_t zone = cr != NULL ? crgetzoneid(cr) : 0;
11953 
11954                 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL &&
11955                     (zone == GLOBAL_ZONEID || getzoneid() == zone)))
11956                         (void) dtrace_enabling_match(enab, NULL);
11957         }
11958 
11959         mutex_exit(&dtrace_lock);
11960         mutex_exit(&cpu_lock);
11961 }
11962 
11963 /*
11964  * If an enabling is to be enabled without having matched probes (that is, if
11965  * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
11966  * enabling must be _primed_ by creating an ECB for every ECB description.
11967  * This must be done to assure that we know the number of speculations, the
11968  * number of aggregations, the minimum buffer size needed, etc. before we
11969  * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
11970  * enabling any probes, we create ECBs for every ECB decription, but with a
11971  * NULL probe -- which is exactly what this function does.
11972  */
11973 static void
11974 dtrace_enabling_prime(dtrace_state_t *state)
11975 {
11976         dtrace_enabling_t *enab;
11977         int i;
11978 
11979         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11980                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11981 
11982                 if (enab->dten_vstate->dtvs_state != state)
11983                         continue;
11984 
11985                 /*
11986                  * We don't want to prime an enabling more than once, lest
11987                  * we allow a malicious user to induce resource exhaustion.
11988                  * (The ECBs that result from priming an enabling aren't
11989                  * leaked -- but they also aren't deallocated until the
11990                  * consumer state is destroyed.)
11991                  */
11992                 if (enab->dten_primed)
11993                         continue;
11994 
11995                 for (i = 0; i < enab->dten_ndesc; i++) {
11996                         enab->dten_current = enab->dten_desc[i];
11997                         (void) dtrace_probe_enable(NULL, enab);
11998                 }
11999 
12000                 enab->dten_primed = 1;
12001         }
12002 }
12003 
12004 /*
12005  * Called to indicate that probes should be provided due to retained
12006  * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
12007  * must take an initial lap through the enabling calling the dtps_provide()
12008  * entry point explicitly to allow for autocreated probes.
12009  */
12010 static void
12011 dtrace_enabling_provide(dtrace_provider_t *prv)
12012 {
12013         int i, all = 0;
12014         dtrace_probedesc_t desc;
12015         dtrace_genid_t gen;
12016 
12017         ASSERT(MUTEX_HELD(&dtrace_lock));
12018         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
12019 
12020         if (prv == NULL) {
12021                 all = 1;
12022                 prv = dtrace_provider;
12023         }
12024 
12025         do {
12026                 dtrace_enabling_t *enab;
12027                 void *parg = prv->dtpv_arg;
12028 
12029 retry:
12030                 gen = dtrace_retained_gen;
12031                 for (enab = dtrace_retained; enab != NULL;
12032                     enab = enab->dten_next) {
12033                         for (i = 0; i < enab->dten_ndesc; i++) {
12034                                 desc = enab->dten_desc[i]->dted_probe;
12035                                 mutex_exit(&dtrace_lock);
12036                                 prv->dtpv_pops.dtps_provide(parg, &desc);
12037                                 mutex_enter(&dtrace_lock);
12038                                 /*
12039                                  * Process the retained enablings again if
12040                                  * they have changed while we weren't holding
12041                                  * dtrace_lock.
12042                                  */
12043                                 if (gen != dtrace_retained_gen)
12044                                         goto retry;
12045                         }
12046                 }
12047         } while (all && (prv = prv->dtpv_next) != NULL);
12048 
12049         mutex_exit(&dtrace_lock);
12050         dtrace_probe_provide(NULL, all ? NULL : prv);
12051         mutex_enter(&dtrace_lock);
12052 }
12053 
12054 /*
12055  * Called to reap ECBs that are attached to probes from defunct providers.
12056  */
12057 static void
12058 dtrace_enabling_reap(void)
12059 {
12060         dtrace_provider_t *prov;
12061         dtrace_probe_t *probe;
12062         dtrace_ecb_t *ecb;
12063         hrtime_t when;
12064         int i;
12065 
12066         mutex_enter(&cpu_lock);
12067         mutex_enter(&dtrace_lock);
12068 
12069         for (i = 0; i < dtrace_nprobes; i++) {
12070                 if ((probe = dtrace_probes[i]) == NULL)
12071                         continue;
12072 
12073                 if (probe->dtpr_ecb == NULL)
12074                         continue;
12075 
12076                 prov = probe->dtpr_provider;
12077 
12078                 if ((when = prov->dtpv_defunct) == 0)
12079                         continue;
12080 
12081                 /*
12082                  * We have ECBs on a defunct provider:  we want to reap these
12083                  * ECBs to allow the provider to unregister.  The destruction
12084                  * of these ECBs must be done carefully:  if we destroy the ECB
12085                  * and the consumer later wishes to consume an EPID that
12086                  * corresponds to the destroyed ECB (and if the EPID metadata
12087                  * has not been previously consumed), the consumer will abort
12088                  * processing on the unknown EPID.  To reduce (but not, sadly,
12089                  * eliminate) the possibility of this, we will only destroy an
12090                  * ECB for a defunct provider if, for the state that
12091                  * corresponds to the ECB:
12092                  *
12093                  *  (a) There is no speculative tracing (which can effectively
12094                  *      cache an EPID for an arbitrary amount of time).
12095                  *
12096                  *  (b) The principal buffers have been switched twice since the
12097                  *      provider became defunct.
12098                  *
12099                  *  (c) The aggregation buffers are of zero size or have been
12100                  *      switched twice since the provider became defunct.
12101                  *
12102                  * We use dts_speculates to determine (a) and call a function
12103                  * (dtrace_buffer_consumed()) to determine (b) and (c).  Note
12104                  * that as soon as we've been unable to destroy one of the ECBs
12105                  * associated with the probe, we quit trying -- reaping is only
12106                  * fruitful in as much as we can destroy all ECBs associated
12107                  * with the defunct provider's probes.
12108                  */
12109                 while ((ecb = probe->dtpr_ecb) != NULL) {
12110                         dtrace_state_t *state = ecb->dte_state;
12111                         dtrace_buffer_t *buf = state->dts_buffer;
12112                         dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
12113 
12114                         if (state->dts_speculates)
12115                                 break;
12116 
12117                         if (!dtrace_buffer_consumed(buf, when))
12118                                 break;
12119 
12120                         if (!dtrace_buffer_consumed(aggbuf, when))
12121                                 break;
12122 
12123                         dtrace_ecb_disable(ecb);
12124                         ASSERT(probe->dtpr_ecb != ecb);
12125                         dtrace_ecb_destroy(ecb);
12126                 }
12127         }
12128 
12129         mutex_exit(&dtrace_lock);
12130         mutex_exit(&cpu_lock);
12131 }
12132 
12133 /*
12134  * DTrace DOF Functions
12135  */
12136 /*ARGSUSED*/
12137 static void
12138 dtrace_dof_error(dof_hdr_t *dof, const char *str)
12139 {
12140         if (dtrace_err_verbose)
12141                 cmn_err(CE_WARN, "failed to process DOF: %s", str);
12142 
12143 #ifdef DTRACE_ERRDEBUG
12144         dtrace_errdebug(str);
12145 #endif
12146 }
12147 
12148 /*
12149  * Create DOF out of a currently enabled state.  Right now, we only create
12150  * DOF containing the run-time options -- but this could be expanded to create
12151  * complete DOF representing the enabled state.
12152  */
12153 static dof_hdr_t *
12154 dtrace_dof_create(dtrace_state_t *state)
12155 {
12156         dof_hdr_t *dof;
12157         dof_sec_t *sec;
12158         dof_optdesc_t *opt;
12159         int i, len = sizeof (dof_hdr_t) +
12160             roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
12161             sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12162 
12163         ASSERT(MUTEX_HELD(&dtrace_lock));
12164 
12165         dof = kmem_zalloc(len, KM_SLEEP);
12166         dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
12167         dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
12168         dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
12169         dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
12170 
12171         dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
12172         dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
12173         dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
12174         dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
12175         dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
12176         dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
12177 
12178         dof->dofh_flags = 0;
12179         dof->dofh_hdrsize = sizeof (dof_hdr_t);
12180         dof->dofh_secsize = sizeof (dof_sec_t);
12181         dof->dofh_secnum = 1;        /* only DOF_SECT_OPTDESC */
12182         dof->dofh_secoff = sizeof (dof_hdr_t);
12183         dof->dofh_loadsz = len;
12184         dof->dofh_filesz = len;
12185         dof->dofh_pad = 0;
12186 
12187         /*
12188          * Fill in the option section header...
12189          */
12190         sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
12191         sec->dofs_type = DOF_SECT_OPTDESC;
12192         sec->dofs_align = sizeof (uint64_t);
12193         sec->dofs_flags = DOF_SECF_LOAD;
12194         sec->dofs_entsize = sizeof (dof_optdesc_t);
12195 
12196         opt = (dof_optdesc_t *)((uintptr_t)sec +
12197             roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
12198 
12199         sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
12200         sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12201 
12202         for (i = 0; i < DTRACEOPT_MAX; i++) {
12203                 opt[i].dofo_option = i;
12204                 opt[i].dofo_strtab = DOF_SECIDX_NONE;
12205                 opt[i].dofo_value = state->dts_options[i];
12206         }
12207 
12208         return (dof);
12209 }
12210 
12211 static dof_hdr_t *
12212 dtrace_dof_copyin(uintptr_t uarg, int *errp)
12213 {
12214         dof_hdr_t hdr, *dof;
12215 
12216         ASSERT(!MUTEX_HELD(&dtrace_lock));
12217 
12218         /*
12219          * First, we're going to copyin() the sizeof (dof_hdr_t).
12220          */
12221         if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
12222                 dtrace_dof_error(NULL, "failed to copyin DOF header");
12223                 *errp = EFAULT;
12224                 return (NULL);
12225         }
12226 
12227         /*
12228          * Now we'll allocate the entire DOF and copy it in -- provided
12229          * that the length isn't outrageous.
12230          */
12231         if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
12232                 dtrace_dof_error(&hdr, "load size exceeds maximum");
12233                 *errp = E2BIG;
12234                 return (NULL);
12235         }
12236 
12237         if (hdr.dofh_loadsz < sizeof (hdr)) {
12238                 dtrace_dof_error(&hdr, "invalid load size");
12239                 *errp = EINVAL;
12240                 return (NULL);
12241         }
12242 
12243         dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
12244 
12245         if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
12246             dof->dofh_loadsz != hdr.dofh_loadsz) {
12247                 kmem_free(dof, hdr.dofh_loadsz);
12248                 *errp = EFAULT;
12249                 return (NULL);
12250         }
12251 
12252         return (dof);
12253 }
12254 
12255 static dof_hdr_t *
12256 dtrace_dof_property(const char *name)
12257 {
12258         uchar_t *buf;
12259         uint64_t loadsz;
12260         unsigned int len, i;
12261         dof_hdr_t *dof;
12262 
12263         /*
12264          * Unfortunately, array of values in .conf files are always (and
12265          * only) interpreted to be integer arrays.  We must read our DOF
12266          * as an integer array, and then squeeze it into a byte array.
12267          */
12268         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
12269             (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
12270                 return (NULL);
12271 
12272         for (i = 0; i < len; i++)
12273                 buf[i] = (uchar_t)(((int *)buf)[i]);
12274 
12275         if (len < sizeof (dof_hdr_t)) {
12276                 ddi_prop_free(buf);
12277                 dtrace_dof_error(NULL, "truncated header");
12278                 return (NULL);
12279         }
12280 
12281         if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
12282                 ddi_prop_free(buf);
12283                 dtrace_dof_error(NULL, "truncated DOF");
12284                 return (NULL);
12285         }
12286 
12287         if (loadsz >= dtrace_dof_maxsize) {
12288                 ddi_prop_free(buf);
12289                 dtrace_dof_error(NULL, "oversized DOF");
12290                 return (NULL);
12291         }
12292 
12293         dof = kmem_alloc(loadsz, KM_SLEEP);
12294         bcopy(buf, dof, loadsz);
12295         ddi_prop_free(buf);
12296 
12297         return (dof);
12298 }
12299 
12300 static void
12301 dtrace_dof_destroy(dof_hdr_t *dof)
12302 {
12303         kmem_free(dof, dof->dofh_loadsz);
12304 }
12305 
12306 /*
12307  * Return the dof_sec_t pointer corresponding to a given section index.  If the
12308  * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
12309  * a type other than DOF_SECT_NONE is specified, the header is checked against
12310  * this type and NULL is returned if the types do not match.
12311  */
12312 static dof_sec_t *
12313 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
12314 {
12315         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
12316             ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
12317 
12318         if (i >= dof->dofh_secnum) {
12319                 dtrace_dof_error(dof, "referenced section index is invalid");
12320                 return (NULL);
12321         }
12322 
12323         if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
12324                 dtrace_dof_error(dof, "referenced section is not loadable");
12325                 return (NULL);
12326         }
12327 
12328         if (type != DOF_SECT_NONE && type != sec->dofs_type) {
12329                 dtrace_dof_error(dof, "referenced section is the wrong type");
12330                 return (NULL);
12331         }
12332 
12333         return (sec);
12334 }
12335 
12336 static dtrace_probedesc_t *
12337 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
12338 {
12339         dof_probedesc_t *probe;
12340         dof_sec_t *strtab;
12341         uintptr_t daddr = (uintptr_t)dof;
12342         uintptr_t str;
12343         size_t size;
12344 
12345         if (sec->dofs_type != DOF_SECT_PROBEDESC) {
12346                 dtrace_dof_error(dof, "invalid probe section");
12347                 return (NULL);
12348         }
12349 
12350         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12351                 dtrace_dof_error(dof, "bad alignment in probe description");
12352                 return (NULL);
12353         }
12354 
12355         if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
12356                 dtrace_dof_error(dof, "truncated probe description");
12357                 return (NULL);
12358         }
12359 
12360         probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
12361         strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
12362 
12363         if (strtab == NULL)
12364                 return (NULL);
12365 
12366         str = daddr + strtab->dofs_offset;
12367         size = strtab->dofs_size;
12368 
12369         if (probe->dofp_provider >= strtab->dofs_size) {
12370                 dtrace_dof_error(dof, "corrupt probe provider");
12371                 return (NULL);
12372         }
12373 
12374         (void) strncpy(desc->dtpd_provider,
12375             (char *)(str + probe->dofp_provider),
12376             MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
12377 
12378         if (probe->dofp_mod >= strtab->dofs_size) {
12379                 dtrace_dof_error(dof, "corrupt probe module");
12380                 return (NULL);
12381         }
12382 
12383         (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
12384             MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
12385 
12386         if (probe->dofp_func >= strtab->dofs_size) {
12387                 dtrace_dof_error(dof, "corrupt probe function");
12388                 return (NULL);
12389         }
12390 
12391         (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
12392             MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
12393 
12394         if (probe->dofp_name >= strtab->dofs_size) {
12395                 dtrace_dof_error(dof, "corrupt probe name");
12396                 return (NULL);
12397         }
12398 
12399         (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
12400             MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
12401 
12402         return (desc);
12403 }
12404 
12405 static dtrace_difo_t *
12406 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12407     cred_t *cr)
12408 {
12409         dtrace_difo_t *dp;
12410         size_t ttl = 0;
12411         dof_difohdr_t *dofd;
12412         uintptr_t daddr = (uintptr_t)dof;
12413         size_t max = dtrace_difo_maxsize;
12414         int i, l, n;
12415 
12416         static const struct {
12417                 int section;
12418                 int bufoffs;
12419                 int lenoffs;
12420                 int entsize;
12421                 int align;
12422                 const char *msg;
12423         } difo[] = {
12424                 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
12425                 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
12426                 sizeof (dif_instr_t), "multiple DIF sections" },
12427 
12428                 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
12429                 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
12430                 sizeof (uint64_t), "multiple integer tables" },
12431 
12432                 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
12433                 offsetof(dtrace_difo_t, dtdo_strlen), 0,
12434                 sizeof (char), "multiple string tables" },
12435 
12436                 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
12437                 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
12438                 sizeof (uint_t), "multiple variable tables" },
12439 
12440                 { DOF_SECT_NONE, 0, 0, 0, NULL }
12441         };
12442 
12443         if (sec->dofs_type != DOF_SECT_DIFOHDR) {
12444                 dtrace_dof_error(dof, "invalid DIFO header section");
12445                 return (NULL);
12446         }
12447 
12448         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12449                 dtrace_dof_error(dof, "bad alignment in DIFO header");
12450                 return (NULL);
12451         }
12452 
12453         if (sec->dofs_size < sizeof (dof_difohdr_t) ||
12454             sec->dofs_size % sizeof (dof_secidx_t)) {
12455                 dtrace_dof_error(dof, "bad size in DIFO header");
12456                 return (NULL);
12457         }
12458 
12459         dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12460         n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
12461 
12462         dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
12463         dp->dtdo_rtype = dofd->dofd_rtype;
12464 
12465         for (l = 0; l < n; l++) {
12466                 dof_sec_t *subsec;
12467                 void **bufp;
12468                 uint32_t *lenp;
12469 
12470                 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
12471                     dofd->dofd_links[l])) == NULL)
12472                         goto err; /* invalid section link */
12473 
12474                 if (ttl + subsec->dofs_size > max) {
12475                         dtrace_dof_error(dof, "exceeds maximum size");
12476                         goto err;
12477                 }
12478 
12479                 ttl += subsec->dofs_size;
12480 
12481                 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
12482                         if (subsec->dofs_type != difo[i].section)
12483                                 continue;
12484 
12485                         if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
12486                                 dtrace_dof_error(dof, "section not loaded");
12487                                 goto err;
12488                         }
12489 
12490                         if (subsec->dofs_align != difo[i].align) {
12491                                 dtrace_dof_error(dof, "bad alignment");
12492                                 goto err;
12493                         }
12494 
12495                         bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
12496                         lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
12497 
12498                         if (*bufp != NULL) {
12499                                 dtrace_dof_error(dof, difo[i].msg);
12500                                 goto err;
12501                         }
12502 
12503                         if (difo[i].entsize != subsec->dofs_entsize) {
12504                                 dtrace_dof_error(dof, "entry size mismatch");
12505                                 goto err;
12506                         }
12507 
12508                         if (subsec->dofs_entsize != 0 &&
12509                             (subsec->dofs_size % subsec->dofs_entsize) != 0) {
12510                                 dtrace_dof_error(dof, "corrupt entry size");
12511                                 goto err;
12512                         }
12513 
12514                         *lenp = subsec->dofs_size;
12515                         *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
12516                         bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
12517                             *bufp, subsec->dofs_size);
12518 
12519                         if (subsec->dofs_entsize != 0)
12520                                 *lenp /= subsec->dofs_entsize;
12521 
12522                         break;
12523                 }
12524 
12525                 /*
12526                  * If we encounter a loadable DIFO sub-section that is not
12527                  * known to us, assume this is a broken program and fail.
12528                  */
12529                 if (difo[i].section == DOF_SECT_NONE &&
12530                     (subsec->dofs_flags & DOF_SECF_LOAD)) {
12531                         dtrace_dof_error(dof, "unrecognized DIFO subsection");
12532                         goto err;
12533                 }
12534         }
12535 
12536         if (dp->dtdo_buf == NULL) {
12537                 /*
12538                  * We can't have a DIF object without DIF text.
12539                  */
12540                 dtrace_dof_error(dof, "missing DIF text");
12541                 goto err;
12542         }
12543 
12544         /*
12545          * Before we validate the DIF object, run through the variable table
12546          * looking for the strings -- if any of their size are under, we'll set
12547          * their size to be the system-wide default string size.  Note that
12548          * this should _not_ happen if the "strsize" option has been set --
12549          * in this case, the compiler should have set the size to reflect the
12550          * setting of the option.
12551          */
12552         for (i = 0; i < dp->dtdo_varlen; i++) {
12553                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
12554                 dtrace_diftype_t *t = &v->dtdv_type;
12555 
12556                 if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
12557                         continue;
12558 
12559                 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
12560                         t->dtdt_size = dtrace_strsize_default;
12561         }
12562 
12563         if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
12564                 goto err;
12565 
12566         dtrace_difo_init(dp, vstate);
12567         return (dp);
12568 
12569 err:
12570         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
12571         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
12572         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
12573         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
12574 
12575         kmem_free(dp, sizeof (dtrace_difo_t));
12576         return (NULL);
12577 }
12578 
12579 static dtrace_predicate_t *
12580 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12581     cred_t *cr)
12582 {
12583         dtrace_difo_t *dp;
12584 
12585         if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
12586                 return (NULL);
12587 
12588         return (dtrace_predicate_create(dp));
12589 }
12590 
12591 static dtrace_actdesc_t *
12592 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12593     cred_t *cr)
12594 {
12595         dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
12596         dof_actdesc_t *desc;
12597         dof_sec_t *difosec;
12598         size_t offs;
12599         uintptr_t daddr = (uintptr_t)dof;
12600         uint64_t arg;
12601         dtrace_actkind_t kind;
12602 
12603         if (sec->dofs_type != DOF_SECT_ACTDESC) {
12604                 dtrace_dof_error(dof, "invalid action section");
12605                 return (NULL);
12606         }
12607 
12608         if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
12609                 dtrace_dof_error(dof, "truncated action description");
12610                 return (NULL);
12611         }
12612 
12613         if (sec->dofs_align != sizeof (uint64_t)) {
12614                 dtrace_dof_error(dof, "bad alignment in action description");
12615                 return (NULL);
12616         }
12617 
12618         if (sec->dofs_size < sec->dofs_entsize) {
12619                 dtrace_dof_error(dof, "section entry size exceeds total size");
12620                 return (NULL);
12621         }
12622 
12623         if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
12624                 dtrace_dof_error(dof, "bad entry size in action description");
12625                 return (NULL);
12626         }
12627 
12628         if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
12629                 dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
12630                 return (NULL);
12631         }
12632 
12633         for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
12634                 desc = (dof_actdesc_t *)(daddr +
12635                     (uintptr_t)sec->dofs_offset + offs);
12636                 kind = (dtrace_actkind_t)desc->dofa_kind;
12637 
12638                 if ((DTRACEACT_ISPRINTFLIKE(kind) &&
12639                     (kind != DTRACEACT_PRINTA ||
12640                     desc->dofa_strtab != DOF_SECIDX_NONE)) ||
12641                     (kind == DTRACEACT_DIFEXPR &&
12642                     desc->dofa_strtab != DOF_SECIDX_NONE)) {
12643                         dof_sec_t *strtab;
12644                         char *str, *fmt;
12645                         uint64_t i;
12646 
12647                         /*
12648                          * The argument to these actions is an index into the
12649                          * DOF string table.  For printf()-like actions, this
12650                          * is the format string.  For print(), this is the
12651                          * CTF type of the expression result.
12652                          */
12653                         if ((strtab = dtrace_dof_sect(dof,
12654                             DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
12655                                 goto err;
12656 
12657                         str = (char *)((uintptr_t)dof +
12658                             (uintptr_t)strtab->dofs_offset);
12659 
12660                         for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
12661                                 if (str[i] == '\0')
12662                                         break;
12663                         }
12664 
12665                         if (i >= strtab->dofs_size) {
12666                                 dtrace_dof_error(dof, "bogus format string");
12667                                 goto err;
12668                         }
12669 
12670                         if (i == desc->dofa_arg) {
12671                                 dtrace_dof_error(dof, "empty format string");
12672                                 goto err;
12673                         }
12674 
12675                         i -= desc->dofa_arg;
12676                         fmt = kmem_alloc(i + 1, KM_SLEEP);
12677                         bcopy(&str[desc->dofa_arg], fmt, i + 1);
12678                         arg = (uint64_t)(uintptr_t)fmt;
12679                 } else {
12680                         if (kind == DTRACEACT_PRINTA) {
12681                                 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
12682                                 arg = 0;
12683                         } else {
12684                                 arg = desc->dofa_arg;
12685                         }
12686                 }
12687 
12688                 act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
12689                     desc->dofa_uarg, arg);
12690 
12691                 if (last != NULL) {
12692                         last->dtad_next = act;
12693                 } else {
12694                         first = act;
12695                 }
12696 
12697                 last = act;
12698 
12699                 if (desc->dofa_difo == DOF_SECIDX_NONE)
12700                         continue;
12701 
12702                 if ((difosec = dtrace_dof_sect(dof,
12703                     DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
12704                         goto err;
12705 
12706                 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
12707 
12708                 if (act->dtad_difo == NULL)
12709                         goto err;
12710         }
12711 
12712         ASSERT(first != NULL);
12713         return (first);
12714 
12715 err:
12716         for (act = first; act != NULL; act = next) {
12717                 next = act->dtad_next;
12718                 dtrace_actdesc_release(act, vstate);
12719         }
12720 
12721         return (NULL);
12722 }
12723 
12724 static dtrace_ecbdesc_t *
12725 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12726     cred_t *cr)
12727 {
12728         dtrace_ecbdesc_t *ep;
12729         dof_ecbdesc_t *ecb;
12730         dtrace_probedesc_t *desc;
12731         dtrace_predicate_t *pred = NULL;
12732 
12733         if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
12734                 dtrace_dof_error(dof, "truncated ECB description");
12735                 return (NULL);
12736         }
12737 
12738         if (sec->dofs_align != sizeof (uint64_t)) {
12739                 dtrace_dof_error(dof, "bad alignment in ECB description");
12740                 return (NULL);
12741         }
12742 
12743         ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
12744         sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
12745 
12746         if (sec == NULL)
12747                 return (NULL);
12748 
12749         ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
12750         ep->dted_uarg = ecb->dofe_uarg;
12751         desc = &ep->dted_probe;
12752 
12753         if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
12754                 goto err;
12755 
12756         if (ecb->dofe_pred != DOF_SECIDX_NONE) {
12757                 if ((sec = dtrace_dof_sect(dof,
12758                     DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
12759                         goto err;
12760 
12761                 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
12762                         goto err;
12763 
12764                 ep->dted_pred.dtpdd_predicate = pred;
12765         }
12766 
12767         if (ecb->dofe_actions != DOF_SECIDX_NONE) {
12768                 if ((sec = dtrace_dof_sect(dof,
12769                     DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
12770                         goto err;
12771 
12772                 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
12773 
12774                 if (ep->dted_action == NULL)
12775                         goto err;
12776         }
12777 
12778         return (ep);
12779 
12780 err:
12781         if (pred != NULL)
12782                 dtrace_predicate_release(pred, vstate);
12783         kmem_free(ep, sizeof (dtrace_ecbdesc_t));
12784         return (NULL);
12785 }
12786 
12787 /*
12788  * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
12789  * specified DOF.  At present, this amounts to simply adding 'ubase' to the
12790  * site of any user SETX relocations to account for load object base address.
12791  * In the future, if we need other relocations, this function can be extended.
12792  */
12793 static int
12794 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
12795 {
12796         uintptr_t daddr = (uintptr_t)dof;
12797         dof_relohdr_t *dofr =
12798             (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12799         dof_sec_t *ss, *rs, *ts;
12800         dof_relodesc_t *r;
12801         uint_t i, n;
12802 
12803         if (sec->dofs_size < sizeof (dof_relohdr_t) ||
12804             sec->dofs_align != sizeof (dof_secidx_t)) {
12805                 dtrace_dof_error(dof, "invalid relocation header");
12806                 return (-1);
12807         }
12808 
12809         ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
12810         rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
12811         ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
12812 
12813         if (ss == NULL || rs == NULL || ts == NULL)
12814                 return (-1); /* dtrace_dof_error() has been called already */
12815 
12816         if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
12817             rs->dofs_align != sizeof (uint64_t)) {
12818                 dtrace_dof_error(dof, "invalid relocation section");
12819                 return (-1);
12820         }
12821 
12822         r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
12823         n = rs->dofs_size / rs->dofs_entsize;
12824 
12825         for (i = 0; i < n; i++) {
12826                 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
12827 
12828                 switch (r->dofr_type) {
12829                 case DOF_RELO_NONE:
12830                         break;
12831                 case DOF_RELO_SETX:
12832                         if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
12833                             sizeof (uint64_t) > ts->dofs_size) {
12834                                 dtrace_dof_error(dof, "bad relocation offset");
12835                                 return (-1);
12836                         }
12837 
12838                         if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
12839                                 dtrace_dof_error(dof, "misaligned setx relo");
12840                                 return (-1);
12841                         }
12842 
12843                         *(uint64_t *)taddr += ubase;
12844                         break;
12845                 default:
12846                         dtrace_dof_error(dof, "invalid relocation type");
12847                         return (-1);
12848                 }
12849 
12850                 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
12851         }
12852 
12853         return (0);
12854 }
12855 
12856 /*
12857  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
12858  * header:  it should be at the front of a memory region that is at least
12859  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
12860  * size.  It need not be validated in any other way.
12861  */
12862 static int
12863 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
12864     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
12865 {
12866         uint64_t len = dof->dofh_loadsz, seclen;
12867         uintptr_t daddr = (uintptr_t)dof;
12868         dtrace_ecbdesc_t *ep;
12869         dtrace_enabling_t *enab;
12870         uint_t i;
12871 
12872         ASSERT(MUTEX_HELD(&dtrace_lock));
12873         ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
12874 
12875         /*
12876          * Check the DOF header identification bytes.  In addition to checking
12877          * valid settings, we also verify that unused bits/bytes are zeroed so
12878          * we can use them later without fear of regressing existing binaries.
12879          */
12880         if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
12881             DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
12882                 dtrace_dof_error(dof, "DOF magic string mismatch");
12883                 return (-1);
12884         }
12885 
12886         if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
12887             dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
12888                 dtrace_dof_error(dof, "DOF has invalid data model");
12889                 return (-1);
12890         }
12891 
12892         if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
12893                 dtrace_dof_error(dof, "DOF encoding mismatch");
12894                 return (-1);
12895         }
12896 
12897         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
12898             dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
12899                 dtrace_dof_error(dof, "DOF version mismatch");
12900                 return (-1);
12901         }
12902 
12903         if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
12904                 dtrace_dof_error(dof, "DOF uses unsupported instruction set");
12905                 return (-1);
12906         }
12907 
12908         if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
12909                 dtrace_dof_error(dof, "DOF uses too many integer registers");
12910                 return (-1);
12911         }
12912 
12913         if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
12914                 dtrace_dof_error(dof, "DOF uses too many tuple registers");
12915                 return (-1);
12916         }
12917 
12918         for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
12919                 if (dof->dofh_ident[i] != 0) {
12920                         dtrace_dof_error(dof, "DOF has invalid ident byte set");
12921                         return (-1);
12922                 }
12923         }
12924 
12925         if (dof->dofh_flags & ~DOF_FL_VALID) {
12926                 dtrace_dof_error(dof, "DOF has invalid flag bits set");
12927                 return (-1);
12928         }
12929 
12930         if (dof->dofh_secsize == 0) {
12931                 dtrace_dof_error(dof, "zero section header size");
12932                 return (-1);
12933         }
12934 
12935         /*
12936          * Check that the section headers don't exceed the amount of DOF
12937          * data.  Note that we cast the section size and number of sections
12938          * to uint64_t's to prevent possible overflow in the multiplication.
12939          */
12940         seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
12941 
12942         if (dof->dofh_secoff > len || seclen > len ||
12943             dof->dofh_secoff + seclen > len) {
12944                 dtrace_dof_error(dof, "truncated section headers");
12945                 return (-1);
12946         }
12947 
12948         if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
12949                 dtrace_dof_error(dof, "misaligned section headers");
12950                 return (-1);
12951         }
12952 
12953         if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
12954                 dtrace_dof_error(dof, "misaligned section size");
12955                 return (-1);
12956         }
12957 
12958         /*
12959          * Take an initial pass through the section headers to be sure that
12960          * the headers don't have stray offsets.  If the 'noprobes' flag is
12961          * set, do not permit sections relating to providers, probes, or args.
12962          */
12963         for (i = 0; i < dof->dofh_secnum; i++) {
12964                 dof_sec_t *sec = (dof_sec_t *)(daddr +
12965                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12966 
12967                 if (noprobes) {
12968                         switch (sec->dofs_type) {
12969                         case DOF_SECT_PROVIDER:
12970                         case DOF_SECT_PROBES:
12971                         case DOF_SECT_PRARGS:
12972                         case DOF_SECT_PROFFS:
12973                                 dtrace_dof_error(dof, "illegal sections "
12974                                     "for enabling");
12975                                 return (-1);
12976                         }
12977                 }
12978 
12979                 if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
12980                     !(sec->dofs_flags & DOF_SECF_LOAD)) {
12981                         dtrace_dof_error(dof, "loadable section with load "
12982                             "flag unset");
12983                         return (-1);
12984                 }
12985 
12986                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
12987                         continue; /* just ignore non-loadable sections */
12988 
12989                 if (sec->dofs_align & (sec->dofs_align - 1)) {
12990                         dtrace_dof_error(dof, "bad section alignment");
12991                         return (-1);
12992                 }
12993 
12994                 if (sec->dofs_offset & (sec->dofs_align - 1)) {
12995                         dtrace_dof_error(dof, "misaligned section");
12996                         return (-1);
12997                 }
12998 
12999                 if (sec->dofs_offset > len || sec->dofs_size > len ||
13000                     sec->dofs_offset + sec->dofs_size > len) {
13001                         dtrace_dof_error(dof, "corrupt section header");
13002                         return (-1);
13003                 }
13004 
13005                 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
13006                     sec->dofs_offset + sec->dofs_size - 1) != '\0') {
13007                         dtrace_dof_error(dof, "non-terminating string table");
13008                         return (-1);
13009                 }
13010         }
13011 
13012         /*
13013          * Take a second pass through the sections and locate and perform any
13014          * relocations that are present.  We do this after the first pass to
13015          * be sure that all sections have had their headers validated.
13016          */
13017         for (i = 0; i < dof->dofh_secnum; i++) {
13018                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13019                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13020 
13021                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
13022                         continue; /* skip sections that are not loadable */
13023 
13024                 switch (sec->dofs_type) {
13025                 case DOF_SECT_URELHDR:
13026                         if (dtrace_dof_relocate(dof, sec, ubase) != 0)
13027                                 return (-1);
13028                         break;
13029                 }
13030         }
13031 
13032         if ((enab = *enabp) == NULL)
13033                 enab = *enabp = dtrace_enabling_create(vstate);
13034 
13035         for (i = 0; i < dof->dofh_secnum; i++) {
13036                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13037                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13038 
13039                 if (sec->dofs_type != DOF_SECT_ECBDESC)
13040                         continue;
13041 
13042                 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
13043                         dtrace_enabling_destroy(enab);
13044                         *enabp = NULL;
13045                         return (-1);
13046                 }
13047 
13048                 dtrace_enabling_add(enab, ep);
13049         }
13050 
13051         return (0);
13052 }
13053 
13054 /*
13055  * Process DOF for any options.  This routine assumes that the DOF has been
13056  * at least processed by dtrace_dof_slurp().
13057  */
13058 static int
13059 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
13060 {
13061         int i, rval;
13062         uint32_t entsize;
13063         size_t offs;
13064         dof_optdesc_t *desc;
13065 
13066         for (i = 0; i < dof->dofh_secnum; i++) {
13067                 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
13068                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13069 
13070                 if (sec->dofs_type != DOF_SECT_OPTDESC)
13071                         continue;
13072 
13073                 if (sec->dofs_align != sizeof (uint64_t)) {
13074                         dtrace_dof_error(dof, "bad alignment in "
13075                             "option description");
13076                         return (EINVAL);
13077                 }
13078 
13079                 if ((entsize = sec->dofs_entsize) == 0) {
13080                         dtrace_dof_error(dof, "zeroed option entry size");
13081                         return (EINVAL);
13082                 }
13083 
13084                 if (entsize < sizeof (dof_optdesc_t)) {
13085                         dtrace_dof_error(dof, "bad option entry size");
13086                         return (EINVAL);
13087                 }
13088 
13089                 for (offs = 0; offs < sec->dofs_size; offs += entsize) {
13090                         desc = (dof_optdesc_t *)((uintptr_t)dof +
13091                             (uintptr_t)sec->dofs_offset + offs);
13092 
13093                         if (desc->dofo_strtab != DOF_SECIDX_NONE) {
13094                                 dtrace_dof_error(dof, "non-zero option string");
13095                                 return (EINVAL);
13096                         }
13097 
13098                         if (desc->dofo_value == DTRACEOPT_UNSET) {
13099                                 dtrace_dof_error(dof, "unset option");
13100                                 return (EINVAL);
13101                         }
13102 
13103                         if ((rval = dtrace_state_option(state,
13104                             desc->dofo_option, desc->dofo_value)) != 0) {
13105                                 dtrace_dof_error(dof, "rejected option");
13106                                 return (rval);
13107                         }
13108                 }
13109         }
13110 
13111         return (0);
13112 }
13113 
13114 /*
13115  * DTrace Consumer State Functions
13116  */
13117 int
13118 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
13119 {
13120         size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
13121         void *base;
13122         uintptr_t limit;
13123         dtrace_dynvar_t *dvar, *next, *start;
13124         int i;
13125 
13126         ASSERT(MUTEX_HELD(&dtrace_lock));
13127         ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
13128 
13129         bzero(dstate, sizeof (dtrace_dstate_t));
13130 
13131         if ((dstate->dtds_chunksize = chunksize) == 0)
13132                 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
13133 
13134         if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
13135                 size = min;
13136 
13137         if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
13138                 return (ENOMEM);
13139 
13140         dstate->dtds_size = size;
13141         dstate->dtds_base = base;
13142         dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
13143         bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
13144 
13145         hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
13146 
13147         if (hashsize != 1 && (hashsize & 1))
13148                 hashsize--;
13149 
13150         dstate->dtds_hashsize = hashsize;
13151         dstate->dtds_hash = dstate->dtds_base;
13152 
13153         /*
13154          * Set all of our hash buckets to point to the single sink, and (if
13155          * it hasn't already been set), set the sink's hash value to be the
13156          * sink sentinel value.  The sink is needed for dynamic variable
13157          * lookups to know that they have iterated over an entire, valid hash
13158          * chain.
13159          */
13160         for (i = 0; i < hashsize; i++)
13161                 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
13162 
13163         if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
13164                 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
13165 
13166         /*
13167          * Determine number of active CPUs.  Divide free list evenly among
13168          * active CPUs.
13169          */
13170         start = (dtrace_dynvar_t *)
13171             ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
13172         limit = (uintptr_t)base + size;
13173 
13174         maxper = (limit - (uintptr_t)start) / NCPU;
13175         maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
13176 
13177         for (i = 0; i < NCPU; i++) {
13178                 dstate->dtds_percpu[i].dtdsc_free = dvar = start;
13179 
13180                 /*
13181                  * If we don't even have enough chunks to make it once through
13182                  * NCPUs, we're just going to allocate everything to the first
13183                  * CPU.  And if we're on the last CPU, we're going to allocate
13184                  * whatever is left over.  In either case, we set the limit to
13185                  * be the limit of the dynamic variable space.
13186                  */
13187                 if (maxper == 0 || i == NCPU - 1) {
13188                         limit = (uintptr_t)base + size;
13189                         start = NULL;
13190                 } else {
13191                         limit = (uintptr_t)start + maxper;
13192                         start = (dtrace_dynvar_t *)limit;
13193                 }
13194 
13195                 ASSERT(limit <= (uintptr_t)base + size);
13196 
13197                 for (;;) {
13198                         next = (dtrace_dynvar_t *)((uintptr_t)dvar +
13199                             dstate->dtds_chunksize);
13200 
13201                         if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
13202                                 break;
13203 
13204                         dvar->dtdv_next = next;
13205                         dvar = next;
13206                 }
13207 
13208                 if (maxper == 0)
13209                         break;
13210         }
13211 
13212         return (0);
13213 }
13214 
13215 void
13216 dtrace_dstate_fini(dtrace_dstate_t *dstate)
13217 {
13218         ASSERT(MUTEX_HELD(&cpu_lock));
13219 
13220         if (dstate->dtds_base == NULL)
13221                 return;
13222 
13223         kmem_free(dstate->dtds_base, dstate->dtds_size);
13224         kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
13225 }
13226 
13227 static void
13228 dtrace_vstate_fini(dtrace_vstate_t *vstate)
13229 {
13230         /*
13231          * Logical XOR, where are you?
13232          */
13233         ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
13234 
13235         if (vstate->dtvs_nglobals > 0) {
13236                 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
13237                     sizeof (dtrace_statvar_t *));
13238         }
13239 
13240         if (vstate->dtvs_ntlocals > 0) {
13241                 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
13242                     sizeof (dtrace_difv_t));
13243         }
13244 
13245         ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
13246 
13247         if (vstate->dtvs_nlocals > 0) {
13248                 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
13249                     sizeof (dtrace_statvar_t *));
13250         }
13251 }
13252 
13253 static void
13254 dtrace_state_clean(dtrace_state_t *state)
13255 {
13256         if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
13257                 return;
13258 
13259         dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
13260         dtrace_speculation_clean(state);
13261 }
13262 
13263 static void
13264 dtrace_state_deadman(dtrace_state_t *state)
13265 {
13266         hrtime_t now;
13267 
13268         dtrace_sync();
13269 
13270         now = dtrace_gethrtime();
13271 
13272         if (state != dtrace_anon.dta_state &&
13273             now - state->dts_laststatus >= dtrace_deadman_user)
13274                 return;
13275 
13276         /*
13277          * We must be sure that dts_alive never appears to be less than the
13278          * value upon entry to dtrace_state_deadman(), and because we lack a
13279          * dtrace_cas64(), we cannot store to it atomically.  We thus instead
13280          * store INT64_MAX to it, followed by a memory barrier, followed by
13281          * the new value.  This assures that dts_alive never appears to be
13282          * less than its true value, regardless of the order in which the
13283          * stores to the underlying storage are issued.
13284          */
13285         state->dts_alive = INT64_MAX;
13286         dtrace_membar_producer();
13287         state->dts_alive = now;
13288 }
13289 
13290 dtrace_state_t *
13291 dtrace_state_create(dev_t *devp, cred_t *cr)
13292 {
13293         minor_t minor;
13294         major_t major;
13295         char c[30];
13296         dtrace_state_t *state;
13297         dtrace_optval_t *opt;
13298         int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
13299 
13300         ASSERT(MUTEX_HELD(&dtrace_lock));
13301         ASSERT(MUTEX_HELD(&cpu_lock));
13302 
13303         minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
13304             VM_BESTFIT | VM_SLEEP);
13305 
13306         if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
13307                 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
13308                 return (NULL);
13309         }
13310 
13311         state = ddi_get_soft_state(dtrace_softstate, minor);
13312         state->dts_epid = DTRACE_EPIDNONE + 1;
13313 
13314         (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
13315         state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
13316             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
13317 
13318         if (devp != NULL) {
13319                 major = getemajor(*devp);
13320         } else {
13321                 major = ddi_driver_major(dtrace_devi);
13322         }
13323 
13324         state->dts_dev = makedevice(major, minor);
13325 
13326         if (devp != NULL)
13327                 *devp = state->dts_dev;
13328 
13329         /*
13330          * We allocate NCPU buffers.  On the one hand, this can be quite
13331          * a bit of memory per instance (nearly 36K on a Starcat).  On the
13332          * other hand, it saves an additional memory reference in the probe
13333          * path.
13334          */
13335         state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
13336         state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
13337         state->dts_cleaner = CYCLIC_NONE;
13338         state->dts_deadman = CYCLIC_NONE;
13339         state->dts_vstate.dtvs_state = state;
13340 
13341         for (i = 0; i < DTRACEOPT_MAX; i++)
13342                 state->dts_options[i] = DTRACEOPT_UNSET;
13343 
13344         /*
13345          * Set the default options.
13346          */
13347         opt = state->dts_options;
13348         opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
13349         opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
13350         opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
13351         opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
13352         opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
13353         opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
13354         opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
13355         opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
13356         opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
13357         opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
13358         opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
13359         opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
13360         opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
13361         opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
13362 
13363         state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
13364 
13365         /*
13366          * Depending on the user credentials, we set flag bits which alter probe
13367          * visibility or the amount of destructiveness allowed.  In the case of
13368          * actual anonymous tracing, or the possession of all privileges, all of
13369          * the normal checks are bypassed.
13370          */
13371         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
13372                 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
13373                 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
13374         } else {
13375                 /*
13376                  * Set up the credentials for this instantiation.  We take a
13377                  * hold on the credential to prevent it from disappearing on
13378                  * us; this in turn prevents the zone_t referenced by this
13379                  * credential from disappearing.  This means that we can
13380                  * examine the credential and the zone from probe context.
13381                  */
13382                 crhold(cr);
13383                 state->dts_cred.dcr_cred = cr;
13384 
13385                 /*
13386                  * CRA_PROC means "we have *some* privilege for dtrace" and
13387                  * unlocks the use of variables like pid, zonename, etc.
13388                  */
13389                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
13390                     PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13391                         state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
13392                 }
13393 
13394                 /*
13395                  * dtrace_user allows use of syscall and profile providers.
13396                  * If the user also has proc_owner and/or proc_zone, we
13397                  * extend the scope to include additional visibility and
13398                  * destructive power.
13399                  */
13400                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
13401                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
13402                                 state->dts_cred.dcr_visible |=
13403                                     DTRACE_CRV_ALLPROC;
13404 
13405                                 state->dts_cred.dcr_action |=
13406                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13407                         }
13408 
13409                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
13410                                 state->dts_cred.dcr_visible |=
13411                                     DTRACE_CRV_ALLZONE;
13412 
13413                                 state->dts_cred.dcr_action |=
13414                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13415                         }
13416 
13417                         /*
13418                          * If we have all privs in whatever zone this is,
13419                          * we can do destructive things to processes which
13420                          * have altered credentials.
13421                          */
13422                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13423                             cr->cr_zone->zone_privset)) {
13424                                 state->dts_cred.dcr_action |=
13425                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13426                         }
13427                 }
13428 
13429                 /*
13430                  * Holding the dtrace_kernel privilege also implies that
13431                  * the user has the dtrace_user privilege from a visibility
13432                  * perspective.  But without further privileges, some
13433                  * destructive actions are not available.
13434                  */
13435                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
13436                         /*
13437                          * Make all probes in all zones visible.  However,
13438                          * this doesn't mean that all actions become available
13439                          * to all zones.
13440                          */
13441                         state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
13442                             DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
13443 
13444                         state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
13445                             DTRACE_CRA_PROC;
13446                         /*
13447                          * Holding proc_owner means that destructive actions
13448                          * for *this* zone are allowed.
13449                          */
13450                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13451                                 state->dts_cred.dcr_action |=
13452                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13453 
13454                         /*
13455                          * Holding proc_zone means that destructive actions
13456                          * for this user/group ID in all zones is allowed.
13457                          */
13458                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13459                                 state->dts_cred.dcr_action |=
13460                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13461 
13462                         /*
13463                          * If we have all privs in whatever zone this is,
13464                          * we can do destructive things to processes which
13465                          * have altered credentials.
13466                          */
13467                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13468                             cr->cr_zone->zone_privset)) {
13469                                 state->dts_cred.dcr_action |=
13470                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13471                         }
13472                 }
13473 
13474                 /*
13475                  * Holding the dtrace_proc privilege gives control over fasttrap
13476                  * and pid providers.  We need to grant wider destructive
13477                  * privileges in the event that the user has proc_owner and/or
13478                  * proc_zone.
13479                  */
13480                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13481                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13482                                 state->dts_cred.dcr_action |=
13483                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13484 
13485                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13486                                 state->dts_cred.dcr_action |=
13487                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13488                 }
13489         }
13490 
13491         return (state);
13492 }
13493 
13494 static int
13495 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
13496 {
13497         dtrace_optval_t *opt = state->dts_options, size;
13498         processorid_t cpu;
13499         int flags = 0, rval, factor, divisor = 1;
13500 
13501         ASSERT(MUTEX_HELD(&dtrace_lock));
13502         ASSERT(MUTEX_HELD(&cpu_lock));
13503         ASSERT(which < DTRACEOPT_MAX);
13504         ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
13505             (state == dtrace_anon.dta_state &&
13506             state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
13507 
13508         if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
13509                 return (0);
13510 
13511         if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
13512                 cpu = opt[DTRACEOPT_CPU];
13513 
13514         if (which == DTRACEOPT_SPECSIZE)
13515                 flags |= DTRACEBUF_NOSWITCH;
13516 
13517         if (which == DTRACEOPT_BUFSIZE) {
13518                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
13519                         flags |= DTRACEBUF_RING;
13520 
13521                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
13522                         flags |= DTRACEBUF_FILL;
13523 
13524                 if (state != dtrace_anon.dta_state ||
13525                     state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
13526                         flags |= DTRACEBUF_INACTIVE;
13527         }
13528 
13529         for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
13530                 /*
13531                  * The size must be 8-byte aligned.  If the size is not 8-byte
13532                  * aligned, drop it down by the difference.
13533                  */
13534                 if (size & (sizeof (uint64_t) - 1))
13535                         size -= size & (sizeof (uint64_t) - 1);
13536 
13537                 if (size < state->dts_reserve) {
13538                         /*
13539                          * Buffers always must be large enough to accommodate
13540                          * their prereserved space.  We return E2BIG instead
13541                          * of ENOMEM in this case to allow for user-level
13542                          * software to differentiate the cases.
13543                          */
13544                         return (E2BIG);
13545                 }
13546 
13547                 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
13548 
13549                 if (rval != ENOMEM) {
13550                         opt[which] = size;
13551                         return (rval);
13552                 }
13553 
13554                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13555                         return (rval);
13556 
13557                 for (divisor = 2; divisor < factor; divisor <<= 1)
13558                         continue;
13559         }
13560 
13561         return (ENOMEM);
13562 }
13563 
13564 static int
13565 dtrace_state_buffers(dtrace_state_t *state)
13566 {
13567         dtrace_speculation_t *spec = state->dts_speculations;
13568         int rval, i;
13569 
13570         if ((rval = dtrace_state_buffer(state, state->dts_buffer,
13571             DTRACEOPT_BUFSIZE)) != 0)
13572                 return (rval);
13573 
13574         if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
13575             DTRACEOPT_AGGSIZE)) != 0)
13576                 return (rval);
13577 
13578         for (i = 0; i < state->dts_nspeculations; i++) {
13579                 if ((rval = dtrace_state_buffer(state,
13580                     spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
13581                         return (rval);
13582         }
13583 
13584         return (0);
13585 }
13586 
13587 static void
13588 dtrace_state_prereserve(dtrace_state_t *state)
13589 {
13590         dtrace_ecb_t *ecb;
13591         dtrace_probe_t *probe;
13592 
13593         state->dts_reserve = 0;
13594 
13595         if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
13596                 return;
13597 
13598         /*
13599          * If our buffer policy is a "fill" buffer policy, we need to set the
13600          * prereserved space to be the space required by the END probes.
13601          */
13602         probe = dtrace_probes[dtrace_probeid_end - 1];
13603         ASSERT(probe != NULL);
13604 
13605         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
13606                 if (ecb->dte_state != state)
13607                         continue;
13608 
13609                 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
13610         }
13611 }
13612 
13613 static int
13614 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
13615 {
13616         dtrace_optval_t *opt = state->dts_options, sz, nspec;
13617         dtrace_speculation_t *spec;
13618         dtrace_buffer_t *buf;
13619         cyc_handler_t hdlr;
13620         cyc_time_t when;
13621         int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
13622         dtrace_icookie_t cookie;
13623 
13624         mutex_enter(&cpu_lock);
13625         mutex_enter(&dtrace_lock);
13626 
13627         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
13628                 rval = EBUSY;
13629                 goto out;
13630         }
13631 
13632         /*
13633          * Before we can perform any checks, we must prime all of the
13634          * retained enablings that correspond to this state.
13635          */
13636         dtrace_enabling_prime(state);
13637 
13638         if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
13639                 rval = EACCES;
13640                 goto out;
13641         }
13642 
13643         dtrace_state_prereserve(state);
13644 
13645         /*
13646          * Now we want to do is try to allocate our speculations.
13647          * We do not automatically resize the number of speculations; if
13648          * this fails, we will fail the operation.
13649          */
13650         nspec = opt[DTRACEOPT_NSPEC];
13651         ASSERT(nspec != DTRACEOPT_UNSET);
13652 
13653         if (nspec > INT_MAX) {
13654                 rval = ENOMEM;
13655                 goto out;
13656         }
13657 
13658         spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
13659             KM_NOSLEEP | KM_NORMALPRI);
13660 
13661         if (spec == NULL) {
13662                 rval = ENOMEM;
13663                 goto out;
13664         }
13665 
13666         state->dts_speculations = spec;
13667         state->dts_nspeculations = (int)nspec;
13668 
13669         for (i = 0; i < nspec; i++) {
13670                 if ((buf = kmem_zalloc(bufsize,
13671                     KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
13672                         rval = ENOMEM;
13673                         goto err;
13674                 }
13675 
13676                 spec[i].dtsp_buffer = buf;
13677         }
13678 
13679         if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
13680                 if (dtrace_anon.dta_state == NULL) {
13681                         rval = ENOENT;
13682                         goto out;
13683                 }
13684 
13685                 if (state->dts_necbs != 0) {
13686                         rval = EALREADY;
13687                         goto out;
13688                 }
13689 
13690                 state->dts_anon = dtrace_anon_grab();
13691                 ASSERT(state->dts_anon != NULL);
13692                 state = state->dts_anon;
13693 
13694                 /*
13695                  * We want "grabanon" to be set in the grabbed state, so we'll
13696                  * copy that option value from the grabbing state into the
13697                  * grabbed state.
13698                  */
13699                 state->dts_options[DTRACEOPT_GRABANON] =
13700                     opt[DTRACEOPT_GRABANON];
13701 
13702                 *cpu = dtrace_anon.dta_beganon;
13703 
13704                 /*
13705                  * If the anonymous state is active (as it almost certainly
13706                  * is if the anonymous enabling ultimately matched anything),
13707                  * we don't allow any further option processing -- but we
13708                  * don't return failure.
13709                  */
13710                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
13711                         goto out;
13712         }
13713 
13714         if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
13715             opt[DTRACEOPT_AGGSIZE] != 0) {
13716                 if (state->dts_aggregations == NULL) {
13717                         /*
13718                          * We're not going to create an aggregation buffer
13719                          * because we don't have any ECBs that contain
13720                          * aggregations -- set this option to 0.
13721                          */
13722                         opt[DTRACEOPT_AGGSIZE] = 0;
13723                 } else {
13724                         /*
13725                          * If we have an aggregation buffer, we must also have
13726                          * a buffer to use as scratch.
13727                          */
13728                         if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
13729                             opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
13730                                 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
13731                         }
13732                 }
13733         }
13734 
13735         if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
13736             opt[DTRACEOPT_SPECSIZE] != 0) {
13737                 if (!state->dts_speculates) {
13738                         /*
13739                          * We're not going to create speculation buffers
13740                          * because we don't have any ECBs that actually
13741                          * speculate -- set the speculation size to 0.
13742                          */
13743                         opt[DTRACEOPT_SPECSIZE] = 0;
13744                 }
13745         }
13746 
13747         /*
13748          * The bare minimum size for any buffer that we're actually going to
13749          * do anything to is sizeof (uint64_t).
13750          */
13751         sz = sizeof (uint64_t);
13752 
13753         if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
13754             (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
13755             (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
13756                 /*
13757                  * A buffer size has been explicitly set to 0 (or to a size
13758                  * that will be adjusted to 0) and we need the space -- we
13759                  * need to return failure.  We return ENOSPC to differentiate
13760                  * it from failing to allocate a buffer due to failure to meet
13761                  * the reserve (for which we return E2BIG).
13762                  */
13763                 rval = ENOSPC;
13764                 goto out;
13765         }
13766 
13767         if ((rval = dtrace_state_buffers(state)) != 0)
13768                 goto err;
13769 
13770         if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
13771                 sz = dtrace_dstate_defsize;
13772 
13773         do {
13774                 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
13775 
13776                 if (rval == 0)
13777                         break;
13778 
13779                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13780                         goto err;
13781         } while (sz >>= 1);
13782 
13783         opt[DTRACEOPT_DYNVARSIZE] = sz;
13784 
13785         if (rval != 0)
13786                 goto err;
13787 
13788         if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
13789                 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
13790 
13791         if (opt[DTRACEOPT_CLEANRATE] == 0)
13792                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13793 
13794         if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
13795                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
13796 
13797         if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
13798                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13799 
13800         hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
13801         hdlr.cyh_arg = state;
13802         hdlr.cyh_level = CY_LOW_LEVEL;
13803 
13804         when.cyt_when = 0;
13805         when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
13806 
13807         state->dts_cleaner = cyclic_add(&hdlr, &when);
13808 
13809         hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
13810         hdlr.cyh_arg = state;
13811         hdlr.cyh_level = CY_LOW_LEVEL;
13812 
13813         when.cyt_when = 0;
13814         when.cyt_interval = dtrace_deadman_interval;
13815 
13816         state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
13817         state->dts_deadman = cyclic_add(&hdlr, &when);
13818 
13819         state->dts_activity = DTRACE_ACTIVITY_WARMUP;
13820 
13821         if (state->dts_getf != 0 &&
13822             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
13823                 /*
13824                  * We don't have kernel privs but we have at least one call
13825                  * to getf(); we need to bump our zone's count, and (if
13826                  * this is the first enabling to have an unprivileged call
13827                  * to getf()) we need to hook into closef().
13828                  */
13829                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++;
13830 
13831                 if (dtrace_getf++ == 0) {
13832                         ASSERT(dtrace_closef == NULL);
13833                         dtrace_closef = dtrace_getf_barrier;
13834                 }
13835         }
13836 
13837         /*
13838          * Now it's time to actually fire the BEGIN probe.  We need to disable
13839          * interrupts here both to record the CPU on which we fired the BEGIN
13840          * probe (the data from this CPU will be processed first at user
13841          * level) and to manually activate the buffer for this CPU.
13842          */
13843         cookie = dtrace_interrupt_disable();
13844         *cpu = CPU->cpu_id;
13845         ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
13846         state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
13847 
13848         dtrace_probe(dtrace_probeid_begin,
13849             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13850         dtrace_interrupt_enable(cookie);
13851         /*
13852          * We may have had an exit action from a BEGIN probe; only change our
13853          * state to ACTIVE if we're still in WARMUP.
13854          */
13855         ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
13856             state->dts_activity == DTRACE_ACTIVITY_DRAINING);
13857 
13858         if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
13859                 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
13860 
13861         /*
13862          * Regardless of whether or not now we're in ACTIVE or DRAINING, we
13863          * want each CPU to transition its principal buffer out of the
13864          * INACTIVE state.  Doing this assures that no CPU will suddenly begin
13865          * processing an ECB halfway down a probe's ECB chain; all CPUs will
13866          * atomically transition from processing none of a state's ECBs to
13867          * processing all of them.
13868          */
13869         dtrace_xcall(DTRACE_CPUALL,
13870             (dtrace_xcall_t)dtrace_buffer_activate, state);
13871         goto out;
13872 
13873 err:
13874         dtrace_buffer_free(state->dts_buffer);
13875         dtrace_buffer_free(state->dts_aggbuffer);
13876 
13877         if ((nspec = state->dts_nspeculations) == 0) {
13878                 ASSERT(state->dts_speculations == NULL);
13879                 goto out;
13880         }
13881 
13882         spec = state->dts_speculations;
13883         ASSERT(spec != NULL);
13884 
13885         for (i = 0; i < state->dts_nspeculations; i++) {
13886                 if ((buf = spec[i].dtsp_buffer) == NULL)
13887                         break;
13888 
13889                 dtrace_buffer_free(buf);
13890                 kmem_free(buf, bufsize);
13891         }
13892 
13893         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13894         state->dts_nspeculations = 0;
13895         state->dts_speculations = NULL;
13896 
13897 out:
13898         mutex_exit(&dtrace_lock);
13899         mutex_exit(&cpu_lock);
13900 
13901         return (rval);
13902 }
13903 
13904 static int
13905 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
13906 {
13907         dtrace_icookie_t cookie;
13908 
13909         ASSERT(MUTEX_HELD(&dtrace_lock));
13910 
13911         if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
13912             state->dts_activity != DTRACE_ACTIVITY_DRAINING)
13913                 return (EINVAL);
13914 
13915         /*
13916          * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
13917          * to be sure that every CPU has seen it.  See below for the details
13918          * on why this is done.
13919          */
13920         state->dts_activity = DTRACE_ACTIVITY_DRAINING;
13921         dtrace_sync();
13922 
13923         /*
13924          * By this point, it is impossible for any CPU to be still processing
13925          * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
13926          * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
13927          * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
13928          * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
13929          * iff we're in the END probe.
13930          */
13931         state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
13932         dtrace_sync();
13933         ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
13934 
13935         /*
13936          * Finally, we can release the reserve and call the END probe.  We
13937          * disable interrupts across calling the END probe to allow us to
13938          * return the CPU on which we actually called the END probe.  This
13939          * allows user-land to be sure that this CPU's principal buffer is
13940          * processed last.
13941          */
13942         state->dts_reserve = 0;
13943 
13944         cookie = dtrace_interrupt_disable();
13945         *cpu = CPU->cpu_id;
13946         dtrace_probe(dtrace_probeid_end,
13947             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13948         dtrace_interrupt_enable(cookie);
13949 
13950         state->dts_activity = DTRACE_ACTIVITY_STOPPED;
13951         dtrace_sync();
13952 
13953         if (state->dts_getf != 0 &&
13954             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
13955                 /*
13956                  * We don't have kernel privs but we have at least one call
13957                  * to getf(); we need to lower our zone's count, and (if
13958                  * this is the last enabling to have an unprivileged call
13959                  * to getf()) we need to clear the closef() hook.
13960                  */
13961                 ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0);
13962                 ASSERT(dtrace_closef == dtrace_getf_barrier);
13963                 ASSERT(dtrace_getf > 0);
13964 
13965                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--;
13966 
13967                 if (--dtrace_getf == 0)
13968                         dtrace_closef = NULL;
13969         }
13970 
13971         return (0);
13972 }
13973 
13974 static int
13975 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
13976     dtrace_optval_t val)
13977 {
13978         ASSERT(MUTEX_HELD(&dtrace_lock));
13979 
13980         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
13981                 return (EBUSY);
13982 
13983         if (option >= DTRACEOPT_MAX)
13984                 return (EINVAL);
13985 
13986         if (option != DTRACEOPT_CPU && val < 0)
13987                 return (EINVAL);
13988 
13989         switch (option) {
13990         case DTRACEOPT_DESTRUCTIVE:
13991                 if (dtrace_destructive_disallow)
13992                         return (EACCES);
13993 
13994                 state->dts_cred.dcr_destructive = 1;
13995                 break;
13996 
13997         case DTRACEOPT_BUFSIZE:
13998         case DTRACEOPT_DYNVARSIZE:
13999         case DTRACEOPT_AGGSIZE:
14000         case DTRACEOPT_SPECSIZE:
14001         case DTRACEOPT_STRSIZE:
14002                 if (val < 0)
14003                         return (EINVAL);
14004 
14005                 if (val >= LONG_MAX) {
14006                         /*
14007                          * If this is an otherwise negative value, set it to
14008                          * the highest multiple of 128m less than LONG_MAX.
14009                          * Technically, we're adjusting the size without
14010                          * regard to the buffer resizing policy, but in fact,
14011                          * this has no effect -- if we set the buffer size to
14012                          * ~LONG_MAX and the buffer policy is ultimately set to
14013                          * be "manual", the buffer allocation is guaranteed to
14014                          * fail, if only because the allocation requires two
14015                          * buffers.  (We set the the size to the highest
14016                          * multiple of 128m because it ensures that the size
14017                          * will remain a multiple of a megabyte when
14018                          * repeatedly halved -- all the way down to 15m.)
14019                          */
14020                         val = LONG_MAX - (1 << 27) + 1;
14021                 }
14022         }
14023 
14024         state->dts_options[option] = val;
14025 
14026         return (0);
14027 }
14028 
14029 static void
14030 dtrace_state_destroy(dtrace_state_t *state)
14031 {
14032         dtrace_ecb_t *ecb;
14033         dtrace_vstate_t *vstate = &state->dts_vstate;
14034         minor_t minor = getminor(state->dts_dev);
14035         int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
14036         dtrace_speculation_t *spec = state->dts_speculations;
14037         int nspec = state->dts_nspeculations;
14038         uint32_t match;
14039 
14040         ASSERT(MUTEX_HELD(&dtrace_lock));
14041         ASSERT(MUTEX_HELD(&cpu_lock));
14042 
14043         /*
14044          * First, retract any retained enablings for this state.
14045          */
14046         dtrace_enabling_retract(state);
14047         ASSERT(state->dts_nretained == 0);
14048 
14049         if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
14050             state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
14051                 /*
14052                  * We have managed to come into dtrace_state_destroy() on a
14053                  * hot enabling -- almost certainly because of a disorderly
14054                  * shutdown of a consumer.  (That is, a consumer that is
14055                  * exiting without having called dtrace_stop().) In this case,
14056                  * we're going to set our activity to be KILLED, and then
14057                  * issue a sync to be sure that everyone is out of probe
14058                  * context before we start blowing away ECBs.
14059                  */
14060                 state->dts_activity = DTRACE_ACTIVITY_KILLED;
14061                 dtrace_sync();
14062         }
14063 
14064         /*
14065          * Release the credential hold we took in dtrace_state_create().
14066          */
14067         if (state->dts_cred.dcr_cred != NULL)
14068                 crfree(state->dts_cred.dcr_cred);
14069 
14070         /*
14071          * Now we can safely disable and destroy any enabled probes.  Because
14072          * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
14073          * (especially if they're all enabled), we take two passes through the
14074          * ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes, and
14075          * in the second we disable whatever is left over.
14076          */
14077         for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
14078                 for (i = 0; i < state->dts_necbs; i++) {
14079                         if ((ecb = state->dts_ecbs[i]) == NULL)
14080                                 continue;
14081 
14082                         if (match && ecb->dte_probe != NULL) {
14083                                 dtrace_probe_t *probe = ecb->dte_probe;
14084                                 dtrace_provider_t *prov = probe->dtpr_provider;
14085 
14086                                 if (!(prov->dtpv_priv.dtpp_flags & match))
14087                                         continue;
14088                         }
14089 
14090                         dtrace_ecb_disable(ecb);
14091                         dtrace_ecb_destroy(ecb);
14092                 }
14093 
14094                 if (!match)
14095                         break;
14096         }
14097 
14098         /*
14099          * Before we free the buffers, perform one more sync to assure that
14100          * every CPU is out of probe context.
14101          */
14102         dtrace_sync();
14103 
14104         dtrace_buffer_free(state->dts_buffer);
14105         dtrace_buffer_free(state->dts_aggbuffer);
14106 
14107         for (i = 0; i < nspec; i++)
14108                 dtrace_buffer_free(spec[i].dtsp_buffer);
14109 
14110         if (state->dts_cleaner != CYCLIC_NONE)
14111                 cyclic_remove(state->dts_cleaner);
14112 
14113         if (state->dts_deadman != CYCLIC_NONE)
14114                 cyclic_remove(state->dts_deadman);
14115 
14116         dtrace_dstate_fini(&vstate->dtvs_dynvars);
14117         dtrace_vstate_fini(vstate);
14118         kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
14119 
14120         if (state->dts_aggregations != NULL) {
14121 #ifdef DEBUG
14122                 for (i = 0; i < state->dts_naggregations; i++)
14123                         ASSERT(state->dts_aggregations[i] == NULL);
14124 #endif
14125                 ASSERT(state->dts_naggregations > 0);
14126                 kmem_free(state->dts_aggregations,
14127                     state->dts_naggregations * sizeof (dtrace_aggregation_t *));
14128         }
14129 
14130         kmem_free(state->dts_buffer, bufsize);
14131         kmem_free(state->dts_aggbuffer, bufsize);
14132 
14133         for (i = 0; i < nspec; i++)
14134                 kmem_free(spec[i].dtsp_buffer, bufsize);
14135 
14136         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
14137 
14138         dtrace_format_destroy(state);
14139 
14140         vmem_destroy(state->dts_aggid_arena);
14141         ddi_soft_state_free(dtrace_softstate, minor);
14142         vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
14143 }
14144 
14145 /*
14146  * DTrace Anonymous Enabling Functions
14147  */
14148 static dtrace_state_t *
14149 dtrace_anon_grab(void)
14150 {
14151         dtrace_state_t *state;
14152 
14153         ASSERT(MUTEX_HELD(&dtrace_lock));
14154 
14155         if ((state = dtrace_anon.dta_state) == NULL) {
14156                 ASSERT(dtrace_anon.dta_enabling == NULL);
14157                 return (NULL);
14158         }
14159 
14160         ASSERT(dtrace_anon.dta_enabling != NULL);
14161         ASSERT(dtrace_retained != NULL);
14162 
14163         dtrace_enabling_destroy(dtrace_anon.dta_enabling);
14164         dtrace_anon.dta_enabling = NULL;
14165         dtrace_anon.dta_state = NULL;
14166 
14167         return (state);
14168 }
14169 
14170 static void
14171 dtrace_anon_property(void)
14172 {
14173         int i, rv;
14174         dtrace_state_t *state;
14175         dof_hdr_t *dof;
14176         char c[32];             /* enough for "dof-data-" + digits */
14177 
14178         ASSERT(MUTEX_HELD(&dtrace_lock));
14179         ASSERT(MUTEX_HELD(&cpu_lock));
14180 
14181         for (i = 0; ; i++) {
14182                 (void) snprintf(c, sizeof (c), "dof-data-%d", i);
14183 
14184                 dtrace_err_verbose = 1;
14185 
14186                 if ((dof = dtrace_dof_property(c)) == NULL) {
14187                         dtrace_err_verbose = 0;
14188                         break;
14189                 }
14190 
14191                 /*
14192                  * We want to create anonymous state, so we need to transition
14193                  * the kernel debugger to indicate that DTrace is active.  If
14194                  * this fails (e.g. because the debugger has modified text in
14195                  * some way), we won't continue with the processing.
14196                  */
14197                 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
14198                         cmn_err(CE_NOTE, "kernel debugger active; anonymous "
14199                             "enabling ignored.");
14200                         dtrace_dof_destroy(dof);
14201                         break;
14202                 }
14203 
14204                 /*
14205                  * If we haven't allocated an anonymous state, we'll do so now.
14206                  */
14207                 if ((state = dtrace_anon.dta_state) == NULL) {
14208                         state = dtrace_state_create(NULL, NULL);
14209                         dtrace_anon.dta_state = state;
14210 
14211                         if (state == NULL) {
14212                                 /*
14213                                  * This basically shouldn't happen:  the only
14214                                  * failure mode from dtrace_state_create() is a
14215                                  * failure of ddi_soft_state_zalloc() that
14216                                  * itself should never happen.  Still, the
14217                                  * interface allows for a failure mode, and
14218                                  * we want to fail as gracefully as possible:
14219                                  * we'll emit an error message and cease
14220                                  * processing anonymous state in this case.
14221                                  */
14222                                 cmn_err(CE_WARN, "failed to create "
14223                                     "anonymous state");
14224                                 dtrace_dof_destroy(dof);
14225                                 break;
14226                         }
14227                 }
14228 
14229                 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
14230                     &dtrace_anon.dta_enabling, 0, B_TRUE);
14231 
14232                 if (rv == 0)
14233                         rv = dtrace_dof_options(dof, state);
14234 
14235                 dtrace_err_verbose = 0;
14236                 dtrace_dof_destroy(dof);
14237 
14238                 if (rv != 0) {
14239                         /*
14240                          * This is malformed DOF; chuck any anonymous state
14241                          * that we created.
14242                          */
14243                         ASSERT(dtrace_anon.dta_enabling == NULL);
14244                         dtrace_state_destroy(state);
14245                         dtrace_anon.dta_state = NULL;
14246                         break;
14247                 }
14248 
14249                 ASSERT(dtrace_anon.dta_enabling != NULL);
14250         }
14251 
14252         if (dtrace_anon.dta_enabling != NULL) {
14253                 int rval;
14254 
14255                 /*
14256                  * dtrace_enabling_retain() can only fail because we are
14257                  * trying to retain more enablings than are allowed -- but
14258                  * we only have one anonymous enabling, and we are guaranteed
14259                  * to be allowed at least one retained enabling; we assert
14260                  * that dtrace_enabling_retain() returns success.
14261                  */
14262                 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
14263                 ASSERT(rval == 0);
14264 
14265                 dtrace_enabling_dump(dtrace_anon.dta_enabling);
14266         }
14267 }
14268 
14269 /*
14270  * DTrace Helper Functions
14271  */
14272 static void
14273 dtrace_helper_trace(dtrace_helper_action_t *helper,
14274     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
14275 {
14276         uint32_t size, next, nnext, i;
14277         dtrace_helptrace_t *ent;
14278         uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14279 
14280         if (!dtrace_helptrace_enabled)
14281                 return;
14282 
14283         ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
14284 
14285         /*
14286          * What would a tracing framework be without its own tracing
14287          * framework?  (Well, a hell of a lot simpler, for starters...)
14288          */
14289         size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
14290             sizeof (uint64_t) - sizeof (uint64_t);
14291 
14292         /*
14293          * Iterate until we can allocate a slot in the trace buffer.
14294          */
14295         do {
14296                 next = dtrace_helptrace_next;
14297 
14298                 if (next + size < dtrace_helptrace_bufsize) {
14299                         nnext = next + size;
14300                 } else {
14301                         nnext = size;
14302                 }
14303         } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
14304 
14305         /*
14306          * We have our slot; fill it in.
14307          */
14308         if (nnext == size)
14309                 next = 0;
14310 
14311         ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
14312         ent->dtht_helper = helper;
14313         ent->dtht_where = where;
14314         ent->dtht_nlocals = vstate->dtvs_nlocals;
14315 
14316         ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
14317             mstate->dtms_fltoffs : -1;
14318         ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
14319         ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
14320 
14321         for (i = 0; i < vstate->dtvs_nlocals; i++) {
14322                 dtrace_statvar_t *svar;
14323 
14324                 if ((svar = vstate->dtvs_locals[i]) == NULL)
14325                         continue;
14326 
14327                 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
14328                 ent->dtht_locals[i] =
14329                     ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
14330         }
14331 }
14332 
14333 static uint64_t
14334 dtrace_helper(int which, dtrace_mstate_t *mstate,
14335     dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
14336 {
14337         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14338         uint64_t sarg0 = mstate->dtms_arg[0];
14339         uint64_t sarg1 = mstate->dtms_arg[1];
14340         uint64_t rval;
14341         dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
14342         dtrace_helper_action_t *helper;
14343         dtrace_vstate_t *vstate;
14344         dtrace_difo_t *pred;
14345         int i, trace = dtrace_helptrace_enabled;
14346 
14347         ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
14348 
14349         if (helpers == NULL)
14350                 return (0);
14351 
14352         if ((helper = helpers->dthps_actions[which]) == NULL)
14353                 return (0);
14354 
14355         vstate = &helpers->dthps_vstate;
14356         mstate->dtms_arg[0] = arg0;
14357         mstate->dtms_arg[1] = arg1;
14358 
14359         /*
14360          * Now iterate over each helper.  If its predicate evaluates to 'true',
14361          * we'll call the corresponding actions.  Note that the below calls
14362          * to dtrace_dif_emulate() may set faults in machine state.  This is
14363          * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
14364          * the stored DIF offset with its own (which is the desired behavior).
14365          * Also, note the calls to dtrace_dif_emulate() may allocate scratch
14366          * from machine state; this is okay, too.
14367          */
14368         for (; helper != NULL; helper = helper->dtha_next) {
14369                 if ((pred = helper->dtha_predicate) != NULL) {
14370                         if (trace)
14371                                 dtrace_helper_trace(helper, mstate, vstate, 0);
14372 
14373                         if (!dtrace_dif_emulate(pred, mstate, vstate, state))
14374                                 goto next;
14375 
14376                         if (*flags & CPU_DTRACE_FAULT)
14377                                 goto err;
14378                 }
14379 
14380                 for (i = 0; i < helper->dtha_nactions; i++) {
14381                         if (trace)
14382                                 dtrace_helper_trace(helper,
14383                                     mstate, vstate, i + 1);
14384 
14385                         rval = dtrace_dif_emulate(helper->dtha_actions[i],
14386                             mstate, vstate, state);
14387 
14388                         if (*flags & CPU_DTRACE_FAULT)
14389                                 goto err;
14390                 }
14391 
14392 next:
14393                 if (trace)
14394                         dtrace_helper_trace(helper, mstate, vstate,
14395                             DTRACE_HELPTRACE_NEXT);
14396         }
14397 
14398         if (trace)
14399                 dtrace_helper_trace(helper, mstate, vstate,
14400                     DTRACE_HELPTRACE_DONE);
14401 
14402         /*
14403          * Restore the arg0 that we saved upon entry.
14404          */
14405         mstate->dtms_arg[0] = sarg0;
14406         mstate->dtms_arg[1] = sarg1;
14407 
14408         return (rval);
14409 
14410 err:
14411         if (trace)
14412                 dtrace_helper_trace(helper, mstate, vstate,
14413                     DTRACE_HELPTRACE_ERR);
14414 
14415         /*
14416          * Restore the arg0 that we saved upon entry.
14417          */
14418         mstate->dtms_arg[0] = sarg0;
14419         mstate->dtms_arg[1] = sarg1;
14420 
14421         return (NULL);
14422 }
14423 
14424 static void
14425 dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
14426     dtrace_vstate_t *vstate)
14427 {
14428         int i;
14429 
14430         if (helper->dtha_predicate != NULL)
14431                 dtrace_difo_release(helper->dtha_predicate, vstate);
14432 
14433         for (i = 0; i < helper->dtha_nactions; i++) {
14434                 ASSERT(helper->dtha_actions[i] != NULL);
14435                 dtrace_difo_release(helper->dtha_actions[i], vstate);
14436         }
14437 
14438         kmem_free(helper->dtha_actions,
14439             helper->dtha_nactions * sizeof (dtrace_difo_t *));
14440         kmem_free(helper, sizeof (dtrace_helper_action_t));
14441 }
14442 
14443 static int
14444 dtrace_helper_destroygen(int gen)
14445 {
14446         proc_t *p = curproc;
14447         dtrace_helpers_t *help = p->p_dtrace_helpers;
14448         dtrace_vstate_t *vstate;
14449         int i;
14450 
14451         ASSERT(MUTEX_HELD(&dtrace_lock));
14452 
14453         if (help == NULL || gen > help->dthps_generation)
14454                 return (EINVAL);
14455 
14456         vstate = &help->dthps_vstate;
14457 
14458         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14459                 dtrace_helper_action_t *last = NULL, *h, *next;
14460 
14461                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
14462                         next = h->dtha_next;
14463 
14464                         if (h->dtha_generation == gen) {
14465                                 if (last != NULL) {
14466                                         last->dtha_next = next;
14467                                 } else {
14468                                         help->dthps_actions[i] = next;
14469                                 }
14470 
14471                                 dtrace_helper_action_destroy(h, vstate);
14472                         } else {
14473                                 last = h;
14474                         }
14475                 }
14476         }
14477 
14478         /*
14479          * Interate until we've cleared out all helper providers with the
14480          * given generation number.
14481          */
14482         for (;;) {
14483                 dtrace_helper_provider_t *prov;
14484 
14485                 /*
14486                  * Look for a helper provider with the right generation. We
14487                  * have to start back at the beginning of the list each time
14488                  * because we drop dtrace_lock. It's unlikely that we'll make
14489                  * more than two passes.
14490                  */
14491                 for (i = 0; i < help->dthps_nprovs; i++) {
14492                         prov = help->dthps_provs[i];
14493 
14494                         if (prov->dthp_generation == gen)
14495                                 break;
14496                 }
14497 
14498                 /*
14499                  * If there were no matches, we're done.
14500                  */
14501                 if (i == help->dthps_nprovs)
14502                         break;
14503 
14504                 /*
14505                  * Move the last helper provider into this slot.
14506                  */
14507                 help->dthps_nprovs--;
14508                 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
14509                 help->dthps_provs[help->dthps_nprovs] = NULL;
14510 
14511                 mutex_exit(&dtrace_lock);
14512 
14513                 /*
14514                  * If we have a meta provider, remove this helper provider.
14515                  */
14516                 mutex_enter(&dtrace_meta_lock);
14517                 if (dtrace_meta_pid != NULL) {
14518                         ASSERT(dtrace_deferred_pid == NULL);
14519                         dtrace_helper_provider_remove(&prov->dthp_prov,
14520                             p->p_pid);
14521                 }
14522                 mutex_exit(&dtrace_meta_lock);
14523 
14524                 dtrace_helper_provider_destroy(prov);
14525 
14526                 mutex_enter(&dtrace_lock);
14527         }
14528 
14529         return (0);
14530 }
14531 
14532 static int
14533 dtrace_helper_validate(dtrace_helper_action_t *helper)
14534 {
14535         int err = 0, i;
14536         dtrace_difo_t *dp;
14537 
14538         if ((dp = helper->dtha_predicate) != NULL)
14539                 err += dtrace_difo_validate_helper(dp);
14540 
14541         for (i = 0; i < helper->dtha_nactions; i++)
14542                 err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
14543 
14544         return (err == 0);
14545 }
14546 
14547 static int
14548 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
14549 {
14550         dtrace_helpers_t *help;
14551         dtrace_helper_action_t *helper, *last;
14552         dtrace_actdesc_t *act;
14553         dtrace_vstate_t *vstate;
14554         dtrace_predicate_t *pred;
14555         int count = 0, nactions = 0, i;
14556 
14557         if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
14558                 return (EINVAL);
14559 
14560         help = curproc->p_dtrace_helpers;
14561         last = help->dthps_actions[which];
14562         vstate = &help->dthps_vstate;
14563 
14564         for (count = 0; last != NULL; last = last->dtha_next) {
14565                 count++;
14566                 if (last->dtha_next == NULL)
14567                         break;
14568         }
14569 
14570         /*
14571          * If we already have dtrace_helper_actions_max helper actions for this
14572          * helper action type, we'll refuse to add a new one.
14573          */
14574         if (count >= dtrace_helper_actions_max)
14575                 return (ENOSPC);
14576 
14577         helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
14578         helper->dtha_generation = help->dthps_generation;
14579 
14580         if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
14581                 ASSERT(pred->dtp_difo != NULL);
14582                 dtrace_difo_hold(pred->dtp_difo);
14583                 helper->dtha_predicate = pred->dtp_difo;
14584         }
14585 
14586         for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
14587                 if (act->dtad_kind != DTRACEACT_DIFEXPR)
14588                         goto err;
14589 
14590                 if (act->dtad_difo == NULL)
14591                         goto err;
14592 
14593                 nactions++;
14594         }
14595 
14596         helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
14597             (helper->dtha_nactions = nactions), KM_SLEEP);
14598 
14599         for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
14600                 dtrace_difo_hold(act->dtad_difo);
14601                 helper->dtha_actions[i++] = act->dtad_difo;
14602         }
14603 
14604         if (!dtrace_helper_validate(helper))
14605                 goto err;
14606 
14607         if (last == NULL) {
14608                 help->dthps_actions[which] = helper;
14609         } else {
14610                 last->dtha_next = helper;
14611         }
14612 
14613         if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
14614                 dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
14615                 dtrace_helptrace_next = 0;
14616         }
14617 
14618         return (0);
14619 err:
14620         dtrace_helper_action_destroy(helper, vstate);
14621         return (EINVAL);
14622 }
14623 
14624 static void
14625 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
14626     dof_helper_t *dofhp)
14627 {
14628         ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
14629 
14630         mutex_enter(&dtrace_meta_lock);
14631         mutex_enter(&dtrace_lock);
14632 
14633         if (!dtrace_attached() || dtrace_meta_pid == NULL) {
14634                 /*
14635                  * If the dtrace module is loaded but not attached, or if
14636                  * there aren't isn't a meta provider registered to deal with
14637                  * these provider descriptions, we need to postpone creating
14638                  * the actual providers until later.
14639                  */
14640 
14641                 if (help->dthps_next == NULL && help->dthps_prev == NULL &&
14642                     dtrace_deferred_pid != help) {
14643                         help->dthps_deferred = 1;
14644                         help->dthps_pid = p->p_pid;
14645                         help->dthps_next = dtrace_deferred_pid;
14646                         help->dthps_prev = NULL;
14647                         if (dtrace_deferred_pid != NULL)
14648                                 dtrace_deferred_pid->dthps_prev = help;
14649                         dtrace_deferred_pid = help;
14650                 }
14651 
14652                 mutex_exit(&dtrace_lock);
14653 
14654         } else if (dofhp != NULL) {
14655                 /*
14656                  * If the dtrace module is loaded and we have a particular
14657                  * helper provider description, pass that off to the
14658                  * meta provider.
14659                  */
14660 
14661                 mutex_exit(&dtrace_lock);
14662 
14663                 dtrace_helper_provide(dofhp, p->p_pid);
14664 
14665         } else {
14666                 /*
14667                  * Otherwise, just pass all the helper provider descriptions
14668                  * off to the meta provider.
14669                  */
14670 
14671                 int i;
14672                 mutex_exit(&dtrace_lock);
14673 
14674                 for (i = 0; i < help->dthps_nprovs; i++) {
14675                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
14676                             p->p_pid);
14677                 }
14678         }
14679 
14680         mutex_exit(&dtrace_meta_lock);
14681 }
14682 
14683 static int
14684 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
14685 {
14686         dtrace_helpers_t *help;
14687         dtrace_helper_provider_t *hprov, **tmp_provs;
14688         uint_t tmp_maxprovs, i;
14689 
14690         ASSERT(MUTEX_HELD(&dtrace_lock));
14691 
14692         help = curproc->p_dtrace_helpers;
14693         ASSERT(help != NULL);
14694 
14695         /*
14696          * If we already have dtrace_helper_providers_max helper providers,
14697          * we're refuse to add a new one.
14698          */
14699         if (help->dthps_nprovs >= dtrace_helper_providers_max)
14700                 return (ENOSPC);
14701 
14702         /*
14703          * Check to make sure this isn't a duplicate.
14704          */
14705         for (i = 0; i < help->dthps_nprovs; i++) {
14706                 if (dofhp->dofhp_dof ==
14707                     help->dthps_provs[i]->dthp_prov.dofhp_dof)
14708                         return (EALREADY);
14709         }
14710 
14711         hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
14712         hprov->dthp_prov = *dofhp;
14713         hprov->dthp_ref = 1;
14714         hprov->dthp_generation = gen;
14715 
14716         /*
14717          * Allocate a bigger table for helper providers if it's already full.
14718          */
14719         if (help->dthps_maxprovs == help->dthps_nprovs) {
14720                 tmp_maxprovs = help->dthps_maxprovs;
14721                 tmp_provs = help->dthps_provs;
14722 
14723                 if (help->dthps_maxprovs == 0)
14724                         help->dthps_maxprovs = 2;
14725                 else
14726                         help->dthps_maxprovs *= 2;
14727                 if (help->dthps_maxprovs > dtrace_helper_providers_max)
14728                         help->dthps_maxprovs = dtrace_helper_providers_max;
14729 
14730                 ASSERT(tmp_maxprovs < help->dthps_maxprovs);
14731 
14732                 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
14733                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
14734 
14735                 if (tmp_provs != NULL) {
14736                         bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
14737                             sizeof (dtrace_helper_provider_t *));
14738                         kmem_free(tmp_provs, tmp_maxprovs *
14739                             sizeof (dtrace_helper_provider_t *));
14740                 }
14741         }
14742 
14743         help->dthps_provs[help->dthps_nprovs] = hprov;
14744         help->dthps_nprovs++;
14745 
14746         return (0);
14747 }
14748 
14749 static void
14750 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
14751 {
14752         mutex_enter(&dtrace_lock);
14753 
14754         if (--hprov->dthp_ref == 0) {
14755                 dof_hdr_t *dof;
14756                 mutex_exit(&dtrace_lock);
14757                 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
14758                 dtrace_dof_destroy(dof);
14759                 kmem_free(hprov, sizeof (dtrace_helper_provider_t));
14760         } else {
14761                 mutex_exit(&dtrace_lock);
14762         }
14763 }
14764 
14765 static int
14766 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
14767 {
14768         uintptr_t daddr = (uintptr_t)dof;
14769         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
14770         dof_provider_t *provider;
14771         dof_probe_t *probe;
14772         uint8_t *arg;
14773         char *strtab, *typestr;
14774         dof_stridx_t typeidx;
14775         size_t typesz;
14776         uint_t nprobes, j, k;
14777 
14778         ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
14779 
14780         if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
14781                 dtrace_dof_error(dof, "misaligned section offset");
14782                 return (-1);
14783         }
14784 
14785         /*
14786          * The section needs to be large enough to contain the DOF provider
14787          * structure appropriate for the given version.
14788          */
14789         if (sec->dofs_size <
14790             ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
14791             offsetof(dof_provider_t, dofpv_prenoffs) :
14792             sizeof (dof_provider_t))) {
14793                 dtrace_dof_error(dof, "provider section too small");
14794                 return (-1);
14795         }
14796 
14797         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
14798         str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
14799         prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
14800         arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
14801         off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
14802 
14803         if (str_sec == NULL || prb_sec == NULL ||
14804             arg_sec == NULL || off_sec == NULL)
14805                 return (-1);
14806 
14807         enoff_sec = NULL;
14808 
14809         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
14810             provider->dofpv_prenoffs != DOF_SECT_NONE &&
14811             (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
14812             provider->dofpv_prenoffs)) == NULL)
14813                 return (-1);
14814 
14815         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
14816 
14817         if (provider->dofpv_name >= str_sec->dofs_size ||
14818             strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
14819                 dtrace_dof_error(dof, "invalid provider name");
14820                 return (-1);
14821         }
14822 
14823         if (prb_sec->dofs_entsize == 0 ||
14824             prb_sec->dofs_entsize > prb_sec->dofs_size) {
14825                 dtrace_dof_error(dof, "invalid entry size");
14826                 return (-1);
14827         }
14828 
14829         if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
14830                 dtrace_dof_error(dof, "misaligned entry size");
14831                 return (-1);
14832         }
14833 
14834         if (off_sec->dofs_entsize != sizeof (uint32_t)) {
14835                 dtrace_dof_error(dof, "invalid entry size");
14836                 return (-1);
14837         }
14838 
14839         if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
14840                 dtrace_dof_error(dof, "misaligned section offset");
14841                 return (-1);
14842         }
14843 
14844         if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
14845                 dtrace_dof_error(dof, "invalid entry size");
14846                 return (-1);
14847         }
14848 
14849         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
14850 
14851         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
14852 
14853         /*
14854          * Take a pass through the probes to check for errors.
14855          */
14856         for (j = 0; j < nprobes; j++) {
14857                 probe = (dof_probe_t *)(uintptr_t)(daddr +
14858                     prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
14859 
14860                 if (probe->dofpr_func >= str_sec->dofs_size) {
14861                         dtrace_dof_error(dof, "invalid function name");
14862                         return (-1);
14863                 }
14864 
14865                 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
14866                         dtrace_dof_error(dof, "function name too long");
14867                         return (-1);
14868                 }
14869 
14870                 if (probe->dofpr_name >= str_sec->dofs_size ||
14871                     strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
14872                         dtrace_dof_error(dof, "invalid probe name");
14873                         return (-1);
14874                 }
14875 
14876                 /*
14877                  * The offset count must not wrap the index, and the offsets
14878                  * must also not overflow the section's data.
14879                  */
14880                 if (probe->dofpr_offidx + probe->dofpr_noffs <
14881                     probe->dofpr_offidx ||
14882                     (probe->dofpr_offidx + probe->dofpr_noffs) *
14883                     off_sec->dofs_entsize > off_sec->dofs_size) {
14884                         dtrace_dof_error(dof, "invalid probe offset");
14885                         return (-1);
14886                 }
14887 
14888                 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
14889                         /*
14890                          * If there's no is-enabled offset section, make sure
14891                          * there aren't any is-enabled offsets. Otherwise
14892                          * perform the same checks as for probe offsets
14893                          * (immediately above).
14894                          */
14895                         if (enoff_sec == NULL) {
14896                                 if (probe->dofpr_enoffidx != 0 ||
14897                                     probe->dofpr_nenoffs != 0) {
14898                                         dtrace_dof_error(dof, "is-enabled "
14899                                             "offsets with null section");
14900                                         return (-1);
14901                                 }
14902                         } else if (probe->dofpr_enoffidx +
14903                             probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
14904                             (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
14905                             enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
14906                                 dtrace_dof_error(dof, "invalid is-enabled "
14907                                     "offset");
14908                                 return (-1);
14909                         }
14910 
14911                         if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
14912                                 dtrace_dof_error(dof, "zero probe and "
14913                                     "is-enabled offsets");
14914                                 return (-1);
14915                         }
14916                 } else if (probe->dofpr_noffs == 0) {
14917                         dtrace_dof_error(dof, "zero probe offsets");
14918                         return (-1);
14919                 }
14920 
14921                 if (probe->dofpr_argidx + probe->dofpr_xargc <
14922                     probe->dofpr_argidx ||
14923                     (probe->dofpr_argidx + probe->dofpr_xargc) *
14924                     arg_sec->dofs_entsize > arg_sec->dofs_size) {
14925                         dtrace_dof_error(dof, "invalid args");
14926                         return (-1);
14927                 }
14928 
14929                 typeidx = probe->dofpr_nargv;
14930                 typestr = strtab + probe->dofpr_nargv;
14931                 for (k = 0; k < probe->dofpr_nargc; k++) {
14932                         if (typeidx >= str_sec->dofs_size) {
14933                                 dtrace_dof_error(dof, "bad "
14934                                     "native argument type");
14935                                 return (-1);
14936                         }
14937 
14938                         typesz = strlen(typestr) + 1;
14939                         if (typesz > DTRACE_ARGTYPELEN) {
14940                                 dtrace_dof_error(dof, "native "
14941                                     "argument type too long");
14942                                 return (-1);
14943                         }
14944                         typeidx += typesz;
14945                         typestr += typesz;
14946                 }
14947 
14948                 typeidx = probe->dofpr_xargv;
14949                 typestr = strtab + probe->dofpr_xargv;
14950                 for (k = 0; k < probe->dofpr_xargc; k++) {
14951                         if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
14952                                 dtrace_dof_error(dof, "bad "
14953                                     "native argument index");
14954                                 return (-1);
14955                         }
14956 
14957                         if (typeidx >= str_sec->dofs_size) {
14958                                 dtrace_dof_error(dof, "bad "
14959                                     "translated argument type");
14960                                 return (-1);
14961                         }
14962 
14963                         typesz = strlen(typestr) + 1;
14964                         if (typesz > DTRACE_ARGTYPELEN) {
14965                                 dtrace_dof_error(dof, "translated argument "
14966                                     "type too long");
14967                                 return (-1);
14968                         }
14969 
14970                         typeidx += typesz;
14971                         typestr += typesz;
14972                 }
14973         }
14974 
14975         return (0);
14976 }
14977 
14978 static int
14979 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
14980 {
14981         dtrace_helpers_t *help;
14982         dtrace_vstate_t *vstate;
14983         dtrace_enabling_t *enab = NULL;
14984         int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
14985         uintptr_t daddr = (uintptr_t)dof;
14986 
14987         ASSERT(MUTEX_HELD(&dtrace_lock));
14988 
14989         if ((help = curproc->p_dtrace_helpers) == NULL)
14990                 help = dtrace_helpers_create(curproc);
14991 
14992         vstate = &help->dthps_vstate;
14993 
14994         if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
14995             dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
14996                 dtrace_dof_destroy(dof);
14997                 return (rv);
14998         }
14999 
15000         /*
15001          * Look for helper providers and validate their descriptions.
15002          */
15003         if (dhp != NULL) {
15004                 for (i = 0; i < dof->dofh_secnum; i++) {
15005                         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
15006                             dof->dofh_secoff + i * dof->dofh_secsize);
15007 
15008                         if (sec->dofs_type != DOF_SECT_PROVIDER)
15009                                 continue;
15010 
15011                         if (dtrace_helper_provider_validate(dof, sec) != 0) {
15012                                 dtrace_enabling_destroy(enab);
15013                                 dtrace_dof_destroy(dof);
15014                                 return (-1);
15015                         }
15016 
15017                         nprovs++;
15018                 }
15019         }
15020 
15021         /*
15022          * Now we need to walk through the ECB descriptions in the enabling.
15023          */
15024         for (i = 0; i < enab->dten_ndesc; i++) {
15025                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
15026                 dtrace_probedesc_t *desc = &ep->dted_probe;
15027 
15028                 if (strcmp(desc->dtpd_provider, "dtrace") != 0)
15029                         continue;
15030 
15031                 if (strcmp(desc->dtpd_mod, "helper") != 0)
15032                         continue;
15033 
15034                 if (strcmp(desc->dtpd_func, "ustack") != 0)
15035                         continue;
15036 
15037                 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
15038                     ep)) != 0) {
15039                         /*
15040                          * Adding this helper action failed -- we are now going
15041                          * to rip out the entire generation and return failure.
15042                          */
15043                         (void) dtrace_helper_destroygen(help->dthps_generation);
15044                         dtrace_enabling_destroy(enab);
15045                         dtrace_dof_destroy(dof);
15046                         return (-1);
15047                 }
15048 
15049                 nhelpers++;
15050         }
15051 
15052         if (nhelpers < enab->dten_ndesc)
15053                 dtrace_dof_error(dof, "unmatched helpers");
15054 
15055         gen = help->dthps_generation++;
15056         dtrace_enabling_destroy(enab);
15057 
15058         if (dhp != NULL && nprovs > 0) {
15059                 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
15060                 if (dtrace_helper_provider_add(dhp, gen) == 0) {
15061                         mutex_exit(&dtrace_lock);
15062                         dtrace_helper_provider_register(curproc, help, dhp);
15063                         mutex_enter(&dtrace_lock);
15064 
15065                         destroy = 0;
15066                 }
15067         }
15068 
15069         if (destroy)
15070                 dtrace_dof_destroy(dof);
15071 
15072         return (gen);
15073 }
15074 
15075 static dtrace_helpers_t *
15076 dtrace_helpers_create(proc_t *p)
15077 {
15078         dtrace_helpers_t *help;
15079 
15080         ASSERT(MUTEX_HELD(&dtrace_lock));
15081         ASSERT(p->p_dtrace_helpers == NULL);
15082 
15083         help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
15084         help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
15085             DTRACE_NHELPER_ACTIONS, KM_SLEEP);
15086 
15087         p->p_dtrace_helpers = help;
15088         dtrace_helpers++;
15089 
15090         return (help);
15091 }
15092 
15093 static void
15094 dtrace_helpers_destroy(void)
15095 {
15096         dtrace_helpers_t *help;
15097         dtrace_vstate_t *vstate;
15098         proc_t *p = curproc;
15099         int i;
15100 
15101         mutex_enter(&dtrace_lock);
15102 
15103         ASSERT(p->p_dtrace_helpers != NULL);
15104         ASSERT(dtrace_helpers > 0);
15105 
15106         help = p->p_dtrace_helpers;
15107         vstate = &help->dthps_vstate;
15108 
15109         /*
15110          * We're now going to lose the help from this process.
15111          */
15112         p->p_dtrace_helpers = NULL;
15113         dtrace_sync();
15114 
15115         /*
15116          * Destory the helper actions.
15117          */
15118         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15119                 dtrace_helper_action_t *h, *next;
15120 
15121                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
15122                         next = h->dtha_next;
15123                         dtrace_helper_action_destroy(h, vstate);
15124                         h = next;
15125                 }
15126         }
15127 
15128         mutex_exit(&dtrace_lock);
15129 
15130         /*
15131          * Destroy the helper providers.
15132          */
15133         if (help->dthps_maxprovs > 0) {
15134                 mutex_enter(&dtrace_meta_lock);
15135                 if (dtrace_meta_pid != NULL) {
15136                         ASSERT(dtrace_deferred_pid == NULL);
15137 
15138                         for (i = 0; i < help->dthps_nprovs; i++) {
15139                                 dtrace_helper_provider_remove(
15140                                     &help->dthps_provs[i]->dthp_prov, p->p_pid);
15141                         }
15142                 } else {
15143                         mutex_enter(&dtrace_lock);
15144                         ASSERT(help->dthps_deferred == 0 ||
15145                             help->dthps_next != NULL ||
15146                             help->dthps_prev != NULL ||
15147                             help == dtrace_deferred_pid);
15148 
15149                         /*
15150                          * Remove the helper from the deferred list.
15151                          */
15152                         if (help->dthps_next != NULL)
15153                                 help->dthps_next->dthps_prev = help->dthps_prev;
15154                         if (help->dthps_prev != NULL)
15155                                 help->dthps_prev->dthps_next = help->dthps_next;
15156                         if (dtrace_deferred_pid == help) {
15157                                 dtrace_deferred_pid = help->dthps_next;
15158                                 ASSERT(help->dthps_prev == NULL);
15159                         }
15160 
15161                         mutex_exit(&dtrace_lock);
15162                 }
15163 
15164                 mutex_exit(&dtrace_meta_lock);
15165 
15166                 for (i = 0; i < help->dthps_nprovs; i++) {
15167                         dtrace_helper_provider_destroy(help->dthps_provs[i]);
15168                 }
15169 
15170                 kmem_free(help->dthps_provs, help->dthps_maxprovs *
15171                     sizeof (dtrace_helper_provider_t *));
15172         }
15173 
15174         mutex_enter(&dtrace_lock);
15175 
15176         dtrace_vstate_fini(&help->dthps_vstate);
15177         kmem_free(help->dthps_actions,
15178             sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
15179         kmem_free(help, sizeof (dtrace_helpers_t));
15180 
15181         --dtrace_helpers;
15182         mutex_exit(&dtrace_lock);
15183 }
15184 
15185 static void
15186 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
15187 {
15188         dtrace_helpers_t *help, *newhelp;
15189         dtrace_helper_action_t *helper, *new, *last;
15190         dtrace_difo_t *dp;
15191         dtrace_vstate_t *vstate;
15192         int i, j, sz, hasprovs = 0;
15193 
15194         mutex_enter(&dtrace_lock);
15195         ASSERT(from->p_dtrace_helpers != NULL);
15196         ASSERT(dtrace_helpers > 0);
15197 
15198         help = from->p_dtrace_helpers;
15199         newhelp = dtrace_helpers_create(to);
15200         ASSERT(to->p_dtrace_helpers != NULL);
15201 
15202         newhelp->dthps_generation = help->dthps_generation;
15203         vstate = &newhelp->dthps_vstate;
15204 
15205         /*
15206          * Duplicate the helper actions.
15207          */
15208         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15209                 if ((helper = help->dthps_actions[i]) == NULL)
15210                         continue;
15211 
15212                 for (last = NULL; helper != NULL; helper = helper->dtha_next) {
15213                         new = kmem_zalloc(sizeof (dtrace_helper_action_t),
15214                             KM_SLEEP);
15215                         new->dtha_generation = helper->dtha_generation;
15216 
15217                         if ((dp = helper->dtha_predicate) != NULL) {
15218                                 dp = dtrace_difo_duplicate(dp, vstate);
15219                                 new->dtha_predicate = dp;
15220                         }
15221 
15222                         new->dtha_nactions = helper->dtha_nactions;
15223                         sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
15224                         new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
15225 
15226                         for (j = 0; j < new->dtha_nactions; j++) {
15227                                 dtrace_difo_t *dp = helper->dtha_actions[j];
15228 
15229                                 ASSERT(dp != NULL);
15230                                 dp = dtrace_difo_duplicate(dp, vstate);
15231                                 new->dtha_actions[j] = dp;
15232                         }
15233 
15234                         if (last != NULL) {
15235                                 last->dtha_next = new;
15236                         } else {
15237                                 newhelp->dthps_actions[i] = new;
15238                         }
15239 
15240                         last = new;
15241                 }
15242         }
15243 
15244         /*
15245          * Duplicate the helper providers and register them with the
15246          * DTrace framework.
15247          */
15248         if (help->dthps_nprovs > 0) {
15249                 newhelp->dthps_nprovs = help->dthps_nprovs;
15250                 newhelp->dthps_maxprovs = help->dthps_nprovs;
15251                 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
15252                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
15253                 for (i = 0; i < newhelp->dthps_nprovs; i++) {
15254                         newhelp->dthps_provs[i] = help->dthps_provs[i];
15255                         newhelp->dthps_provs[i]->dthp_ref++;
15256                 }
15257 
15258                 hasprovs = 1;
15259         }
15260 
15261         mutex_exit(&dtrace_lock);
15262 
15263         if (hasprovs)
15264                 dtrace_helper_provider_register(to, newhelp, NULL);
15265 }
15266 
15267 /*
15268  * DTrace Hook Functions
15269  */
15270 static void
15271 dtrace_module_loaded(struct modctl *ctl)
15272 {
15273         dtrace_provider_t *prv;
15274 
15275         mutex_enter(&dtrace_provider_lock);
15276         mutex_enter(&mod_lock);
15277 
15278         ASSERT(ctl->mod_busy);
15279 
15280         /*
15281          * We're going to call each providers per-module provide operation
15282          * specifying only this module.
15283          */
15284         for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
15285                 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
15286 
15287         mutex_exit(&mod_lock);
15288         mutex_exit(&dtrace_provider_lock);
15289 
15290         /*
15291          * If we have any retained enablings, we need to match against them.
15292          * Enabling probes requires that cpu_lock be held, and we cannot hold
15293          * cpu_lock here -- it is legal for cpu_lock to be held when loading a
15294          * module.  (In particular, this happens when loading scheduling
15295          * classes.)  So if we have any retained enablings, we need to dispatch
15296          * our task queue to do the match for us.
15297          */
15298         mutex_enter(&dtrace_lock);
15299 
15300         if (dtrace_retained == NULL) {
15301                 mutex_exit(&dtrace_lock);
15302                 return;
15303         }
15304 
15305         (void) taskq_dispatch(dtrace_taskq,
15306             (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
15307 
15308         mutex_exit(&dtrace_lock);
15309 
15310         /*
15311          * And now, for a little heuristic sleaze:  in general, we want to
15312          * match modules as soon as they load.  However, we cannot guarantee
15313          * this, because it would lead us to the lock ordering violation
15314          * outlined above.  The common case, of course, is that cpu_lock is
15315          * _not_ held -- so we delay here for a clock tick, hoping that that's
15316          * long enough for the task queue to do its work.  If it's not, it's
15317          * not a serious problem -- it just means that the module that we
15318          * just loaded may not be immediately instrumentable.
15319          */
15320         delay(1);
15321 }
15322 
15323 static void
15324 dtrace_module_unloaded(struct modctl *ctl)
15325 {
15326         dtrace_probe_t template, *probe, *first, *next;
15327         dtrace_provider_t *prov;
15328 
15329         template.dtpr_mod = ctl->mod_modname;
15330 
15331         mutex_enter(&dtrace_provider_lock);
15332         mutex_enter(&mod_lock);
15333         mutex_enter(&dtrace_lock);
15334 
15335         if (dtrace_bymod == NULL) {
15336                 /*
15337                  * The DTrace module is loaded (obviously) but not attached;
15338                  * we don't have any work to do.
15339                  */
15340                 mutex_exit(&dtrace_provider_lock);
15341                 mutex_exit(&mod_lock);
15342                 mutex_exit(&dtrace_lock);
15343                 return;
15344         }
15345 
15346         for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
15347             probe != NULL; probe = probe->dtpr_nextmod) {
15348                 if (probe->dtpr_ecb != NULL) {
15349                         mutex_exit(&dtrace_provider_lock);
15350                         mutex_exit(&mod_lock);
15351                         mutex_exit(&dtrace_lock);
15352 
15353                         /*
15354                          * This shouldn't _actually_ be possible -- we're
15355                          * unloading a module that has an enabled probe in it.
15356                          * (It's normally up to the provider to make sure that
15357                          * this can't happen.)  However, because dtps_enable()
15358                          * doesn't have a failure mode, there can be an
15359                          * enable/unload race.  Upshot:  we don't want to
15360                          * assert, but we're not going to disable the
15361                          * probe, either.
15362                          */
15363                         if (dtrace_err_verbose) {
15364                                 cmn_err(CE_WARN, "unloaded module '%s' had "
15365                                     "enabled probes", ctl->mod_modname);
15366                         }
15367 
15368                         return;
15369                 }
15370         }
15371 
15372         probe = first;
15373 
15374         for (first = NULL; probe != NULL; probe = next) {
15375                 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
15376 
15377                 dtrace_probes[probe->dtpr_id - 1] = NULL;
15378 
15379                 next = probe->dtpr_nextmod;
15380                 dtrace_hash_remove(dtrace_bymod, probe);
15381                 dtrace_hash_remove(dtrace_byfunc, probe);
15382                 dtrace_hash_remove(dtrace_byname, probe);
15383 
15384                 if (first == NULL) {
15385                         first = probe;
15386                         probe->dtpr_nextmod = NULL;
15387                 } else {
15388                         probe->dtpr_nextmod = first;
15389                         first = probe;
15390                 }
15391         }
15392 
15393         /*
15394          * We've removed all of the module's probes from the hash chains and
15395          * from the probe array.  Now issue a dtrace_sync() to be sure that
15396          * everyone has cleared out from any probe array processing.
15397          */
15398         dtrace_sync();
15399 
15400         for (probe = first; probe != NULL; probe = first) {
15401                 first = probe->dtpr_nextmod;
15402                 prov = probe->dtpr_provider;
15403                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
15404                     probe->dtpr_arg);
15405                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
15406                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
15407                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
15408                 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
15409                 kmem_free(probe, sizeof (dtrace_probe_t));
15410         }
15411 
15412         mutex_exit(&dtrace_lock);
15413         mutex_exit(&mod_lock);
15414         mutex_exit(&dtrace_provider_lock);
15415 }
15416 
15417 void
15418 dtrace_suspend(void)
15419 {
15420         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
15421 }
15422 
15423 void
15424 dtrace_resume(void)
15425 {
15426         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
15427 }
15428 
15429 static int
15430 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
15431 {
15432         ASSERT(MUTEX_HELD(&cpu_lock));
15433         mutex_enter(&dtrace_lock);
15434 
15435         switch (what) {
15436         case CPU_CONFIG: {
15437                 dtrace_state_t *state;
15438                 dtrace_optval_t *opt, rs, c;
15439 
15440                 /*
15441                  * For now, we only allocate a new buffer for anonymous state.
15442                  */
15443                 if ((state = dtrace_anon.dta_state) == NULL)
15444                         break;
15445 
15446                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
15447                         break;
15448 
15449                 opt = state->dts_options;
15450                 c = opt[DTRACEOPT_CPU];
15451 
15452                 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
15453                         break;
15454 
15455                 /*
15456                  * Regardless of what the actual policy is, we're going to
15457                  * temporarily set our resize policy to be manual.  We're
15458                  * also going to temporarily set our CPU option to denote
15459                  * the newly configured CPU.
15460                  */
15461                 rs = opt[DTRACEOPT_BUFRESIZE];
15462                 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
15463                 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
15464 
15465                 (void) dtrace_state_buffers(state);
15466 
15467                 opt[DTRACEOPT_BUFRESIZE] = rs;
15468                 opt[DTRACEOPT_CPU] = c;
15469 
15470                 break;
15471         }
15472 
15473         case CPU_UNCONFIG:
15474                 /*
15475                  * We don't free the buffer in the CPU_UNCONFIG case.  (The
15476                  * buffer will be freed when the consumer exits.)
15477                  */
15478                 break;
15479 
15480         default:
15481                 break;
15482         }
15483 
15484         mutex_exit(&dtrace_lock);
15485         return (0);
15486 }
15487 
15488 static void
15489 dtrace_cpu_setup_initial(processorid_t cpu)
15490 {
15491         (void) dtrace_cpu_setup(CPU_CONFIG, cpu);
15492 }
15493 
15494 static void
15495 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
15496 {
15497         if (dtrace_toxranges >= dtrace_toxranges_max) {
15498                 int osize, nsize;
15499                 dtrace_toxrange_t *range;
15500 
15501                 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15502 
15503                 if (osize == 0) {
15504                         ASSERT(dtrace_toxrange == NULL);
15505                         ASSERT(dtrace_toxranges_max == 0);
15506                         dtrace_toxranges_max = 1;
15507                 } else {
15508                         dtrace_toxranges_max <<= 1;
15509                 }
15510 
15511                 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15512                 range = kmem_zalloc(nsize, KM_SLEEP);
15513 
15514                 if (dtrace_toxrange != NULL) {
15515                         ASSERT(osize != 0);
15516                         bcopy(dtrace_toxrange, range, osize);
15517                         kmem_free(dtrace_toxrange, osize);
15518                 }
15519 
15520                 dtrace_toxrange = range;
15521         }
15522 
15523         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
15524         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
15525 
15526         dtrace_toxrange[dtrace_toxranges].dtt_base = base;
15527         dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
15528         dtrace_toxranges++;
15529 }
15530 
15531 static void
15532 dtrace_getf_barrier()
15533 {
15534         /*
15535          * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
15536          * that contain calls to getf(), this routine will be called on every
15537          * closef() before either the underlying vnode is released or the
15538          * file_t itself is freed.  By the time we are here, it is essential
15539          * that the file_t can no longer be accessed from a call to getf()
15540          * in probe context -- that assures that a dtrace_sync() can be used
15541          * to clear out any enablings referring to the old structures.
15542          */
15543         if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 ||
15544             kcred->cr_zone->zone_dtrace_getf != 0)
15545                 dtrace_sync();
15546 }
15547 
15548 /*
15549  * DTrace Driver Cookbook Functions
15550  */
15551 /*ARGSUSED*/
15552 static int
15553 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
15554 {
15555         dtrace_provider_id_t id;
15556         dtrace_state_t *state = NULL;
15557         dtrace_enabling_t *enab;
15558 
15559         mutex_enter(&cpu_lock);
15560         mutex_enter(&dtrace_provider_lock);
15561         mutex_enter(&dtrace_lock);
15562 
15563         if (ddi_soft_state_init(&dtrace_softstate,
15564             sizeof (dtrace_state_t), 0) != 0) {
15565                 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
15566                 mutex_exit(&cpu_lock);
15567                 mutex_exit(&dtrace_provider_lock);
15568                 mutex_exit(&dtrace_lock);
15569                 return (DDI_FAILURE);
15570         }
15571 
15572         if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
15573             DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
15574             ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
15575             DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
15576                 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
15577                 ddi_remove_minor_node(devi, NULL);
15578                 ddi_soft_state_fini(&dtrace_softstate);
15579                 mutex_exit(&cpu_lock);
15580                 mutex_exit(&dtrace_provider_lock);
15581                 mutex_exit(&dtrace_lock);
15582                 return (DDI_FAILURE);
15583         }
15584 
15585         ddi_report_dev(devi);
15586         dtrace_devi = devi;
15587 
15588         dtrace_modload = dtrace_module_loaded;
15589         dtrace_modunload = dtrace_module_unloaded;
15590         dtrace_cpu_init = dtrace_cpu_setup_initial;
15591         dtrace_helpers_cleanup = dtrace_helpers_destroy;
15592         dtrace_helpers_fork = dtrace_helpers_duplicate;
15593         dtrace_cpustart_init = dtrace_suspend;
15594         dtrace_cpustart_fini = dtrace_resume;
15595         dtrace_debugger_init = dtrace_suspend;
15596         dtrace_debugger_fini = dtrace_resume;
15597 
15598         register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
15599 
15600         ASSERT(MUTEX_HELD(&cpu_lock));
15601 
15602         dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
15603             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
15604         dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
15605             UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
15606             VM_SLEEP | VMC_IDENTIFIER);
15607         dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
15608             1, INT_MAX, 0);
15609 
15610         dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
15611             sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
15612             NULL, NULL, NULL, NULL, NULL, 0);
15613 
15614         ASSERT(MUTEX_HELD(&cpu_lock));
15615         dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
15616             offsetof(dtrace_probe_t, dtpr_nextmod),
15617             offsetof(dtrace_probe_t, dtpr_prevmod));
15618 
15619         dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
15620             offsetof(dtrace_probe_t, dtpr_nextfunc),
15621             offsetof(dtrace_probe_t, dtpr_prevfunc));
15622 
15623         dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
15624             offsetof(dtrace_probe_t, dtpr_nextname),
15625             offsetof(dtrace_probe_t, dtpr_prevname));
15626 
15627         if (dtrace_retain_max < 1) {
15628                 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
15629                     "setting to 1", dtrace_retain_max);
15630                 dtrace_retain_max = 1;
15631         }
15632 
15633         /*
15634          * Now discover our toxic ranges.
15635          */
15636         dtrace_toxic_ranges(dtrace_toxrange_add);
15637 
15638         /*
15639          * Before we register ourselves as a provider to our own framework,
15640          * we would like to assert that dtrace_provider is NULL -- but that's
15641          * not true if we were loaded as a dependency of a DTrace provider.
15642          * Once we've registered, we can assert that dtrace_provider is our
15643          * pseudo provider.
15644          */
15645         (void) dtrace_register("dtrace", &dtrace_provider_attr,
15646             DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
15647 
15648         ASSERT(dtrace_provider != NULL);
15649         ASSERT((dtrace_provider_id_t)dtrace_provider == id);
15650 
15651         dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
15652             dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
15653         dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
15654             dtrace_provider, NULL, NULL, "END", 0, NULL);
15655         dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
15656             dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
15657 
15658         dtrace_anon_property();
15659         mutex_exit(&cpu_lock);
15660 
15661         /*
15662          * If DTrace helper tracing is enabled, we need to allocate the
15663          * trace buffer and initialize the values.
15664          */
15665         if (dtrace_helptrace_enabled) {
15666                 ASSERT(dtrace_helptrace_buffer == NULL);
15667                 dtrace_helptrace_buffer =
15668                     kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
15669                 dtrace_helptrace_next = 0;
15670         }
15671 
15672         /*
15673          * If there are already providers, we must ask them to provide their
15674          * probes, and then match any anonymous enabling against them.  Note
15675          * that there should be no other retained enablings at this time:
15676          * the only retained enablings at this time should be the anonymous
15677          * enabling.
15678          */
15679         if (dtrace_anon.dta_enabling != NULL) {
15680                 ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
15681 
15682                 dtrace_enabling_provide(NULL);
15683                 state = dtrace_anon.dta_state;
15684 
15685                 /*
15686                  * We couldn't hold cpu_lock across the above call to
15687                  * dtrace_enabling_provide(), but we must hold it to actually
15688                  * enable the probes.  We have to drop all of our locks, pick
15689                  * up cpu_lock, and regain our locks before matching the
15690                  * retained anonymous enabling.
15691                  */
15692                 mutex_exit(&dtrace_lock);
15693                 mutex_exit(&dtrace_provider_lock);
15694 
15695                 mutex_enter(&cpu_lock);
15696                 mutex_enter(&dtrace_provider_lock);
15697                 mutex_enter(&dtrace_lock);
15698 
15699                 if ((enab = dtrace_anon.dta_enabling) != NULL)
15700                         (void) dtrace_enabling_match(enab, NULL);
15701 
15702                 mutex_exit(&cpu_lock);
15703         }
15704 
15705         mutex_exit(&dtrace_lock);
15706         mutex_exit(&dtrace_provider_lock);
15707 
15708         if (state != NULL) {
15709                 /*
15710                  * If we created any anonymous state, set it going now.
15711                  */
15712                 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
15713         }
15714 
15715         return (DDI_SUCCESS);
15716 }
15717 
15718 /*ARGSUSED*/
15719 static int
15720 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
15721 {
15722         dtrace_state_t *state;
15723         uint32_t priv;
15724         uid_t uid;
15725         zoneid_t zoneid;
15726 
15727         if (getminor(*devp) == DTRACEMNRN_HELPER)
15728                 return (0);
15729 
15730         /*
15731          * If this wasn't an open with the "helper" minor, then it must be
15732          * the "dtrace" minor.
15733          */
15734         if (getminor(*devp) != DTRACEMNRN_DTRACE)
15735                 return (ENXIO);
15736 
15737         /*
15738          * If no DTRACE_PRIV_* bits are set in the credential, then the
15739          * caller lacks sufficient permission to do anything with DTrace.
15740          */
15741         dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
15742         if (priv == DTRACE_PRIV_NONE)
15743                 return (EACCES);
15744 
15745         /*
15746          * Ask all providers to provide all their probes.
15747          */
15748         mutex_enter(&dtrace_provider_lock);
15749         dtrace_probe_provide(NULL, NULL);
15750         mutex_exit(&dtrace_provider_lock);
15751 
15752         mutex_enter(&cpu_lock);
15753         mutex_enter(&dtrace_lock);
15754         dtrace_opens++;
15755         dtrace_membar_producer();
15756 
15757         /*
15758          * If the kernel debugger is active (that is, if the kernel debugger
15759          * modified text in some way), we won't allow the open.
15760          */
15761         if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
15762                 dtrace_opens--;
15763                 mutex_exit(&cpu_lock);
15764                 mutex_exit(&dtrace_lock);
15765                 return (EBUSY);
15766         }
15767 
15768         state = dtrace_state_create(devp, cred_p);
15769         mutex_exit(&cpu_lock);
15770 
15771         if (state == NULL) {
15772                 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15773                         (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15774                 mutex_exit(&dtrace_lock);
15775                 return (EAGAIN);
15776         }
15777 
15778         mutex_exit(&dtrace_lock);
15779 
15780         return (0);
15781 }
15782 
15783 /*ARGSUSED*/
15784 static int
15785 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
15786 {
15787         minor_t minor = getminor(dev);
15788         dtrace_state_t *state;
15789 
15790         if (minor == DTRACEMNRN_HELPER)
15791                 return (0);
15792 
15793         state = ddi_get_soft_state(dtrace_softstate, minor);
15794 
15795         mutex_enter(&cpu_lock);
15796         mutex_enter(&dtrace_lock);
15797 
15798         if (state->dts_anon) {
15799                 /*
15800                  * There is anonymous state. Destroy that first.
15801                  */
15802                 ASSERT(dtrace_anon.dta_state == NULL);
15803                 dtrace_state_destroy(state->dts_anon);
15804         }
15805 
15806         dtrace_state_destroy(state);
15807         ASSERT(dtrace_opens > 0);
15808 
15809         /*
15810          * Only relinquish control of the kernel debugger interface when there
15811          * are no consumers and no anonymous enablings.
15812          */
15813         if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15814                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15815 
15816         mutex_exit(&dtrace_lock);
15817         mutex_exit(&cpu_lock);
15818 
15819         return (0);
15820 }
15821 
15822 /*ARGSUSED*/
15823 static int
15824 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
15825 {
15826         int rval;
15827         dof_helper_t help, *dhp = NULL;
15828 
15829         switch (cmd) {
15830         case DTRACEHIOC_ADDDOF:
15831                 if (copyin((void *)arg, &help, sizeof (help)) != 0) {
15832                         dtrace_dof_error(NULL, "failed to copyin DOF helper");
15833                         return (EFAULT);
15834                 }
15835 
15836                 dhp = &help;
15837                 arg = (intptr_t)help.dofhp_dof;
15838                 /*FALLTHROUGH*/
15839 
15840         case DTRACEHIOC_ADD: {
15841                 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
15842 
15843                 if (dof == NULL)
15844                         return (rval);
15845 
15846                 mutex_enter(&dtrace_lock);
15847 
15848                 /*
15849                  * dtrace_helper_slurp() takes responsibility for the dof --
15850                  * it may free it now or it may save it and free it later.
15851                  */
15852                 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
15853                         *rv = rval;
15854                         rval = 0;
15855                 } else {
15856                         rval = EINVAL;
15857                 }
15858 
15859                 mutex_exit(&dtrace_lock);
15860                 return (rval);
15861         }
15862 
15863         case DTRACEHIOC_REMOVE: {
15864                 mutex_enter(&dtrace_lock);
15865                 rval = dtrace_helper_destroygen(arg);
15866                 mutex_exit(&dtrace_lock);
15867 
15868                 return (rval);
15869         }
15870 
15871         default:
15872                 break;
15873         }
15874 
15875         return (ENOTTY);
15876 }
15877 
15878 /*ARGSUSED*/
15879 static int
15880 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
15881 {
15882         minor_t minor = getminor(dev);
15883         dtrace_state_t *state;
15884         int rval;
15885 
15886         if (minor == DTRACEMNRN_HELPER)
15887                 return (dtrace_ioctl_helper(cmd, arg, rv));
15888 
15889         state = ddi_get_soft_state(dtrace_softstate, minor);
15890 
15891         if (state->dts_anon) {
15892                 ASSERT(dtrace_anon.dta_state == NULL);
15893                 state = state->dts_anon;
15894         }
15895 
15896         switch (cmd) {
15897         case DTRACEIOC_PROVIDER: {
15898                 dtrace_providerdesc_t pvd;
15899                 dtrace_provider_t *pvp;
15900 
15901                 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
15902                         return (EFAULT);
15903 
15904                 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
15905                 mutex_enter(&dtrace_provider_lock);
15906 
15907                 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
15908                         if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
15909                                 break;
15910                 }
15911 
15912                 mutex_exit(&dtrace_provider_lock);
15913 
15914                 if (pvp == NULL)
15915                         return (ESRCH);
15916 
15917                 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
15918                 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
15919                 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
15920                         return (EFAULT);
15921 
15922                 return (0);
15923         }
15924 
15925         case DTRACEIOC_EPROBE: {
15926                 dtrace_eprobedesc_t epdesc;
15927                 dtrace_ecb_t *ecb;
15928                 dtrace_action_t *act;
15929                 void *buf;
15930                 size_t size;
15931                 uintptr_t dest;
15932                 int nrecs;
15933 
15934                 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
15935                         return (EFAULT);
15936 
15937                 mutex_enter(&dtrace_lock);
15938 
15939                 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
15940                         mutex_exit(&dtrace_lock);
15941                         return (EINVAL);
15942                 }
15943 
15944                 if (ecb->dte_probe == NULL) {
15945                         mutex_exit(&dtrace_lock);
15946                         return (EINVAL);
15947                 }
15948 
15949                 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
15950                 epdesc.dtepd_uarg = ecb->dte_uarg;
15951                 epdesc.dtepd_size = ecb->dte_size;
15952 
15953                 nrecs = epdesc.dtepd_nrecs;
15954                 epdesc.dtepd_nrecs = 0;
15955                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15956                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15957                                 continue;
15958 
15959                         epdesc.dtepd_nrecs++;
15960                 }
15961 
15962                 /*
15963                  * Now that we have the size, we need to allocate a temporary
15964                  * buffer in which to store the complete description.  We need
15965                  * the temporary buffer to be able to drop dtrace_lock()
15966                  * across the copyout(), below.
15967                  */
15968                 size = sizeof (dtrace_eprobedesc_t) +
15969                     (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
15970 
15971                 buf = kmem_alloc(size, KM_SLEEP);
15972                 dest = (uintptr_t)buf;
15973 
15974                 bcopy(&epdesc, (void *)dest, sizeof (epdesc));
15975                 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
15976 
15977                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15978                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15979                                 continue;
15980 
15981                         if (nrecs-- == 0)
15982                                 break;
15983 
15984                         bcopy(&act->dta_rec, (void *)dest,
15985                             sizeof (dtrace_recdesc_t));
15986                         dest += sizeof (dtrace_recdesc_t);
15987                 }
15988 
15989                 mutex_exit(&dtrace_lock);
15990 
15991                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
15992                         kmem_free(buf, size);
15993                         return (EFAULT);
15994                 }
15995 
15996                 kmem_free(buf, size);
15997                 return (0);
15998         }
15999 
16000         case DTRACEIOC_AGGDESC: {
16001                 dtrace_aggdesc_t aggdesc;
16002                 dtrace_action_t *act;
16003                 dtrace_aggregation_t *agg;
16004                 int nrecs;
16005                 uint32_t offs;
16006                 dtrace_recdesc_t *lrec;
16007                 void *buf;
16008                 size_t size;
16009                 uintptr_t dest;
16010 
16011                 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
16012                         return (EFAULT);
16013 
16014                 mutex_enter(&dtrace_lock);
16015 
16016                 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
16017                         mutex_exit(&dtrace_lock);
16018                         return (EINVAL);
16019                 }
16020 
16021                 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
16022 
16023                 nrecs = aggdesc.dtagd_nrecs;
16024                 aggdesc.dtagd_nrecs = 0;
16025 
16026                 offs = agg->dtag_base;
16027                 lrec = &agg->dtag_action.dta_rec;
16028                 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
16029 
16030                 for (act = agg->dtag_first; ; act = act->dta_next) {
16031                         ASSERT(act->dta_intuple ||
16032                             DTRACEACT_ISAGG(act->dta_kind));
16033 
16034                         /*
16035                          * If this action has a record size of zero, it
16036                          * denotes an argument to the aggregating action.
16037                          * Because the presence of this record doesn't (or
16038                          * shouldn't) affect the way the data is interpreted,
16039                          * we don't copy it out to save user-level the
16040                          * confusion of dealing with a zero-length record.
16041                          */
16042                         if (act->dta_rec.dtrd_size == 0) {
16043                                 ASSERT(agg->dtag_hasarg);
16044                                 continue;
16045                         }
16046 
16047                         aggdesc.dtagd_nrecs++;
16048 
16049                         if (act == &agg->dtag_action)
16050                                 break;
16051                 }
16052 
16053                 /*
16054                  * Now that we have the size, we need to allocate a temporary
16055                  * buffer in which to store the complete description.  We need
16056                  * the temporary buffer to be able to drop dtrace_lock()
16057                  * across the copyout(), below.
16058                  */
16059                 size = sizeof (dtrace_aggdesc_t) +
16060                     (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
16061 
16062                 buf = kmem_alloc(size, KM_SLEEP);
16063                 dest = (uintptr_t)buf;
16064 
16065                 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
16066                 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
16067 
16068                 for (act = agg->dtag_first; ; act = act->dta_next) {
16069                         dtrace_recdesc_t rec = act->dta_rec;
16070 
16071                         /*
16072                          * See the comment in the above loop for why we pass
16073                          * over zero-length records.
16074                          */
16075                         if (rec.dtrd_size == 0) {
16076                                 ASSERT(agg->dtag_hasarg);
16077                                 continue;
16078                         }
16079 
16080                         if (nrecs-- == 0)
16081                                 break;
16082 
16083                         rec.dtrd_offset -= offs;
16084                         bcopy(&rec, (void *)dest, sizeof (rec));
16085                         dest += sizeof (dtrace_recdesc_t);
16086 
16087                         if (act == &agg->dtag_action)
16088                                 break;
16089                 }
16090 
16091                 mutex_exit(&dtrace_lock);
16092 
16093                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
16094                         kmem_free(buf, size);
16095                         return (EFAULT);
16096                 }
16097 
16098                 kmem_free(buf, size);
16099                 return (0);
16100         }
16101 
16102         case DTRACEIOC_ENABLE: {
16103                 dof_hdr_t *dof;
16104                 dtrace_enabling_t *enab = NULL;
16105                 dtrace_vstate_t *vstate;
16106                 int err = 0;
16107 
16108                 *rv = 0;
16109 
16110                 /*
16111                  * If a NULL argument has been passed, we take this as our
16112                  * cue to reevaluate our enablings.
16113                  */
16114                 if (arg == NULL) {
16115                         dtrace_enabling_matchall();
16116 
16117                         return (0);
16118                 }
16119 
16120                 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
16121                         return (rval);
16122 
16123                 mutex_enter(&cpu_lock);
16124                 mutex_enter(&dtrace_lock);
16125                 vstate = &state->dts_vstate;
16126 
16127                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
16128                         mutex_exit(&dtrace_lock);
16129                         mutex_exit(&cpu_lock);
16130                         dtrace_dof_destroy(dof);
16131                         return (EBUSY);
16132                 }
16133 
16134                 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
16135                         mutex_exit(&dtrace_lock);
16136                         mutex_exit(&cpu_lock);
16137                         dtrace_dof_destroy(dof);
16138                         return (EINVAL);
16139                 }
16140 
16141                 if ((rval = dtrace_dof_options(dof, state)) != 0) {
16142                         dtrace_enabling_destroy(enab);
16143                         mutex_exit(&dtrace_lock);
16144                         mutex_exit(&cpu_lock);
16145                         dtrace_dof_destroy(dof);
16146                         return (rval);
16147                 }
16148 
16149                 if ((err = dtrace_enabling_match(enab, rv)) == 0) {
16150                         err = dtrace_enabling_retain(enab);
16151                 } else {
16152                         dtrace_enabling_destroy(enab);
16153                 }
16154 
16155                 mutex_exit(&cpu_lock);
16156                 mutex_exit(&dtrace_lock);
16157                 dtrace_dof_destroy(dof);
16158 
16159                 return (err);
16160         }
16161 
16162         case DTRACEIOC_REPLICATE: {
16163                 dtrace_repldesc_t desc;
16164                 dtrace_probedesc_t *match = &desc.dtrpd_match;
16165                 dtrace_probedesc_t *create = &desc.dtrpd_create;
16166                 int err;
16167 
16168                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16169                         return (EFAULT);
16170 
16171                 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16172                 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16173                 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16174                 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16175 
16176                 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16177                 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16178                 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16179                 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16180 
16181                 mutex_enter(&dtrace_lock);
16182                 err = dtrace_enabling_replicate(state, match, create);
16183                 mutex_exit(&dtrace_lock);
16184 
16185                 return (err);
16186         }
16187 
16188         case DTRACEIOC_PROBEMATCH:
16189         case DTRACEIOC_PROBES: {
16190                 dtrace_probe_t *probe = NULL;
16191                 dtrace_probedesc_t desc;
16192                 dtrace_probekey_t pkey;
16193                 dtrace_id_t i;
16194                 int m = 0;
16195                 uint32_t priv;
16196                 uid_t uid;
16197                 zoneid_t zoneid;
16198 
16199                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16200                         return (EFAULT);
16201 
16202                 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16203                 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16204                 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16205                 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16206 
16207                 /*
16208                  * Before we attempt to match this probe, we want to give
16209                  * all providers the opportunity to provide it.
16210                  */
16211                 if (desc.dtpd_id == DTRACE_IDNONE) {
16212                         mutex_enter(&dtrace_provider_lock);
16213                         dtrace_probe_provide(&desc, NULL);
16214                         mutex_exit(&dtrace_provider_lock);
16215                         desc.dtpd_id++;
16216                 }
16217 
16218                 if (cmd == DTRACEIOC_PROBEMATCH)  {
16219                         dtrace_probekey(&desc, &pkey);
16220                         pkey.dtpk_id = DTRACE_IDNONE;
16221                 }
16222 
16223                 dtrace_cred2priv(cr, &priv, &uid, &zoneid);
16224 
16225                 mutex_enter(&dtrace_lock);
16226 
16227                 if (cmd == DTRACEIOC_PROBEMATCH) {
16228                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16229                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16230                                     (m = dtrace_match_probe(probe, &pkey,
16231                                     priv, uid, zoneid)) != 0)
16232                                         break;
16233                         }
16234 
16235                         if (m < 0) {
16236                                 mutex_exit(&dtrace_lock);
16237                                 return (EINVAL);
16238                         }
16239 
16240                 } else {
16241                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16242                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16243                                     dtrace_match_priv(probe, priv, uid, zoneid))
16244                                         break;
16245                         }
16246                 }
16247 
16248                 if (probe == NULL) {
16249                         mutex_exit(&dtrace_lock);
16250                         return (ESRCH);
16251                 }
16252 
16253                 dtrace_probe_description(probe, &desc);
16254                 mutex_exit(&dtrace_lock);
16255 
16256                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16257                         return (EFAULT);
16258 
16259                 return (0);
16260         }
16261 
16262         case DTRACEIOC_PROBEARG: {
16263                 dtrace_argdesc_t desc;
16264                 dtrace_probe_t *probe;
16265                 dtrace_provider_t *prov;
16266 
16267                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16268                         return (EFAULT);
16269 
16270                 if (desc.dtargd_id == DTRACE_IDNONE)
16271                         return (EINVAL);
16272 
16273                 if (desc.dtargd_ndx == DTRACE_ARGNONE)
16274                         return (EINVAL);
16275 
16276                 mutex_enter(&dtrace_provider_lock);
16277                 mutex_enter(&mod_lock);
16278                 mutex_enter(&dtrace_lock);
16279 
16280                 if (desc.dtargd_id > dtrace_nprobes) {
16281                         mutex_exit(&dtrace_lock);
16282                         mutex_exit(&mod_lock);
16283                         mutex_exit(&dtrace_provider_lock);
16284                         return (EINVAL);
16285                 }
16286 
16287                 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
16288                         mutex_exit(&dtrace_lock);
16289                         mutex_exit(&mod_lock);
16290                         mutex_exit(&dtrace_provider_lock);
16291                         return (EINVAL);
16292                 }
16293 
16294                 mutex_exit(&dtrace_lock);
16295 
16296                 prov = probe->dtpr_provider;
16297 
16298                 if (prov->dtpv_pops.dtps_getargdesc == NULL) {
16299                         /*
16300                          * There isn't any typed information for this probe.
16301                          * Set the argument number to DTRACE_ARGNONE.
16302                          */
16303                         desc.dtargd_ndx = DTRACE_ARGNONE;
16304                 } else {
16305                         desc.dtargd_native[0] = '\0';
16306                         desc.dtargd_xlate[0] = '\0';
16307                         desc.dtargd_mapping = desc.dtargd_ndx;
16308 
16309                         prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
16310                             probe->dtpr_id, probe->dtpr_arg, &desc);
16311                 }
16312 
16313                 mutex_exit(&mod_lock);
16314                 mutex_exit(&dtrace_provider_lock);
16315 
16316                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16317                         return (EFAULT);
16318 
16319                 return (0);
16320         }
16321 
16322         case DTRACEIOC_GO: {
16323                 processorid_t cpuid;
16324                 rval = dtrace_state_go(state, &cpuid);
16325 
16326                 if (rval != 0)
16327                         return (rval);
16328 
16329                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16330                         return (EFAULT);
16331 
16332                 return (0);
16333         }
16334 
16335         case DTRACEIOC_STOP: {
16336                 processorid_t cpuid;
16337 
16338                 mutex_enter(&dtrace_lock);
16339                 rval = dtrace_state_stop(state, &cpuid);
16340                 mutex_exit(&dtrace_lock);
16341 
16342                 if (rval != 0)
16343                         return (rval);
16344 
16345                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16346                         return (EFAULT);
16347 
16348                 return (0);
16349         }
16350 
16351         case DTRACEIOC_DOFGET: {
16352                 dof_hdr_t hdr, *dof;
16353                 uint64_t len;
16354 
16355                 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
16356                         return (EFAULT);
16357 
16358                 mutex_enter(&dtrace_lock);
16359                 dof = dtrace_dof_create(state);
16360                 mutex_exit(&dtrace_lock);
16361 
16362                 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
16363                 rval = copyout(dof, (void *)arg, len);
16364                 dtrace_dof_destroy(dof);
16365 
16366                 return (rval == 0 ? 0 : EFAULT);
16367         }
16368 
16369         case DTRACEIOC_AGGSNAP:
16370         case DTRACEIOC_BUFSNAP: {
16371                 dtrace_bufdesc_t desc;
16372                 caddr_t cached;
16373                 dtrace_buffer_t *buf;
16374 
16375                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16376                         return (EFAULT);
16377 
16378                 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
16379                         return (EINVAL);
16380 
16381                 mutex_enter(&dtrace_lock);
16382 
16383                 if (cmd == DTRACEIOC_BUFSNAP) {
16384                         buf = &state->dts_buffer[desc.dtbd_cpu];
16385                 } else {
16386                         buf = &state->dts_aggbuffer[desc.dtbd_cpu];
16387                 }
16388 
16389                 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
16390                         size_t sz = buf->dtb_offset;
16391 
16392                         if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
16393                                 mutex_exit(&dtrace_lock);
16394                                 return (EBUSY);
16395                         }
16396 
16397                         /*
16398                          * If this buffer has already been consumed, we're
16399                          * going to indicate that there's nothing left here
16400                          * to consume.
16401                          */
16402                         if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
16403                                 mutex_exit(&dtrace_lock);
16404 
16405                                 desc.dtbd_size = 0;
16406                                 desc.dtbd_drops = 0;
16407                                 desc.dtbd_errors = 0;
16408                                 desc.dtbd_oldest = 0;
16409                                 sz = sizeof (desc);
16410 
16411                                 if (copyout(&desc, (void *)arg, sz) != 0)
16412                                         return (EFAULT);
16413 
16414                                 return (0);
16415                         }
16416 
16417                         /*
16418                          * If this is a ring buffer that has wrapped, we want
16419                          * to copy the whole thing out.
16420                          */
16421                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
16422                                 dtrace_buffer_polish(buf);
16423                                 sz = buf->dtb_size;
16424                         }
16425 
16426                         if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
16427                                 mutex_exit(&dtrace_lock);
16428                                 return (EFAULT);
16429                         }
16430 
16431                         desc.dtbd_size = sz;
16432                         desc.dtbd_drops = buf->dtb_drops;
16433                         desc.dtbd_errors = buf->dtb_errors;
16434                         desc.dtbd_oldest = buf->dtb_xamot_offset;
16435                         desc.dtbd_timestamp = dtrace_gethrtime();
16436 
16437                         mutex_exit(&dtrace_lock);
16438 
16439                         if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16440                                 return (EFAULT);
16441 
16442                         buf->dtb_flags |= DTRACEBUF_CONSUMED;
16443 
16444                         return (0);
16445                 }
16446 
16447                 if (buf->dtb_tomax == NULL) {
16448                         ASSERT(buf->dtb_xamot == NULL);
16449                         mutex_exit(&dtrace_lock);
16450                         return (ENOENT);
16451                 }
16452 
16453                 cached = buf->dtb_tomax;
16454                 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
16455 
16456                 dtrace_xcall(desc.dtbd_cpu,
16457                     (dtrace_xcall_t)dtrace_buffer_switch, buf);
16458 
16459                 state->dts_errors += buf->dtb_xamot_errors;
16460 
16461                 /*
16462                  * If the buffers did not actually switch, then the cross call
16463                  * did not take place -- presumably because the given CPU is
16464                  * not in the ready set.  If this is the case, we'll return
16465                  * ENOENT.
16466                  */
16467                 if (buf->dtb_tomax == cached) {
16468                         ASSERT(buf->dtb_xamot != cached);
16469                         mutex_exit(&dtrace_lock);
16470                         return (ENOENT);
16471                 }
16472 
16473                 ASSERT(cached == buf->dtb_xamot);
16474 
16475                 /*
16476                  * We have our snapshot; now copy it out.
16477                  */
16478                 if (copyout(buf->dtb_xamot, desc.dtbd_data,
16479                     buf->dtb_xamot_offset) != 0) {
16480                         mutex_exit(&dtrace_lock);
16481                         return (EFAULT);
16482                 }
16483 
16484                 desc.dtbd_size = buf->dtb_xamot_offset;
16485                 desc.dtbd_drops = buf->dtb_xamot_drops;
16486                 desc.dtbd_errors = buf->dtb_xamot_errors;
16487                 desc.dtbd_oldest = 0;
16488                 desc.dtbd_timestamp = buf->dtb_switched;
16489 
16490                 mutex_exit(&dtrace_lock);
16491 
16492                 /*
16493                  * Finally, copy out the buffer description.
16494                  */
16495                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16496                         return (EFAULT);
16497 
16498                 return (0);
16499         }
16500 
16501         case DTRACEIOC_CONF: {
16502                 dtrace_conf_t conf;
16503 
16504                 bzero(&conf, sizeof (conf));
16505                 conf.dtc_difversion = DIF_VERSION;
16506                 conf.dtc_difintregs = DIF_DIR_NREGS;
16507                 conf.dtc_diftupregs = DIF_DTR_NREGS;
16508                 conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
16509 
16510                 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
16511                         return (EFAULT);
16512 
16513                 return (0);
16514         }
16515 
16516         case DTRACEIOC_STATUS: {
16517                 dtrace_status_t stat;
16518                 dtrace_dstate_t *dstate;
16519                 int i, j;
16520                 uint64_t nerrs;
16521 
16522                 /*
16523                  * See the comment in dtrace_state_deadman() for the reason
16524                  * for setting dts_laststatus to INT64_MAX before setting
16525                  * it to the correct value.
16526                  */
16527                 state->dts_laststatus = INT64_MAX;
16528                 dtrace_membar_producer();
16529                 state->dts_laststatus = dtrace_gethrtime();
16530 
16531                 bzero(&stat, sizeof (stat));
16532 
16533                 mutex_enter(&dtrace_lock);
16534 
16535                 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
16536                         mutex_exit(&dtrace_lock);
16537                         return (ENOENT);
16538                 }
16539 
16540                 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
16541                         stat.dtst_exiting = 1;
16542 
16543                 nerrs = state->dts_errors;
16544                 dstate = &state->dts_vstate.dtvs_dynvars;
16545 
16546                 for (i = 0; i < NCPU; i++) {
16547                         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
16548 
16549                         stat.dtst_dyndrops += dcpu->dtdsc_drops;
16550                         stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
16551                         stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
16552 
16553                         if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
16554                                 stat.dtst_filled++;
16555 
16556                         nerrs += state->dts_buffer[i].dtb_errors;
16557 
16558                         for (j = 0; j < state->dts_nspeculations; j++) {
16559                                 dtrace_speculation_t *spec;
16560                                 dtrace_buffer_t *buf;
16561 
16562                                 spec = &state->dts_speculations[j];
16563                                 buf = &spec->dtsp_buffer[i];
16564                                 stat.dtst_specdrops += buf->dtb_xamot_drops;
16565                         }
16566                 }
16567 
16568                 stat.dtst_specdrops_busy = state->dts_speculations_busy;
16569                 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
16570                 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
16571                 stat.dtst_dblerrors = state->dts_dblerrors;
16572                 stat.dtst_killed =
16573                     (state->dts_activity == DTRACE_ACTIVITY_KILLED);
16574                 stat.dtst_errors = nerrs;
16575 
16576                 mutex_exit(&dtrace_lock);
16577 
16578                 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
16579                         return (EFAULT);
16580 
16581                 return (0);
16582         }
16583 
16584         case DTRACEIOC_FORMAT: {
16585                 dtrace_fmtdesc_t fmt;
16586                 char *str;
16587                 int len;
16588 
16589                 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
16590                         return (EFAULT);
16591 
16592                 mutex_enter(&dtrace_lock);
16593 
16594                 if (fmt.dtfd_format == 0 ||
16595                     fmt.dtfd_format > state->dts_nformats) {
16596                         mutex_exit(&dtrace_lock);
16597                         return (EINVAL);
16598                 }
16599 
16600                 /*
16601                  * Format strings are allocated contiguously and they are
16602                  * never freed; if a format index is less than the number
16603                  * of formats, we can assert that the format map is non-NULL
16604                  * and that the format for the specified index is non-NULL.
16605                  */
16606                 ASSERT(state->dts_formats != NULL);
16607                 str = state->dts_formats[fmt.dtfd_format - 1];
16608                 ASSERT(str != NULL);
16609 
16610                 len = strlen(str) + 1;
16611 
16612                 if (len > fmt.dtfd_length) {
16613                         fmt.dtfd_length = len;
16614 
16615                         if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
16616                                 mutex_exit(&dtrace_lock);
16617                                 return (EINVAL);
16618                         }
16619                 } else {
16620                         if (copyout(str, fmt.dtfd_string, len) != 0) {
16621                                 mutex_exit(&dtrace_lock);
16622                                 return (EINVAL);
16623                         }
16624                 }
16625 
16626                 mutex_exit(&dtrace_lock);
16627                 return (0);
16628         }
16629 
16630         default:
16631                 break;
16632         }
16633 
16634         return (ENOTTY);
16635 }
16636 
16637 /*ARGSUSED*/
16638 static int
16639 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
16640 {
16641         dtrace_state_t *state;
16642 
16643         switch (cmd) {
16644         case DDI_DETACH:
16645                 break;
16646 
16647         case DDI_SUSPEND:
16648                 return (DDI_SUCCESS);
16649 
16650         default:
16651                 return (DDI_FAILURE);
16652         }
16653 
16654         mutex_enter(&cpu_lock);
16655         mutex_enter(&dtrace_provider_lock);
16656         mutex_enter(&dtrace_lock);
16657 
16658         ASSERT(dtrace_opens == 0);
16659 
16660         if (dtrace_helpers > 0) {
16661                 mutex_exit(&dtrace_provider_lock);
16662                 mutex_exit(&dtrace_lock);
16663                 mutex_exit(&cpu_lock);
16664                 return (DDI_FAILURE);
16665         }
16666 
16667         if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
16668                 mutex_exit(&dtrace_provider_lock);
16669                 mutex_exit(&dtrace_lock);
16670                 mutex_exit(&cpu_lock);
16671                 return (DDI_FAILURE);
16672         }
16673 
16674         dtrace_provider = NULL;
16675 
16676         if ((state = dtrace_anon_grab()) != NULL) {
16677                 /*
16678                  * If there were ECBs on this state, the provider should
16679                  * have not been allowed to detach; assert that there is
16680                  * none.
16681                  */
16682                 ASSERT(state->dts_necbs == 0);
16683                 dtrace_state_destroy(state);
16684 
16685                 /*
16686                  * If we're being detached with anonymous state, we need to
16687                  * indicate to the kernel debugger that DTrace is now inactive.
16688                  */
16689                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
16690         }
16691 
16692         bzero(&dtrace_anon, sizeof (dtrace_anon_t));
16693         unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
16694         dtrace_cpu_init = NULL;
16695         dtrace_helpers_cleanup = NULL;
16696         dtrace_helpers_fork = NULL;
16697         dtrace_cpustart_init = NULL;
16698         dtrace_cpustart_fini = NULL;
16699         dtrace_debugger_init = NULL;
16700         dtrace_debugger_fini = NULL;
16701         dtrace_modload = NULL;
16702         dtrace_modunload = NULL;
16703 
16704         ASSERT(dtrace_getf == 0);
16705         ASSERT(dtrace_closef == NULL);
16706 
16707         mutex_exit(&cpu_lock);
16708 
16709         if (dtrace_helptrace_enabled) {
16710                 kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
16711                 dtrace_helptrace_buffer = NULL;
16712         }
16713 
16714         kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
16715         dtrace_probes = NULL;
16716         dtrace_nprobes = 0;
16717 
16718         dtrace_hash_destroy(dtrace_bymod);
16719         dtrace_hash_destroy(dtrace_byfunc);
16720         dtrace_hash_destroy(dtrace_byname);
16721         dtrace_bymod = NULL;
16722         dtrace_byfunc = NULL;
16723         dtrace_byname = NULL;
16724 
16725         kmem_cache_destroy(dtrace_state_cache);
16726         vmem_destroy(dtrace_minor);
16727         vmem_destroy(dtrace_arena);
16728 
16729         if (dtrace_toxrange != NULL) {
16730                 kmem_free(dtrace_toxrange,
16731                     dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
16732                 dtrace_toxrange = NULL;
16733                 dtrace_toxranges = 0;
16734                 dtrace_toxranges_max = 0;
16735         }
16736 
16737         ddi_remove_minor_node(dtrace_devi, NULL);
16738         dtrace_devi = NULL;
16739 
16740         ddi_soft_state_fini(&dtrace_softstate);
16741 
16742         ASSERT(dtrace_vtime_references == 0);
16743         ASSERT(dtrace_opens == 0);
16744         ASSERT(dtrace_retained == NULL);
16745 
16746         mutex_exit(&dtrace_lock);
16747         mutex_exit(&dtrace_provider_lock);
16748 
16749         /*
16750          * We don't destroy the task queue until after we have dropped our
16751          * locks (taskq_destroy() may block on running tasks).  To prevent
16752          * attempting to do work after we have effectively detached but before
16753          * the task queue has been destroyed, all tasks dispatched via the
16754          * task queue must check that DTrace is still attached before
16755          * performing any operation.
16756          */
16757         taskq_destroy(dtrace_taskq);
16758         dtrace_taskq = NULL;
16759 
16760         return (DDI_SUCCESS);
16761 }
16762 
16763 /*ARGSUSED*/
16764 static int
16765 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
16766 {
16767         int error;
16768 
16769         switch (infocmd) {
16770         case DDI_INFO_DEVT2DEVINFO:
16771                 *result = (void *)dtrace_devi;
16772                 error = DDI_SUCCESS;
16773                 break;
16774         case DDI_INFO_DEVT2INSTANCE:
16775                 *result = (void *)0;
16776                 error = DDI_SUCCESS;
16777                 break;
16778         default:
16779                 error = DDI_FAILURE;
16780         }
16781         return (error);
16782 }
16783 
16784 static struct cb_ops dtrace_cb_ops = {
16785         dtrace_open,            /* open */
16786         dtrace_close,           /* close */
16787         nulldev,                /* strategy */
16788         nulldev,                /* print */
16789         nodev,                  /* dump */
16790         nodev,                  /* read */
16791         nodev,                  /* write */
16792         dtrace_ioctl,           /* ioctl */
16793         nodev,                  /* devmap */
16794         nodev,                  /* mmap */
16795         nodev,                  /* segmap */
16796         nochpoll,               /* poll */
16797         ddi_prop_op,            /* cb_prop_op */
16798         0,                      /* streamtab  */
16799         D_NEW | D_MP            /* Driver compatibility flag */
16800 };
16801 
16802 static struct dev_ops dtrace_ops = {
16803         DEVO_REV,               /* devo_rev */
16804         0,                      /* refcnt */
16805         dtrace_info,            /* get_dev_info */
16806         nulldev,                /* identify */
16807         nulldev,                /* probe */
16808         dtrace_attach,          /* attach */
16809         dtrace_detach,          /* detach */
16810         nodev,                  /* reset */
16811         &dtrace_cb_ops,             /* driver operations */
16812         NULL,                   /* bus operations */
16813         nodev,                  /* dev power */
16814         ddi_quiesce_not_needed,         /* quiesce */
16815 };
16816 
16817 static struct modldrv modldrv = {
16818         &mod_driverops,             /* module type (this is a pseudo driver) */
16819         "Dynamic Tracing",      /* name of module */
16820         &dtrace_ops,                /* driver ops */
16821 };
16822 
16823 static struct modlinkage modlinkage = {
16824         MODREV_1,
16825         (void *)&modldrv,
16826         NULL
16827 };
16828 
16829 int
16830 _init(void)
16831 {
16832         return (mod_install(&modlinkage));
16833 }
16834 
16835 int
16836 _info(struct modinfo *modinfop)
16837 {
16838         return (mod_info(&modlinkage, modinfop));
16839 }
16840 
16841 int
16842 _fini(void)
16843 {
16844         return (mod_remove(&modlinkage));
16845 }