1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  25  * Copyright (c) 2012 by Delphix. All rights reserved.
  26  */
  27 
  28 /*
  29  * DTrace - Dynamic Tracing for Solaris
  30  *
  31  * This is the implementation of the Solaris Dynamic Tracing framework
  32  * (DTrace).  The user-visible interface to DTrace is described at length in
  33  * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
  34  * library, the in-kernel DTrace framework, and the DTrace providers are
  35  * described in the block comments in the <sys/dtrace.h> header file.  The
  36  * internal architecture of DTrace is described in the block comments in the
  37  * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
  38  * implementation very much assume mastery of all of these sources; if one has
  39  * an unanswered question about the implementation, one should consult them
  40  * first.
  41  *
  42  * The functions here are ordered roughly as follows:
  43  *
  44  *   - Probe context functions
  45  *   - Probe hashing functions
  46  *   - Non-probe context utility functions
  47  *   - Matching functions
  48  *   - Provider-to-Framework API functions
  49  *   - Probe management functions
  50  *   - DIF object functions
  51  *   - Format functions
  52  *   - Predicate functions
  53  *   - ECB functions
  54  *   - Buffer functions
  55  *   - Enabling functions
  56  *   - DOF functions
  57  *   - Anonymous enabling functions
  58  *   - Consumer state functions
  59  *   - Helper functions
  60  *   - Hook functions
  61  *   - Driver cookbook functions
  62  *
  63  * Each group of functions begins with a block comment labelled the "DTrace
  64  * [Group] Functions", allowing one to find each block by searching forward
  65  * on capital-f functions.
  66  */
  67 #include <sys/errno.h>
  68 #include <sys/stat.h>
  69 #include <sys/modctl.h>
  70 #include <sys/conf.h>
  71 #include <sys/systm.h>
  72 #include <sys/ddi.h>
  73 #include <sys/sunddi.h>
  74 #include <sys/cpuvar.h>
  75 #include <sys/kmem.h>
  76 #include <sys/strsubr.h>
  77 #include <sys/sysmacros.h>
  78 #include <sys/dtrace_impl.h>
  79 #include <sys/atomic.h>
  80 #include <sys/cmn_err.h>
  81 #include <sys/mutex_impl.h>
  82 #include <sys/rwlock_impl.h>
  83 #include <sys/ctf_api.h>
  84 #include <sys/panic.h>
  85 #include <sys/priv_impl.h>
  86 #include <sys/policy.h>
  87 #include <sys/cred_impl.h>
  88 #include <sys/procfs_isa.h>
  89 #include <sys/taskq.h>
  90 #include <sys/mkdev.h>
  91 #include <sys/kdi.h>
  92 #include <sys/zone.h>
  93 #include <sys/socket.h>
  94 #include <netinet/in.h>
  95 #include "strtolctype.h"
  96 
  97 /*
  98  * DTrace Tunable Variables
  99  *
 100  * The following variables may be tuned by adding a line to /etc/system that
 101  * includes both the name of the DTrace module ("dtrace") and the name of the
 102  * variable.  For example:
 103  *
 104  *   set dtrace:dtrace_destructive_disallow = 1
 105  *
 106  * In general, the only variables that one should be tuning this way are those
 107  * that affect system-wide DTrace behavior, and for which the default behavior
 108  * is undesirable.  Most of these variables are tunable on a per-consumer
 109  * basis using DTrace options, and need not be tuned on a system-wide basis.
 110  * When tuning these variables, avoid pathological values; while some attempt
 111  * is made to verify the integrity of these variables, they are not considered
 112  * part of the supported interface to DTrace, and they are therefore not
 113  * checked comprehensively.  Further, these variables should not be tuned
 114  * dynamically via "mdb -kw" or other means; they should only be tuned via
 115  * /etc/system.
 116  */
 117 int             dtrace_destructive_disallow = 0;
 118 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 119 size_t          dtrace_difo_maxsize = (256 * 1024);
 120 dtrace_optval_t dtrace_dof_maxsize = (256 * 1024);
 121 size_t          dtrace_global_maxsize = (16 * 1024);
 122 size_t          dtrace_actions_max = (16 * 1024);
 123 size_t          dtrace_retain_max = 1024;
 124 dtrace_optval_t dtrace_helper_actions_max = 1024;
 125 dtrace_optval_t dtrace_helper_providers_max = 32;
 126 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
 127 size_t          dtrace_strsize_default = 256;
 128 dtrace_optval_t dtrace_cleanrate_default = 9900990;             /* 101 hz */
 129 dtrace_optval_t dtrace_cleanrate_min = 200000;                  /* 5000 hz */
 130 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;  /* 1/minute */
 131 dtrace_optval_t dtrace_aggrate_default = NANOSEC;               /* 1 hz */
 132 dtrace_optval_t dtrace_statusrate_default = NANOSEC;            /* 1 hz */
 133 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;  /* 6/minute */
 134 dtrace_optval_t dtrace_switchrate_default = NANOSEC;            /* 1 hz */
 135 dtrace_optval_t dtrace_nspec_default = 1;
 136 dtrace_optval_t dtrace_specsize_default = 32 * 1024;
 137 dtrace_optval_t dtrace_stackframes_default = 20;
 138 dtrace_optval_t dtrace_ustackframes_default = 20;
 139 dtrace_optval_t dtrace_jstackframes_default = 50;
 140 dtrace_optval_t dtrace_jstackstrsize_default = 512;
 141 int             dtrace_msgdsize_max = 128;
 142 hrtime_t        dtrace_chill_max = 500 * (NANOSEC / MILLISEC);  /* 500 ms */
 143 hrtime_t        dtrace_chill_interval = NANOSEC;                /* 1000 ms */
 144 int             dtrace_devdepth_max = 32;
 145 int             dtrace_err_verbose;
 146 hrtime_t        dtrace_deadman_interval = NANOSEC;
 147 hrtime_t        dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
 148 hrtime_t        dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
 149 hrtime_t        dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
 150 
 151 /*
 152  * DTrace External Variables
 153  *
 154  * As dtrace(7D) is a kernel module, any DTrace variables are obviously
 155  * available to DTrace consumers via the backtick (`) syntax.  One of these,
 156  * dtrace_zero, is made deliberately so:  it is provided as a source of
 157  * well-known, zero-filled memory.  While this variable is not documented,
 158  * it is used by some translators as an implementation detail.
 159  */
 160 const char      dtrace_zero[256] = { 0 };       /* zero-filled memory */
 161 
 162 /*
 163  * DTrace Internal Variables
 164  */
 165 static dev_info_t       *dtrace_devi;           /* device info */
 166 static vmem_t           *dtrace_arena;          /* probe ID arena */
 167 static vmem_t           *dtrace_minor;          /* minor number arena */
 168 static taskq_t          *dtrace_taskq;          /* task queue */
 169 static dtrace_probe_t   **dtrace_probes;        /* array of all probes */
 170 static int              dtrace_nprobes;         /* number of probes */
 171 static dtrace_provider_t *dtrace_provider;      /* provider list */
 172 static dtrace_meta_t    *dtrace_meta_pid;       /* user-land meta provider */
 173 static int              dtrace_opens;           /* number of opens */
 174 static int              dtrace_helpers;         /* number of helpers */
 175 static int              dtrace_getf;            /* number of unpriv getf()s */
 176 static void             *dtrace_softstate;      /* softstate pointer */
 177 static dtrace_hash_t    *dtrace_bymod;          /* probes hashed by module */
 178 static dtrace_hash_t    *dtrace_byfunc;         /* probes hashed by function */
 179 static dtrace_hash_t    *dtrace_byname;         /* probes hashed by name */
 180 static dtrace_toxrange_t *dtrace_toxrange;      /* toxic range array */
 181 static int              dtrace_toxranges;       /* number of toxic ranges */
 182 static int              dtrace_toxranges_max;   /* size of toxic range array */
 183 static dtrace_anon_t    dtrace_anon;            /* anonymous enabling */
 184 static kmem_cache_t     *dtrace_state_cache;    /* cache for dynamic state */
 185 static uint64_t         dtrace_vtime_references; /* number of vtimestamp refs */
 186 static kthread_t        *dtrace_panicked;       /* panicking thread */
 187 static dtrace_ecb_t     *dtrace_ecb_create_cache; /* cached created ECB */
 188 static dtrace_genid_t   dtrace_probegen;        /* current probe generation */
 189 static dtrace_helpers_t *dtrace_deferred_pid;   /* deferred helper list */
 190 static dtrace_enabling_t *dtrace_retained;      /* list of retained enablings */
 191 static dtrace_genid_t   dtrace_retained_gen;    /* current retained enab gen */
 192 static dtrace_dynvar_t  dtrace_dynhash_sink;    /* end of dynamic hash chains */
 193 static int              dtrace_dynvar_failclean; /* dynvars failed to clean */
 194 
 195 /*
 196  * DTrace Locking
 197  * DTrace is protected by three (relatively coarse-grained) locks:
 198  *
 199  * (1) dtrace_lock is required to manipulate essentially any DTrace state,
 200  *     including enabling state, probes, ECBs, consumer state, helper state,
 201  *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
 202  *     probe context is lock-free -- synchronization is handled via the
 203  *     dtrace_sync() cross call mechanism.
 204  *
 205  * (2) dtrace_provider_lock is required when manipulating provider state, or
 206  *     when provider state must be held constant.
 207  *
 208  * (3) dtrace_meta_lock is required when manipulating meta provider state, or
 209  *     when meta provider state must be held constant.
 210  *
 211  * The lock ordering between these three locks is dtrace_meta_lock before
 212  * dtrace_provider_lock before dtrace_lock.  (In particular, there are
 213  * several places where dtrace_provider_lock is held by the framework as it
 214  * calls into the providers -- which then call back into the framework,
 215  * grabbing dtrace_lock.)
 216  *
 217  * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
 218  * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
 219  * role as a coarse-grained lock; it is acquired before both of these locks.
 220  * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
 221  * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
 222  * mod_lock is similar with respect to dtrace_provider_lock in that it must be
 223  * acquired _between_ dtrace_provider_lock and dtrace_lock.
 224  */
 225 static kmutex_t         dtrace_lock;            /* probe state lock */
 226 static kmutex_t         dtrace_provider_lock;   /* provider state lock */
 227 static kmutex_t         dtrace_meta_lock;       /* meta-provider state lock */
 228 
 229 /*
 230  * DTrace Provider Variables
 231  *
 232  * These are the variables relating to DTrace as a provider (that is, the
 233  * provider of the BEGIN, END, and ERROR probes).
 234  */
 235 static dtrace_pattr_t   dtrace_provider_attr = {
 236 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 237 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 238 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 239 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 240 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 241 };
 242 
 243 static void
 244 dtrace_nullop(void)
 245 {}
 246 
 247 static int
 248 dtrace_enable_nullop(void)
 249 {
 250         return (0);
 251 }
 252 
 253 static dtrace_pops_t    dtrace_provider_ops = {
 254         (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
 255         (void (*)(void *, struct modctl *))dtrace_nullop,
 256         (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
 257         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 258         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 259         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 260         NULL,
 261         NULL,
 262         NULL,
 263         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
 264 };
 265 
 266 static dtrace_id_t      dtrace_probeid_begin;   /* special BEGIN probe */
 267 static dtrace_id_t      dtrace_probeid_end;     /* special END probe */
 268 dtrace_id_t             dtrace_probeid_error;   /* special ERROR probe */
 269 
 270 /*
 271  * DTrace Helper Tracing Variables
 272  */
 273 uint32_t dtrace_helptrace_next = 0;
 274 uint32_t dtrace_helptrace_nlocals;
 275 char    *dtrace_helptrace_buffer;
 276 int     dtrace_helptrace_bufsize = 512 * 1024;
 277 
 278 #ifdef DEBUG
 279 int     dtrace_helptrace_enabled = 1;
 280 #else
 281 int     dtrace_helptrace_enabled = 0;
 282 #endif
 283 
 284 /*
 285  * DTrace Error Hashing
 286  *
 287  * On DEBUG kernels, DTrace will track the errors that has seen in a hash
 288  * table.  This is very useful for checking coverage of tests that are
 289  * expected to induce DIF or DOF processing errors, and may be useful for
 290  * debugging problems in the DIF code generator or in DOF generation .  The
 291  * error hash may be examined with the ::dtrace_errhash MDB dcmd.
 292  */
 293 #ifdef DEBUG
 294 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
 295 static const char *dtrace_errlast;
 296 static kthread_t *dtrace_errthread;
 297 static kmutex_t dtrace_errlock;
 298 #endif
 299 
 300 /*
 301  * DTrace Macros and Constants
 302  *
 303  * These are various macros that are useful in various spots in the
 304  * implementation, along with a few random constants that have no meaning
 305  * outside of the implementation.  There is no real structure to this cpp
 306  * mishmash -- but is there ever?
 307  */
 308 #define DTRACE_HASHSTR(hash, probe)     \
 309         dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
 310 
 311 #define DTRACE_HASHNEXT(hash, probe)    \
 312         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
 313 
 314 #define DTRACE_HASHPREV(hash, probe)    \
 315         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
 316 
 317 #define DTRACE_HASHEQ(hash, lhs, rhs)   \
 318         (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
 319             *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
 320 
 321 #define DTRACE_AGGHASHSIZE_SLEW         17
 322 
 323 #define DTRACE_V4MAPPED_OFFSET          (sizeof (uint32_t) * 3)
 324 
 325 /*
 326  * The key for a thread-local variable consists of the lower 61 bits of the
 327  * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
 328  * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
 329  * equal to a variable identifier.  This is necessary (but not sufficient) to
 330  * assure that global associative arrays never collide with thread-local
 331  * variables.  To guarantee that they cannot collide, we must also define the
 332  * order for keying dynamic variables.  That order is:
 333  *
 334  *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
 335  *
 336  * Because the variable-key and the tls-key are in orthogonal spaces, there is
 337  * no way for a global variable key signature to match a thread-local key
 338  * signature.
 339  */
 340 #define DTRACE_TLS_THRKEY(where) { \
 341         uint_t intr = 0; \
 342         uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
 343         for (; actv; actv >>= 1) \
 344                 intr++; \
 345         ASSERT(intr < (1 << 3)); \
 346         (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
 347             (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
 348 }
 349 
 350 #define DT_BSWAP_8(x)   ((x) & 0xff)
 351 #define DT_BSWAP_16(x)  ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
 352 #define DT_BSWAP_32(x)  ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
 353 #define DT_BSWAP_64(x)  ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
 354 
 355 #define DT_MASK_LO 0x00000000FFFFFFFFULL
 356 
 357 #define DTRACE_STORE(type, tomax, offset, what) \
 358         *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
 359 
 360 #ifndef __x86
 361 #define DTRACE_ALIGNCHECK(addr, size, flags)                            \
 362         if (addr & (size - 1)) {                                    \
 363                 *flags |= CPU_DTRACE_BADALIGN;                          \
 364                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 365                 return (0);                                             \
 366         }
 367 #else
 368 #define DTRACE_ALIGNCHECK(addr, size, flags)
 369 #endif
 370 
 371 /*
 372  * Test whether a range of memory starting at testaddr of size testsz falls
 373  * within the range of memory described by addr, sz.  We take care to avoid
 374  * problems with overflow and underflow of the unsigned quantities, and
 375  * disallow all negative sizes.  Ranges of size 0 are allowed.
 376  */
 377 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
 378         ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
 379         (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
 380         (testaddr) + (testsz) >= (testaddr))
 381 
 382 /*
 383  * Test whether alloc_sz bytes will fit in the scratch region.  We isolate
 384  * alloc_sz on the righthand side of the comparison in order to avoid overflow
 385  * or underflow in the comparison with it.  This is simpler than the INRANGE
 386  * check above, because we know that the dtms_scratch_ptr is valid in the
 387  * range.  Allocations of size zero are allowed.
 388  */
 389 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
 390         ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
 391         (mstate)->dtms_scratch_ptr >= (alloc_sz))
 392 
 393 #define DTRACE_LOADFUNC(bits)                                           \
 394 /*CSTYLED*/                                                             \
 395 uint##bits##_t                                                          \
 396 dtrace_load##bits(uintptr_t addr)                                       \
 397 {                                                                       \
 398         size_t size = bits / NBBY;                                      \
 399         /*CSTYLED*/                                                     \
 400         uint##bits##_t rval;                                            \
 401         int i;                                                          \
 402         volatile uint16_t *flags = (volatile uint16_t *)                \
 403             &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;                    \
 404                                                                         \
 405         DTRACE_ALIGNCHECK(addr, size, flags);                           \
 406                                                                         \
 407         for (i = 0; i < dtrace_toxranges; i++) {                     \
 408                 if (addr >= dtrace_toxrange[i].dtt_limit)            \
 409                         continue;                                       \
 410                                                                         \
 411                 if (addr + size <= dtrace_toxrange[i].dtt_base)              \
 412                         continue;                                       \
 413                                                                         \
 414                 /*                                                      \
 415                  * This address falls within a toxic region; return 0.  \
 416                  */                                                     \
 417                 *flags |= CPU_DTRACE_BADADDR;                           \
 418                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 419                 return (0);                                             \
 420         }                                                               \
 421                                                                         \
 422         *flags |= CPU_DTRACE_NOFAULT;                                   \
 423         /*CSTYLED*/                                                     \
 424         rval = *((volatile uint##bits##_t *)addr);                      \
 425         *flags &= ~CPU_DTRACE_NOFAULT;                                      \
 426                                                                         \
 427         return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0);           \
 428 }
 429 
 430 #ifdef _LP64
 431 #define dtrace_loadptr  dtrace_load64
 432 #else
 433 #define dtrace_loadptr  dtrace_load32
 434 #endif
 435 
 436 #define DTRACE_DYNHASH_FREE     0
 437 #define DTRACE_DYNHASH_SINK     1
 438 #define DTRACE_DYNHASH_VALID    2
 439 
 440 #define DTRACE_MATCH_FAIL       -1
 441 #define DTRACE_MATCH_NEXT       0
 442 #define DTRACE_MATCH_DONE       1
 443 #define DTRACE_ANCHORED(probe)  ((probe)->dtpr_func[0] != '\0')
 444 #define DTRACE_STATE_ALIGN      64
 445 
 446 #define DTRACE_FLAGS2FLT(flags)                                         \
 447         (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :               \
 448         ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :            \
 449         ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :                \
 450         ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :            \
 451         ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :            \
 452         ((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :             \
 453         ((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :             \
 454         ((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :   \
 455         ((flags) & CPU_DTRACE_BADSTACK) ?  DTRACEFLT_BADSTACK :             \
 456         DTRACEFLT_UNKNOWN)
 457 
 458 #define DTRACEACT_ISSTRING(act)                                         \
 459         ((act)->dta_kind == DTRACEACT_DIFEXPR &&                     \
 460         (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
 461 
 462 static size_t dtrace_strlen(const char *, size_t);
 463 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
 464 static void dtrace_enabling_provide(dtrace_provider_t *);
 465 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
 466 static void dtrace_enabling_matchall(void);
 467 static void dtrace_enabling_reap(void);
 468 static dtrace_state_t *dtrace_anon_grab(void);
 469 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
 470     dtrace_state_t *, uint64_t, uint64_t);
 471 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
 472 static void dtrace_buffer_drop(dtrace_buffer_t *);
 473 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
 474 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
 475     dtrace_state_t *, dtrace_mstate_t *);
 476 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
 477     dtrace_optval_t);
 478 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
 479 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
 480 static int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *);
 481 static void dtrace_getf_barrier(void);
 482 
 483 /*
 484  * DTrace Probe Context Functions
 485  *
 486  * These functions are called from probe context.  Because probe context is
 487  * any context in which C may be called, arbitrarily locks may be held,
 488  * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
 489  * As a result, functions called from probe context may only call other DTrace
 490  * support functions -- they may not interact at all with the system at large.
 491  * (Note that the ASSERT macro is made probe-context safe by redefining it in
 492  * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
 493  * loads are to be performed from probe context, they _must_ be in terms of
 494  * the safe dtrace_load*() variants.
 495  *
 496  * Some functions in this block are not actually called from probe context;
 497  * for these functions, there will be a comment above the function reading
 498  * "Note:  not called from probe context."
 499  */
 500 void
 501 dtrace_panic(const char *format, ...)
 502 {
 503         va_list alist;
 504 
 505         va_start(alist, format);
 506         dtrace_vpanic(format, alist);
 507         va_end(alist);
 508 }
 509 
 510 int
 511 dtrace_assfail(const char *a, const char *f, int l)
 512 {
 513         dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
 514 
 515         /*
 516          * We just need something here that even the most clever compiler
 517          * cannot optimize away.
 518          */
 519         return (a[(uintptr_t)f]);
 520 }
 521 
 522 /*
 523  * Atomically increment a specified error counter from probe context.
 524  */
 525 static void
 526 dtrace_error(uint32_t *counter)
 527 {
 528         /*
 529          * Most counters stored to in probe context are per-CPU counters.
 530          * However, there are some error conditions that are sufficiently
 531          * arcane that they don't merit per-CPU storage.  If these counters
 532          * are incremented concurrently on different CPUs, scalability will be
 533          * adversely affected -- but we don't expect them to be white-hot in a
 534          * correctly constructed enabling...
 535          */
 536         uint32_t oval, nval;
 537 
 538         do {
 539                 oval = *counter;
 540 
 541                 if ((nval = oval + 1) == 0) {
 542                         /*
 543                          * If the counter would wrap, set it to 1 -- assuring
 544                          * that the counter is never zero when we have seen
 545                          * errors.  (The counter must be 32-bits because we
 546                          * aren't guaranteed a 64-bit compare&swap operation.)
 547                          * To save this code both the infamy of being fingered
 548                          * by a priggish news story and the indignity of being
 549                          * the target of a neo-puritan witch trial, we're
 550                          * carefully avoiding any colorful description of the
 551                          * likelihood of this condition -- but suffice it to
 552                          * say that it is only slightly more likely than the
 553                          * overflow of predicate cache IDs, as discussed in
 554                          * dtrace_predicate_create().
 555                          */
 556                         nval = 1;
 557                 }
 558         } while (dtrace_cas32(counter, oval, nval) != oval);
 559 }
 560 
 561 /*
 562  * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
 563  * uint8_t, a uint16_t, a uint32_t and a uint64_t.
 564  */
 565 DTRACE_LOADFUNC(8)
 566 DTRACE_LOADFUNC(16)
 567 DTRACE_LOADFUNC(32)
 568 DTRACE_LOADFUNC(64)
 569 
 570 static int
 571 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
 572 {
 573         if (dest < mstate->dtms_scratch_base)
 574                 return (0);
 575 
 576         if (dest + size < dest)
 577                 return (0);
 578 
 579         if (dest + size > mstate->dtms_scratch_ptr)
 580                 return (0);
 581 
 582         return (1);
 583 }
 584 
 585 static int
 586 dtrace_canstore_statvar(uint64_t addr, size_t sz,
 587     dtrace_statvar_t **svars, int nsvars)
 588 {
 589         int i;
 590 
 591         for (i = 0; i < nsvars; i++) {
 592                 dtrace_statvar_t *svar = svars[i];
 593 
 594                 if (svar == NULL || svar->dtsv_size == 0)
 595                         continue;
 596 
 597                 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
 598                         return (1);
 599         }
 600 
 601         return (0);
 602 }
 603 
 604 /*
 605  * Check to see if the address is within a memory region to which a store may
 606  * be issued.  This includes the DTrace scratch areas, and any DTrace variable
 607  * region.  The caller of dtrace_canstore() is responsible for performing any
 608  * alignment checks that are needed before stores are actually executed.
 609  */
 610 static int
 611 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 612     dtrace_vstate_t *vstate)
 613 {
 614         /*
 615          * First, check to see if the address is in scratch space...
 616          */
 617         if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
 618             mstate->dtms_scratch_size))
 619                 return (1);
 620 
 621         /*
 622          * Now check to see if it's a dynamic variable.  This check will pick
 623          * up both thread-local variables and any global dynamically-allocated
 624          * variables.
 625          */
 626         if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
 627             vstate->dtvs_dynvars.dtds_size)) {
 628                 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
 629                 uintptr_t base = (uintptr_t)dstate->dtds_base +
 630                     (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
 631                 uintptr_t chunkoffs;
 632 
 633                 /*
 634                  * Before we assume that we can store here, we need to make
 635                  * sure that it isn't in our metadata -- storing to our
 636                  * dynamic variable metadata would corrupt our state.  For
 637                  * the range to not include any dynamic variable metadata,
 638                  * it must:
 639                  *
 640                  *      (1) Start above the hash table that is at the base of
 641                  *      the dynamic variable space
 642                  *
 643                  *      (2) Have a starting chunk offset that is beyond the
 644                  *      dtrace_dynvar_t that is at the base of every chunk
 645                  *
 646                  *      (3) Not span a chunk boundary
 647                  *
 648                  */
 649                 if (addr < base)
 650                         return (0);
 651 
 652                 chunkoffs = (addr - base) % dstate->dtds_chunksize;
 653 
 654                 if (chunkoffs < sizeof (dtrace_dynvar_t))
 655                         return (0);
 656 
 657                 if (chunkoffs + sz > dstate->dtds_chunksize)
 658                         return (0);
 659 
 660                 return (1);
 661         }
 662 
 663         /*
 664          * Finally, check the static local and global variables.  These checks
 665          * take the longest, so we perform them last.
 666          */
 667         if (dtrace_canstore_statvar(addr, sz,
 668             vstate->dtvs_locals, vstate->dtvs_nlocals))
 669                 return (1);
 670 
 671         if (dtrace_canstore_statvar(addr, sz,
 672             vstate->dtvs_globals, vstate->dtvs_nglobals))
 673                 return (1);
 674 
 675         return (0);
 676 }
 677 
 678 
 679 /*
 680  * Convenience routine to check to see if the address is within a memory
 681  * region in which a load may be issued given the user's privilege level;
 682  * if not, it sets the appropriate error flags and loads 'addr' into the
 683  * illegal value slot.
 684  *
 685  * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
 686  * appropriate memory access protection.
 687  */
 688 static int
 689 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 690     dtrace_vstate_t *vstate)
 691 {
 692         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
 693         file_t *fp;
 694 
 695         /*
 696          * If we hold the privilege to read from kernel memory, then
 697          * everything is readable.
 698          */
 699         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 700                 return (1);
 701 
 702         /*
 703          * You can obviously read that which you can store.
 704          */
 705         if (dtrace_canstore(addr, sz, mstate, vstate))
 706                 return (1);
 707 
 708         /*
 709          * We're allowed to read from our own string table.
 710          */
 711         if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
 712             mstate->dtms_difo->dtdo_strlen))
 713                 return (1);
 714 
 715         if (vstate->dtvs_state != NULL &&
 716             dtrace_priv_proc(vstate->dtvs_state, mstate)) {
 717                 proc_t *p;
 718 
 719                 /*
 720                  * When we have privileges to the current process, there are
 721                  * several context-related kernel structures that are safe to
 722                  * read, even absent the privilege to read from kernel memory.
 723                  * These reads are safe because these structures contain only
 724                  * state that (1) we're permitted to read, (2) is harmless or
 725                  * (3) contains pointers to additional kernel state that we're
 726                  * not permitted to read (and as such, do not present an
 727                  * opportunity for privilege escalation).  Finally (and
 728                  * critically), because of the nature of their relation with
 729                  * the current thread context, the memory associated with these
 730                  * structures cannot change over the duration of probe context,
 731                  * and it is therefore impossible for this memory to be
 732                  * deallocated and reallocated as something else while it's
 733                  * being operated upon.
 734                  */
 735                 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t)))
 736                         return (1);
 737 
 738                 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
 739                     sz, curthread->t_procp, sizeof (proc_t))) {
 740                         return (1);
 741                 }
 742 
 743                 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
 744                     curthread->t_cred, sizeof (cred_t))) {
 745                         return (1);
 746                 }
 747 
 748                 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
 749                     &(p->p_pidp->pid_id), sizeof (pid_t))) {
 750                         return (1);
 751                 }
 752 
 753                 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
 754                     curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
 755                         return (1);
 756                 }
 757         }
 758 
 759         if ((fp = mstate->dtms_getf) != NULL) {
 760                 uintptr_t psz = sizeof (void *);
 761                 vnode_t *vp;
 762                 vnodeops_t *op;
 763 
 764                 /*
 765                  * When getf() returns a file_t, the enabling is implicitly
 766                  * granted the (transient) right to read the returned file_t
 767                  * as well as the v_path and v_op->vnop_name of the underlying
 768                  * vnode.  These accesses are allowed after a successful
 769                  * getf() because the members that they refer to cannot change
 770                  * once set -- and the barrier logic in the kernel's closef()
 771                  * path assures that the file_t and its referenced vode_t
 772                  * cannot themselves be stale (that is, it impossible for
 773                  * either dtms_getf itself or its f_vnode member to reference
 774                  * freed memory).
 775                  */
 776                 if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t)))
 777                         return (1);
 778 
 779                 if ((vp = fp->f_vnode) != NULL) {
 780                         if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz))
 781                                 return (1);
 782 
 783                         if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz,
 784                             vp->v_path, strlen(vp->v_path) + 1)) {
 785                                 return (1);
 786                         }
 787 
 788                         if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz))
 789                                 return (1);
 790 
 791                         if ((op = vp->v_op) != NULL &&
 792                             DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) {
 793                                 return (1);
 794                         }
 795 
 796                         if (op != NULL && op->vnop_name != NULL &&
 797                             DTRACE_INRANGE(addr, sz, op->vnop_name,
 798                             strlen(op->vnop_name) + 1)) {
 799                                 return (1);
 800                         }
 801                 }
 802         }
 803 
 804         DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
 805         *illval = addr;
 806         return (0);
 807 }
 808 
 809 /*
 810  * Convenience routine to check to see if a given string is within a memory
 811  * region in which a load may be issued given the user's privilege level;
 812  * this exists so that we don't need to issue unnecessary dtrace_strlen()
 813  * calls in the event that the user has all privileges.
 814  */
 815 static int
 816 dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 817     dtrace_vstate_t *vstate)
 818 {
 819         size_t strsz;
 820 
 821         /*
 822          * If we hold the privilege to read from kernel memory, then
 823          * everything is readable.
 824          */
 825         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 826                 return (1);
 827 
 828         strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
 829         if (dtrace_canload(addr, strsz, mstate, vstate))
 830                 return (1);
 831 
 832         return (0);
 833 }
 834 
 835 /*
 836  * Convenience routine to check to see if a given variable is within a memory
 837  * region in which a load may be issued given the user's privilege level.
 838  */
 839 static int
 840 dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
 841     dtrace_vstate_t *vstate)
 842 {
 843         size_t sz;
 844         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
 845 
 846         /*
 847          * If we hold the privilege to read from kernel memory, then
 848          * everything is readable.
 849          */
 850         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 851                 return (1);
 852 
 853         if (type->dtdt_kind == DIF_TYPE_STRING)
 854                 sz = dtrace_strlen(src,
 855                     vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
 856         else
 857                 sz = type->dtdt_size;
 858 
 859         return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
 860 }
 861 
 862 /*
 863  * Convert a string to a signed integer using safe loads.
 864  *
 865  * NOTE: This function uses various macros from strtolctype.h to manipulate
 866  * digit values, etc -- these have all been checked to ensure they make
 867  * no additional function calls.
 868  */
 869 static int64_t
 870 dtrace_strtoll(char *input, int base, size_t limit)
 871 {
 872         uintptr_t pos = (uintptr_t)input;
 873         int64_t val = 0;
 874         int x;
 875         boolean_t neg = B_FALSE;
 876         char c, cc, ccc;
 877         uintptr_t end = pos + limit;
 878 
 879         /*
 880          * Consume any whitespace preceding digits.
 881          */
 882         while ((c = dtrace_load8(pos)) == ' ' || c == '\t')
 883                 pos++;
 884 
 885         /*
 886          * Handle an explicit sign if one is present.
 887          */
 888         if (c == '-' || c == '+') {
 889                 if (c == '-')
 890                         neg = B_TRUE;
 891                 c = dtrace_load8(++pos);
 892         }
 893 
 894         /*
 895          * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it
 896          * if present.
 897          */
 898         if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' ||
 899             cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) {
 900                 pos += 2;
 901                 c = ccc;
 902         }
 903 
 904         /*
 905          * Read in contiguous digits until the first non-digit character.
 906          */
 907         for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base;
 908             c = dtrace_load8(++pos))
 909                 val = val * base + x;
 910 
 911         return (neg ? -val : val);
 912 }
 913 
 914 /*
 915  * Compare two strings using safe loads.
 916  */
 917 static int
 918 dtrace_strncmp(char *s1, char *s2, size_t limit)
 919 {
 920         uint8_t c1, c2;
 921         volatile uint16_t *flags;
 922 
 923         if (s1 == s2 || limit == 0)
 924                 return (0);
 925 
 926         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 927 
 928         do {
 929                 if (s1 == NULL) {
 930                         c1 = '\0';
 931                 } else {
 932                         c1 = dtrace_load8((uintptr_t)s1++);
 933                 }
 934 
 935                 if (s2 == NULL) {
 936                         c2 = '\0';
 937                 } else {
 938                         c2 = dtrace_load8((uintptr_t)s2++);
 939                 }
 940 
 941                 if (c1 != c2)
 942                         return (c1 - c2);
 943         } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
 944 
 945         return (0);
 946 }
 947 
 948 /*
 949  * Compute strlen(s) for a string using safe memory accesses.  The additional
 950  * len parameter is used to specify a maximum length to ensure completion.
 951  */
 952 static size_t
 953 dtrace_strlen(const char *s, size_t lim)
 954 {
 955         uint_t len;
 956 
 957         for (len = 0; len != lim; len++) {
 958                 if (dtrace_load8((uintptr_t)s++) == '\0')
 959                         break;
 960         }
 961 
 962         return (len);
 963 }
 964 
 965 /*
 966  * Check if an address falls within a toxic region.
 967  */
 968 static int
 969 dtrace_istoxic(uintptr_t kaddr, size_t size)
 970 {
 971         uintptr_t taddr, tsize;
 972         int i;
 973 
 974         for (i = 0; i < dtrace_toxranges; i++) {
 975                 taddr = dtrace_toxrange[i].dtt_base;
 976                 tsize = dtrace_toxrange[i].dtt_limit - taddr;
 977 
 978                 if (kaddr - taddr < tsize) {
 979                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 980                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
 981                         return (1);
 982                 }
 983 
 984                 if (taddr - kaddr < size) {
 985                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 986                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
 987                         return (1);
 988                 }
 989         }
 990 
 991         return (0);
 992 }
 993 
 994 /*
 995  * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
 996  * memory specified by the DIF program.  The dst is assumed to be safe memory
 997  * that we can store to directly because it is managed by DTrace.  As with
 998  * standard bcopy, overlapping copies are handled properly.
 999  */
1000 static void
1001 dtrace_bcopy(const void *src, void *dst, size_t len)
1002 {
1003         if (len != 0) {
1004                 uint8_t *s1 = dst;
1005                 const uint8_t *s2 = src;
1006 
1007                 if (s1 <= s2) {
1008                         do {
1009                                 *s1++ = dtrace_load8((uintptr_t)s2++);
1010                         } while (--len != 0);
1011                 } else {
1012                         s2 += len;
1013                         s1 += len;
1014 
1015                         do {
1016                                 *--s1 = dtrace_load8((uintptr_t)--s2);
1017                         } while (--len != 0);
1018                 }
1019         }
1020 }
1021 
1022 /*
1023  * Copy src to dst using safe memory accesses, up to either the specified
1024  * length, or the point that a nul byte is encountered.  The src is assumed to
1025  * be unsafe memory specified by the DIF program.  The dst is assumed to be
1026  * safe memory that we can store to directly because it is managed by DTrace.
1027  * Unlike dtrace_bcopy(), overlapping regions are not handled.
1028  */
1029 static void
1030 dtrace_strcpy(const void *src, void *dst, size_t len)
1031 {
1032         if (len != 0) {
1033                 uint8_t *s1 = dst, c;
1034                 const uint8_t *s2 = src;
1035 
1036                 do {
1037                         *s1++ = c = dtrace_load8((uintptr_t)s2++);
1038                 } while (--len != 0 && c != '\0');
1039         }
1040 }
1041 
1042 /*
1043  * Copy src to dst, deriving the size and type from the specified (BYREF)
1044  * variable type.  The src is assumed to be unsafe memory specified by the DIF
1045  * program.  The dst is assumed to be DTrace variable memory that is of the
1046  * specified type; we assume that we can store to directly.
1047  */
1048 static void
1049 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
1050 {
1051         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
1052 
1053         if (type->dtdt_kind == DIF_TYPE_STRING) {
1054                 dtrace_strcpy(src, dst, type->dtdt_size);
1055         } else {
1056                 dtrace_bcopy(src, dst, type->dtdt_size);
1057         }
1058 }
1059 
1060 /*
1061  * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
1062  * unsafe memory specified by the DIF program.  The s2 data is assumed to be
1063  * safe memory that we can access directly because it is managed by DTrace.
1064  */
1065 static int
1066 dtrace_bcmp(const void *s1, const void *s2, size_t len)
1067 {
1068         volatile uint16_t *flags;
1069 
1070         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
1071 
1072         if (s1 == s2)
1073                 return (0);
1074 
1075         if (s1 == NULL || s2 == NULL)
1076                 return (1);
1077 
1078         if (s1 != s2 && len != 0) {
1079                 const uint8_t *ps1 = s1;
1080                 const uint8_t *ps2 = s2;
1081 
1082                 do {
1083                         if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
1084                                 return (1);
1085                 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
1086         }
1087         return (0);
1088 }
1089 
1090 /*
1091  * Zero the specified region using a simple byte-by-byte loop.  Note that this
1092  * is for safe DTrace-managed memory only.
1093  */
1094 static void
1095 dtrace_bzero(void *dst, size_t len)
1096 {
1097         uchar_t *cp;
1098 
1099         for (cp = dst; len != 0; len--)
1100                 *cp++ = 0;
1101 }
1102 
1103 static void
1104 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
1105 {
1106         uint64_t result[2];
1107 
1108         result[0] = addend1[0] + addend2[0];
1109         result[1] = addend1[1] + addend2[1] +
1110             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
1111 
1112         sum[0] = result[0];
1113         sum[1] = result[1];
1114 }
1115 
1116 /*
1117  * Shift the 128-bit value in a by b. If b is positive, shift left.
1118  * If b is negative, shift right.
1119  */
1120 static void
1121 dtrace_shift_128(uint64_t *a, int b)
1122 {
1123         uint64_t mask;
1124 
1125         if (b == 0)
1126                 return;
1127 
1128         if (b < 0) {
1129                 b = -b;
1130                 if (b >= 64) {
1131                         a[0] = a[1] >> (b - 64);
1132                         a[1] = 0;
1133                 } else {
1134                         a[0] >>= b;
1135                         mask = 1LL << (64 - b);
1136                         mask -= 1;
1137                         a[0] |= ((a[1] & mask) << (64 - b));
1138                         a[1] >>= b;
1139                 }
1140         } else {
1141                 if (b >= 64) {
1142                         a[1] = a[0] << (b - 64);
1143                         a[0] = 0;
1144                 } else {
1145                         a[1] <<= b;
1146                         mask = a[0] >> (64 - b);
1147                         a[1] |= mask;
1148                         a[0] <<= b;
1149                 }
1150         }
1151 }
1152 
1153 /*
1154  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1155  * use native multiplication on those, and then re-combine into the
1156  * resulting 128-bit value.
1157  *
1158  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1159  *     hi1 * hi2 << 64 +
1160  *     hi1 * lo2 << 32 +
1161  *     hi2 * lo1 << 32 +
1162  *     lo1 * lo2
1163  */
1164 static void
1165 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
1166 {
1167         uint64_t hi1, hi2, lo1, lo2;
1168         uint64_t tmp[2];
1169 
1170         hi1 = factor1 >> 32;
1171         hi2 = factor2 >> 32;
1172 
1173         lo1 = factor1 & DT_MASK_LO;
1174         lo2 = factor2 & DT_MASK_LO;
1175 
1176         product[0] = lo1 * lo2;
1177         product[1] = hi1 * hi2;
1178 
1179         tmp[0] = hi1 * lo2;
1180         tmp[1] = 0;
1181         dtrace_shift_128(tmp, 32);
1182         dtrace_add_128(product, tmp, product);
1183 
1184         tmp[0] = hi2 * lo1;
1185         tmp[1] = 0;
1186         dtrace_shift_128(tmp, 32);
1187         dtrace_add_128(product, tmp, product);
1188 }
1189 
1190 /*
1191  * This privilege check should be used by actions and subroutines to
1192  * verify that the user credentials of the process that enabled the
1193  * invoking ECB match the target credentials
1194  */
1195 static int
1196 dtrace_priv_proc_common_user(dtrace_state_t *state)
1197 {
1198         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1199 
1200         /*
1201          * We should always have a non-NULL state cred here, since if cred
1202          * is null (anonymous tracing), we fast-path bypass this routine.
1203          */
1204         ASSERT(s_cr != NULL);
1205 
1206         if ((cr = CRED()) != NULL &&
1207             s_cr->cr_uid == cr->cr_uid &&
1208             s_cr->cr_uid == cr->cr_ruid &&
1209             s_cr->cr_uid == cr->cr_suid &&
1210             s_cr->cr_gid == cr->cr_gid &&
1211             s_cr->cr_gid == cr->cr_rgid &&
1212             s_cr->cr_gid == cr->cr_sgid)
1213                 return (1);
1214 
1215         return (0);
1216 }
1217 
1218 /*
1219  * This privilege check should be used by actions and subroutines to
1220  * verify that the zone of the process that enabled the invoking ECB
1221  * matches the target credentials
1222  */
1223 static int
1224 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1225 {
1226         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1227 
1228         /*
1229          * We should always have a non-NULL state cred here, since if cred
1230          * is null (anonymous tracing), we fast-path bypass this routine.
1231          */
1232         ASSERT(s_cr != NULL);
1233 
1234         if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone)
1235                 return (1);
1236 
1237         return (0);
1238 }
1239 
1240 /*
1241  * This privilege check should be used by actions and subroutines to
1242  * verify that the process has not setuid or changed credentials.
1243  */
1244 static int
1245 dtrace_priv_proc_common_nocd()
1246 {
1247         proc_t *proc;
1248 
1249         if ((proc = ttoproc(curthread)) != NULL &&
1250             !(proc->p_flag & SNOCD))
1251                 return (1);
1252 
1253         return (0);
1254 }
1255 
1256 static int
1257 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate)
1258 {
1259         int action = state->dts_cred.dcr_action;
1260 
1261         if (!(mstate->dtms_access & DTRACE_ACCESS_PROC))
1262                 goto bad;
1263 
1264         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
1265             dtrace_priv_proc_common_zone(state) == 0)
1266                 goto bad;
1267 
1268         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
1269             dtrace_priv_proc_common_user(state) == 0)
1270                 goto bad;
1271 
1272         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
1273             dtrace_priv_proc_common_nocd() == 0)
1274                 goto bad;
1275 
1276         return (1);
1277 
1278 bad:
1279         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1280 
1281         return (0);
1282 }
1283 
1284 static int
1285 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate)
1286 {
1287         if (mstate->dtms_access & DTRACE_ACCESS_PROC) {
1288                 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1289                         return (1);
1290 
1291                 if (dtrace_priv_proc_common_zone(state) &&
1292                     dtrace_priv_proc_common_user(state) &&
1293                     dtrace_priv_proc_common_nocd())
1294                         return (1);
1295         }
1296 
1297         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1298 
1299         return (0);
1300 }
1301 
1302 static int
1303 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate)
1304 {
1305         if ((mstate->dtms_access & DTRACE_ACCESS_PROC) &&
1306             (state->dts_cred.dcr_action & DTRACE_CRA_PROC))
1307                 return (1);
1308 
1309         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1310 
1311         return (0);
1312 }
1313 
1314 static int
1315 dtrace_priv_kernel(dtrace_state_t *state)
1316 {
1317         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1318                 return (1);
1319 
1320         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1321 
1322         return (0);
1323 }
1324 
1325 static int
1326 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1327 {
1328         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1329                 return (1);
1330 
1331         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1332 
1333         return (0);
1334 }
1335 
1336 /*
1337  * Determine if the dte_cond of the specified ECB allows for processing of
1338  * the current probe to continue.  Note that this routine may allow continued
1339  * processing, but with access(es) stripped from the mstate's dtms_access
1340  * field.
1341  */
1342 static int
1343 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
1344     dtrace_ecb_t *ecb)
1345 {
1346         dtrace_probe_t *probe = ecb->dte_probe;
1347         dtrace_provider_t *prov = probe->dtpr_provider;
1348         dtrace_pops_t *pops = &prov->dtpv_pops;
1349         int mode = DTRACE_MODE_NOPRIV_DROP;
1350 
1351         ASSERT(ecb->dte_cond);
1352 
1353         if (pops->dtps_mode != NULL) {
1354                 mode = pops->dtps_mode(prov->dtpv_arg,
1355                     probe->dtpr_id, probe->dtpr_arg);
1356 
1357                 ASSERT(mode & (DTRACE_MODE_USER | DTRACE_MODE_KERNEL));
1358                 ASSERT(mode & (DTRACE_MODE_NOPRIV_RESTRICT |
1359                     DTRACE_MODE_NOPRIV_DROP));
1360         }
1361 
1362         /*
1363          * If the dte_cond bits indicate that this consumer is only allowed to
1364          * see user-mode firings of this probe, check that the probe was fired
1365          * while in a user context.  If that's not the case, use the policy
1366          * specified by the provider to determine if we drop the probe or
1367          * merely restrict operation.
1368          */
1369         if (ecb->dte_cond & DTRACE_COND_USERMODE) {
1370                 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
1371 
1372                 if (!(mode & DTRACE_MODE_USER)) {
1373                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1374                                 return (0);
1375 
1376                         mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1377                 }
1378         }
1379 
1380         /*
1381          * This is more subtle than it looks. We have to be absolutely certain
1382          * that CRED() isn't going to change out from under us so it's only
1383          * legit to examine that structure if we're in constrained situations.
1384          * Currently, the only times we'll this check is if a non-super-user
1385          * has enabled the profile or syscall providers -- providers that
1386          * allow visibility of all processes. For the profile case, the check
1387          * above will ensure that we're examining a user context.
1388          */
1389         if (ecb->dte_cond & DTRACE_COND_OWNER) {
1390                 cred_t *cr;
1391                 cred_t *s_cr = state->dts_cred.dcr_cred;
1392                 proc_t *proc;
1393 
1394                 ASSERT(s_cr != NULL);
1395 
1396                 if ((cr = CRED()) == NULL ||
1397                     s_cr->cr_uid != cr->cr_uid ||
1398                     s_cr->cr_uid != cr->cr_ruid ||
1399                     s_cr->cr_uid != cr->cr_suid ||
1400                     s_cr->cr_gid != cr->cr_gid ||
1401                     s_cr->cr_gid != cr->cr_rgid ||
1402                     s_cr->cr_gid != cr->cr_sgid ||
1403                     (proc = ttoproc(curthread)) == NULL ||
1404                     (proc->p_flag & SNOCD)) {
1405                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1406                                 return (0);
1407 
1408                         mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
1409                 }
1410         }
1411 
1412         /*
1413          * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1414          * in our zone, check to see if our mode policy is to restrict rather
1415          * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1416          * and DTRACE_ACCESS_ARGS
1417          */
1418         if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
1419                 cred_t *cr;
1420                 cred_t *s_cr = state->dts_cred.dcr_cred;
1421 
1422                 ASSERT(s_cr != NULL);
1423 
1424                 if ((cr = CRED()) == NULL ||
1425                     s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
1426                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1427                                 return (0);
1428 
1429                         mstate->dtms_access &=
1430                             ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
1431                 }
1432         }
1433 
1434         /*
1435          * By merits of being in this code path at all, we have limited
1436          * privileges.  If the provider has indicated that limited privileges
1437          * are to denote restricted operation, strip off the ability to access
1438          * arguments.
1439          */
1440         if (mode & DTRACE_MODE_LIMITEDPRIV_RESTRICT)
1441                 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1442 
1443         return (1);
1444 }
1445 
1446 /*
1447  * Note:  not called from probe context.  This function is called
1448  * asynchronously (and at a regular interval) from outside of probe context to
1449  * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
1450  * cleaning is explained in detail in <sys/dtrace_impl.h>.
1451  */
1452 void
1453 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
1454 {
1455         dtrace_dynvar_t *dirty;
1456         dtrace_dstate_percpu_t *dcpu;
1457         dtrace_dynvar_t **rinsep;
1458         int i, j, work = 0;
1459 
1460         for (i = 0; i < NCPU; i++) {
1461                 dcpu = &dstate->dtds_percpu[i];
1462                 rinsep = &dcpu->dtdsc_rinsing;
1463 
1464                 /*
1465                  * If the dirty list is NULL, there is no dirty work to do.
1466                  */
1467                 if (dcpu->dtdsc_dirty == NULL)
1468                         continue;
1469 
1470                 if (dcpu->dtdsc_rinsing != NULL) {
1471                         /*
1472                          * If the rinsing list is non-NULL, then it is because
1473                          * this CPU was selected to accept another CPU's
1474                          * dirty list -- and since that time, dirty buffers
1475                          * have accumulated.  This is a highly unlikely
1476                          * condition, but we choose to ignore the dirty
1477                          * buffers -- they'll be picked up a future cleanse.
1478                          */
1479                         continue;
1480                 }
1481 
1482                 if (dcpu->dtdsc_clean != NULL) {
1483                         /*
1484                          * If the clean list is non-NULL, then we're in a
1485                          * situation where a CPU has done deallocations (we
1486                          * have a non-NULL dirty list) but no allocations (we
1487                          * also have a non-NULL clean list).  We can't simply
1488                          * move the dirty list into the clean list on this
1489                          * CPU, yet we also don't want to allow this condition
1490                          * to persist, lest a short clean list prevent a
1491                          * massive dirty list from being cleaned (which in
1492                          * turn could lead to otherwise avoidable dynamic
1493                          * drops).  To deal with this, we look for some CPU
1494                          * with a NULL clean list, NULL dirty list, and NULL
1495                          * rinsing list -- and then we borrow this CPU to
1496                          * rinse our dirty list.
1497                          */
1498                         for (j = 0; j < NCPU; j++) {
1499                                 dtrace_dstate_percpu_t *rinser;
1500 
1501                                 rinser = &dstate->dtds_percpu[j];
1502 
1503                                 if (rinser->dtdsc_rinsing != NULL)
1504                                         continue;
1505 
1506                                 if (rinser->dtdsc_dirty != NULL)
1507                                         continue;
1508 
1509                                 if (rinser->dtdsc_clean != NULL)
1510                                         continue;
1511 
1512                                 rinsep = &rinser->dtdsc_rinsing;
1513                                 break;
1514                         }
1515 
1516                         if (j == NCPU) {
1517                                 /*
1518                                  * We were unable to find another CPU that
1519                                  * could accept this dirty list -- we are
1520                                  * therefore unable to clean it now.
1521                                  */
1522                                 dtrace_dynvar_failclean++;
1523                                 continue;
1524                         }
1525                 }
1526 
1527                 work = 1;
1528 
1529                 /*
1530                  * Atomically move the dirty list aside.
1531                  */
1532                 do {
1533                         dirty = dcpu->dtdsc_dirty;
1534 
1535                         /*
1536                          * Before we zap the dirty list, set the rinsing list.
1537                          * (This allows for a potential assertion in
1538                          * dtrace_dynvar():  if a free dynamic variable appears
1539                          * on a hash chain, either the dirty list or the
1540                          * rinsing list for some CPU must be non-NULL.)
1541                          */
1542                         *rinsep = dirty;
1543                         dtrace_membar_producer();
1544                 } while (dtrace_casptr(&dcpu->dtdsc_dirty,
1545                     dirty, NULL) != dirty);
1546         }
1547 
1548         if (!work) {
1549                 /*
1550                  * We have no work to do; we can simply return.
1551                  */
1552                 return;
1553         }
1554 
1555         dtrace_sync();
1556 
1557         for (i = 0; i < NCPU; i++) {
1558                 dcpu = &dstate->dtds_percpu[i];
1559 
1560                 if (dcpu->dtdsc_rinsing == NULL)
1561                         continue;
1562 
1563                 /*
1564                  * We are now guaranteed that no hash chain contains a pointer
1565                  * into this dirty list; we can make it clean.
1566                  */
1567                 ASSERT(dcpu->dtdsc_clean == NULL);
1568                 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
1569                 dcpu->dtdsc_rinsing = NULL;
1570         }
1571 
1572         /*
1573          * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1574          * sure that all CPUs have seen all of the dtdsc_clean pointers.
1575          * This prevents a race whereby a CPU incorrectly decides that
1576          * the state should be something other than DTRACE_DSTATE_CLEAN
1577          * after dtrace_dynvar_clean() has completed.
1578          */
1579         dtrace_sync();
1580 
1581         dstate->dtds_state = DTRACE_DSTATE_CLEAN;
1582 }
1583 
1584 /*
1585  * Depending on the value of the op parameter, this function looks-up,
1586  * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
1587  * allocation is requested, this function will return a pointer to a
1588  * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1589  * variable can be allocated.  If NULL is returned, the appropriate counter
1590  * will be incremented.
1591  */
1592 dtrace_dynvar_t *
1593 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
1594     dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
1595     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
1596 {
1597         uint64_t hashval = DTRACE_DYNHASH_VALID;
1598         dtrace_dynhash_t *hash = dstate->dtds_hash;
1599         dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
1600         processorid_t me = CPU->cpu_id, cpu = me;
1601         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
1602         size_t bucket, ksize;
1603         size_t chunksize = dstate->dtds_chunksize;
1604         uintptr_t kdata, lock, nstate;
1605         uint_t i;
1606 
1607         ASSERT(nkeys != 0);
1608 
1609         /*
1610          * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
1611          * algorithm.  For the by-value portions, we perform the algorithm in
1612          * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
1613          * bit, and seems to have only a minute effect on distribution.  For
1614          * the by-reference data, we perform "One-at-a-time" iterating (safely)
1615          * over each referenced byte.  It's painful to do this, but it's much
1616          * better than pathological hash distribution.  The efficacy of the
1617          * hashing algorithm (and a comparison with other algorithms) may be
1618          * found by running the ::dtrace_dynstat MDB dcmd.
1619          */
1620         for (i = 0; i < nkeys; i++) {
1621                 if (key[i].dttk_size == 0) {
1622                         uint64_t val = key[i].dttk_value;
1623 
1624                         hashval += (val >> 48) & 0xffff;
1625                         hashval += (hashval << 10);
1626                         hashval ^= (hashval >> 6);
1627 
1628                         hashval += (val >> 32) & 0xffff;
1629                         hashval += (hashval << 10);
1630                         hashval ^= (hashval >> 6);
1631 
1632                         hashval += (val >> 16) & 0xffff;
1633                         hashval += (hashval << 10);
1634                         hashval ^= (hashval >> 6);
1635 
1636                         hashval += val & 0xffff;
1637                         hashval += (hashval << 10);
1638                         hashval ^= (hashval >> 6);
1639                 } else {
1640                         /*
1641                          * This is incredibly painful, but it beats the hell
1642                          * out of the alternative.
1643                          */
1644                         uint64_t j, size = key[i].dttk_size;
1645                         uintptr_t base = (uintptr_t)key[i].dttk_value;
1646 
1647                         if (!dtrace_canload(base, size, mstate, vstate))
1648                                 break;
1649 
1650                         for (j = 0; j < size; j++) {
1651                                 hashval += dtrace_load8(base + j);
1652                                 hashval += (hashval << 10);
1653                                 hashval ^= (hashval >> 6);
1654                         }
1655                 }
1656         }
1657 
1658         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
1659                 return (NULL);
1660 
1661         hashval += (hashval << 3);
1662         hashval ^= (hashval >> 11);
1663         hashval += (hashval << 15);
1664 
1665         /*
1666          * There is a remote chance (ideally, 1 in 2^31) that our hashval
1667          * comes out to be one of our two sentinel hash values.  If this
1668          * actually happens, we set the hashval to be a value known to be a
1669          * non-sentinel value.
1670          */
1671         if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
1672                 hashval = DTRACE_DYNHASH_VALID;
1673 
1674         /*
1675          * Yes, it's painful to do a divide here.  If the cycle count becomes
1676          * important here, tricks can be pulled to reduce it.  (However, it's
1677          * critical that hash collisions be kept to an absolute minimum;
1678          * they're much more painful than a divide.)  It's better to have a
1679          * solution that generates few collisions and still keeps things
1680          * relatively simple.
1681          */
1682         bucket = hashval % dstate->dtds_hashsize;
1683 
1684         if (op == DTRACE_DYNVAR_DEALLOC) {
1685                 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1686 
1687                 for (;;) {
1688                         while ((lock = *lockp) & 1)
1689                                 continue;
1690 
1691                         if (dtrace_casptr((void *)lockp,
1692                             (void *)lock, (void *)(lock + 1)) == (void *)lock)
1693                                 break;
1694                 }
1695 
1696                 dtrace_membar_producer();
1697         }
1698 
1699 top:
1700         prev = NULL;
1701         lock = hash[bucket].dtdh_lock;
1702 
1703         dtrace_membar_consumer();
1704 
1705         start = hash[bucket].dtdh_chain;
1706         ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
1707             start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
1708             op != DTRACE_DYNVAR_DEALLOC));
1709 
1710         for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1711                 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1712                 dtrace_key_t *dkey = &dtuple->dtt_key[0];
1713 
1714                 if (dvar->dtdv_hashval != hashval) {
1715                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
1716                                 /*
1717                                  * We've reached the sink, and therefore the
1718                                  * end of the hash chain; we can kick out of
1719                                  * the loop knowing that we have seen a valid
1720                                  * snapshot of state.
1721                                  */
1722                                 ASSERT(dvar->dtdv_next == NULL);
1723                                 ASSERT(dvar == &dtrace_dynhash_sink);
1724                                 break;
1725                         }
1726 
1727                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
1728                                 /*
1729                                  * We've gone off the rails:  somewhere along
1730                                  * the line, one of the members of this hash
1731                                  * chain was deleted.  Note that we could also
1732                                  * detect this by simply letting this loop run
1733                                  * to completion, as we would eventually hit
1734                                  * the end of the dirty list.  However, we
1735                                  * want to avoid running the length of the
1736                                  * dirty list unnecessarily (it might be quite
1737                                  * long), so we catch this as early as
1738                                  * possible by detecting the hash marker.  In
1739                                  * this case, we simply set dvar to NULL and
1740                                  * break; the conditional after the loop will
1741                                  * send us back to top.
1742                                  */
1743                                 dvar = NULL;
1744                                 break;
1745                         }
1746 
1747                         goto next;
1748                 }
1749 
1750                 if (dtuple->dtt_nkeys != nkeys)
1751                         goto next;
1752 
1753                 for (i = 0; i < nkeys; i++, dkey++) {
1754                         if (dkey->dttk_size != key[i].dttk_size)
1755                                 goto next; /* size or type mismatch */
1756 
1757                         if (dkey->dttk_size != 0) {
1758                                 if (dtrace_bcmp(
1759                                     (void *)(uintptr_t)key[i].dttk_value,
1760                                     (void *)(uintptr_t)dkey->dttk_value,
1761                                     dkey->dttk_size))
1762                                         goto next;
1763                         } else {
1764                                 if (dkey->dttk_value != key[i].dttk_value)
1765                                         goto next;
1766                         }
1767                 }
1768 
1769                 if (op != DTRACE_DYNVAR_DEALLOC)
1770                         return (dvar);
1771 
1772                 ASSERT(dvar->dtdv_next == NULL ||
1773                     dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
1774 
1775                 if (prev != NULL) {
1776                         ASSERT(hash[bucket].dtdh_chain != dvar);
1777                         ASSERT(start != dvar);
1778                         ASSERT(prev->dtdv_next == dvar);
1779                         prev->dtdv_next = dvar->dtdv_next;
1780                 } else {
1781                         if (dtrace_casptr(&hash[bucket].dtdh_chain,
1782                             start, dvar->dtdv_next) != start) {
1783                                 /*
1784                                  * We have failed to atomically swing the
1785                                  * hash table head pointer, presumably because
1786                                  * of a conflicting allocation on another CPU.
1787                                  * We need to reread the hash chain and try
1788                                  * again.
1789                                  */
1790                                 goto top;
1791                         }
1792                 }
1793 
1794                 dtrace_membar_producer();
1795 
1796                 /*
1797                  * Now set the hash value to indicate that it's free.
1798                  */
1799                 ASSERT(hash[bucket].dtdh_chain != dvar);
1800                 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1801 
1802                 dtrace_membar_producer();
1803 
1804                 /*
1805                  * Set the next pointer to point at the dirty list, and
1806                  * atomically swing the dirty pointer to the newly freed dvar.
1807                  */
1808                 do {
1809                         next = dcpu->dtdsc_dirty;
1810                         dvar->dtdv_next = next;
1811                 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1812 
1813                 /*
1814                  * Finally, unlock this hash bucket.
1815                  */
1816                 ASSERT(hash[bucket].dtdh_lock == lock);
1817                 ASSERT(lock & 1);
1818                 hash[bucket].dtdh_lock++;
1819 
1820                 return (NULL);
1821 next:
1822                 prev = dvar;
1823                 continue;
1824         }
1825 
1826         if (dvar == NULL) {
1827                 /*
1828                  * If dvar is NULL, it is because we went off the rails:
1829                  * one of the elements that we traversed in the hash chain
1830                  * was deleted while we were traversing it.  In this case,
1831                  * we assert that we aren't doing a dealloc (deallocs lock
1832                  * the hash bucket to prevent themselves from racing with
1833                  * one another), and retry the hash chain traversal.
1834                  */
1835                 ASSERT(op != DTRACE_DYNVAR_DEALLOC);
1836                 goto top;
1837         }
1838 
1839         if (op != DTRACE_DYNVAR_ALLOC) {
1840                 /*
1841                  * If we are not to allocate a new variable, we want to
1842                  * return NULL now.  Before we return, check that the value
1843                  * of the lock word hasn't changed.  If it has, we may have
1844                  * seen an inconsistent snapshot.
1845                  */
1846                 if (op == DTRACE_DYNVAR_NOALLOC) {
1847                         if (hash[bucket].dtdh_lock != lock)
1848                                 goto top;
1849                 } else {
1850                         ASSERT(op == DTRACE_DYNVAR_DEALLOC);
1851                         ASSERT(hash[bucket].dtdh_lock == lock);
1852                         ASSERT(lock & 1);
1853                         hash[bucket].dtdh_lock++;
1854                 }
1855 
1856                 return (NULL);
1857         }
1858 
1859         /*
1860          * We need to allocate a new dynamic variable.  The size we need is the
1861          * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
1862          * size of any auxiliary key data (rounded up to 8-byte alignment) plus
1863          * the size of any referred-to data (dsize).  We then round the final
1864          * size up to the chunksize for allocation.
1865          */
1866         for (ksize = 0, i = 0; i < nkeys; i++)
1867                 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
1868 
1869         /*
1870          * This should be pretty much impossible, but could happen if, say,
1871          * strange DIF specified the tuple.  Ideally, this should be an
1872          * assertion and not an error condition -- but that requires that the
1873          * chunksize calculation in dtrace_difo_chunksize() be absolutely
1874          * bullet-proof.  (That is, it must not be able to be fooled by
1875          * malicious DIF.)  Given the lack of backwards branches in DIF,
1876          * solving this would presumably not amount to solving the Halting
1877          * Problem -- but it still seems awfully hard.
1878          */
1879         if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
1880             ksize + dsize > chunksize) {
1881                 dcpu->dtdsc_drops++;
1882                 return (NULL);
1883         }
1884 
1885         nstate = DTRACE_DSTATE_EMPTY;
1886 
1887         do {
1888 retry:
1889                 free = dcpu->dtdsc_free;
1890 
1891                 if (free == NULL) {
1892                         dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
1893                         void *rval;
1894 
1895                         if (clean == NULL) {
1896                                 /*
1897                                  * We're out of dynamic variable space on
1898                                  * this CPU.  Unless we have tried all CPUs,
1899                                  * we'll try to allocate from a different
1900                                  * CPU.
1901                                  */
1902                                 switch (dstate->dtds_state) {
1903                                 case DTRACE_DSTATE_CLEAN: {
1904                                         void *sp = &dstate->dtds_state;
1905 
1906                                         if (++cpu >= NCPU)
1907                                                 cpu = 0;
1908 
1909                                         if (dcpu->dtdsc_dirty != NULL &&
1910                                             nstate == DTRACE_DSTATE_EMPTY)
1911                                                 nstate = DTRACE_DSTATE_DIRTY;
1912 
1913                                         if (dcpu->dtdsc_rinsing != NULL)
1914                                                 nstate = DTRACE_DSTATE_RINSING;
1915 
1916                                         dcpu = &dstate->dtds_percpu[cpu];
1917 
1918                                         if (cpu != me)
1919                                                 goto retry;
1920 
1921                                         (void) dtrace_cas32(sp,
1922                                             DTRACE_DSTATE_CLEAN, nstate);
1923 
1924                                         /*
1925                                          * To increment the correct bean
1926                                          * counter, take another lap.
1927                                          */
1928                                         goto retry;
1929                                 }
1930 
1931                                 case DTRACE_DSTATE_DIRTY:
1932                                         dcpu->dtdsc_dirty_drops++;
1933                                         break;
1934 
1935                                 case DTRACE_DSTATE_RINSING:
1936                                         dcpu->dtdsc_rinsing_drops++;
1937                                         break;
1938 
1939                                 case DTRACE_DSTATE_EMPTY:
1940                                         dcpu->dtdsc_drops++;
1941                                         break;
1942                                 }
1943 
1944                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
1945                                 return (NULL);
1946                         }
1947 
1948                         /*
1949                          * The clean list appears to be non-empty.  We want to
1950                          * move the clean list to the free list; we start by
1951                          * moving the clean pointer aside.
1952                          */
1953                         if (dtrace_casptr(&dcpu->dtdsc_clean,
1954                             clean, NULL) != clean) {
1955                                 /*
1956                                  * We are in one of two situations:
1957                                  *
1958                                  *  (a) The clean list was switched to the
1959                                  *      free list by another CPU.
1960                                  *
1961                                  *  (b) The clean list was added to by the
1962                                  *      cleansing cyclic.
1963                                  *
1964                                  * In either of these situations, we can
1965                                  * just reattempt the free list allocation.
1966                                  */
1967                                 goto retry;
1968                         }
1969 
1970                         ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
1971 
1972                         /*
1973                          * Now we'll move the clean list to our free list.
1974                          * It's impossible for this to fail:  the only way
1975                          * the free list can be updated is through this
1976                          * code path, and only one CPU can own the clean list.
1977                          * Thus, it would only be possible for this to fail if
1978                          * this code were racing with dtrace_dynvar_clean().
1979                          * (That is, if dtrace_dynvar_clean() updated the clean
1980                          * list, and we ended up racing to update the free
1981                          * list.)  This race is prevented by the dtrace_sync()
1982                          * in dtrace_dynvar_clean() -- which flushes the
1983                          * owners of the clean lists out before resetting
1984                          * the clean lists.
1985                          */
1986                         dcpu = &dstate->dtds_percpu[me];
1987                         rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
1988                         ASSERT(rval == NULL);
1989                         goto retry;
1990                 }
1991 
1992                 dvar = free;
1993                 new_free = dvar->dtdv_next;
1994         } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
1995 
1996         /*
1997          * We have now allocated a new chunk.  We copy the tuple keys into the
1998          * tuple array and copy any referenced key data into the data space
1999          * following the tuple array.  As we do this, we relocate dttk_value
2000          * in the final tuple to point to the key data address in the chunk.
2001          */
2002         kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
2003         dvar->dtdv_data = (void *)(kdata + ksize);
2004         dvar->dtdv_tuple.dtt_nkeys = nkeys;
2005 
2006         for (i = 0; i < nkeys; i++) {
2007                 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
2008                 size_t kesize = key[i].dttk_size;
2009 
2010                 if (kesize != 0) {
2011                         dtrace_bcopy(
2012                             (const void *)(uintptr_t)key[i].dttk_value,
2013                             (void *)kdata, kesize);
2014                         dkey->dttk_value = kdata;
2015                         kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
2016                 } else {
2017                         dkey->dttk_value = key[i].dttk_value;
2018                 }
2019 
2020                 dkey->dttk_size = kesize;
2021         }
2022 
2023         ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
2024         dvar->dtdv_hashval = hashval;
2025         dvar->dtdv_next = start;
2026 
2027         if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
2028                 return (dvar);
2029 
2030         /*
2031          * The cas has failed.  Either another CPU is adding an element to
2032          * this hash chain, or another CPU is deleting an element from this
2033          * hash chain.  The simplest way to deal with both of these cases
2034          * (though not necessarily the most efficient) is to free our
2035          * allocated block and tail-call ourselves.  Note that the free is
2036          * to the dirty list and _not_ to the free list.  This is to prevent
2037          * races with allocators, above.
2038          */
2039         dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
2040 
2041         dtrace_membar_producer();
2042 
2043         do {
2044                 free = dcpu->dtdsc_dirty;
2045                 dvar->dtdv_next = free;
2046         } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
2047 
2048         return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
2049 }
2050 
2051 /*ARGSUSED*/
2052 static void
2053 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
2054 {
2055         if ((int64_t)nval < (int64_t)*oval)
2056                 *oval = nval;
2057 }
2058 
2059 /*ARGSUSED*/
2060 static void
2061 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
2062 {
2063         if ((int64_t)nval > (int64_t)*oval)
2064                 *oval = nval;
2065 }
2066 
2067 static void
2068 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
2069 {
2070         int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
2071         int64_t val = (int64_t)nval;
2072 
2073         if (val < 0) {
2074                 for (i = 0; i < zero; i++) {
2075                         if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
2076                                 quanta[i] += incr;
2077                                 return;
2078                         }
2079                 }
2080         } else {
2081                 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
2082                         if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
2083                                 quanta[i - 1] += incr;
2084                                 return;
2085                         }
2086                 }
2087 
2088                 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
2089                 return;
2090         }
2091 
2092         ASSERT(0);
2093 }
2094 
2095 static void
2096 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
2097 {
2098         uint64_t arg = *lquanta++;
2099         int32_t base = DTRACE_LQUANTIZE_BASE(arg);
2100         uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
2101         uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
2102         int32_t val = (int32_t)nval, level;
2103 
2104         ASSERT(step != 0);
2105         ASSERT(levels != 0);
2106 
2107         if (val < base) {
2108                 /*
2109                  * This is an underflow.
2110                  */
2111                 lquanta[0] += incr;
2112                 return;
2113         }
2114 
2115         level = (val - base) / step;
2116 
2117         if (level < levels) {
2118                 lquanta[level + 1] += incr;
2119                 return;
2120         }
2121 
2122         /*
2123          * This is an overflow.
2124          */
2125         lquanta[levels + 1] += incr;
2126 }
2127 
2128 static int
2129 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
2130     uint16_t high, uint16_t nsteps, int64_t value)
2131 {
2132         int64_t this = 1, last, next;
2133         int base = 1, order;
2134 
2135         ASSERT(factor <= nsteps);
2136         ASSERT(nsteps % factor == 0);
2137 
2138         for (order = 0; order < low; order++)
2139                 this *= factor;
2140 
2141         /*
2142          * If our value is less than our factor taken to the power of the
2143          * low order of magnitude, it goes into the zeroth bucket.
2144          */
2145         if (value < (last = this))
2146                 return (0);
2147 
2148         for (this *= factor; order <= high; order++) {
2149                 int nbuckets = this > nsteps ? nsteps : this;
2150 
2151                 if ((next = this * factor) < this) {
2152                         /*
2153                          * We should not generally get log/linear quantizations
2154                          * with a high magnitude that allows 64-bits to
2155                          * overflow, but we nonetheless protect against this
2156                          * by explicitly checking for overflow, and clamping
2157                          * our value accordingly.
2158                          */
2159                         value = this - 1;
2160                 }
2161 
2162                 if (value < this) {
2163                         /*
2164                          * If our value lies within this order of magnitude,
2165                          * determine its position by taking the offset within
2166                          * the order of magnitude, dividing by the bucket
2167                          * width, and adding to our (accumulated) base.
2168                          */
2169                         return (base + (value - last) / (this / nbuckets));
2170                 }
2171 
2172                 base += nbuckets - (nbuckets / factor);
2173                 last = this;
2174                 this = next;
2175         }
2176 
2177         /*
2178          * Our value is greater than or equal to our factor taken to the
2179          * power of one plus the high magnitude -- return the top bucket.
2180          */
2181         return (base);
2182 }
2183 
2184 static void
2185 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
2186 {
2187         uint64_t arg = *llquanta++;
2188         uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2189         uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2190         uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2191         uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2192 
2193         llquanta[dtrace_aggregate_llquantize_bucket(factor,
2194             low, high, nsteps, nval)] += incr;
2195 }
2196 
2197 /*ARGSUSED*/
2198 static void
2199 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
2200 {
2201         data[0]++;
2202         data[1] += nval;
2203 }
2204 
2205 /*ARGSUSED*/
2206 static void
2207 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
2208 {
2209         int64_t snval = (int64_t)nval;
2210         uint64_t tmp[2];
2211 
2212         data[0]++;
2213         data[1] += nval;
2214 
2215         /*
2216          * What we want to say here is:
2217          *
2218          * data[2] += nval * nval;
2219          *
2220          * But given that nval is 64-bit, we could easily overflow, so
2221          * we do this as 128-bit arithmetic.
2222          */
2223         if (snval < 0)
2224                 snval = -snval;
2225 
2226         dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
2227         dtrace_add_128(data + 2, tmp, data + 2);
2228 }
2229 
2230 /*ARGSUSED*/
2231 static void
2232 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
2233 {
2234         *oval = *oval + 1;
2235 }
2236 
2237 /*ARGSUSED*/
2238 static void
2239 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
2240 {
2241         *oval += nval;
2242 }
2243 
2244 /*
2245  * Aggregate given the tuple in the principal data buffer, and the aggregating
2246  * action denoted by the specified dtrace_aggregation_t.  The aggregation
2247  * buffer is specified as the buf parameter.  This routine does not return
2248  * failure; if there is no space in the aggregation buffer, the data will be
2249  * dropped, and a corresponding counter incremented.
2250  */
2251 static void
2252 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
2253     intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
2254 {
2255         dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
2256         uint32_t i, ndx, size, fsize;
2257         uint32_t align = sizeof (uint64_t) - 1;
2258         dtrace_aggbuffer_t *agb;
2259         dtrace_aggkey_t *key;
2260         uint32_t hashval = 0, limit, isstr;
2261         caddr_t tomax, data, kdata;
2262         dtrace_actkind_t action;
2263         dtrace_action_t *act;
2264         uintptr_t offs;
2265 
2266         if (buf == NULL)
2267                 return;
2268 
2269         if (!agg->dtag_hasarg) {
2270                 /*
2271                  * Currently, only quantize() and lquantize() take additional
2272                  * arguments, and they have the same semantics:  an increment
2273                  * value that defaults to 1 when not present.  If additional
2274                  * aggregating actions take arguments, the setting of the
2275                  * default argument value will presumably have to become more
2276                  * sophisticated...
2277                  */
2278                 arg = 1;
2279         }
2280 
2281         action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
2282         size = rec->dtrd_offset - agg->dtag_base;
2283         fsize = size + rec->dtrd_size;
2284 
2285         ASSERT(dbuf->dtb_tomax != NULL);
2286         data = dbuf->dtb_tomax + offset + agg->dtag_base;
2287 
2288         if ((tomax = buf->dtb_tomax) == NULL) {
2289                 dtrace_buffer_drop(buf);
2290                 return;
2291         }
2292 
2293         /*
2294          * The metastructure is always at the bottom of the buffer.
2295          */
2296         agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
2297             sizeof (dtrace_aggbuffer_t));
2298 
2299         if (buf->dtb_offset == 0) {
2300                 /*
2301                  * We just kludge up approximately 1/8th of the size to be
2302                  * buckets.  If this guess ends up being routinely
2303                  * off-the-mark, we may need to dynamically readjust this
2304                  * based on past performance.
2305                  */
2306                 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
2307 
2308                 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
2309                     (uintptr_t)tomax || hashsize == 0) {
2310                         /*
2311                          * We've been given a ludicrously small buffer;
2312                          * increment our drop count and leave.
2313                          */
2314                         dtrace_buffer_drop(buf);
2315                         return;
2316                 }
2317 
2318                 /*
2319                  * And now, a pathetic attempt to try to get a an odd (or
2320                  * perchance, a prime) hash size for better hash distribution.
2321                  */
2322                 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
2323                         hashsize -= DTRACE_AGGHASHSIZE_SLEW;
2324 
2325                 agb->dtagb_hashsize = hashsize;
2326                 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
2327                     agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
2328                 agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
2329 
2330                 for (i = 0; i < agb->dtagb_hashsize; i++)
2331                         agb->dtagb_hash[i] = NULL;
2332         }
2333 
2334         ASSERT(agg->dtag_first != NULL);
2335         ASSERT(agg->dtag_first->dta_intuple);
2336 
2337         /*
2338          * Calculate the hash value based on the key.  Note that we _don't_
2339          * include the aggid in the hashing (but we will store it as part of
2340          * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
2341          * algorithm: a simple, quick algorithm that has no known funnels, and
2342          * gets good distribution in practice.  The efficacy of the hashing
2343          * algorithm (and a comparison with other algorithms) may be found by
2344          * running the ::dtrace_aggstat MDB dcmd.
2345          */
2346         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2347                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2348                 limit = i + act->dta_rec.dtrd_size;
2349                 ASSERT(limit <= size);
2350                 isstr = DTRACEACT_ISSTRING(act);
2351 
2352                 for (; i < limit; i++) {
2353                         hashval += data[i];
2354                         hashval += (hashval << 10);
2355                         hashval ^= (hashval >> 6);
2356 
2357                         if (isstr && data[i] == '\0')
2358                                 break;
2359                 }
2360         }
2361 
2362         hashval += (hashval << 3);
2363         hashval ^= (hashval >> 11);
2364         hashval += (hashval << 15);
2365 
2366         /*
2367          * Yes, the divide here is expensive -- but it's generally the least
2368          * of the performance issues given the amount of data that we iterate
2369          * over to compute hash values, compare data, etc.
2370          */
2371         ndx = hashval % agb->dtagb_hashsize;
2372 
2373         for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
2374                 ASSERT((caddr_t)key >= tomax);
2375                 ASSERT((caddr_t)key < tomax + buf->dtb_size);
2376 
2377                 if (hashval != key->dtak_hashval || key->dtak_size != size)
2378                         continue;
2379 
2380                 kdata = key->dtak_data;
2381                 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
2382 
2383                 for (act = agg->dtag_first; act->dta_intuple;
2384                     act = act->dta_next) {
2385                         i = act->dta_rec.dtrd_offset - agg->dtag_base;
2386                         limit = i + act->dta_rec.dtrd_size;
2387                         ASSERT(limit <= size);
2388                         isstr = DTRACEACT_ISSTRING(act);
2389 
2390                         for (; i < limit; i++) {
2391                                 if (kdata[i] != data[i])
2392                                         goto next;
2393 
2394                                 if (isstr && data[i] == '\0')
2395                                         break;
2396                         }
2397                 }
2398 
2399                 if (action != key->dtak_action) {
2400                         /*
2401                          * We are aggregating on the same value in the same
2402                          * aggregation with two different aggregating actions.
2403                          * (This should have been picked up in the compiler,
2404                          * so we may be dealing with errant or devious DIF.)
2405                          * This is an error condition; we indicate as much,
2406                          * and return.
2407                          */
2408                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2409                         return;
2410                 }
2411 
2412                 /*
2413                  * This is a hit:  we need to apply the aggregator to
2414                  * the value at this key.
2415                  */
2416                 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
2417                 return;
2418 next:
2419                 continue;
2420         }
2421 
2422         /*
2423          * We didn't find it.  We need to allocate some zero-filled space,
2424          * link it into the hash table appropriately, and apply the aggregator
2425          * to the (zero-filled) value.
2426          */
2427         offs = buf->dtb_offset;
2428         while (offs & (align - 1))
2429                 offs += sizeof (uint32_t);
2430 
2431         /*
2432          * If we don't have enough room to both allocate a new key _and_
2433          * its associated data, increment the drop count and return.
2434          */
2435         if ((uintptr_t)tomax + offs + fsize >
2436             agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
2437                 dtrace_buffer_drop(buf);
2438                 return;
2439         }
2440 
2441         /*CONSTCOND*/
2442         ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
2443         key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
2444         agb->dtagb_free -= sizeof (dtrace_aggkey_t);
2445 
2446         key->dtak_data = kdata = tomax + offs;
2447         buf->dtb_offset = offs + fsize;
2448 
2449         /*
2450          * Now copy the data across.
2451          */
2452         *((dtrace_aggid_t *)kdata) = agg->dtag_id;
2453 
2454         for (i = sizeof (dtrace_aggid_t); i < size; i++)
2455                 kdata[i] = data[i];
2456 
2457         /*
2458          * Because strings are not zeroed out by default, we need to iterate
2459          * looking for actions that store strings, and we need to explicitly
2460          * pad these strings out with zeroes.
2461          */
2462         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2463                 int nul;
2464 
2465                 if (!DTRACEACT_ISSTRING(act))
2466                         continue;
2467 
2468                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2469                 limit = i + act->dta_rec.dtrd_size;
2470                 ASSERT(limit <= size);
2471 
2472                 for (nul = 0; i < limit; i++) {
2473                         if (nul) {
2474                                 kdata[i] = '\0';
2475                                 continue;
2476                         }
2477 
2478                         if (data[i] != '\0')
2479                                 continue;
2480 
2481                         nul = 1;
2482                 }
2483         }
2484 
2485         for (i = size; i < fsize; i++)
2486                 kdata[i] = 0;
2487 
2488         key->dtak_hashval = hashval;
2489         key->dtak_size = size;
2490         key->dtak_action = action;
2491         key->dtak_next = agb->dtagb_hash[ndx];
2492         agb->dtagb_hash[ndx] = key;
2493 
2494         /*
2495          * Finally, apply the aggregator.
2496          */
2497         *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
2498         agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
2499 }
2500 
2501 /*
2502  * Given consumer state, this routine finds a speculation in the INACTIVE
2503  * state and transitions it into the ACTIVE state.  If there is no speculation
2504  * in the INACTIVE state, 0 is returned.  In this case, no error counter is
2505  * incremented -- it is up to the caller to take appropriate action.
2506  */
2507 static int
2508 dtrace_speculation(dtrace_state_t *state)
2509 {
2510         int i = 0;
2511         dtrace_speculation_state_t current;
2512         uint32_t *stat = &state->dts_speculations_unavail, count;
2513 
2514         while (i < state->dts_nspeculations) {
2515                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2516 
2517                 current = spec->dtsp_state;
2518 
2519                 if (current != DTRACESPEC_INACTIVE) {
2520                         if (current == DTRACESPEC_COMMITTINGMANY ||
2521                             current == DTRACESPEC_COMMITTING ||
2522                             current == DTRACESPEC_DISCARDING)
2523                                 stat = &state->dts_speculations_busy;
2524                         i++;
2525                         continue;
2526                 }
2527 
2528                 if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2529                     current, DTRACESPEC_ACTIVE) == current)
2530                         return (i + 1);
2531         }
2532 
2533         /*
2534          * We couldn't find a speculation.  If we found as much as a single
2535          * busy speculation buffer, we'll attribute this failure as "busy"
2536          * instead of "unavail".
2537          */
2538         do {
2539                 count = *stat;
2540         } while (dtrace_cas32(stat, count, count + 1) != count);
2541 
2542         return (0);
2543 }
2544 
2545 /*
2546  * This routine commits an active speculation.  If the specified speculation
2547  * is not in a valid state to perform a commit(), this routine will silently do
2548  * nothing.  The state of the specified speculation is transitioned according
2549  * to the state transition diagram outlined in <sys/dtrace_impl.h>
2550  */
2551 static void
2552 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2553     dtrace_specid_t which)
2554 {
2555         dtrace_speculation_t *spec;
2556         dtrace_buffer_t *src, *dest;
2557         uintptr_t daddr, saddr, dlimit, slimit;
2558         dtrace_speculation_state_t current, new;
2559         intptr_t offs;
2560         uint64_t timestamp;
2561 
2562         if (which == 0)
2563                 return;
2564 
2565         if (which > state->dts_nspeculations) {
2566                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2567                 return;
2568         }
2569 
2570         spec = &state->dts_speculations[which - 1];
2571         src = &spec->dtsp_buffer[cpu];
2572         dest = &state->dts_buffer[cpu];
2573 
2574         do {
2575                 current = spec->dtsp_state;
2576 
2577                 if (current == DTRACESPEC_COMMITTINGMANY)
2578                         break;
2579 
2580                 switch (current) {
2581                 case DTRACESPEC_INACTIVE:
2582                 case DTRACESPEC_DISCARDING:
2583                         return;
2584 
2585                 case DTRACESPEC_COMMITTING:
2586                         /*
2587                          * This is only possible if we are (a) commit()'ing
2588                          * without having done a prior speculate() on this CPU
2589                          * and (b) racing with another commit() on a different
2590                          * CPU.  There's nothing to do -- we just assert that
2591                          * our offset is 0.
2592                          */
2593                         ASSERT(src->dtb_offset == 0);
2594                         return;
2595 
2596                 case DTRACESPEC_ACTIVE:
2597                         new = DTRACESPEC_COMMITTING;
2598                         break;
2599 
2600                 case DTRACESPEC_ACTIVEONE:
2601                         /*
2602                          * This speculation is active on one CPU.  If our
2603                          * buffer offset is non-zero, we know that the one CPU
2604                          * must be us.  Otherwise, we are committing on a
2605                          * different CPU from the speculate(), and we must
2606                          * rely on being asynchronously cleaned.
2607                          */
2608                         if (src->dtb_offset != 0) {
2609                                 new = DTRACESPEC_COMMITTING;
2610                                 break;
2611                         }
2612                         /*FALLTHROUGH*/
2613 
2614                 case DTRACESPEC_ACTIVEMANY:
2615                         new = DTRACESPEC_COMMITTINGMANY;
2616                         break;
2617 
2618                 default:
2619                         ASSERT(0);
2620                 }
2621         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2622             current, new) != current);
2623 
2624         /*
2625          * We have set the state to indicate that we are committing this
2626          * speculation.  Now reserve the necessary space in the destination
2627          * buffer.
2628          */
2629         if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
2630             sizeof (uint64_t), state, NULL)) < 0) {
2631                 dtrace_buffer_drop(dest);
2632                 goto out;
2633         }
2634 
2635         /*
2636          * We have sufficient space to copy the speculative buffer into the
2637          * primary buffer.  First, modify the speculative buffer, filling
2638          * in the timestamp of all entries with the current time.  The data
2639          * must have the commit() time rather than the time it was traced,
2640          * so that all entries in the primary buffer are in timestamp order.
2641          */
2642         timestamp = dtrace_gethrtime();
2643         saddr = (uintptr_t)src->dtb_tomax;
2644         slimit = saddr + src->dtb_offset;
2645         while (saddr < slimit) {
2646                 size_t size;
2647                 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
2648 
2649                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2650                         saddr += sizeof (dtrace_epid_t);
2651                         continue;
2652                 }
2653                 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
2654                 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
2655 
2656                 ASSERT3U(saddr + size, <=, slimit);
2657                 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
2658                 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
2659 
2660                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
2661 
2662                 saddr += size;
2663         }
2664 
2665         /*
2666          * Copy the buffer across.  (Note that this is a
2667          * highly subobtimal bcopy(); in the unlikely event that this becomes
2668          * a serious performance issue, a high-performance DTrace-specific
2669          * bcopy() should obviously be invented.)
2670          */
2671         daddr = (uintptr_t)dest->dtb_tomax + offs;
2672         dlimit = daddr + src->dtb_offset;
2673         saddr = (uintptr_t)src->dtb_tomax;
2674 
2675         /*
2676          * First, the aligned portion.
2677          */
2678         while (dlimit - daddr >= sizeof (uint64_t)) {
2679                 *((uint64_t *)daddr) = *((uint64_t *)saddr);
2680 
2681                 daddr += sizeof (uint64_t);
2682                 saddr += sizeof (uint64_t);
2683         }
2684 
2685         /*
2686          * Now any left-over bit...
2687          */
2688         while (dlimit - daddr)
2689                 *((uint8_t *)daddr++) = *((uint8_t *)saddr++);
2690 
2691         /*
2692          * Finally, commit the reserved space in the destination buffer.
2693          */
2694         dest->dtb_offset = offs + src->dtb_offset;
2695 
2696 out:
2697         /*
2698          * If we're lucky enough to be the only active CPU on this speculation
2699          * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2700          */
2701         if (current == DTRACESPEC_ACTIVE ||
2702             (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
2703                 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
2704                     DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
2705 
2706                 ASSERT(rval == DTRACESPEC_COMMITTING);
2707         }
2708 
2709         src->dtb_offset = 0;
2710         src->dtb_xamot_drops += src->dtb_drops;
2711         src->dtb_drops = 0;
2712 }
2713 
2714 /*
2715  * This routine discards an active speculation.  If the specified speculation
2716  * is not in a valid state to perform a discard(), this routine will silently
2717  * do nothing.  The state of the specified speculation is transitioned
2718  * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2719  */
2720 static void
2721 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2722     dtrace_specid_t which)
2723 {
2724         dtrace_speculation_t *spec;
2725         dtrace_speculation_state_t current, new;
2726         dtrace_buffer_t *buf;
2727 
2728         if (which == 0)
2729                 return;
2730 
2731         if (which > state->dts_nspeculations) {
2732                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2733                 return;
2734         }
2735 
2736         spec = &state->dts_speculations[which - 1];
2737         buf = &spec->dtsp_buffer[cpu];
2738 
2739         do {
2740                 current = spec->dtsp_state;
2741 
2742                 switch (current) {
2743                 case DTRACESPEC_INACTIVE:
2744                 case DTRACESPEC_COMMITTINGMANY:
2745                 case DTRACESPEC_COMMITTING:
2746                 case DTRACESPEC_DISCARDING:
2747                         return;
2748 
2749                 case DTRACESPEC_ACTIVE:
2750                 case DTRACESPEC_ACTIVEMANY:
2751                         new = DTRACESPEC_DISCARDING;
2752                         break;
2753 
2754                 case DTRACESPEC_ACTIVEONE:
2755                         if (buf->dtb_offset != 0) {
2756                                 new = DTRACESPEC_INACTIVE;
2757                         } else {
2758                                 new = DTRACESPEC_DISCARDING;
2759                         }
2760                         break;
2761 
2762                 default:
2763                         ASSERT(0);
2764                 }
2765         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2766             current, new) != current);
2767 
2768         buf->dtb_offset = 0;
2769         buf->dtb_drops = 0;
2770 }
2771 
2772 /*
2773  * Note:  not called from probe context.  This function is called
2774  * asynchronously from cross call context to clean any speculations that are
2775  * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
2776  * transitioned back to the INACTIVE state until all CPUs have cleaned the
2777  * speculation.
2778  */
2779 static void
2780 dtrace_speculation_clean_here(dtrace_state_t *state)
2781 {
2782         dtrace_icookie_t cookie;
2783         processorid_t cpu = CPU->cpu_id;
2784         dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2785         dtrace_specid_t i;
2786 
2787         cookie = dtrace_interrupt_disable();
2788 
2789         if (dest->dtb_tomax == NULL) {
2790                 dtrace_interrupt_enable(cookie);
2791                 return;
2792         }
2793 
2794         for (i = 0; i < state->dts_nspeculations; i++) {
2795                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2796                 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
2797 
2798                 if (src->dtb_tomax == NULL)
2799                         continue;
2800 
2801                 if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
2802                         src->dtb_offset = 0;
2803                         continue;
2804                 }
2805 
2806                 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2807                         continue;
2808 
2809                 if (src->dtb_offset == 0)
2810                         continue;
2811 
2812                 dtrace_speculation_commit(state, cpu, i + 1);
2813         }
2814 
2815         dtrace_interrupt_enable(cookie);
2816 }
2817 
2818 /*
2819  * Note:  not called from probe context.  This function is called
2820  * asynchronously (and at a regular interval) to clean any speculations that
2821  * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
2822  * is work to be done, it cross calls all CPUs to perform that work;
2823  * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2824  * INACTIVE state until they have been cleaned by all CPUs.
2825  */
2826 static void
2827 dtrace_speculation_clean(dtrace_state_t *state)
2828 {
2829         int work = 0, rv;
2830         dtrace_specid_t i;
2831 
2832         for (i = 0; i < state->dts_nspeculations; i++) {
2833                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2834 
2835                 ASSERT(!spec->dtsp_cleaning);
2836 
2837                 if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
2838                     spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2839                         continue;
2840 
2841                 work++;
2842                 spec->dtsp_cleaning = 1;
2843         }
2844 
2845         if (!work)
2846                 return;
2847 
2848         dtrace_xcall(DTRACE_CPUALL,
2849             (dtrace_xcall_t)dtrace_speculation_clean_here, state);
2850 
2851         /*
2852          * We now know that all CPUs have committed or discarded their
2853          * speculation buffers, as appropriate.  We can now set the state
2854          * to inactive.
2855          */
2856         for (i = 0; i < state->dts_nspeculations; i++) {
2857                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2858                 dtrace_speculation_state_t current, new;
2859 
2860                 if (!spec->dtsp_cleaning)
2861                         continue;
2862 
2863                 current = spec->dtsp_state;
2864                 ASSERT(current == DTRACESPEC_DISCARDING ||
2865                     current == DTRACESPEC_COMMITTINGMANY);
2866 
2867                 new = DTRACESPEC_INACTIVE;
2868 
2869                 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
2870                 ASSERT(rv == current);
2871                 spec->dtsp_cleaning = 0;
2872         }
2873 }
2874 
2875 /*
2876  * Called as part of a speculate() to get the speculative buffer associated
2877  * with a given speculation.  Returns NULL if the specified speculation is not
2878  * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
2879  * the active CPU is not the specified CPU -- the speculation will be
2880  * atomically transitioned into the ACTIVEMANY state.
2881  */
2882 static dtrace_buffer_t *
2883 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2884     dtrace_specid_t which)
2885 {
2886         dtrace_speculation_t *spec;
2887         dtrace_speculation_state_t current, new;
2888         dtrace_buffer_t *buf;
2889 
2890         if (which == 0)
2891                 return (NULL);
2892 
2893         if (which > state->dts_nspeculations) {
2894                 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2895                 return (NULL);
2896         }
2897 
2898         spec = &state->dts_speculations[which - 1];
2899         buf = &spec->dtsp_buffer[cpuid];
2900 
2901         do {
2902                 current = spec->dtsp_state;
2903 
2904                 switch (current) {
2905                 case DTRACESPEC_INACTIVE:
2906                 case DTRACESPEC_COMMITTINGMANY:
2907                 case DTRACESPEC_DISCARDING:
2908                         return (NULL);
2909 
2910                 case DTRACESPEC_COMMITTING:
2911                         ASSERT(buf->dtb_offset == 0);
2912                         return (NULL);
2913 
2914                 case DTRACESPEC_ACTIVEONE:
2915                         /*
2916                          * This speculation is currently active on one CPU.
2917                          * Check the offset in the buffer; if it's non-zero,
2918                          * that CPU must be us (and we leave the state alone).
2919                          * If it's zero, assume that we're starting on a new
2920                          * CPU -- and change the state to indicate that the
2921                          * speculation is active on more than one CPU.
2922                          */
2923                         if (buf->dtb_offset != 0)
2924                                 return (buf);
2925 
2926                         new = DTRACESPEC_ACTIVEMANY;
2927                         break;
2928 
2929                 case DTRACESPEC_ACTIVEMANY:
2930                         return (buf);
2931 
2932                 case DTRACESPEC_ACTIVE:
2933                         new = DTRACESPEC_ACTIVEONE;
2934                         break;
2935 
2936                 default:
2937                         ASSERT(0);
2938                 }
2939         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2940             current, new) != current);
2941 
2942         ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
2943         return (buf);
2944 }
2945 
2946 /*
2947  * Return a string.  In the event that the user lacks the privilege to access
2948  * arbitrary kernel memory, we copy the string out to scratch memory so that we
2949  * don't fail access checking.
2950  *
2951  * dtrace_dif_variable() uses this routine as a helper for various
2952  * builtin values such as 'execname' and 'probefunc.'
2953  */
2954 uintptr_t
2955 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
2956     dtrace_mstate_t *mstate)
2957 {
2958         uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2959         uintptr_t ret;
2960         size_t strsz;
2961 
2962         /*
2963          * The easy case: this probe is allowed to read all of memory, so
2964          * we can just return this as a vanilla pointer.
2965          */
2966         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
2967                 return (addr);
2968 
2969         /*
2970          * This is the tougher case: we copy the string in question from
2971          * kernel memory into scratch memory and return it that way: this
2972          * ensures that we won't trip up when access checking tests the
2973          * BYREF return value.
2974          */
2975         strsz = dtrace_strlen((char *)addr, size) + 1;
2976 
2977         if (mstate->dtms_scratch_ptr + strsz >
2978             mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2979                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2980                 return (NULL);
2981         }
2982 
2983         dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
2984             strsz);
2985         ret = mstate->dtms_scratch_ptr;
2986         mstate->dtms_scratch_ptr += strsz;
2987         return (ret);
2988 }
2989 
2990 /*
2991  * This function implements the DIF emulator's variable lookups.  The emulator
2992  * passes a reserved variable identifier and optional built-in array index.
2993  */
2994 static uint64_t
2995 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
2996     uint64_t ndx)
2997 {
2998         /*
2999          * If we're accessing one of the uncached arguments, we'll turn this
3000          * into a reference in the args array.
3001          */
3002         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
3003                 ndx = v - DIF_VAR_ARG0;
3004                 v = DIF_VAR_ARGS;
3005         }
3006 
3007         switch (v) {
3008         case DIF_VAR_ARGS:
3009                 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) {
3010                         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |=
3011                             CPU_DTRACE_KPRIV;
3012                         return (0);
3013                 }
3014 
3015                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
3016                 if (ndx >= sizeof (mstate->dtms_arg) /
3017                     sizeof (mstate->dtms_arg[0])) {
3018                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3019                         dtrace_provider_t *pv;
3020                         uint64_t val;
3021 
3022                         pv = mstate->dtms_probe->dtpr_provider;
3023                         if (pv->dtpv_pops.dtps_getargval != NULL)
3024                                 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
3025                                     mstate->dtms_probe->dtpr_id,
3026                                     mstate->dtms_probe->dtpr_arg, ndx, aframes);
3027                         else
3028                                 val = dtrace_getarg(ndx, aframes);
3029 
3030                         /*
3031                          * This is regrettably required to keep the compiler
3032                          * from tail-optimizing the call to dtrace_getarg().
3033                          * The condition always evaluates to true, but the
3034                          * compiler has no way of figuring that out a priori.
3035                          * (None of this would be necessary if the compiler
3036                          * could be relied upon to _always_ tail-optimize
3037                          * the call to dtrace_getarg() -- but it can't.)
3038                          */
3039                         if (mstate->dtms_probe != NULL)
3040                                 return (val);
3041 
3042                         ASSERT(0);
3043                 }
3044 
3045                 return (mstate->dtms_arg[ndx]);
3046 
3047         case DIF_VAR_UREGS: {
3048                 klwp_t *lwp;
3049 
3050                 if (!dtrace_priv_proc(state, mstate))
3051                         return (0);
3052 
3053                 if ((lwp = curthread->t_lwp) == NULL) {
3054                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
3055                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
3056                         return (0);
3057                 }
3058 
3059                 return (dtrace_getreg(lwp->lwp_regs, ndx));
3060         }
3061 
3062         case DIF_VAR_VMREGS: {
3063                 uint64_t rval;
3064 
3065                 if (!dtrace_priv_kernel(state))
3066                         return (0);
3067 
3068                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3069 
3070                 rval = dtrace_getvmreg(ndx,
3071                     &cpu_core[CPU->cpu_id].cpuc_dtrace_flags);
3072 
3073                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3074 
3075                 return (rval);
3076         }
3077 
3078         case DIF_VAR_CURTHREAD:
3079                 if (!dtrace_priv_proc(state, mstate))
3080                         return (0);
3081                 return ((uint64_t)(uintptr_t)curthread);
3082 
3083         case DIF_VAR_TIMESTAMP:
3084                 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
3085                         mstate->dtms_timestamp = dtrace_gethrtime();
3086                         mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
3087                 }
3088                 return (mstate->dtms_timestamp);
3089 
3090         case DIF_VAR_VTIMESTAMP:
3091                 ASSERT(dtrace_vtime_references != 0);
3092                 return (curthread->t_dtrace_vtime);
3093 
3094         case DIF_VAR_WALLTIMESTAMP:
3095                 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
3096                         mstate->dtms_walltimestamp = dtrace_gethrestime();
3097                         mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
3098                 }
3099                 return (mstate->dtms_walltimestamp);
3100 
3101         case DIF_VAR_IPL:
3102                 if (!dtrace_priv_kernel(state))
3103                         return (0);
3104                 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
3105                         mstate->dtms_ipl = dtrace_getipl();
3106                         mstate->dtms_present |= DTRACE_MSTATE_IPL;
3107                 }
3108                 return (mstate->dtms_ipl);
3109 
3110         case DIF_VAR_EPID:
3111                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
3112                 return (mstate->dtms_epid);
3113 
3114         case DIF_VAR_ID:
3115                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3116                 return (mstate->dtms_probe->dtpr_id);
3117 
3118         case DIF_VAR_STACKDEPTH:
3119                 if (!dtrace_priv_kernel(state))
3120                         return (0);
3121                 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
3122                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3123 
3124                         mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
3125                         mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
3126                 }
3127                 return (mstate->dtms_stackdepth);
3128 
3129         case DIF_VAR_USTACKDEPTH:
3130                 if (!dtrace_priv_proc(state, mstate))
3131                         return (0);
3132                 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
3133                         /*
3134                          * See comment in DIF_VAR_PID.
3135                          */
3136                         if (DTRACE_ANCHORED(mstate->dtms_probe) &&
3137                             CPU_ON_INTR(CPU)) {
3138                                 mstate->dtms_ustackdepth = 0;
3139                         } else {
3140                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3141                                 mstate->dtms_ustackdepth =
3142                                     dtrace_getustackdepth();
3143                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3144                         }
3145                         mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
3146                 }
3147                 return (mstate->dtms_ustackdepth);
3148 
3149         case DIF_VAR_CALLER:
3150                 if (!dtrace_priv_kernel(state))
3151                         return (0);
3152                 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
3153                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3154 
3155                         if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
3156                                 /*
3157                                  * If this is an unanchored probe, we are
3158                                  * required to go through the slow path:
3159                                  * dtrace_caller() only guarantees correct
3160                                  * results for anchored probes.
3161                                  */
3162                                 pc_t caller[2];
3163 
3164                                 dtrace_getpcstack(caller, 2, aframes,
3165                                     (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
3166                                 mstate->dtms_caller = caller[1];
3167                         } else if ((mstate->dtms_caller =
3168                             dtrace_caller(aframes)) == -1) {
3169                                 /*
3170                                  * We have failed to do this the quick way;
3171                                  * we must resort to the slower approach of
3172                                  * calling dtrace_getpcstack().
3173                                  */
3174                                 pc_t caller;
3175 
3176                                 dtrace_getpcstack(&caller, 1, aframes, NULL);
3177                                 mstate->dtms_caller = caller;
3178                         }
3179 
3180                         mstate->dtms_present |= DTRACE_MSTATE_CALLER;
3181                 }
3182                 return (mstate->dtms_caller);
3183 
3184         case DIF_VAR_UCALLER:
3185                 if (!dtrace_priv_proc(state, mstate))
3186                         return (0);
3187 
3188                 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
3189                         uint64_t ustack[3];
3190 
3191                         /*
3192                          * dtrace_getupcstack() fills in the first uint64_t
3193                          * with the current PID.  The second uint64_t will
3194                          * be the program counter at user-level.  The third
3195                          * uint64_t will contain the caller, which is what
3196                          * we're after.
3197                          */
3198                         ustack[2] = NULL;
3199                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3200                         dtrace_getupcstack(ustack, 3);
3201                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3202                         mstate->dtms_ucaller = ustack[2];
3203                         mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
3204                 }
3205 
3206                 return (mstate->dtms_ucaller);
3207 
3208         case DIF_VAR_PROBEPROV:
3209                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3210                 return (dtrace_dif_varstr(
3211                     (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
3212                     state, mstate));
3213 
3214         case DIF_VAR_PROBEMOD:
3215                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3216                 return (dtrace_dif_varstr(
3217                     (uintptr_t)mstate->dtms_probe->dtpr_mod,
3218                     state, mstate));
3219 
3220         case DIF_VAR_PROBEFUNC:
3221                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3222                 return (dtrace_dif_varstr(
3223                     (uintptr_t)mstate->dtms_probe->dtpr_func,
3224                     state, mstate));
3225 
3226         case DIF_VAR_PROBENAME:
3227                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3228                 return (dtrace_dif_varstr(
3229                     (uintptr_t)mstate->dtms_probe->dtpr_name,
3230                     state, mstate));
3231 
3232         case DIF_VAR_PID:
3233                 if (!dtrace_priv_proc(state, mstate))
3234                         return (0);
3235 
3236                 /*
3237                  * Note that we are assuming that an unanchored probe is
3238                  * always due to a high-level interrupt.  (And we're assuming
3239                  * that there is only a single high level interrupt.)
3240                  */
3241                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3242                         return (pid0.pid_id);
3243 
3244                 /*
3245                  * It is always safe to dereference one's own t_procp pointer:
3246                  * it always points to a valid, allocated proc structure.
3247                  * Further, it is always safe to dereference the p_pidp member
3248                  * of one's own proc structure.  (These are truisms becuase
3249                  * threads and processes don't clean up their own state --
3250                  * they leave that task to whomever reaps them.)
3251                  */
3252                 return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
3253 
3254         case DIF_VAR_PPID:
3255                 if (!dtrace_priv_proc(state, mstate))
3256                         return (0);
3257 
3258                 /*
3259                  * See comment in DIF_VAR_PID.
3260                  */
3261                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3262                         return (pid0.pid_id);
3263 
3264                 /*
3265                  * It is always safe to dereference one's own t_procp pointer:
3266                  * it always points to a valid, allocated proc structure.
3267                  * (This is true because threads don't clean up their own
3268                  * state -- they leave that task to whomever reaps them.)
3269                  */
3270                 return ((uint64_t)curthread->t_procp->p_ppid);
3271 
3272         case DIF_VAR_TID:
3273                 /*
3274                  * See comment in DIF_VAR_PID.
3275                  */
3276                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3277                         return (0);
3278 
3279                 return ((uint64_t)curthread->t_tid);
3280 
3281         case DIF_VAR_EXECNAME:
3282                 if (!dtrace_priv_proc(state, mstate))
3283                         return (0);
3284 
3285                 /*
3286                  * See comment in DIF_VAR_PID.
3287                  */
3288                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3289                         return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
3290 
3291                 /*
3292                  * It is always safe to dereference one's own t_procp pointer:
3293                  * it always points to a valid, allocated proc structure.
3294                  * (This is true because threads don't clean up their own
3295                  * state -- they leave that task to whomever reaps them.)
3296                  */
3297                 return (dtrace_dif_varstr(
3298                     (uintptr_t)curthread->t_procp->p_user.u_comm,
3299                     state, mstate));
3300 
3301         case DIF_VAR_ZONENAME:
3302                 if (!dtrace_priv_proc(state, mstate))
3303                         return (0);
3304 
3305                 /*
3306                  * See comment in DIF_VAR_PID.
3307                  */
3308                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3309                         return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
3310 
3311                 /*
3312                  * It is always safe to dereference one's own t_procp pointer:
3313                  * it always points to a valid, allocated proc structure.
3314                  * (This is true because threads don't clean up their own
3315                  * state -- they leave that task to whomever reaps them.)
3316                  */
3317                 return (dtrace_dif_varstr(
3318                     (uintptr_t)curthread->t_procp->p_zone->zone_name,
3319                     state, mstate));
3320 
3321         case DIF_VAR_UID:
3322                 if (!dtrace_priv_proc(state, mstate))
3323                         return (0);
3324 
3325                 /*
3326                  * See comment in DIF_VAR_PID.
3327                  */
3328                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3329                         return ((uint64_t)p0.p_cred->cr_uid);
3330 
3331                 /*
3332                  * It is always safe to dereference one's own t_procp pointer:
3333                  * it always points to a valid, allocated proc structure.
3334                  * (This is true because threads don't clean up their own
3335                  * state -- they leave that task to whomever reaps them.)
3336                  *
3337                  * Additionally, it is safe to dereference one's own process
3338                  * credential, since this is never NULL after process birth.
3339                  */
3340                 return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
3341 
3342         case DIF_VAR_GID:
3343                 if (!dtrace_priv_proc(state, mstate))
3344                         return (0);
3345 
3346                 /*
3347                  * See comment in DIF_VAR_PID.
3348                  */
3349                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3350                         return ((uint64_t)p0.p_cred->cr_gid);
3351 
3352                 /*
3353                  * It is always safe to dereference one's own t_procp pointer:
3354                  * it always points to a valid, allocated proc structure.
3355                  * (This is true because threads don't clean up their own
3356                  * state -- they leave that task to whomever reaps them.)
3357                  *
3358                  * Additionally, it is safe to dereference one's own process
3359                  * credential, since this is never NULL after process birth.
3360                  */
3361                 return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
3362 
3363         case DIF_VAR_ERRNO: {
3364                 klwp_t *lwp;
3365                 if (!dtrace_priv_proc(state, mstate))
3366                         return (0);
3367 
3368                 /*
3369                  * See comment in DIF_VAR_PID.
3370                  */
3371                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3372                         return (0);
3373 
3374                 /*
3375                  * It is always safe to dereference one's own t_lwp pointer in
3376                  * the event that this pointer is non-NULL.  (This is true
3377                  * because threads and lwps don't clean up their own state --
3378                  * they leave that task to whomever reaps them.)
3379                  */
3380                 if ((lwp = curthread->t_lwp) == NULL)
3381                         return (0);
3382 
3383                 return ((uint64_t)lwp->lwp_errno);
3384         }
3385         default:
3386                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3387                 return (0);
3388         }
3389 }
3390 
3391 
3392 typedef enum dtrace_json_state {
3393         DTRACE_JSON_REST = 1,
3394         DTRACE_JSON_OBJECT,
3395         DTRACE_JSON_STRING,
3396         DTRACE_JSON_STRING_ESCAPE,
3397         DTRACE_JSON_STRING_ESCAPE_UNICODE,
3398         DTRACE_JSON_COLON,
3399         DTRACE_JSON_COMMA,
3400         DTRACE_JSON_VALUE,
3401         DTRACE_JSON_IDENTIFIER,
3402         DTRACE_JSON_NUMBER,
3403         DTRACE_JSON_NUMBER_FRAC,
3404         DTRACE_JSON_NUMBER_EXP,
3405         DTRACE_JSON_COLLECT_OBJECT
3406 } dtrace_json_state_t;
3407 
3408 /*
3409  * This function possesses just enough knowledge about JSON to extract a single
3410  * value from a JSON string and store it in the scratch buffer.  It is able
3411  * to extract nested object values, and members of arrays by index.
3412  *
3413  * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to
3414  * be looked up as we descend into the object tree.  e.g.
3415  *
3416  *    foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL
3417  *       with nelems = 5.
3418  *
3419  * The run time of this function must be bounded above by strsize to limit the
3420  * amount of work done in probe context.  As such, it is implemented as a
3421  * simple state machine, reading one character at a time using safe loads
3422  * until we find the requested element, hit a parsing error or run off the
3423  * end of the object or string.
3424  *
3425  * As there is no way for a subroutine to return an error without interrupting
3426  * clause execution, we simply return NULL in the event of a missing key or any
3427  * other error condition.  Each NULL return in this function is commented with
3428  * the error condition it represents -- parsing or otherwise.
3429  *
3430  * The set of states for the state machine closely matches the JSON
3431  * specification (http://json.org/).  Briefly:
3432  *
3433  *   DTRACE_JSON_REST:
3434  *     Skip whitespace until we find either a top-level Object, moving
3435  *     to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE.
3436  *
3437  *   DTRACE_JSON_OBJECT:
3438  *     Locate the next key String in an Object.  Sets a flag to denote
3439  *     the next String as a key string and moves to DTRACE_JSON_STRING.
3440  *
3441  *   DTRACE_JSON_COLON:
3442  *     Skip whitespace until we find the colon that separates key Strings
3443  *     from their values.  Once found, move to DTRACE_JSON_VALUE.
3444  *
3445  *   DTRACE_JSON_VALUE:
3446  *     Detects the type of the next value (String, Number, Identifier, Object
3447  *     or Array) and routes to the states that process that type.  Here we also
3448  *     deal with the element selector list if we are requested to traverse down
3449  *     into the object tree.
3450  *
3451  *   DTRACE_JSON_COMMA:
3452  *     Skip whitespace until we find the comma that separates key-value pairs
3453  *     in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays
3454  *     (similarly DTRACE_JSON_VALUE).  All following literal value processing
3455  *     states return to this state at the end of their value, unless otherwise
3456  *     noted.
3457  *
3458  *   DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP:
3459  *     Processes a Number literal from the JSON, including any exponent
3460  *     component that may be present.  Numbers are returned as strings, which
3461  *     may be passed to strtoll() if an integer is required.
3462  *
3463  *   DTRACE_JSON_IDENTIFIER:
3464  *     Processes a "true", "false" or "null" literal in the JSON.
3465  *
3466  *   DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE,
3467  *   DTRACE_JSON_STRING_ESCAPE_UNICODE:
3468  *     Processes a String literal from the JSON, whether the String denotes
3469  *     a key, a value or part of a larger Object.  Handles all escape sequences
3470  *     present in the specification, including four-digit unicode characters,
3471  *     but merely includes the escape sequence without converting it to the
3472  *     actual escaped character.  If the String is flagged as a key, we
3473  *     move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA.
3474  *
3475  *   DTRACE_JSON_COLLECT_OBJECT:
3476  *     This state collects an entire Object (or Array), correctly handling
3477  *     embedded strings.  If the full element selector list matches this nested
3478  *     object, we return the Object in full as a string.  If not, we use this
3479  *     state to skip to the next value at this level and continue processing.
3480  *
3481  * NOTE: This function uses various macros from strtolctype.h to manipulate
3482  * digit values, etc -- these have all been checked to ensure they make
3483  * no additional function calls.
3484  */
3485 static char *
3486 dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems,
3487     char *dest)
3488 {
3489         dtrace_json_state_t state = DTRACE_JSON_REST;
3490         int64_t array_elem = INT64_MIN;
3491         int64_t array_pos = 0;
3492         uint8_t escape_unicount = 0;
3493         boolean_t string_is_key = B_FALSE;
3494         boolean_t collect_object = B_FALSE;
3495         boolean_t found_key = B_FALSE;
3496         boolean_t in_array = B_FALSE;
3497         uint32_t braces = 0, brackets = 0;
3498         char *elem = elemlist;
3499         char *dd = dest;
3500         uintptr_t cur;
3501 
3502         for (cur = json; cur < json + size; cur++) {
3503                 char cc = dtrace_load8(cur);
3504                 if (cc == '\0')
3505                         return (NULL);
3506 
3507                 switch (state) {
3508                 case DTRACE_JSON_REST:
3509                         if (isspace(cc))
3510                                 break;
3511 
3512                         if (cc == '{') {
3513                                 state = DTRACE_JSON_OBJECT;
3514                                 break;
3515                         }
3516 
3517                         if (cc == '[') {
3518                                 in_array = B_TRUE;
3519                                 array_pos = 0;
3520                                 array_elem = dtrace_strtoll(elem, 10, size);
3521                                 found_key = array_elem == 0 ? B_TRUE : B_FALSE;
3522                                 state = DTRACE_JSON_VALUE;
3523                                 break;
3524                         }
3525 
3526                         /*
3527                          * ERROR: expected to find a top-level object or array.
3528                          */
3529                         return (NULL);
3530                 case DTRACE_JSON_OBJECT:
3531                         if (isspace(cc))
3532                                 break;
3533 
3534                         if (cc == '"') {
3535                                 state = DTRACE_JSON_STRING;
3536                                 string_is_key = B_TRUE;
3537                                 break;
3538                         }
3539 
3540                         /*
3541                          * ERROR: either the object did not start with a key
3542                          * string, or we've run off the end of the object
3543                          * without finding the requested key.
3544                          */
3545                         return (NULL);
3546                 case DTRACE_JSON_STRING:
3547                         if (cc == '\\') {
3548                                 *dd++ = '\\';
3549                                 state = DTRACE_JSON_STRING_ESCAPE;
3550                                 break;
3551                         }
3552 
3553                         if (cc == '"') {
3554                                 if (collect_object) {
3555                                         /*
3556                                          * We don't reset the dest here, as
3557                                          * the string is part of a larger
3558                                          * object being collected.
3559                                          */
3560                                         *dd++ = cc;
3561                                         collect_object = B_FALSE;
3562                                         state = DTRACE_JSON_COLLECT_OBJECT;
3563                                         break;
3564                                 }
3565                                 *dd = '\0';
3566                                 dd = dest; /* reset string buffer */
3567                                 if (string_is_key) {
3568                                         if (dtrace_strncmp(dest, elem,
3569                                             size) == 0)
3570                                                 found_key = B_TRUE;
3571                                 } else if (found_key) {
3572                                         if (nelems > 1) {
3573                                                 /*
3574                                                  * We expected an object, not
3575                                                  * this string.
3576                                                  */
3577                                                 return (NULL);
3578                                         }
3579                                         return (dest);
3580                                 }
3581                                 state = string_is_key ? DTRACE_JSON_COLON :
3582                                     DTRACE_JSON_COMMA;
3583                                 string_is_key = B_FALSE;
3584                                 break;
3585                         }
3586 
3587                         *dd++ = cc;
3588                         break;
3589                 case DTRACE_JSON_STRING_ESCAPE:
3590                         *dd++ = cc;
3591                         if (cc == 'u') {
3592                                 escape_unicount = 0;
3593                                 state = DTRACE_JSON_STRING_ESCAPE_UNICODE;
3594                         } else {
3595                                 state = DTRACE_JSON_STRING;
3596                         }
3597                         break;
3598                 case DTRACE_JSON_STRING_ESCAPE_UNICODE:
3599                         if (!isxdigit(cc)) {
3600                                 /*
3601                                  * ERROR: invalid unicode escape, expected
3602                                  * four valid hexidecimal digits.
3603                                  */
3604                                 return (NULL);
3605                         }
3606 
3607                         *dd++ = cc;
3608                         if (++escape_unicount == 4)
3609                                 state = DTRACE_JSON_STRING;
3610                         break;
3611                 case DTRACE_JSON_COLON:
3612                         if (isspace(cc))
3613                                 break;
3614 
3615                         if (cc == ':') {
3616                                 state = DTRACE_JSON_VALUE;
3617                                 break;
3618                         }
3619 
3620                         /*
3621                          * ERROR: expected a colon.
3622                          */
3623                         return (NULL);
3624                 case DTRACE_JSON_COMMA:
3625                         if (isspace(cc))
3626                                 break;
3627 
3628                         if (cc == ',') {
3629                                 if (in_array) {
3630                                         state = DTRACE_JSON_VALUE;
3631                                         if (++array_pos == array_elem)
3632                                                 found_key = B_TRUE;
3633                                 } else {
3634                                         state = DTRACE_JSON_OBJECT;
3635                                 }
3636                                 break;
3637                         }
3638 
3639                         /*
3640                          * ERROR: either we hit an unexpected character, or
3641                          * we reached the end of the object or array without
3642                          * finding the requested key.
3643                          */
3644                         return (NULL);
3645                 case DTRACE_JSON_IDENTIFIER:
3646                         if (islower(cc)) {
3647                                 *dd++ = cc;
3648                                 break;
3649                         }
3650 
3651                         *dd = '\0';
3652                         dd = dest; /* reset string buffer */
3653 
3654                         if (dtrace_strncmp(dest, "true", 5) == 0 ||
3655                             dtrace_strncmp(dest, "false", 6) == 0 ||
3656                             dtrace_strncmp(dest, "null", 5) == 0) {
3657                                 if (found_key) {
3658                                         if (nelems > 1) {
3659                                                 /*
3660                                                  * ERROR: We expected an object,
3661                                                  * not this identifier.
3662                                                  */
3663                                                 return (NULL);
3664                                         }
3665                                         return (dest);
3666                                 } else {
3667                                         cur--;
3668                                         state = DTRACE_JSON_COMMA;
3669                                         break;
3670                                 }
3671                         }
3672 
3673                         /*
3674                          * ERROR: we did not recognise the identifier as one
3675                          * of those in the JSON specification.
3676                          */
3677                         return (NULL);
3678                 case DTRACE_JSON_NUMBER:
3679                         if (cc == '.') {
3680                                 *dd++ = cc;
3681                                 state = DTRACE_JSON_NUMBER_FRAC;
3682                                 break;
3683                         }
3684 
3685                         if (cc == 'x' || cc == 'X') {
3686                                 /*
3687                                  * ERROR: specification explicitly excludes
3688                                  * hexidecimal or octal numbers.
3689                                  */
3690                                 return (NULL);
3691                         }
3692 
3693                         /* FALLTHRU */
3694                 case DTRACE_JSON_NUMBER_FRAC:
3695                         if (cc == 'e' || cc == 'E') {
3696                                 *dd++ = cc;
3697                                 state = DTRACE_JSON_NUMBER_EXP;
3698                                 break;
3699                         }
3700 
3701                         if (cc == '+' || cc == '-') {
3702                                 /*
3703                                  * ERROR: expect sign as part of exponent only.
3704                                  */
3705                                 return (NULL);
3706                         }
3707                         /* FALLTHRU */
3708                 case DTRACE_JSON_NUMBER_EXP:
3709                         if (isdigit(cc) || cc == '+' || cc == '-') {
3710                                 *dd++ = cc;
3711                                 break;
3712                         }
3713 
3714                         *dd = '\0';
3715                         dd = dest; /* reset string buffer */
3716                         if (found_key) {
3717                                 if (nelems > 1) {
3718                                         /*
3719                                          * ERROR: We expected an object, not
3720                                          * this number.
3721                                          */
3722                                         return (NULL);
3723                                 }
3724                                 return (dest);
3725                         }
3726 
3727                         cur--;
3728                         state = DTRACE_JSON_COMMA;
3729                         break;
3730                 case DTRACE_JSON_VALUE:
3731                         if (isspace(cc))
3732                                 break;
3733 
3734                         if (cc == '{' || cc == '[') {
3735                                 if (nelems > 1 && found_key) {
3736                                         in_array = cc == '[' ? B_TRUE : B_FALSE;
3737                                         /*
3738                                          * If our element selector directs us
3739                                          * to descend into this nested object,
3740                                          * then move to the next selector
3741                                          * element in the list and restart the
3742                                          * state machine.
3743                                          */
3744                                         while (*elem != '\0')
3745                                                 elem++;
3746                                         elem++; /* skip the inter-element NUL */
3747                                         nelems--;
3748                                         dd = dest;
3749                                         if (in_array) {
3750                                                 state = DTRACE_JSON_VALUE;
3751                                                 array_pos = 0;
3752                                                 array_elem = dtrace_strtoll(
3753                                                     elem, 10, size);
3754                                                 found_key = array_elem == 0 ?
3755                                                     B_TRUE : B_FALSE;
3756                                         } else {
3757                                                 found_key = B_FALSE;
3758                                                 state = DTRACE_JSON_OBJECT;
3759                                         }
3760                                         break;
3761                                 }
3762 
3763                                 /*
3764                                  * Otherwise, we wish to either skip this
3765                                  * nested object or return it in full.
3766                                  */
3767                                 if (cc == '[')
3768                                         brackets = 1;
3769                                 else
3770                                         braces = 1;
3771                                 *dd++ = cc;
3772                                 state = DTRACE_JSON_COLLECT_OBJECT;
3773                                 break;
3774                         }
3775 
3776                         if (cc == '"') {
3777                                 state = DTRACE_JSON_STRING;
3778                                 break;
3779                         }
3780 
3781                         if (islower(cc)) {
3782                                 /*
3783                                  * Here we deal with true, false and null.
3784                                  */
3785                                 *dd++ = cc;
3786                                 state = DTRACE_JSON_IDENTIFIER;
3787                                 break;
3788                         }
3789 
3790                         if (cc == '-' || isdigit(cc)) {
3791                                 *dd++ = cc;
3792                                 state = DTRACE_JSON_NUMBER;
3793                                 break;
3794                         }
3795 
3796                         /*
3797                          * ERROR: unexpected character at start of value.
3798                          */
3799                         return (NULL);
3800                 case DTRACE_JSON_COLLECT_OBJECT:
3801                         if (cc == '\0')
3802                                 /*
3803                                  * ERROR: unexpected end of input.
3804                                  */
3805                                 return (NULL);
3806 
3807                         *dd++ = cc;
3808                         if (cc == '"') {
3809                                 collect_object = B_TRUE;
3810                                 state = DTRACE_JSON_STRING;
3811                                 break;
3812                         }
3813 
3814                         if (cc == ']') {
3815                                 if (brackets-- == 0) {
3816                                         /*
3817                                          * ERROR: unbalanced brackets.
3818                                          */
3819                                         return (NULL);
3820                                 }
3821                         } else if (cc == '}') {
3822                                 if (braces-- == 0) {
3823                                         /*
3824                                          * ERROR: unbalanced braces.
3825                                          */
3826                                         return (NULL);
3827                                 }
3828                         } else if (cc == '{') {
3829                                 braces++;
3830                         } else if (cc == '[') {
3831                                 brackets++;
3832                         }
3833 
3834                         if (brackets == 0 && braces == 0) {
3835                                 if (found_key) {
3836                                         *dd = '\0';
3837                                         return (dest);
3838                                 }
3839                                 dd = dest; /* reset string buffer */
3840                                 state = DTRACE_JSON_COMMA;
3841                         }
3842                         break;
3843                 }
3844         }
3845         return (NULL);
3846 }
3847 
3848 /*
3849  * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
3850  * Notice that we don't bother validating the proper number of arguments or
3851  * their types in the tuple stack.  This isn't needed because all argument
3852  * interpretation is safe because of our load safety -- the worst that can
3853  * happen is that a bogus program can obtain bogus results.
3854  */
3855 static void
3856 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
3857     dtrace_key_t *tupregs, int nargs,
3858     dtrace_mstate_t *mstate, dtrace_state_t *state)
3859 {
3860         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
3861         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
3862         dtrace_vstate_t *vstate = &state->dts_vstate;
3863 
3864         union {
3865                 mutex_impl_t mi;
3866                 uint64_t mx;
3867         } m;
3868 
3869         union {
3870                 krwlock_t ri;
3871                 uintptr_t rw;
3872         } r;
3873 
3874         switch (subr) {
3875         case DIF_SUBR_RAND:
3876                 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
3877                 break;
3878 
3879         case DIF_SUBR_MUTEX_OWNED:
3880                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3881                     mstate, vstate)) {
3882                         regs[rd] = NULL;
3883                         break;
3884                 }
3885 
3886                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3887                 if (MUTEX_TYPE_ADAPTIVE(&m.mi))
3888                         regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
3889                 else
3890                         regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
3891                 break;
3892 
3893         case DIF_SUBR_MUTEX_OWNER:
3894                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3895                     mstate, vstate)) {
3896                         regs[rd] = NULL;
3897                         break;
3898                 }
3899 
3900                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3901                 if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
3902                     MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
3903                         regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
3904                 else
3905                         regs[rd] = 0;
3906                 break;
3907 
3908         case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
3909                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3910                     mstate, vstate)) {
3911                         regs[rd] = NULL;
3912                         break;
3913                 }
3914 
3915                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3916                 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
3917                 break;
3918 
3919         case DIF_SUBR_MUTEX_TYPE_SPIN:
3920                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3921                     mstate, vstate)) {
3922                         regs[rd] = NULL;
3923                         break;
3924                 }
3925 
3926                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3927                 regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
3928                 break;
3929 
3930         case DIF_SUBR_RW_READ_HELD: {
3931                 uintptr_t tmp;
3932 
3933                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
3934                     mstate, vstate)) {
3935                         regs[rd] = NULL;
3936                         break;
3937                 }
3938 
3939                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3940                 regs[rd] = _RW_READ_HELD(&r.ri, tmp);
3941                 break;
3942         }
3943 
3944         case DIF_SUBR_RW_WRITE_HELD:
3945                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3946                     mstate, vstate)) {
3947                         regs[rd] = NULL;
3948                         break;
3949                 }
3950 
3951                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3952                 regs[rd] = _RW_WRITE_HELD(&r.ri);
3953                 break;
3954 
3955         case DIF_SUBR_RW_ISWRITER:
3956                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3957                     mstate, vstate)) {
3958                         regs[rd] = NULL;
3959                         break;
3960                 }
3961 
3962                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3963                 regs[rd] = _RW_ISWRITER(&r.ri);
3964                 break;
3965 
3966         case DIF_SUBR_BCOPY: {
3967                 /*
3968                  * We need to be sure that the destination is in the scratch
3969                  * region -- no other region is allowed.
3970                  */
3971                 uintptr_t src = tupregs[0].dttk_value;
3972                 uintptr_t dest = tupregs[1].dttk_value;
3973                 size_t size = tupregs[2].dttk_value;
3974 
3975                 if (!dtrace_inscratch(dest, size, mstate)) {
3976                         *flags |= CPU_DTRACE_BADADDR;
3977                         *illval = regs[rd];
3978                         break;
3979                 }
3980 
3981                 if (!dtrace_canload(src, size, mstate, vstate)) {
3982                         regs[rd] = NULL;
3983                         break;
3984                 }
3985 
3986                 dtrace_bcopy((void *)src, (void *)dest, size);
3987                 break;
3988         }
3989 
3990         case DIF_SUBR_ALLOCA:
3991         case DIF_SUBR_COPYIN: {
3992                 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
3993                 uint64_t size =
3994                     tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
3995                 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
3996 
3997                 /*
3998                  * This action doesn't require any credential checks since
3999                  * probes will not activate in user contexts to which the
4000                  * enabling user does not have permissions.
4001                  */
4002 
4003                 /*
4004                  * Rounding up the user allocation size could have overflowed
4005                  * a large, bogus allocation (like -1ULL) to 0.
4006                  */
4007                 if (scratch_size < size ||
4008                     !DTRACE_INSCRATCH(mstate, scratch_size)) {
4009                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4010                         regs[rd] = NULL;
4011                         break;
4012                 }
4013 
4014                 if (subr == DIF_SUBR_COPYIN) {
4015                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4016                         dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4017                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4018                 }
4019 
4020                 mstate->dtms_scratch_ptr += scratch_size;
4021                 regs[rd] = dest;
4022                 break;
4023         }
4024 
4025         case DIF_SUBR_COPYINTO: {
4026                 uint64_t size = tupregs[1].dttk_value;
4027                 uintptr_t dest = tupregs[2].dttk_value;
4028 
4029                 /*
4030                  * This action doesn't require any credential checks since
4031                  * probes will not activate in user contexts to which the
4032                  * enabling user does not have permissions.
4033                  */
4034                 if (!dtrace_inscratch(dest, size, mstate)) {
4035                         *flags |= CPU_DTRACE_BADADDR;
4036                         *illval = regs[rd];
4037                         break;
4038                 }
4039 
4040                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4041                 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4042                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4043                 break;
4044         }
4045 
4046         case DIF_SUBR_COPYINSTR: {
4047                 uintptr_t dest = mstate->dtms_scratch_ptr;
4048                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4049 
4050                 if (nargs > 1 && tupregs[1].dttk_value < size)
4051                         size = tupregs[1].dttk_value + 1;
4052 
4053                 /*
4054                  * This action doesn't require any credential checks since
4055                  * probes will not activate in user contexts to which the
4056                  * enabling user does not have permissions.
4057                  */
4058                 if (!DTRACE_INSCRATCH(mstate, size)) {
4059                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4060                         regs[rd] = NULL;
4061                         break;
4062                 }
4063 
4064                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4065                 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
4066                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4067 
4068                 ((char *)dest)[size - 1] = '\0';
4069                 mstate->dtms_scratch_ptr += size;
4070                 regs[rd] = dest;
4071                 break;
4072         }
4073 
4074         case DIF_SUBR_MSGSIZE:
4075         case DIF_SUBR_MSGDSIZE: {
4076                 uintptr_t baddr = tupregs[0].dttk_value, daddr;
4077                 uintptr_t wptr, rptr;
4078                 size_t count = 0;
4079                 int cont = 0;
4080 
4081                 while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4082 
4083                         if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
4084                             vstate)) {
4085                                 regs[rd] = NULL;
4086                                 break;
4087                         }
4088 
4089                         wptr = dtrace_loadptr(baddr +
4090                             offsetof(mblk_t, b_wptr));
4091 
4092                         rptr = dtrace_loadptr(baddr +
4093                             offsetof(mblk_t, b_rptr));
4094 
4095                         if (wptr < rptr) {
4096                                 *flags |= CPU_DTRACE_BADADDR;
4097                                 *illval = tupregs[0].dttk_value;
4098                                 break;
4099                         }
4100 
4101                         daddr = dtrace_loadptr(baddr +
4102                             offsetof(mblk_t, b_datap));
4103 
4104                         baddr = dtrace_loadptr(baddr +
4105                             offsetof(mblk_t, b_cont));
4106 
4107                         /*
4108                          * We want to prevent against denial-of-service here,
4109                          * so we're only going to search the list for
4110                          * dtrace_msgdsize_max mblks.
4111                          */
4112                         if (cont++ > dtrace_msgdsize_max) {
4113                                 *flags |= CPU_DTRACE_ILLOP;
4114                                 break;
4115                         }
4116 
4117                         if (subr == DIF_SUBR_MSGDSIZE) {
4118                                 if (dtrace_load8(daddr +
4119                                     offsetof(dblk_t, db_type)) != M_DATA)
4120                                         continue;
4121                         }
4122 
4123                         count += wptr - rptr;
4124                 }
4125 
4126                 if (!(*flags & CPU_DTRACE_FAULT))
4127                         regs[rd] = count;
4128 
4129                 break;
4130         }
4131 
4132         case DIF_SUBR_PROGENYOF: {
4133                 pid_t pid = tupregs[0].dttk_value;
4134                 proc_t *p;
4135                 int rval = 0;
4136 
4137                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4138 
4139                 for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
4140                         if (p->p_pidp->pid_id == pid) {
4141                                 rval = 1;
4142                                 break;
4143                         }
4144                 }
4145 
4146                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4147 
4148                 regs[rd] = rval;
4149                 break;
4150         }
4151 
4152         case DIF_SUBR_SPECULATION:
4153                 regs[rd] = dtrace_speculation(state);
4154                 break;
4155 
4156         case DIF_SUBR_COPYOUT: {
4157                 uintptr_t kaddr = tupregs[0].dttk_value;
4158                 uintptr_t uaddr = tupregs[1].dttk_value;
4159                 uint64_t size = tupregs[2].dttk_value;
4160 
4161                 if (!dtrace_destructive_disallow &&
4162                     dtrace_priv_proc_control(state, mstate) &&
4163                     !dtrace_istoxic(kaddr, size)) {
4164                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4165                         dtrace_copyout(kaddr, uaddr, size, flags);
4166                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4167                 }
4168                 break;
4169         }
4170 
4171         case DIF_SUBR_COPYOUTSTR: {
4172                 uintptr_t kaddr = tupregs[0].dttk_value;
4173                 uintptr_t uaddr = tupregs[1].dttk_value;
4174                 uint64_t size = tupregs[2].dttk_value;
4175 
4176                 if (!dtrace_destructive_disallow &&
4177                     dtrace_priv_proc_control(state, mstate) &&
4178                     !dtrace_istoxic(kaddr, size)) {
4179                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4180                         dtrace_copyoutstr(kaddr, uaddr, size, flags);
4181                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4182                 }
4183                 break;
4184         }
4185 
4186         case DIF_SUBR_STRLEN: {
4187                 size_t sz;
4188                 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
4189                 sz = dtrace_strlen((char *)addr,
4190                     state->dts_options[DTRACEOPT_STRSIZE]);
4191 
4192                 if (!dtrace_canload(addr, sz + 1, mstate, vstate)) {
4193                         regs[rd] = NULL;
4194                         break;
4195                 }
4196 
4197                 regs[rd] = sz;
4198 
4199                 break;
4200         }
4201 
4202         case DIF_SUBR_STRCHR:
4203         case DIF_SUBR_STRRCHR: {
4204                 /*
4205                  * We're going to iterate over the string looking for the
4206                  * specified character.  We will iterate until we have reached
4207                  * the string length or we have found the character.  If this
4208                  * is DIF_SUBR_STRRCHR, we will look for the last occurrence
4209                  * of the specified character instead of the first.
4210                  */
4211                 uintptr_t saddr = tupregs[0].dttk_value;
4212                 uintptr_t addr = tupregs[0].dttk_value;
4213                 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
4214                 char c, target = (char)tupregs[1].dttk_value;
4215 
4216                 for (regs[rd] = NULL; addr < limit; addr++) {
4217                         if ((c = dtrace_load8(addr)) == target) {
4218                                 regs[rd] = addr;
4219 
4220                                 if (subr == DIF_SUBR_STRCHR)
4221                                         break;
4222                         }
4223 
4224                         if (c == '\0')
4225                                 break;
4226                 }
4227 
4228                 if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) {
4229                         regs[rd] = NULL;
4230                         break;
4231                 }
4232 
4233                 break;
4234         }
4235 
4236         case DIF_SUBR_STRSTR:
4237         case DIF_SUBR_INDEX:
4238         case DIF_SUBR_RINDEX: {
4239                 /*
4240                  * We're going to iterate over the string looking for the
4241                  * specified string.  We will iterate until we have reached
4242                  * the string length or we have found the string.  (Yes, this
4243                  * is done in the most naive way possible -- but considering
4244                  * that the string we're searching for is likely to be
4245                  * relatively short, the complexity of Rabin-Karp or similar
4246                  * hardly seems merited.)
4247                  */
4248                 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
4249                 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
4250                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4251                 size_t len = dtrace_strlen(addr, size);
4252                 size_t sublen = dtrace_strlen(substr, size);
4253                 char *limit = addr + len, *orig = addr;
4254                 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
4255                 int inc = 1;
4256 
4257                 regs[rd] = notfound;
4258 
4259                 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
4260                         regs[rd] = NULL;
4261                         break;
4262                 }
4263 
4264                 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
4265                     vstate)) {
4266                         regs[rd] = NULL;
4267                         break;
4268                 }
4269 
4270                 /*
4271                  * strstr() and index()/rindex() have similar semantics if
4272                  * both strings are the empty string: strstr() returns a
4273                  * pointer to the (empty) string, and index() and rindex()
4274                  * both return index 0 (regardless of any position argument).
4275                  */
4276                 if (sublen == 0 && len == 0) {
4277                         if (subr == DIF_SUBR_STRSTR)
4278                                 regs[rd] = (uintptr_t)addr;
4279                         else
4280                                 regs[rd] = 0;
4281                         break;
4282                 }
4283 
4284                 if (subr != DIF_SUBR_STRSTR) {
4285                         if (subr == DIF_SUBR_RINDEX) {
4286                                 limit = orig - 1;
4287                                 addr += len;
4288                                 inc = -1;
4289                         }
4290 
4291                         /*
4292                          * Both index() and rindex() take an optional position
4293                          * argument that denotes the starting position.
4294                          */
4295                         if (nargs == 3) {
4296                                 int64_t pos = (int64_t)tupregs[2].dttk_value;
4297 
4298                                 /*
4299                                  * If the position argument to index() is
4300                                  * negative, Perl implicitly clamps it at
4301                                  * zero.  This semantic is a little surprising
4302                                  * given the special meaning of negative
4303                                  * positions to similar Perl functions like
4304                                  * substr(), but it appears to reflect a
4305                                  * notion that index() can start from a
4306                                  * negative index and increment its way up to
4307                                  * the string.  Given this notion, Perl's
4308                                  * rindex() is at least self-consistent in
4309                                  * that it implicitly clamps positions greater
4310                                  * than the string length to be the string
4311                                  * length.  Where Perl completely loses
4312                                  * coherence, however, is when the specified
4313                                  * substring is the empty string ("").  In
4314                                  * this case, even if the position is
4315                                  * negative, rindex() returns 0 -- and even if
4316                                  * the position is greater than the length,
4317                                  * index() returns the string length.  These
4318                                  * semantics violate the notion that index()
4319                                  * should never return a value less than the
4320                                  * specified position and that rindex() should
4321                                  * never return a value greater than the
4322                                  * specified position.  (One assumes that
4323                                  * these semantics are artifacts of Perl's
4324                                  * implementation and not the results of
4325                                  * deliberate design -- it beggars belief that
4326                                  * even Larry Wall could desire such oddness.)
4327                                  * While in the abstract one would wish for
4328                                  * consistent position semantics across
4329                                  * substr(), index() and rindex() -- or at the
4330                                  * very least self-consistent position
4331                                  * semantics for index() and rindex() -- we
4332                                  * instead opt to keep with the extant Perl
4333                                  * semantics, in all their broken glory.  (Do
4334                                  * we have more desire to maintain Perl's
4335                                  * semantics than Perl does?  Probably.)
4336                                  */
4337                                 if (subr == DIF_SUBR_RINDEX) {
4338                                         if (pos < 0) {
4339                                                 if (sublen == 0)
4340                                                         regs[rd] = 0;
4341                                                 break;
4342                                         }
4343 
4344                                         if (pos > len)
4345                                                 pos = len;
4346                                 } else {
4347                                         if (pos < 0)
4348                                                 pos = 0;
4349 
4350                                         if (pos >= len) {
4351                                                 if (sublen == 0)
4352                                                         regs[rd] = len;
4353                                                 break;
4354                                         }
4355                                 }
4356 
4357                                 addr = orig + pos;
4358                         }
4359                 }
4360 
4361                 for (regs[rd] = notfound; addr != limit; addr += inc) {
4362                         if (dtrace_strncmp(addr, substr, sublen) == 0) {
4363                                 if (subr != DIF_SUBR_STRSTR) {
4364                                         /*
4365                                          * As D index() and rindex() are
4366                                          * modeled on Perl (and not on awk),
4367                                          * we return a zero-based (and not a
4368                                          * one-based) index.  (For you Perl
4369                                          * weenies: no, we're not going to add
4370                                          * $[ -- and shouldn't you be at a con
4371                                          * or something?)
4372                                          */
4373                                         regs[rd] = (uintptr_t)(addr - orig);
4374                                         break;
4375                                 }
4376 
4377                                 ASSERT(subr == DIF_SUBR_STRSTR);
4378                                 regs[rd] = (uintptr_t)addr;
4379                                 break;
4380                         }
4381                 }
4382 
4383                 break;
4384         }
4385 
4386         case DIF_SUBR_STRTOK: {
4387                 uintptr_t addr = tupregs[0].dttk_value;
4388                 uintptr_t tokaddr = tupregs[1].dttk_value;
4389                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4390                 uintptr_t limit, toklimit = tokaddr + size;
4391                 uint8_t c, tokmap[32];   /* 256 / 8 */
4392                 char *dest = (char *)mstate->dtms_scratch_ptr;
4393                 int i;
4394 
4395                 /*
4396                  * Check both the token buffer and (later) the input buffer,
4397                  * since both could be non-scratch addresses.
4398                  */
4399                 if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) {
4400                         regs[rd] = NULL;
4401                         break;
4402                 }
4403 
4404                 if (!DTRACE_INSCRATCH(mstate, size)) {
4405                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4406                         regs[rd] = NULL;
4407                         break;
4408                 }
4409 
4410                 if (addr == NULL) {
4411                         /*
4412                          * If the address specified is NULL, we use our saved
4413                          * strtok pointer from the mstate.  Note that this
4414                          * means that the saved strtok pointer is _only_
4415                          * valid within multiple enablings of the same probe --
4416                          * it behaves like an implicit clause-local variable.
4417                          */
4418                         addr = mstate->dtms_strtok;
4419                 } else {
4420                         /*
4421                          * If the user-specified address is non-NULL we must
4422                          * access check it.  This is the only time we have
4423                          * a chance to do so, since this address may reside
4424                          * in the string table of this clause-- future calls
4425                          * (when we fetch addr from mstate->dtms_strtok)
4426                          * would fail this access check.
4427                          */
4428                         if (!dtrace_strcanload(addr, size, mstate, vstate)) {
4429                                 regs[rd] = NULL;
4430                                 break;
4431                         }
4432                 }
4433 
4434                 /*
4435                  * First, zero the token map, and then process the token
4436                  * string -- setting a bit in the map for every character
4437                  * found in the token string.
4438                  */
4439                 for (i = 0; i < sizeof (tokmap); i++)
4440                         tokmap[i] = 0;
4441 
4442                 for (; tokaddr < toklimit; tokaddr++) {
4443                         if ((c = dtrace_load8(tokaddr)) == '\0')
4444                                 break;
4445 
4446                         ASSERT((c >> 3) < sizeof (tokmap));
4447                         tokmap[c >> 3] |= (1 << (c & 0x7));
4448                 }
4449 
4450                 for (limit = addr + size; addr < limit; addr++) {
4451                         /*
4452                          * We're looking for a character that is _not_ contained
4453                          * in the token string.
4454                          */
4455                         if ((c = dtrace_load8(addr)) == '\0')
4456                                 break;
4457 
4458                         if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
4459                                 break;
4460                 }
4461 
4462                 if (c == '\0') {
4463                         /*
4464                          * We reached the end of the string without finding
4465                          * any character that was not in the token string.
4466                          * We return NULL in this case, and we set the saved
4467                          * address to NULL as well.
4468                          */
4469                         regs[rd] = NULL;
4470                         mstate->dtms_strtok = NULL;
4471                         break;
4472                 }
4473 
4474                 /*
4475                  * From here on, we're copying into the destination string.
4476                  */
4477                 for (i = 0; addr < limit && i < size - 1; addr++) {
4478                         if ((c = dtrace_load8(addr)) == '\0')
4479                                 break;
4480 
4481                         if (tokmap[c >> 3] & (1 << (c & 0x7)))
4482                                 break;
4483 
4484                         ASSERT(i < size);
4485                         dest[i++] = c;
4486                 }
4487 
4488                 ASSERT(i < size);
4489                 dest[i] = '\0';
4490                 regs[rd] = (uintptr_t)dest;
4491                 mstate->dtms_scratch_ptr += size;
4492                 mstate->dtms_strtok = addr;
4493                 break;
4494         }
4495 
4496         case DIF_SUBR_SUBSTR: {
4497                 uintptr_t s = tupregs[0].dttk_value;
4498                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4499                 char *d = (char *)mstate->dtms_scratch_ptr;
4500                 int64_t index = (int64_t)tupregs[1].dttk_value;
4501                 int64_t remaining = (int64_t)tupregs[2].dttk_value;
4502                 size_t len = dtrace_strlen((char *)s, size);
4503                 int64_t i;
4504 
4505                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4506                         regs[rd] = NULL;
4507                         break;
4508                 }
4509 
4510                 if (!DTRACE_INSCRATCH(mstate, size)) {
4511                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4512                         regs[rd] = NULL;
4513                         break;
4514                 }
4515 
4516                 if (nargs <= 2)
4517                         remaining = (int64_t)size;
4518 
4519                 if (index < 0) {
4520                         index += len;
4521 
4522                         if (index < 0 && index + remaining > 0) {
4523                                 remaining += index;
4524                                 index = 0;
4525                         }
4526                 }
4527 
4528                 if (index >= len || index < 0) {
4529                         remaining = 0;
4530                 } else if (remaining < 0) {
4531                         remaining += len - index;
4532                 } else if (index + remaining > size) {
4533                         remaining = size - index;
4534                 }
4535 
4536                 for (i = 0; i < remaining; i++) {
4537                         if ((d[i] = dtrace_load8(s + index + i)) == '\0')
4538                                 break;
4539                 }
4540 
4541                 d[i] = '\0';
4542 
4543                 mstate->dtms_scratch_ptr += size;
4544                 regs[rd] = (uintptr_t)d;
4545                 break;
4546         }
4547 
4548         case DIF_SUBR_JSON: {
4549                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4550                 uintptr_t json = tupregs[0].dttk_value;
4551                 size_t jsonlen = dtrace_strlen((char *)json, size);
4552                 uintptr_t elem = tupregs[1].dttk_value;
4553                 size_t elemlen = dtrace_strlen((char *)elem, size);
4554 
4555                 char *dest = (char *)mstate->dtms_scratch_ptr;
4556                 char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1;
4557                 char *ee = elemlist;
4558                 int nelems = 1;
4559                 uintptr_t cur;
4560 
4561                 if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) ||
4562                     !dtrace_canload(elem, elemlen + 1, mstate, vstate)) {
4563                         regs[rd] = NULL;
4564                         break;
4565                 }
4566 
4567                 if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) {
4568                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4569                         regs[rd] = NULL;
4570                         break;
4571                 }
4572 
4573                 /*
4574                  * Read the element selector and split it up into a packed list
4575                  * of strings.
4576                  */
4577                 for (cur = elem; cur < elem + elemlen; cur++) {
4578                         char cc = dtrace_load8(cur);
4579 
4580                         if (cur == elem && cc == '[') {
4581                                 /*
4582                                  * If the first element selector key is
4583                                  * actually an array index then ignore the
4584                                  * bracket.
4585                                  */
4586                                 continue;
4587                         }
4588 
4589                         if (cc == ']')
4590                                 continue;
4591 
4592                         if (cc == '.' || cc == '[') {
4593                                 nelems++;
4594                                 cc = '\0';
4595                         }
4596 
4597                         *ee++ = cc;
4598                 }
4599                 *ee++ = '\0';
4600 
4601                 if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist,
4602                     nelems, dest)) != NULL)
4603                         mstate->dtms_scratch_ptr += jsonlen + 1;
4604                 break;
4605         }
4606 
4607         case DIF_SUBR_TOUPPER:
4608         case DIF_SUBR_TOLOWER: {
4609                 uintptr_t s = tupregs[0].dttk_value;
4610                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4611                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
4612                 size_t len = dtrace_strlen((char *)s, size);
4613                 char lower, upper, convert;
4614                 int64_t i;
4615 
4616                 if (subr == DIF_SUBR_TOUPPER) {
4617                         lower = 'a';
4618                         upper = 'z';
4619                         convert = 'A';
4620                 } else {
4621                         lower = 'A';
4622                         upper = 'Z';
4623                         convert = 'a';
4624                 }
4625 
4626                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4627                         regs[rd] = NULL;
4628                         break;
4629                 }
4630 
4631                 if (!DTRACE_INSCRATCH(mstate, size)) {
4632                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4633                         regs[rd] = NULL;
4634                         break;
4635                 }
4636 
4637                 for (i = 0; i < size - 1; i++) {
4638                         if ((c = dtrace_load8(s + i)) == '\0')
4639                                 break;
4640 
4641                         if (c >= lower && c <= upper)
4642                                 c = convert + (c - lower);
4643 
4644                         dest[i] = c;
4645                 }
4646 
4647                 ASSERT(i < size);
4648                 dest[i] = '\0';
4649                 regs[rd] = (uintptr_t)dest;
4650                 mstate->dtms_scratch_ptr += size;
4651                 break;
4652         }
4653 
4654 case DIF_SUBR_GETMAJOR:
4655 #ifdef _LP64
4656                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
4657 #else
4658                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
4659 #endif
4660                 break;
4661 
4662         case DIF_SUBR_GETMINOR:
4663 #ifdef _LP64
4664                 regs[rd] = tupregs[0].dttk_value & MAXMIN64;
4665 #else
4666                 regs[rd] = tupregs[0].dttk_value & MAXMIN;
4667 #endif
4668                 break;
4669 
4670         case DIF_SUBR_DDI_PATHNAME: {
4671                 /*
4672                  * This one is a galactic mess.  We are going to roughly
4673                  * emulate ddi_pathname(), but it's made more complicated
4674                  * by the fact that we (a) want to include the minor name and
4675                  * (b) must proceed iteratively instead of recursively.
4676                  */
4677                 uintptr_t dest = mstate->dtms_scratch_ptr;
4678                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4679                 char *start = (char *)dest, *end = start + size - 1;
4680                 uintptr_t daddr = tupregs[0].dttk_value;
4681                 int64_t minor = (int64_t)tupregs[1].dttk_value;
4682                 char *s;
4683                 int i, len, depth = 0;
4684 
4685                 /*
4686                  * Due to all the pointer jumping we do and context we must
4687                  * rely upon, we just mandate that the user must have kernel
4688                  * read privileges to use this routine.
4689                  */
4690                 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
4691                         *flags |= CPU_DTRACE_KPRIV;
4692                         *illval = daddr;
4693                         regs[rd] = NULL;
4694                 }
4695 
4696                 if (!DTRACE_INSCRATCH(mstate, size)) {
4697                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4698                         regs[rd] = NULL;
4699                         break;
4700                 }
4701 
4702                 *end = '\0';
4703 
4704                 /*
4705                  * We want to have a name for the minor.  In order to do this,
4706                  * we need to walk the minor list from the devinfo.  We want
4707                  * to be sure that we don't infinitely walk a circular list,
4708                  * so we check for circularity by sending a scout pointer
4709                  * ahead two elements for every element that we iterate over;
4710                  * if the list is circular, these will ultimately point to the
4711                  * same element.  You may recognize this little trick as the
4712                  * answer to a stupid interview question -- one that always
4713                  * seems to be asked by those who had to have it laboriously
4714                  * explained to them, and who can't even concisely describe
4715                  * the conditions under which one would be forced to resort to
4716                  * this technique.  Needless to say, those conditions are
4717                  * found here -- and probably only here.  Is this the only use
4718                  * of this infamous trick in shipping, production code?  If it
4719                  * isn't, it probably should be...
4720                  */
4721                 if (minor != -1) {
4722                         uintptr_t maddr = dtrace_loadptr(daddr +
4723                             offsetof(struct dev_info, devi_minor));
4724 
4725                         uintptr_t next = offsetof(struct ddi_minor_data, next);
4726                         uintptr_t name = offsetof(struct ddi_minor_data,
4727                             d_minor) + offsetof(struct ddi_minor, name);
4728                         uintptr_t dev = offsetof(struct ddi_minor_data,
4729                             d_minor) + offsetof(struct ddi_minor, dev);
4730                         uintptr_t scout;
4731 
4732                         if (maddr != NULL)
4733                                 scout = dtrace_loadptr(maddr + next);
4734 
4735                         while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4736                                 uint64_t m;
4737 #ifdef _LP64
4738                                 m = dtrace_load64(maddr + dev) & MAXMIN64;
4739 #else
4740                                 m = dtrace_load32(maddr + dev) & MAXMIN;
4741 #endif
4742                                 if (m != minor) {
4743                                         maddr = dtrace_loadptr(maddr + next);
4744 
4745                                         if (scout == NULL)
4746                                                 continue;
4747 
4748                                         scout = dtrace_loadptr(scout + next);
4749 
4750                                         if (scout == NULL)
4751                                                 continue;
4752 
4753                                         scout = dtrace_loadptr(scout + next);
4754 
4755                                         if (scout == NULL)
4756                                                 continue;
4757 
4758                                         if (scout == maddr) {
4759                                                 *flags |= CPU_DTRACE_ILLOP;
4760                                                 break;
4761                                         }
4762 
4763                                         continue;
4764                                 }
4765 
4766                                 /*
4767                                  * We have the minor data.  Now we need to
4768                                  * copy the minor's name into the end of the
4769                                  * pathname.
4770                                  */
4771                                 s = (char *)dtrace_loadptr(maddr + name);
4772                                 len = dtrace_strlen(s, size);
4773 
4774                                 if (*flags & CPU_DTRACE_FAULT)
4775                                         break;
4776 
4777                                 if (len != 0) {
4778                                         if ((end -= (len + 1)) < start)
4779                                                 break;
4780 
4781                                         *end = ':';
4782                                 }
4783 
4784                                 for (i = 1; i <= len; i++)
4785                                         end[i] = dtrace_load8((uintptr_t)s++);
4786                                 break;
4787                         }
4788                 }
4789 
4790                 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4791                         ddi_node_state_t devi_state;
4792 
4793                         devi_state = dtrace_load32(daddr +
4794                             offsetof(struct dev_info, devi_node_state));
4795 
4796                         if (*flags & CPU_DTRACE_FAULT)
4797                                 break;
4798 
4799                         if (devi_state >= DS_INITIALIZED) {
4800                                 s = (char *)dtrace_loadptr(daddr +
4801                                     offsetof(struct dev_info, devi_addr));
4802                                 len = dtrace_strlen(s, size);
4803 
4804                                 if (*flags & CPU_DTRACE_FAULT)
4805                                         break;
4806 
4807                                 if (len != 0) {
4808                                         if ((end -= (len + 1)) < start)
4809                                                 break;
4810 
4811                                         *end = '@';
4812                                 }
4813 
4814                                 for (i = 1; i <= len; i++)
4815                                         end[i] = dtrace_load8((uintptr_t)s++);
4816                         }
4817 
4818                         /*
4819                          * Now for the node name...
4820                          */
4821                         s = (char *)dtrace_loadptr(daddr +
4822                             offsetof(struct dev_info, devi_node_name));
4823 
4824                         daddr = dtrace_loadptr(daddr +
4825                             offsetof(struct dev_info, devi_parent));
4826 
4827                         /*
4828                          * If our parent is NULL (that is, if we're the root
4829                          * node), we're going to use the special path
4830                          * "devices".
4831                          */
4832                         if (daddr == NULL)
4833                                 s = "devices";
4834 
4835                         len = dtrace_strlen(s, size);
4836                         if (*flags & CPU_DTRACE_FAULT)
4837                                 break;
4838 
4839                         if ((end -= (len + 1)) < start)
4840                                 break;
4841 
4842                         for (i = 1; i <= len; i++)
4843                                 end[i] = dtrace_load8((uintptr_t)s++);
4844                         *end = '/';
4845 
4846                         if (depth++ > dtrace_devdepth_max) {
4847                                 *flags |= CPU_DTRACE_ILLOP;
4848                                 break;
4849                         }
4850                 }
4851 
4852                 if (end < start)
4853                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4854 
4855                 if (daddr == NULL) {
4856                         regs[rd] = (uintptr_t)end;
4857                         mstate->dtms_scratch_ptr += size;
4858                 }
4859 
4860                 break;
4861         }
4862 
4863         case DIF_SUBR_STRJOIN: {
4864                 char *d = (char *)mstate->dtms_scratch_ptr;
4865                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4866                 uintptr_t s1 = tupregs[0].dttk_value;
4867                 uintptr_t s2 = tupregs[1].dttk_value;
4868                 int i = 0;
4869 
4870                 if (!dtrace_strcanload(s1, size, mstate, vstate) ||
4871                     !dtrace_strcanload(s2, size, mstate, vstate)) {
4872                         regs[rd] = NULL;
4873                         break;
4874                 }
4875 
4876                 if (!DTRACE_INSCRATCH(mstate, size)) {
4877                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4878                         regs[rd] = NULL;
4879                         break;
4880                 }
4881 
4882                 for (;;) {
4883                         if (i >= size) {
4884                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4885                                 regs[rd] = NULL;
4886                                 break;
4887                         }
4888 
4889                         if ((d[i++] = dtrace_load8(s1++)) == '\0') {
4890                                 i--;
4891                                 break;
4892                         }
4893                 }
4894 
4895                 for (;;) {
4896                         if (i >= size) {
4897                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4898                                 regs[rd] = NULL;
4899                                 break;
4900                         }
4901 
4902                         if ((d[i++] = dtrace_load8(s2++)) == '\0')
4903                                 break;
4904                 }
4905 
4906                 if (i < size) {
4907                         mstate->dtms_scratch_ptr += i;
4908                         regs[rd] = (uintptr_t)d;
4909                 }
4910 
4911                 break;
4912         }
4913 
4914         case DIF_SUBR_STRTOLL: {
4915                 uintptr_t s = tupregs[0].dttk_value;
4916                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4917                 int base = 10;
4918 
4919                 if (nargs > 1) {
4920                         if ((base = tupregs[1].dttk_value) <= 1 ||
4921                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4922                                 *flags |= CPU_DTRACE_ILLOP;
4923                                 break;
4924                         }
4925                 }
4926 
4927                 if (!dtrace_strcanload(s, size, mstate, vstate)) {
4928                         regs[rd] = INT64_MIN;
4929                         break;
4930                 }
4931 
4932                 regs[rd] = dtrace_strtoll((char *)s, base, size);
4933                 break;
4934         }
4935 
4936         case DIF_SUBR_LLTOSTR: {
4937                 int64_t i = (int64_t)tupregs[0].dttk_value;
4938                 uint64_t val, digit;
4939                 uint64_t size = 65;     /* enough room for 2^64 in binary */
4940                 char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
4941                 int base = 10;
4942 
4943                 if (nargs > 1) {
4944                         if ((base = tupregs[1].dttk_value) <= 1 ||
4945                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4946                                 *flags |= CPU_DTRACE_ILLOP;
4947                                 break;
4948                         }
4949                 }
4950 
4951                 val = (base == 10 && i < 0) ? i * -1 : i;
4952 
4953                 if (!DTRACE_INSCRATCH(mstate, size)) {
4954                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4955                         regs[rd] = NULL;
4956                         break;
4957                 }
4958 
4959                 for (*end-- = '\0'; val; val /= base) {
4960                         if ((digit = val % base) <= '9' - '0') {
4961                                 *end-- = '0' + digit;
4962                         } else {
4963                                 *end-- = 'a' + (digit - ('9' - '0') - 1);
4964                         }
4965                 }
4966 
4967                 if (i == 0 && base == 16)
4968                         *end-- = '0';
4969 
4970                 if (base == 16)
4971                         *end-- = 'x';
4972 
4973                 if (i == 0 || base == 8 || base == 16)
4974                         *end-- = '0';
4975 
4976                 if (i < 0 && base == 10)
4977                         *end-- = '-';
4978 
4979                 regs[rd] = (uintptr_t)end + 1;
4980                 mstate->dtms_scratch_ptr += size;
4981                 break;
4982         }
4983 
4984         case DIF_SUBR_HTONS:
4985         case DIF_SUBR_NTOHS:
4986 #ifdef _BIG_ENDIAN
4987                 regs[rd] = (uint16_t)tupregs[0].dttk_value;
4988 #else
4989                 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
4990 #endif
4991                 break;
4992 
4993 
4994         case DIF_SUBR_HTONL:
4995         case DIF_SUBR_NTOHL:
4996 #ifdef _BIG_ENDIAN
4997                 regs[rd] = (uint32_t)tupregs[0].dttk_value;
4998 #else
4999                 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
5000 #endif
5001                 break;
5002 
5003 
5004         case DIF_SUBR_HTONLL:
5005         case DIF_SUBR_NTOHLL:
5006 #ifdef _BIG_ENDIAN
5007                 regs[rd] = (uint64_t)tupregs[0].dttk_value;
5008 #else
5009                 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
5010 #endif
5011                 break;
5012 
5013 
5014         case DIF_SUBR_DIRNAME:
5015         case DIF_SUBR_BASENAME: {
5016                 char *dest = (char *)mstate->dtms_scratch_ptr;
5017                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5018                 uintptr_t src = tupregs[0].dttk_value;
5019                 int i, j, len = dtrace_strlen((char *)src, size);
5020                 int lastbase = -1, firstbase = -1, lastdir = -1;
5021                 int start, end;
5022 
5023                 if (!dtrace_canload(src, len + 1, mstate, vstate)) {
5024                         regs[rd] = NULL;
5025                         break;
5026                 }
5027 
5028                 if (!DTRACE_INSCRATCH(mstate, size)) {
5029                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5030                         regs[rd] = NULL;
5031                         break;
5032                 }
5033 
5034                 /*
5035                  * The basename and dirname for a zero-length string is
5036                  * defined to be "."
5037                  */
5038                 if (len == 0) {
5039                         len = 1;
5040                         src = (uintptr_t)".";
5041                 }
5042 
5043                 /*
5044                  * Start from the back of the string, moving back toward the
5045                  * front until we see a character that isn't a slash.  That
5046                  * character is the last character in the basename.
5047                  */
5048                 for (i = len - 1; i >= 0; i--) {
5049                         if (dtrace_load8(src + i) != '/')
5050                                 break;
5051                 }
5052 
5053                 if (i >= 0)
5054                         lastbase = i;
5055 
5056                 /*
5057                  * Starting from the last character in the basename, move
5058                  * towards the front until we find a slash.  The character
5059                  * that we processed immediately before that is the first
5060                  * character in the basename.
5061                  */
5062                 for (; i >= 0; i--) {
5063                         if (dtrace_load8(src + i) == '/')
5064                                 break;
5065                 }
5066 
5067                 if (i >= 0)
5068                         firstbase = i + 1;
5069 
5070                 /*
5071                  * Now keep going until we find a non-slash character.  That
5072                  * character is the last character in the dirname.
5073                  */
5074                 for (; i >= 0; i--) {
5075                         if (dtrace_load8(src + i) != '/')
5076                                 break;
5077                 }
5078 
5079                 if (i >= 0)
5080                         lastdir = i;
5081 
5082                 ASSERT(!(lastbase == -1 && firstbase != -1));
5083                 ASSERT(!(firstbase == -1 && lastdir != -1));
5084 
5085                 if (lastbase == -1) {
5086                         /*
5087                          * We didn't find a non-slash character.  We know that
5088                          * the length is non-zero, so the whole string must be
5089                          * slashes.  In either the dirname or the basename
5090                          * case, we return '/'.
5091                          */
5092                         ASSERT(firstbase == -1);
5093                         firstbase = lastbase = lastdir = 0;
5094                 }
5095 
5096                 if (firstbase == -1) {
5097                         /*
5098                          * The entire string consists only of a basename
5099                          * component.  If we're looking for dirname, we need
5100                          * to change our string to be just "."; if we're
5101                          * looking for a basename, we'll just set the first
5102                          * character of the basename to be 0.
5103                          */
5104                         if (subr == DIF_SUBR_DIRNAME) {
5105                                 ASSERT(lastdir == -1);
5106                                 src = (uintptr_t)".";
5107                                 lastdir = 0;
5108                         } else {
5109                                 firstbase = 0;
5110                         }
5111                 }
5112 
5113                 if (subr == DIF_SUBR_DIRNAME) {
5114                         if (lastdir == -1) {
5115                                 /*
5116                                  * We know that we have a slash in the name --
5117                                  * or lastdir would be set to 0, above.  And
5118                                  * because lastdir is -1, we know that this
5119                                  * slash must be the first character.  (That
5120                                  * is, the full string must be of the form
5121                                  * "/basename".)  In this case, the last
5122                                  * character of the directory name is 0.
5123                                  */
5124                                 lastdir = 0;
5125                         }
5126 
5127                         start = 0;
5128                         end = lastdir;
5129                 } else {
5130                         ASSERT(subr == DIF_SUBR_BASENAME);
5131                         ASSERT(firstbase != -1 && lastbase != -1);
5132                         start = firstbase;
5133                         end = lastbase;
5134                 }
5135 
5136                 for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
5137                         dest[j] = dtrace_load8(src + i);
5138 
5139                 dest[j] = '\0';
5140                 regs[rd] = (uintptr_t)dest;
5141                 mstate->dtms_scratch_ptr += size;
5142                 break;
5143         }
5144 
5145         case DIF_SUBR_GETF: {
5146                 uintptr_t fd = tupregs[0].dttk_value;
5147                 uf_info_t *finfo = &curthread->t_procp->p_user.u_finfo;
5148                 file_t *fp;
5149 
5150                 if (!dtrace_priv_proc(state, mstate)) {
5151                         regs[rd] = NULL;
5152                         break;
5153                 }
5154 
5155                 /*
5156                  * This is safe because fi_nfiles only increases, and the
5157                  * fi_list array is not freed when the array size doubles.
5158                  * (See the comment in flist_grow() for details on the
5159                  * management of the u_finfo structure.)
5160                  */
5161                 fp = fd < finfo->fi_nfiles ? finfo->fi_list[fd].uf_file : NULL;
5162 
5163                 mstate->dtms_getf = fp;
5164                 regs[rd] = (uintptr_t)fp;
5165                 break;
5166         }
5167 
5168         case DIF_SUBR_CLEANPATH: {
5169                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
5170                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5171                 uintptr_t src = tupregs[0].dttk_value;
5172                 int i = 0, j = 0;
5173                 zone_t *z;
5174 
5175                 if (!dtrace_strcanload(src, size, mstate, vstate)) {
5176                         regs[rd] = NULL;
5177                         break;
5178                 }
5179 
5180                 if (!DTRACE_INSCRATCH(mstate, size)) {
5181                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5182                         regs[rd] = NULL;
5183                         break;
5184                 }
5185 
5186                 /*
5187                  * Move forward, loading each character.
5188                  */
5189                 do {
5190                         c = dtrace_load8(src + i++);
5191 next:
5192                         if (j + 5 >= size)   /* 5 = strlen("/..c\0") */
5193                                 break;
5194 
5195                         if (c != '/') {
5196                                 dest[j++] = c;
5197                                 continue;
5198                         }
5199 
5200                         c = dtrace_load8(src + i++);
5201 
5202                         if (c == '/') {
5203                                 /*
5204                                  * We have two slashes -- we can just advance
5205                                  * to the next character.
5206                                  */
5207                                 goto next;
5208                         }
5209 
5210                         if (c != '.') {
5211                                 /*
5212                                  * This is not "." and it's not ".." -- we can
5213                                  * just store the "/" and this character and
5214                                  * drive on.
5215                                  */
5216                                 dest[j++] = '/';
5217                                 dest[j++] = c;
5218                                 continue;
5219                         }
5220 
5221                         c = dtrace_load8(src + i++);
5222 
5223                         if (c == '/') {
5224                                 /*
5225                                  * This is a "/./" component.  We're not going
5226                                  * to store anything in the destination buffer;
5227                                  * we're just going to go to the next component.
5228                                  */
5229                                 goto next;
5230                         }
5231 
5232                         if (c != '.') {
5233                                 /*
5234                                  * This is not ".." -- we can just store the
5235                                  * "/." and this character and continue
5236                                  * processing.
5237                                  */
5238                                 dest[j++] = '/';
5239                                 dest[j++] = '.';
5240                                 dest[j++] = c;
5241                                 continue;
5242                         }
5243 
5244                         c = dtrace_load8(src + i++);
5245 
5246                         if (c != '/' && c != '\0') {
5247                                 /*
5248                                  * This is not ".." -- it's "..[mumble]".
5249                                  * We'll store the "/.." and this character
5250                                  * and continue processing.
5251                                  */
5252                                 dest[j++] = '/';
5253                                 dest[j++] = '.';
5254                                 dest[j++] = '.';
5255                                 dest[j++] = c;
5256                                 continue;
5257                         }
5258 
5259                         /*
5260                          * This is "/../" or "/..\0".  We need to back up
5261                          * our destination pointer until we find a "/".
5262                          */
5263                         i--;
5264                         while (j != 0 && dest[--j] != '/')
5265                                 continue;
5266 
5267                         if (c == '\0')
5268                                 dest[++j] = '/';
5269                 } while (c != '\0');
5270 
5271                 dest[j] = '\0';
5272 
5273                 if (mstate->dtms_getf != NULL &&
5274                     !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) &&
5275                     (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) {
5276                         /*
5277                          * If we've done a getf() as a part of this ECB and we
5278                          * don't have kernel access (and we're not in the global
5279                          * zone), check if the path we cleaned up begins with
5280                          * the zone's root path, and trim it off if so.  Note
5281                          * that this is an output cleanliness issue, not a
5282                          * security issue: knowing one's zone root path does
5283                          * not enable privilege escalation.
5284                          */
5285                         if (strstr(dest, z->zone_rootpath) == dest)
5286                                 dest += strlen(z->zone_rootpath) - 1;
5287                 }
5288 
5289                 regs[rd] = (uintptr_t)dest;
5290                 mstate->dtms_scratch_ptr += size;
5291                 break;
5292         }
5293 
5294         case DIF_SUBR_INET_NTOA:
5295         case DIF_SUBR_INET_NTOA6:
5296         case DIF_SUBR_INET_NTOP: {
5297                 size_t size;
5298                 int af, argi, i;
5299                 char *base, *end;
5300 
5301                 if (subr == DIF_SUBR_INET_NTOP) {
5302                         af = (int)tupregs[0].dttk_value;
5303                         argi = 1;
5304                 } else {
5305                         af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
5306                         argi = 0;
5307                 }
5308 
5309                 if (af == AF_INET) {
5310                         ipaddr_t ip4;
5311                         uint8_t *ptr8, val;
5312 
5313                         /*
5314                          * Safely load the IPv4 address.
5315                          */
5316                         ip4 = dtrace_load32(tupregs[argi].dttk_value);
5317 
5318                         /*
5319                          * Check an IPv4 string will fit in scratch.
5320                          */
5321                         size = INET_ADDRSTRLEN;
5322                         if (!DTRACE_INSCRATCH(mstate, size)) {
5323                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5324                                 regs[rd] = NULL;
5325                                 break;
5326                         }
5327                         base = (char *)mstate->dtms_scratch_ptr;
5328                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5329 
5330                         /*
5331                          * Stringify as a dotted decimal quad.
5332                          */
5333                         *end-- = '\0';
5334                         ptr8 = (uint8_t *)&ip4;
5335                         for (i = 3; i >= 0; i--) {
5336                                 val = ptr8[i];
5337 
5338                                 if (val == 0) {
5339                                         *end-- = '0';
5340                                 } else {
5341                                         for (; val; val /= 10) {
5342                                                 *end-- = '0' + (val % 10);
5343                                         }
5344                                 }
5345 
5346                                 if (i > 0)
5347                                         *end-- = '.';
5348                         }
5349                         ASSERT(end + 1 >= base);
5350 
5351                 } else if (af == AF_INET6) {
5352                         struct in6_addr ip6;
5353                         int firstzero, tryzero, numzero, v6end;
5354                         uint16_t val;
5355                         const char digits[] = "0123456789abcdef";
5356 
5357                         /*
5358                          * Stringify using RFC 1884 convention 2 - 16 bit
5359                          * hexadecimal values with a zero-run compression.
5360                          * Lower case hexadecimal digits are used.
5361                          *      eg, fe80::214:4fff:fe0b:76c8.
5362                          * The IPv4 embedded form is returned for inet_ntop,
5363                          * just the IPv4 string is returned for inet_ntoa6.
5364                          */
5365 
5366                         /*
5367                          * Safely load the IPv6 address.
5368                          */
5369                         dtrace_bcopy(
5370                             (void *)(uintptr_t)tupregs[argi].dttk_value,
5371                             (void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
5372 
5373                         /*
5374                          * Check an IPv6 string will fit in scratch.
5375                          */
5376                         size = INET6_ADDRSTRLEN;
5377                         if (!DTRACE_INSCRATCH(mstate, size)) {
5378                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5379                                 regs[rd] = NULL;
5380                                 break;
5381                         }
5382                         base = (char *)mstate->dtms_scratch_ptr;
5383                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5384                         *end-- = '\0';
5385 
5386                         /*
5387                          * Find the longest run of 16 bit zero values
5388                          * for the single allowed zero compression - "::".
5389                          */
5390                         firstzero = -1;
5391                         tryzero = -1;
5392                         numzero = 1;
5393                         for (i = 0; i < sizeof (struct in6_addr); i++) {
5394                                 if (ip6._S6_un._S6_u8[i] == 0 &&
5395                                     tryzero == -1 && i % 2 == 0) {
5396                                         tryzero = i;
5397                                         continue;
5398                                 }
5399 
5400                                 if (tryzero != -1 &&
5401                                     (ip6._S6_un._S6_u8[i] != 0 ||
5402                                     i == sizeof (struct in6_addr) - 1)) {
5403 
5404                                         if (i - tryzero <= numzero) {
5405                                                 tryzero = -1;
5406                                                 continue;
5407                                         }
5408 
5409                                         firstzero = tryzero;
5410                                         numzero = i - i % 2 - tryzero;
5411                                         tryzero = -1;
5412 
5413                                         if (ip6._S6_un._S6_u8[i] == 0 &&
5414                                             i == sizeof (struct in6_addr) - 1)
5415                                                 numzero += 2;
5416                                 }
5417                         }
5418                         ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
5419 
5420                         /*
5421                          * Check for an IPv4 embedded address.
5422                          */
5423                         v6end = sizeof (struct in6_addr) - 2;
5424                         if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
5425                             IN6_IS_ADDR_V4COMPAT(&ip6)) {
5426                                 for (i = sizeof (struct in6_addr) - 1;
5427                                     i >= DTRACE_V4MAPPED_OFFSET; i--) {
5428                                         ASSERT(end >= base);
5429 
5430                                         val = ip6._S6_un._S6_u8[i];
5431 
5432                                         if (val == 0) {
5433                                                 *end-- = '0';
5434                                         } else {
5435                                                 for (; val; val /= 10) {
5436                                                         *end-- = '0' + val % 10;
5437                                                 }
5438                                         }
5439 
5440                                         if (i > DTRACE_V4MAPPED_OFFSET)
5441                                                 *end-- = '.';
5442                                 }
5443 
5444                                 if (subr == DIF_SUBR_INET_NTOA6)
5445                                         goto inetout;
5446 
5447                                 /*
5448                                  * Set v6end to skip the IPv4 address that
5449                                  * we have already stringified.
5450                                  */
5451                                 v6end = 10;
5452                         }
5453 
5454                         /*
5455                          * Build the IPv6 string by working through the
5456                          * address in reverse.
5457                          */
5458                         for (i = v6end; i >= 0; i -= 2) {
5459                                 ASSERT(end >= base);
5460 
5461                                 if (i == firstzero + numzero - 2) {
5462                                         *end-- = ':';
5463                                         *end-- = ':';
5464                                         i -= numzero - 2;
5465                                         continue;
5466                                 }
5467 
5468                                 if (i < 14 && i != firstzero - 2)
5469                                         *end-- = ':';
5470 
5471                                 val = (ip6._S6_un._S6_u8[i] << 8) +
5472                                     ip6._S6_un._S6_u8[i + 1];
5473 
5474                                 if (val == 0) {
5475                                         *end-- = '0';
5476                                 } else {
5477                                         for (; val; val /= 16) {
5478                                                 *end-- = digits[val % 16];
5479                                         }
5480                                 }
5481                         }
5482                         ASSERT(end + 1 >= base);
5483 
5484                 } else {
5485                         /*
5486                          * The user didn't use AH_INET or AH_INET6.
5487                          */
5488                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
5489                         regs[rd] = NULL;
5490                         break;
5491                 }
5492 
5493 inetout:        regs[rd] = (uintptr_t)end + 1;
5494                 mstate->dtms_scratch_ptr += size;
5495                 break;
5496         }
5497 
5498         }
5499 }
5500 
5501 /*
5502  * Emulate the execution of DTrace IR instructions specified by the given
5503  * DIF object.  This function is deliberately void of assertions as all of
5504  * the necessary checks are handled by a call to dtrace_difo_validate().
5505  */
5506 static uint64_t
5507 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
5508     dtrace_vstate_t *vstate, dtrace_state_t *state)
5509 {
5510         const dif_instr_t *text = difo->dtdo_buf;
5511         const uint_t textlen = difo->dtdo_len;
5512         const char *strtab = difo->dtdo_strtab;
5513         const uint64_t *inttab = difo->dtdo_inttab;
5514 
5515         uint64_t rval = 0;
5516         dtrace_statvar_t *svar;
5517         dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
5518         dtrace_difv_t *v;
5519         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5520         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
5521 
5522         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
5523         uint64_t regs[DIF_DIR_NREGS];
5524         uint64_t *tmp;
5525 
5526         uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
5527         int64_t cc_r;
5528         uint_t pc = 0, id, opc;
5529         uint8_t ttop = 0;
5530         dif_instr_t instr;
5531         uint_t r1, r2, rd;
5532 
5533         /*
5534          * We stash the current DIF object into the machine state: we need it
5535          * for subsequent access checking.
5536          */
5537         mstate->dtms_difo = difo;
5538 
5539         regs[DIF_REG_R0] = 0;           /* %r0 is fixed at zero */
5540 
5541         while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
5542                 opc = pc;
5543 
5544                 instr = text[pc++];
5545                 r1 = DIF_INSTR_R1(instr);
5546                 r2 = DIF_INSTR_R2(instr);
5547                 rd = DIF_INSTR_RD(instr);
5548 
5549                 switch (DIF_INSTR_OP(instr)) {
5550                 case DIF_OP_OR:
5551                         regs[rd] = regs[r1] | regs[r2];
5552                         break;
5553                 case DIF_OP_XOR:
5554                         regs[rd] = regs[r1] ^ regs[r2];
5555                         break;
5556                 case DIF_OP_AND:
5557                         regs[rd] = regs[r1] & regs[r2];
5558                         break;
5559                 case DIF_OP_SLL:
5560                         regs[rd] = regs[r1] << regs[r2];
5561                         break;
5562                 case DIF_OP_SRL:
5563                         regs[rd] = regs[r1] >> regs[r2];
5564                         break;
5565                 case DIF_OP_SUB:
5566                         regs[rd] = regs[r1] - regs[r2];
5567                         break;
5568                 case DIF_OP_ADD:
5569                         regs[rd] = regs[r1] + regs[r2];
5570                         break;
5571                 case DIF_OP_MUL:
5572                         regs[rd] = regs[r1] * regs[r2];
5573                         break;
5574                 case DIF_OP_SDIV:
5575                         if (regs[r2] == 0) {
5576                                 regs[rd] = 0;
5577                                 *flags |= CPU_DTRACE_DIVZERO;
5578                         } else {
5579                                 regs[rd] = (int64_t)regs[r1] /
5580                                     (int64_t)regs[r2];
5581                         }
5582                         break;
5583 
5584                 case DIF_OP_UDIV:
5585                         if (regs[r2] == 0) {
5586                                 regs[rd] = 0;
5587                                 *flags |= CPU_DTRACE_DIVZERO;
5588                         } else {
5589                                 regs[rd] = regs[r1] / regs[r2];
5590                         }
5591                         break;
5592 
5593                 case DIF_OP_SREM:
5594                         if (regs[r2] == 0) {
5595                                 regs[rd] = 0;
5596                                 *flags |= CPU_DTRACE_DIVZERO;
5597                         } else {
5598                                 regs[rd] = (int64_t)regs[r1] %
5599                                     (int64_t)regs[r2];
5600                         }
5601                         break;
5602 
5603                 case DIF_OP_UREM:
5604                         if (regs[r2] == 0) {
5605                                 regs[rd] = 0;
5606                                 *flags |= CPU_DTRACE_DIVZERO;
5607                         } else {
5608                                 regs[rd] = regs[r1] % regs[r2];
5609                         }
5610                         break;
5611 
5612                 case DIF_OP_NOT:
5613                         regs[rd] = ~regs[r1];
5614                         break;
5615                 case DIF_OP_MOV:
5616                         regs[rd] = regs[r1];
5617                         break;
5618                 case DIF_OP_CMP:
5619                         cc_r = regs[r1] - regs[r2];
5620                         cc_n = cc_r < 0;
5621                         cc_z = cc_r == 0;
5622                         cc_v = 0;
5623                         cc_c = regs[r1] < regs[r2];
5624                         break;
5625                 case DIF_OP_TST:
5626                         cc_n = cc_v = cc_c = 0;
5627                         cc_z = regs[r1] == 0;
5628                         break;
5629                 case DIF_OP_BA:
5630                         pc = DIF_INSTR_LABEL(instr);
5631                         break;
5632                 case DIF_OP_BE:
5633                         if (cc_z)
5634                                 pc = DIF_INSTR_LABEL(instr);
5635                         break;
5636                 case DIF_OP_BNE:
5637                         if (cc_z == 0)
5638                                 pc = DIF_INSTR_LABEL(instr);
5639                         break;
5640                 case DIF_OP_BG:
5641                         if ((cc_z | (cc_n ^ cc_v)) == 0)
5642                                 pc = DIF_INSTR_LABEL(instr);
5643                         break;
5644                 case DIF_OP_BGU:
5645                         if ((cc_c | cc_z) == 0)
5646                                 pc = DIF_INSTR_LABEL(instr);
5647                         break;
5648                 case DIF_OP_BGE:
5649                         if ((cc_n ^ cc_v) == 0)
5650                                 pc = DIF_INSTR_LABEL(instr);
5651                         break;
5652                 case DIF_OP_BGEU:
5653                         if (cc_c == 0)
5654                                 pc = DIF_INSTR_LABEL(instr);
5655                         break;
5656                 case DIF_OP_BL:
5657                         if (cc_n ^ cc_v)
5658                                 pc = DIF_INSTR_LABEL(instr);
5659                         break;
5660                 case DIF_OP_BLU:
5661                         if (cc_c)
5662                                 pc = DIF_INSTR_LABEL(instr);
5663                         break;
5664                 case DIF_OP_BLE:
5665                         if (cc_z | (cc_n ^ cc_v))
5666                                 pc = DIF_INSTR_LABEL(instr);
5667                         break;
5668                 case DIF_OP_BLEU:
5669                         if (cc_c | cc_z)
5670                                 pc = DIF_INSTR_LABEL(instr);
5671                         break;
5672                 case DIF_OP_RLDSB:
5673                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5674                                 break;
5675                         /*FALLTHROUGH*/
5676                 case DIF_OP_LDSB:
5677                         regs[rd] = (int8_t)dtrace_load8(regs[r1]);
5678                         break;
5679                 case DIF_OP_RLDSH:
5680                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5681                                 break;
5682                         /*FALLTHROUGH*/
5683                 case DIF_OP_LDSH:
5684                         regs[rd] = (int16_t)dtrace_load16(regs[r1]);
5685                         break;
5686                 case DIF_OP_RLDSW:
5687                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5688                                 break;
5689                         /*FALLTHROUGH*/
5690                 case DIF_OP_LDSW:
5691                         regs[rd] = (int32_t)dtrace_load32(regs[r1]);
5692                         break;
5693                 case DIF_OP_RLDUB:
5694                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5695                                 break;
5696                         /*FALLTHROUGH*/
5697                 case DIF_OP_LDUB:
5698                         regs[rd] = dtrace_load8(regs[r1]);
5699                         break;
5700                 case DIF_OP_RLDUH:
5701                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5702                                 break;
5703                         /*FALLTHROUGH*/
5704                 case DIF_OP_LDUH:
5705                         regs[rd] = dtrace_load16(regs[r1]);
5706                         break;
5707                 case DIF_OP_RLDUW:
5708                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5709                                 break;
5710                         /*FALLTHROUGH*/
5711                 case DIF_OP_LDUW:
5712                         regs[rd] = dtrace_load32(regs[r1]);
5713                         break;
5714                 case DIF_OP_RLDX:
5715                         if (!dtrace_canload(regs[r1], 8, mstate, vstate))
5716                                 break;
5717                         /*FALLTHROUGH*/
5718                 case DIF_OP_LDX:
5719                         regs[rd] = dtrace_load64(regs[r1]);
5720                         break;
5721                 case DIF_OP_ULDSB:
5722                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5723                         regs[rd] = (int8_t)
5724                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5725                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5726                         break;
5727                 case DIF_OP_ULDSH:
5728                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5729                         regs[rd] = (int16_t)
5730                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5731                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5732                         break;
5733                 case DIF_OP_ULDSW:
5734                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5735                         regs[rd] = (int32_t)
5736                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5737                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5738                         break;
5739                 case DIF_OP_ULDUB:
5740                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5741                         regs[rd] =
5742                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5743                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5744                         break;
5745                 case DIF_OP_ULDUH:
5746                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5747                         regs[rd] =
5748                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5749                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5750                         break;
5751                 case DIF_OP_ULDUW:
5752                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5753                         regs[rd] =
5754                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5755                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5756                         break;
5757                 case DIF_OP_ULDX:
5758                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5759                         regs[rd] =
5760                             dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5761                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5762                         break;
5763                 case DIF_OP_RET:
5764                         rval = regs[rd];
5765                         pc = textlen;
5766                         break;
5767                 case DIF_OP_NOP:
5768                         break;
5769                 case DIF_OP_SETX:
5770                         regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5771                         break;
5772                 case DIF_OP_SETS:
5773                         regs[rd] = (uint64_t)(uintptr_t)
5774                             (strtab + DIF_INSTR_STRING(instr));
5775                         break;
5776                 case DIF_OP_SCMP: {
5777                         size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5778                         uintptr_t s1 = regs[r1];
5779                         uintptr_t s2 = regs[r2];
5780 
5781                         if (s1 != NULL &&
5782                             !dtrace_strcanload(s1, sz, mstate, vstate))
5783                                 break;
5784                         if (s2 != NULL &&
5785                             !dtrace_strcanload(s2, sz, mstate, vstate))
5786                                 break;
5787 
5788                         cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz);
5789 
5790                         cc_n = cc_r < 0;
5791                         cc_z = cc_r == 0;
5792                         cc_v = cc_c = 0;
5793                         break;
5794                 }
5795                 case DIF_OP_LDGA:
5796                         regs[rd] = dtrace_dif_variable(mstate, state,
5797                             r1, regs[r2]);
5798                         break;
5799                 case DIF_OP_LDGS:
5800                         id = DIF_INSTR_VAR(instr);
5801 
5802                         if (id >= DIF_VAR_OTHER_UBASE) {
5803                                 uintptr_t a;
5804 
5805                                 id -= DIF_VAR_OTHER_UBASE;
5806                                 svar = vstate->dtvs_globals[id];
5807                                 ASSERT(svar != NULL);
5808                                 v = &svar->dtsv_var;
5809 
5810                                 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
5811                                         regs[rd] = svar->dtsv_data;
5812                                         break;
5813                                 }
5814 
5815                                 a = (uintptr_t)svar->dtsv_data;
5816 
5817                                 if (*(uint8_t *)a == UINT8_MAX) {
5818                                         /*
5819                                          * If the 0th byte is set to UINT8_MAX
5820                                          * then this is to be treated as a
5821                                          * reference to a NULL variable.
5822                                          */
5823                                         regs[rd] = NULL;
5824                                 } else {
5825                                         regs[rd] = a + sizeof (uint64_t);
5826                                 }
5827 
5828                                 break;
5829                         }
5830 
5831                         regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
5832                         break;
5833 
5834                 case DIF_OP_STGS:
5835                         id = DIF_INSTR_VAR(instr);
5836 
5837                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5838                         id -= DIF_VAR_OTHER_UBASE;
5839 
5840                         svar = vstate->dtvs_globals[id];
5841                         ASSERT(svar != NULL);
5842                         v = &svar->dtsv_var;
5843 
5844                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5845                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5846 
5847                                 ASSERT(a != NULL);
5848                                 ASSERT(svar->dtsv_size != 0);
5849 
5850                                 if (regs[rd] == NULL) {
5851                                         *(uint8_t *)a = UINT8_MAX;
5852                                         break;
5853                                 } else {
5854                                         *(uint8_t *)a = 0;
5855                                         a += sizeof (uint64_t);
5856                                 }
5857                                 if (!dtrace_vcanload(
5858                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5859                                     mstate, vstate))
5860                                         break;
5861 
5862                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5863                                     (void *)a, &v->dtdv_type);
5864                                 break;
5865                         }
5866 
5867                         svar->dtsv_data = regs[rd];
5868                         break;
5869 
5870                 case DIF_OP_LDTA:
5871                         /*
5872                          * There are no DTrace built-in thread-local arrays at
5873                          * present.  This opcode is saved for future work.
5874                          */
5875                         *flags |= CPU_DTRACE_ILLOP;
5876                         regs[rd] = 0;
5877                         break;
5878 
5879                 case DIF_OP_LDLS:
5880                         id = DIF_INSTR_VAR(instr);
5881 
5882                         if (id < DIF_VAR_OTHER_UBASE) {
5883                                 /*
5884                                  * For now, this has no meaning.
5885                                  */
5886                                 regs[rd] = 0;
5887                                 break;
5888                         }
5889 
5890                         id -= DIF_VAR_OTHER_UBASE;
5891 
5892                         ASSERT(id < vstate->dtvs_nlocals);
5893                         ASSERT(vstate->dtvs_locals != NULL);
5894 
5895                         svar = vstate->dtvs_locals[id];
5896                         ASSERT(svar != NULL);
5897                         v = &svar->dtsv_var;
5898 
5899                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5900                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5901                                 size_t sz = v->dtdv_type.dtdt_size;
5902 
5903                                 sz += sizeof (uint64_t);
5904                                 ASSERT(svar->dtsv_size == NCPU * sz);
5905                                 a += CPU->cpu_id * sz;
5906 
5907                                 if (*(uint8_t *)a == UINT8_MAX) {
5908                                         /*
5909                                          * If the 0th byte is set to UINT8_MAX
5910                                          * then this is to be treated as a
5911                                          * reference to a NULL variable.
5912                                          */
5913                                         regs[rd] = NULL;
5914                                 } else {
5915                                         regs[rd] = a + sizeof (uint64_t);
5916                                 }
5917 
5918                                 break;
5919                         }
5920 
5921                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5922                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5923                         regs[rd] = tmp[CPU->cpu_id];
5924                         break;
5925 
5926                 case DIF_OP_STLS:
5927                         id = DIF_INSTR_VAR(instr);
5928 
5929                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5930                         id -= DIF_VAR_OTHER_UBASE;
5931                         ASSERT(id < vstate->dtvs_nlocals);
5932 
5933                         ASSERT(vstate->dtvs_locals != NULL);
5934                         svar = vstate->dtvs_locals[id];
5935                         ASSERT(svar != NULL);
5936                         v = &svar->dtsv_var;
5937 
5938                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5939                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5940                                 size_t sz = v->dtdv_type.dtdt_size;
5941 
5942                                 sz += sizeof (uint64_t);
5943                                 ASSERT(svar->dtsv_size == NCPU * sz);
5944                                 a += CPU->cpu_id * sz;
5945 
5946                                 if (regs[rd] == NULL) {
5947                                         *(uint8_t *)a = UINT8_MAX;
5948                                         break;
5949                                 } else {
5950                                         *(uint8_t *)a = 0;
5951                                         a += sizeof (uint64_t);
5952                                 }
5953 
5954                                 if (!dtrace_vcanload(
5955                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5956                                     mstate, vstate))
5957                                         break;
5958 
5959                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5960                                     (void *)a, &v->dtdv_type);
5961                                 break;
5962                         }
5963 
5964                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5965                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5966                         tmp[CPU->cpu_id] = regs[rd];
5967                         break;
5968 
5969                 case DIF_OP_LDTS: {
5970                         dtrace_dynvar_t *dvar;
5971                         dtrace_key_t *key;
5972 
5973                         id = DIF_INSTR_VAR(instr);
5974                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5975                         id -= DIF_VAR_OTHER_UBASE;
5976                         v = &vstate->dtvs_tlocals[id];
5977 
5978                         key = &tupregs[DIF_DTR_NREGS];
5979                         key[0].dttk_value = (uint64_t)id;
5980                         key[0].dttk_size = 0;
5981                         DTRACE_TLS_THRKEY(key[1].dttk_value);
5982                         key[1].dttk_size = 0;
5983 
5984                         dvar = dtrace_dynvar(dstate, 2, key,
5985                             sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
5986                             mstate, vstate);
5987 
5988                         if (dvar == NULL) {
5989                                 regs[rd] = 0;
5990                                 break;
5991                         }
5992 
5993                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5994                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5995                         } else {
5996                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5997                         }
5998 
5999                         break;
6000                 }
6001 
6002                 case DIF_OP_STTS: {
6003                         dtrace_dynvar_t *dvar;
6004                         dtrace_key_t *key;
6005 
6006                         id = DIF_INSTR_VAR(instr);
6007                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6008                         id -= DIF_VAR_OTHER_UBASE;
6009 
6010                         key = &tupregs[DIF_DTR_NREGS];
6011                         key[0].dttk_value = (uint64_t)id;
6012                         key[0].dttk_size = 0;
6013                         DTRACE_TLS_THRKEY(key[1].dttk_value);
6014                         key[1].dttk_size = 0;
6015                         v = &vstate->dtvs_tlocals[id];
6016 
6017                         dvar = dtrace_dynvar(dstate, 2, key,
6018                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6019                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6020                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6021                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6022 
6023                         /*
6024                          * Given that we're storing to thread-local data,
6025                          * we need to flush our predicate cache.
6026                          */
6027                         curthread->t_predcache = NULL;
6028 
6029                         if (dvar == NULL)
6030                                 break;
6031 
6032                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6033                                 if (!dtrace_vcanload(
6034                                     (void *)(uintptr_t)regs[rd],
6035                                     &v->dtdv_type, mstate, vstate))
6036                                         break;
6037 
6038                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6039                                     dvar->dtdv_data, &v->dtdv_type);
6040                         } else {
6041                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6042                         }
6043 
6044                         break;
6045                 }
6046 
6047                 case DIF_OP_SRA:
6048                         regs[rd] = (int64_t)regs[r1] >> regs[r2];
6049                         break;
6050 
6051                 case DIF_OP_CALL:
6052                         dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
6053                             regs, tupregs, ttop, mstate, state);
6054                         break;
6055 
6056                 case DIF_OP_PUSHTR:
6057                         if (ttop == DIF_DTR_NREGS) {
6058                                 *flags |= CPU_DTRACE_TUPOFLOW;
6059                                 break;
6060                         }
6061 
6062                         if (r1 == DIF_TYPE_STRING) {
6063                                 /*
6064                                  * If this is a string type and the size is 0,
6065                                  * we'll use the system-wide default string
6066                                  * size.  Note that we are _not_ looking at
6067                                  * the value of the DTRACEOPT_STRSIZE option;
6068                                  * had this been set, we would expect to have
6069                                  * a non-zero size value in the "pushtr".
6070                                  */
6071                                 tupregs[ttop].dttk_size =
6072                                     dtrace_strlen((char *)(uintptr_t)regs[rd],
6073                                     regs[r2] ? regs[r2] :
6074                                     dtrace_strsize_default) + 1;
6075                         } else {
6076                                 tupregs[ttop].dttk_size = regs[r2];
6077                         }
6078 
6079                         tupregs[ttop++].dttk_value = regs[rd];
6080                         break;
6081 
6082                 case DIF_OP_PUSHTV:
6083                         if (ttop == DIF_DTR_NREGS) {
6084                                 *flags |= CPU_DTRACE_TUPOFLOW;
6085                                 break;
6086                         }
6087 
6088                         tupregs[ttop].dttk_value = regs[rd];
6089                         tupregs[ttop++].dttk_size = 0;
6090                         break;
6091 
6092                 case DIF_OP_POPTS:
6093                         if (ttop != 0)
6094                                 ttop--;
6095                         break;
6096 
6097                 case DIF_OP_FLUSHTS:
6098                         ttop = 0;
6099                         break;
6100 
6101                 case DIF_OP_LDGAA:
6102                 case DIF_OP_LDTAA: {
6103                         dtrace_dynvar_t *dvar;
6104                         dtrace_key_t *key = tupregs;
6105                         uint_t nkeys = ttop;
6106 
6107                         id = DIF_INSTR_VAR(instr);
6108                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6109                         id -= DIF_VAR_OTHER_UBASE;
6110 
6111                         key[nkeys].dttk_value = (uint64_t)id;
6112                         key[nkeys++].dttk_size = 0;
6113 
6114                         if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
6115                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6116                                 key[nkeys++].dttk_size = 0;
6117                                 v = &vstate->dtvs_tlocals[id];
6118                         } else {
6119                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6120                         }
6121 
6122                         dvar = dtrace_dynvar(dstate, nkeys, key,
6123                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6124                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6125                             DTRACE_DYNVAR_NOALLOC, mstate, vstate);
6126 
6127                         if (dvar == NULL) {
6128                                 regs[rd] = 0;
6129                                 break;
6130                         }
6131 
6132                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6133                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
6134                         } else {
6135                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
6136                         }
6137 
6138                         break;
6139                 }
6140 
6141                 case DIF_OP_STGAA:
6142                 case DIF_OP_STTAA: {
6143                         dtrace_dynvar_t *dvar;
6144                         dtrace_key_t *key = tupregs;
6145                         uint_t nkeys = ttop;
6146 
6147                         id = DIF_INSTR_VAR(instr);
6148                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6149                         id -= DIF_VAR_OTHER_UBASE;
6150 
6151                         key[nkeys].dttk_value = (uint64_t)id;
6152                         key[nkeys++].dttk_size = 0;
6153 
6154                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
6155                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6156                                 key[nkeys++].dttk_size = 0;
6157                                 v = &vstate->dtvs_tlocals[id];
6158                         } else {
6159                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6160                         }
6161 
6162                         dvar = dtrace_dynvar(dstate, nkeys, key,
6163                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6164                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6165                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6166                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6167 
6168                         if (dvar == NULL)
6169                                 break;
6170 
6171                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6172                                 if (!dtrace_vcanload(
6173                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
6174                                     mstate, vstate))
6175                                         break;
6176 
6177                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6178                                     dvar->dtdv_data, &v->dtdv_type);
6179                         } else {
6180                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6181                         }
6182 
6183                         break;
6184                 }
6185 
6186                 case DIF_OP_ALLOCS: {
6187                         uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6188                         size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
6189 
6190                         /*
6191                          * Rounding up the user allocation size could have
6192                          * overflowed large, bogus allocations (like -1ULL) to
6193                          * 0.
6194                          */
6195                         if (size < regs[r1] ||
6196                             !DTRACE_INSCRATCH(mstate, size)) {
6197                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6198                                 regs[rd] = NULL;
6199                                 break;
6200                         }
6201 
6202                         dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
6203                         mstate->dtms_scratch_ptr += size;
6204                         regs[rd] = ptr;
6205                         break;
6206                 }
6207 
6208                 case DIF_OP_COPYS:
6209                         if (!dtrace_canstore(regs[rd], regs[r2],
6210                             mstate, vstate)) {
6211                                 *flags |= CPU_DTRACE_BADADDR;
6212                                 *illval = regs[rd];
6213                                 break;
6214                         }
6215 
6216                         if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
6217                                 break;
6218 
6219                         dtrace_bcopy((void *)(uintptr_t)regs[r1],
6220                             (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
6221                         break;
6222 
6223                 case DIF_OP_STB:
6224                         if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
6225                                 *flags |= CPU_DTRACE_BADADDR;
6226                                 *illval = regs[rd];
6227                                 break;
6228                         }
6229                         *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
6230                         break;
6231 
6232                 case DIF_OP_STH:
6233                         if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
6234                                 *flags |= CPU_DTRACE_BADADDR;
6235                                 *illval = regs[rd];
6236                                 break;
6237                         }
6238                         if (regs[rd] & 1) {
6239                                 *flags |= CPU_DTRACE_BADALIGN;
6240                                 *illval = regs[rd];
6241                                 break;
6242                         }
6243                         *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
6244                         break;
6245 
6246                 case DIF_OP_STW:
6247                         if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
6248                                 *flags |= CPU_DTRACE_BADADDR;
6249                                 *illval = regs[rd];
6250                                 break;
6251                         }
6252                         if (regs[rd] & 3) {
6253                                 *flags |= CPU_DTRACE_BADALIGN;
6254                                 *illval = regs[rd];
6255                                 break;
6256                         }
6257                         *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
6258                         break;
6259 
6260                 case DIF_OP_STX:
6261                         if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
6262                                 *flags |= CPU_DTRACE_BADADDR;
6263                                 *illval = regs[rd];
6264                                 break;
6265                         }
6266                         if (regs[rd] & 7) {
6267                                 *flags |= CPU_DTRACE_BADALIGN;
6268                                 *illval = regs[rd];
6269                                 break;
6270                         }
6271                         *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
6272                         break;
6273                 }
6274         }
6275 
6276         if (!(*flags & CPU_DTRACE_FAULT))
6277                 return (rval);
6278 
6279         mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
6280         mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
6281 
6282         return (0);
6283 }
6284 
6285 static void
6286 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
6287 {
6288         dtrace_probe_t *probe = ecb->dte_probe;
6289         dtrace_provider_t *prov = probe->dtpr_provider;
6290         char c[DTRACE_FULLNAMELEN + 80], *str;
6291         char *msg = "dtrace: breakpoint action at probe ";
6292         char *ecbmsg = " (ecb ";
6293         uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
6294         uintptr_t val = (uintptr_t)ecb;
6295         int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
6296 
6297         if (dtrace_destructive_disallow)
6298                 return;
6299 
6300         /*
6301          * It's impossible to be taking action on the NULL probe.
6302          */
6303         ASSERT(probe != NULL);
6304 
6305         /*
6306          * This is a poor man's (destitute man's?) sprintf():  we want to
6307          * print the provider name, module name, function name and name of
6308          * the probe, along with the hex address of the ECB with the breakpoint
6309          * action -- all of which we must place in the character buffer by
6310          * hand.
6311          */
6312         while (*msg != '\0')
6313                 c[i++] = *msg++;
6314 
6315         for (str = prov->dtpv_name; *str != '\0'; str++)
6316                 c[i++] = *str;
6317         c[i++] = ':';
6318 
6319         for (str = probe->dtpr_mod; *str != '\0'; str++)
6320                 c[i++] = *str;
6321         c[i++] = ':';
6322 
6323         for (str = probe->dtpr_func; *str != '\0'; str++)
6324                 c[i++] = *str;
6325         c[i++] = ':';
6326 
6327         for (str = probe->dtpr_name; *str != '\0'; str++)
6328                 c[i++] = *str;
6329 
6330         while (*ecbmsg != '\0')
6331                 c[i++] = *ecbmsg++;
6332 
6333         while (shift >= 0) {
6334                 mask = (uintptr_t)0xf << shift;
6335 
6336                 if (val >= ((uintptr_t)1 << shift))
6337                         c[i++] = "0123456789abcdef"[(val & mask) >> shift];
6338                 shift -= 4;
6339         }
6340 
6341         c[i++] = ')';
6342         c[i] = '\0';
6343 
6344         debug_enter(c);
6345 }
6346 
6347 static void
6348 dtrace_action_panic(dtrace_ecb_t *ecb)
6349 {
6350         dtrace_probe_t *probe = ecb->dte_probe;
6351 
6352         /*
6353          * It's impossible to be taking action on the NULL probe.
6354          */
6355         ASSERT(probe != NULL);
6356 
6357         if (dtrace_destructive_disallow)
6358                 return;
6359 
6360         if (dtrace_panicked != NULL)
6361                 return;
6362 
6363         if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
6364                 return;
6365 
6366         /*
6367          * We won the right to panic.  (We want to be sure that only one
6368          * thread calls panic() from dtrace_probe(), and that panic() is
6369          * called exactly once.)
6370          */
6371         dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
6372             probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
6373             probe->dtpr_func, probe->dtpr_name, (void *)ecb);
6374 }
6375 
6376 static void
6377 dtrace_action_raise(uint64_t sig)
6378 {
6379         if (dtrace_destructive_disallow)
6380                 return;
6381 
6382         if (sig >= NSIG) {
6383                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
6384                 return;
6385         }
6386 
6387         /*
6388          * raise() has a queue depth of 1 -- we ignore all subsequent
6389          * invocations of the raise() action.
6390          */
6391         if (curthread->t_dtrace_sig == 0)
6392                 curthread->t_dtrace_sig = (uint8_t)sig;
6393 
6394         curthread->t_sig_check = 1;
6395         aston(curthread);
6396 }
6397 
6398 static void
6399 dtrace_action_stop(void)
6400 {
6401         if (dtrace_destructive_disallow)
6402                 return;
6403 
6404         if (!curthread->t_dtrace_stop) {
6405                 curthread->t_dtrace_stop = 1;
6406                 curthread->t_sig_check = 1;
6407                 aston(curthread);
6408         }
6409 }
6410 
6411 static void
6412 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
6413 {
6414         hrtime_t now;
6415         volatile uint16_t *flags;
6416         cpu_t *cpu = CPU;
6417 
6418         if (dtrace_destructive_disallow)
6419                 return;
6420 
6421         flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
6422 
6423         now = dtrace_gethrtime();
6424 
6425         if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
6426                 /*
6427                  * We need to advance the mark to the current time.
6428                  */
6429                 cpu->cpu_dtrace_chillmark = now;
6430                 cpu->cpu_dtrace_chilled = 0;
6431         }
6432 
6433         /*
6434          * Now check to see if the requested chill time would take us over
6435          * the maximum amount of time allowed in the chill interval.  (Or
6436          * worse, if the calculation itself induces overflow.)
6437          */
6438         if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
6439             cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
6440                 *flags |= CPU_DTRACE_ILLOP;
6441                 return;
6442         }
6443 
6444         while (dtrace_gethrtime() - now < val)
6445                 continue;
6446 
6447         /*
6448          * Normally, we assure that the value of the variable "timestamp" does
6449          * not change within an ECB.  The presence of chill() represents an
6450          * exception to this rule, however.
6451          */
6452         mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
6453         cpu->cpu_dtrace_chilled += val;
6454 }
6455 
6456 static void
6457 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
6458     uint64_t *buf, uint64_t arg)
6459 {
6460         int nframes = DTRACE_USTACK_NFRAMES(arg);
6461         int strsize = DTRACE_USTACK_STRSIZE(arg);
6462         uint64_t *pcs = &buf[1], *fps;
6463         char *str = (char *)&pcs[nframes];
6464         int size, offs = 0, i, j;
6465         uintptr_t old = mstate->dtms_scratch_ptr, saved;
6466         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6467         char *sym;
6468 
6469         /*
6470          * Should be taking a faster path if string space has not been
6471          * allocated.
6472          */
6473         ASSERT(strsize != 0);
6474 
6475         /*
6476          * We will first allocate some temporary space for the frame pointers.
6477          */
6478         fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6479         size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
6480             (nframes * sizeof (uint64_t));
6481 
6482         if (!DTRACE_INSCRATCH(mstate, size)) {
6483                 /*
6484                  * Not enough room for our frame pointers -- need to indicate
6485                  * that we ran out of scratch space.
6486                  */
6487                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6488                 return;
6489         }
6490 
6491         mstate->dtms_scratch_ptr += size;
6492         saved = mstate->dtms_scratch_ptr;
6493 
6494         /*
6495          * Now get a stack with both program counters and frame pointers.
6496          */
6497         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6498         dtrace_getufpstack(buf, fps, nframes + 1);
6499         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6500 
6501         /*
6502          * If that faulted, we're cooked.
6503          */
6504         if (*flags & CPU_DTRACE_FAULT)
6505                 goto out;
6506 
6507         /*
6508          * Now we want to walk up the stack, calling the USTACK helper.  For
6509          * each iteration, we restore the scratch pointer.
6510          */
6511         for (i = 0; i < nframes; i++) {
6512                 mstate->dtms_scratch_ptr = saved;
6513 
6514                 if (offs >= strsize)
6515                         break;
6516 
6517                 sym = (char *)(uintptr_t)dtrace_helper(
6518                     DTRACE_HELPER_ACTION_USTACK,
6519                     mstate, state, pcs[i], fps[i]);
6520 
6521                 /*
6522                  * If we faulted while running the helper, we're going to
6523                  * clear the fault and null out the corresponding string.
6524                  */
6525                 if (*flags & CPU_DTRACE_FAULT) {
6526                         *flags &= ~CPU_DTRACE_FAULT;
6527                         str[offs++] = '\0';
6528                         continue;
6529                 }
6530 
6531                 if (sym == NULL) {
6532                         str[offs++] = '\0';
6533                         continue;
6534                 }
6535 
6536                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6537 
6538                 /*
6539                  * Now copy in the string that the helper returned to us.
6540                  */
6541                 for (j = 0; offs + j < strsize; j++) {
6542                         if ((str[offs + j] = sym[j]) == '\0')
6543                                 break;
6544                 }
6545 
6546                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6547 
6548                 offs += j + 1;
6549         }
6550 
6551         if (offs >= strsize) {
6552                 /*
6553                  * If we didn't have room for all of the strings, we don't
6554                  * abort processing -- this needn't be a fatal error -- but we
6555                  * still want to increment a counter (dts_stkstroverflows) to
6556                  * allow this condition to be warned about.  (If this is from
6557                  * a jstack() action, it is easily tuned via jstackstrsize.)
6558                  */
6559                 dtrace_error(&state->dts_stkstroverflows);
6560         }
6561 
6562         while (offs < strsize)
6563                 str[offs++] = '\0';
6564 
6565 out:
6566         mstate->dtms_scratch_ptr = old;
6567 }
6568 
6569 static void
6570 dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size,
6571     size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind)
6572 {
6573         volatile uint16_t *flags;
6574         uint64_t val = *valp;
6575         size_t valoffs = *valoffsp;
6576 
6577         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6578         ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF);
6579 
6580         /*
6581          * If this is a string, we're going to only load until we find the zero
6582          * byte -- after which we'll store zero bytes.
6583          */
6584         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
6585                 char c = '\0' + 1;
6586                 size_t s;
6587 
6588                 for (s = 0; s < size; s++) {
6589                         if (c != '\0' && dtkind == DIF_TF_BYREF) {
6590                                 c = dtrace_load8(val++);
6591                         } else if (c != '\0' && dtkind == DIF_TF_BYUREF) {
6592                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6593                                 c = dtrace_fuword8((void *)(uintptr_t)val++);
6594                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6595                                 if (*flags & CPU_DTRACE_FAULT)
6596                                         break;
6597                         }
6598 
6599                         DTRACE_STORE(uint8_t, tomax, valoffs++, c);
6600 
6601                         if (c == '\0' && intuple)
6602                                 break;
6603                 }
6604         } else {
6605                 uint8_t c;
6606                 while (valoffs < end) {
6607                         if (dtkind == DIF_TF_BYREF) {
6608                                 c = dtrace_load8(val++);
6609                         } else if (dtkind == DIF_TF_BYUREF) {
6610                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6611                                 c = dtrace_fuword8((void *)(uintptr_t)val++);
6612                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6613                                 if (*flags & CPU_DTRACE_FAULT)
6614                                         break;
6615                         }
6616 
6617                         DTRACE_STORE(uint8_t, tomax,
6618                             valoffs++, c);
6619                 }
6620         }
6621 
6622         *valp = val;
6623         *valoffsp = valoffs;
6624 }
6625 
6626 /*
6627  * If you're looking for the epicenter of DTrace, you just found it.  This
6628  * is the function called by the provider to fire a probe -- from which all
6629  * subsequent probe-context DTrace activity emanates.
6630  */
6631 void
6632 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
6633     uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
6634 {
6635         processorid_t cpuid;
6636         dtrace_icookie_t cookie;
6637         dtrace_probe_t *probe;
6638         dtrace_mstate_t mstate;
6639         dtrace_ecb_t *ecb;
6640         dtrace_action_t *act;
6641         intptr_t offs;
6642         size_t size;
6643         int vtime, onintr;
6644         volatile uint16_t *flags;
6645         hrtime_t now, end;
6646 
6647         /*
6648          * Kick out immediately if this CPU is still being born (in which case
6649          * curthread will be set to -1) or the current thread can't allow
6650          * probes in its current context.
6651          */
6652         if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
6653                 return;
6654 
6655         cookie = dtrace_interrupt_disable();
6656         probe = dtrace_probes[id - 1];
6657         cpuid = CPU->cpu_id;
6658         onintr = CPU_ON_INTR(CPU);
6659 
6660         CPU->cpu_dtrace_probes++;
6661 
6662         if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
6663             probe->dtpr_predcache == curthread->t_predcache) {
6664                 /*
6665                  * We have hit in the predicate cache; we know that
6666                  * this predicate would evaluate to be false.
6667                  */
6668                 dtrace_interrupt_enable(cookie);
6669                 return;
6670         }
6671 
6672         if (panic_quiesce) {
6673                 /*
6674                  * We don't trace anything if we're panicking.
6675                  */
6676                 dtrace_interrupt_enable(cookie);
6677                 return;
6678         }
6679 
6680         now = dtrace_gethrtime();
6681         vtime = dtrace_vtime_references != 0;
6682 
6683         if (vtime && curthread->t_dtrace_start)
6684                 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
6685 
6686         mstate.dtms_difo = NULL;
6687         mstate.dtms_probe = probe;
6688         mstate.dtms_strtok = NULL;
6689         mstate.dtms_arg[0] = arg0;
6690         mstate.dtms_arg[1] = arg1;
6691         mstate.dtms_arg[2] = arg2;
6692         mstate.dtms_arg[3] = arg3;
6693         mstate.dtms_arg[4] = arg4;
6694 
6695         flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
6696 
6697         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
6698                 dtrace_predicate_t *pred = ecb->dte_predicate;
6699                 dtrace_state_t *state = ecb->dte_state;
6700                 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
6701                 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
6702                 dtrace_vstate_t *vstate = &state->dts_vstate;
6703                 dtrace_provider_t *prov = probe->dtpr_provider;
6704                 uint64_t tracememsize = 0;
6705                 int committed = 0;
6706                 caddr_t tomax;
6707 
6708                 /*
6709                  * A little subtlety with the following (seemingly innocuous)
6710                  * declaration of the automatic 'val':  by looking at the
6711                  * code, you might think that it could be declared in the
6712                  * action processing loop, below.  (That is, it's only used in
6713                  * the action processing loop.)  However, it must be declared
6714                  * out of that scope because in the case of DIF expression
6715                  * arguments to aggregating actions, one iteration of the
6716                  * action loop will use the last iteration's value.
6717                  */
6718 #ifdef lint
6719                 uint64_t val = 0;
6720 #else
6721                 uint64_t val;
6722 #endif
6723 
6724                 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
6725                 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC;
6726                 mstate.dtms_getf = NULL;
6727 
6728                 *flags &= ~CPU_DTRACE_ERROR;
6729 
6730                 if (prov == dtrace_provider) {
6731                         /*
6732                          * If dtrace itself is the provider of this probe,
6733                          * we're only going to continue processing the ECB if
6734                          * arg0 (the dtrace_state_t) is equal to the ECB's
6735                          * creating state.  (This prevents disjoint consumers
6736                          * from seeing one another's metaprobes.)
6737                          */
6738                         if (arg0 != (uint64_t)(uintptr_t)state)
6739                                 continue;
6740                 }
6741 
6742                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
6743                         /*
6744                          * We're not currently active.  If our provider isn't
6745                          * the dtrace pseudo provider, we're not interested.
6746                          */
6747                         if (prov != dtrace_provider)
6748                                 continue;
6749 
6750                         /*
6751                          * Now we must further check if we are in the BEGIN
6752                          * probe.  If we are, we will only continue processing
6753                          * if we're still in WARMUP -- if one BEGIN enabling
6754                          * has invoked the exit() action, we don't want to
6755                          * evaluate subsequent BEGIN enablings.
6756                          */
6757                         if (probe->dtpr_id == dtrace_probeid_begin &&
6758                             state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
6759                                 ASSERT(state->dts_activity ==
6760                                     DTRACE_ACTIVITY_DRAINING);
6761                                 continue;
6762                         }
6763                 }
6764 
6765                 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb))
6766                         continue;
6767 
6768                 if (now - state->dts_alive > dtrace_deadman_timeout) {
6769                         /*
6770                          * We seem to be dead.  Unless we (a) have kernel
6771                          * destructive permissions (b) have explicitly enabled
6772                          * destructive actions and (c) destructive actions have
6773                          * not been disabled, we're going to transition into
6774                          * the KILLED state, from which no further processing
6775                          * on this state will be performed.
6776                          */
6777                         if (!dtrace_priv_kernel_destructive(state) ||
6778                             !state->dts_cred.dcr_destructive ||
6779                             dtrace_destructive_disallow) {
6780                                 void *activity = &state->dts_activity;
6781                                 dtrace_activity_t current;
6782 
6783                                 do {
6784                                         current = state->dts_activity;
6785                                 } while (dtrace_cas32(activity, current,
6786                                     DTRACE_ACTIVITY_KILLED) != current);
6787 
6788                                 continue;
6789                         }
6790                 }
6791 
6792                 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
6793                     ecb->dte_alignment, state, &mstate)) < 0)
6794                         continue;
6795 
6796                 tomax = buf->dtb_tomax;
6797                 ASSERT(tomax != NULL);
6798 
6799                 if (ecb->dte_size != 0) {
6800                         dtrace_rechdr_t dtrh;
6801                         if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
6802                                 mstate.dtms_timestamp = dtrace_gethrtime();
6803                                 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
6804                         }
6805                         ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
6806                         dtrh.dtrh_epid = ecb->dte_epid;
6807                         DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
6808                             mstate.dtms_timestamp);
6809                         *((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
6810                 }
6811 
6812                 mstate.dtms_epid = ecb->dte_epid;
6813                 mstate.dtms_present |= DTRACE_MSTATE_EPID;
6814 
6815                 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
6816                         mstate.dtms_access |= DTRACE_ACCESS_KERNEL;
6817 
6818                 if (pred != NULL) {
6819                         dtrace_difo_t *dp = pred->dtp_difo;
6820                         int rval;
6821 
6822                         rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
6823 
6824                         if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
6825                                 dtrace_cacheid_t cid = probe->dtpr_predcache;
6826 
6827                                 if (cid != DTRACE_CACHEIDNONE && !onintr) {
6828                                         /*
6829                                          * Update the predicate cache...
6830                                          */
6831                                         ASSERT(cid == pred->dtp_cacheid);
6832                                         curthread->t_predcache = cid;
6833                                 }
6834 
6835                                 continue;
6836                         }
6837                 }
6838 
6839                 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
6840                     act != NULL; act = act->dta_next) {
6841                         size_t valoffs;
6842                         dtrace_difo_t *dp;
6843                         dtrace_recdesc_t *rec = &act->dta_rec;
6844 
6845                         size = rec->dtrd_size;
6846                         valoffs = offs + rec->dtrd_offset;
6847 
6848                         if (DTRACEACT_ISAGG(act->dta_kind)) {
6849                                 uint64_t v = 0xbad;
6850                                 dtrace_aggregation_t *agg;
6851 
6852                                 agg = (dtrace_aggregation_t *)act;
6853 
6854                                 if ((dp = act->dta_difo) != NULL)
6855                                         v = dtrace_dif_emulate(dp,
6856                                             &mstate, vstate, state);
6857 
6858                                 if (*flags & CPU_DTRACE_ERROR)
6859                                         continue;
6860 
6861                                 /*
6862                                  * Note that we always pass the expression
6863                                  * value from the previous iteration of the
6864                                  * action loop.  This value will only be used
6865                                  * if there is an expression argument to the
6866                                  * aggregating action, denoted by the
6867                                  * dtag_hasarg field.
6868                                  */
6869                                 dtrace_aggregate(agg, buf,
6870                                     offs, aggbuf, v, val);
6871                                 continue;
6872                         }
6873 
6874                         switch (act->dta_kind) {
6875                         case DTRACEACT_STOP:
6876                                 if (dtrace_priv_proc_destructive(state,
6877                                     &mstate))
6878                                         dtrace_action_stop();
6879                                 continue;
6880 
6881                         case DTRACEACT_BREAKPOINT:
6882                                 if (dtrace_priv_kernel_destructive(state))
6883                                         dtrace_action_breakpoint(ecb);
6884                                 continue;
6885 
6886                         case DTRACEACT_PANIC:
6887                                 if (dtrace_priv_kernel_destructive(state))
6888                                         dtrace_action_panic(ecb);
6889                                 continue;
6890 
6891                         case DTRACEACT_STACK:
6892                                 if (!dtrace_priv_kernel(state))
6893                                         continue;
6894 
6895                                 dtrace_getpcstack((pc_t *)(tomax + valoffs),
6896                                     size / sizeof (pc_t), probe->dtpr_aframes,
6897                                     DTRACE_ANCHORED(probe) ? NULL :
6898                                     (uint32_t *)arg0);
6899 
6900                                 continue;
6901 
6902                         case DTRACEACT_JSTACK:
6903                         case DTRACEACT_USTACK:
6904                                 if (!dtrace_priv_proc(state, &mstate))
6905                                         continue;
6906 
6907                                 /*
6908                                  * See comment in DIF_VAR_PID.
6909                                  */
6910                                 if (DTRACE_ANCHORED(mstate.dtms_probe) &&
6911                                     CPU_ON_INTR(CPU)) {
6912                                         int depth = DTRACE_USTACK_NFRAMES(
6913                                             rec->dtrd_arg) + 1;
6914 
6915                                         dtrace_bzero((void *)(tomax + valoffs),
6916                                             DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
6917                                             + depth * sizeof (uint64_t));
6918 
6919                                         continue;
6920                                 }
6921 
6922                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
6923                                     curproc->p_dtrace_helpers != NULL) {
6924                                         /*
6925                                          * This is the slow path -- we have
6926                                          * allocated string space, and we're
6927                                          * getting the stack of a process that
6928                                          * has helpers.  Call into a separate
6929                                          * routine to perform this processing.
6930                                          */
6931                                         dtrace_action_ustack(&mstate, state,
6932                                             (uint64_t *)(tomax + valoffs),
6933                                             rec->dtrd_arg);
6934                                         continue;
6935                                 }
6936 
6937                                 /*
6938                                  * Clear the string space, since there's no
6939                                  * helper to do it for us.
6940                                  */
6941                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) {
6942                                         int depth = DTRACE_USTACK_NFRAMES(
6943                                             rec->dtrd_arg);
6944                                         size_t strsize = DTRACE_USTACK_STRSIZE(
6945                                             rec->dtrd_arg);
6946                                         uint64_t *buf = (uint64_t *)(tomax +
6947                                             valoffs);
6948                                         void *strspace = &buf[depth + 1];
6949 
6950                                         dtrace_bzero(strspace,
6951                                             MIN(depth, strsize));
6952                                 }
6953 
6954                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6955                                 dtrace_getupcstack((uint64_t *)
6956                                     (tomax + valoffs),
6957                                     DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
6958                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6959                                 continue;
6960 
6961                         default:
6962                                 break;
6963                         }
6964 
6965                         dp = act->dta_difo;
6966                         ASSERT(dp != NULL);
6967 
6968                         val = dtrace_dif_emulate(dp, &mstate, vstate, state);
6969 
6970                         if (*flags & CPU_DTRACE_ERROR)
6971                                 continue;
6972 
6973                         switch (act->dta_kind) {
6974                         case DTRACEACT_SPECULATE: {
6975                                 dtrace_rechdr_t *dtrh;
6976 
6977                                 ASSERT(buf == &state->dts_buffer[cpuid]);
6978                                 buf = dtrace_speculation_buffer(state,
6979                                     cpuid, val);
6980 
6981                                 if (buf == NULL) {
6982                                         *flags |= CPU_DTRACE_DROP;
6983                                         continue;
6984                                 }
6985 
6986                                 offs = dtrace_buffer_reserve(buf,
6987                                     ecb->dte_needed, ecb->dte_alignment,
6988                                     state, NULL);
6989 
6990                                 if (offs < 0) {
6991                                         *flags |= CPU_DTRACE_DROP;
6992                                         continue;
6993                                 }
6994 
6995                                 tomax = buf->dtb_tomax;
6996                                 ASSERT(tomax != NULL);
6997 
6998                                 if (ecb->dte_size == 0)
6999                                         continue;
7000 
7001                                 ASSERT3U(ecb->dte_size, >=,
7002                                     sizeof (dtrace_rechdr_t));
7003                                 dtrh = ((void *)(tomax + offs));
7004                                 dtrh->dtrh_epid = ecb->dte_epid;
7005                                 /*
7006                                  * When the speculation is committed, all of
7007                                  * the records in the speculative buffer will
7008                                  * have their timestamps set to the commit
7009                                  * time.  Until then, it is set to a sentinel
7010                                  * value, for debugability.
7011                                  */
7012                                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
7013                                 continue;
7014                         }
7015 
7016                         case DTRACEACT_CHILL:
7017                                 if (dtrace_priv_kernel_destructive(state))
7018                                         dtrace_action_chill(&mstate, val);
7019                                 continue;
7020 
7021                         case DTRACEACT_RAISE:
7022                                 if (dtrace_priv_proc_destructive(state,
7023                                     &mstate))
7024                                         dtrace_action_raise(val);
7025                                 continue;
7026 
7027                         case DTRACEACT_COMMIT:
7028                                 ASSERT(!committed);
7029 
7030                                 /*
7031                                  * We need to commit our buffer state.
7032                                  */
7033                                 if (ecb->dte_size)
7034                                         buf->dtb_offset = offs + ecb->dte_size;
7035                                 buf = &state->dts_buffer[cpuid];
7036                                 dtrace_speculation_commit(state, cpuid, val);
7037                                 committed = 1;
7038                                 continue;
7039 
7040                         case DTRACEACT_DISCARD:
7041                                 dtrace_speculation_discard(state, cpuid, val);
7042                                 continue;
7043 
7044                         case DTRACEACT_DIFEXPR:
7045                         case DTRACEACT_LIBACT:
7046                         case DTRACEACT_PRINTF:
7047                         case DTRACEACT_PRINTA:
7048                         case DTRACEACT_SYSTEM:
7049                         case DTRACEACT_FREOPEN:
7050                         case DTRACEACT_TRACEMEM:
7051                                 break;
7052 
7053                         case DTRACEACT_TRACEMEM_DYNSIZE:
7054                                 tracememsize = val;
7055                                 break;
7056 
7057                         case DTRACEACT_SYM:
7058                         case DTRACEACT_MOD:
7059                                 if (!dtrace_priv_kernel(state))
7060                                         continue;
7061                                 break;
7062 
7063                         case DTRACEACT_USYM:
7064                         case DTRACEACT_UMOD:
7065                         case DTRACEACT_UADDR: {
7066                                 struct pid *pid = curthread->t_procp->p_pidp;
7067 
7068                                 if (!dtrace_priv_proc(state, &mstate))
7069                                         continue;
7070 
7071                                 DTRACE_STORE(uint64_t, tomax,
7072                                     valoffs, (uint64_t)pid->pid_id);
7073                                 DTRACE_STORE(uint64_t, tomax,
7074                                     valoffs + sizeof (uint64_t), val);
7075 
7076                                 continue;
7077                         }
7078 
7079                         case DTRACEACT_EXIT: {
7080                                 /*
7081                                  * For the exit action, we are going to attempt
7082                                  * to atomically set our activity to be
7083                                  * draining.  If this fails (either because
7084                                  * another CPU has beat us to the exit action,
7085                                  * or because our current activity is something
7086                                  * other than ACTIVE or WARMUP), we will
7087                                  * continue.  This assures that the exit action
7088                                  * can be successfully recorded at most once
7089                                  * when we're in the ACTIVE state.  If we're
7090                                  * encountering the exit() action while in
7091                                  * COOLDOWN, however, we want to honor the new
7092                                  * status code.  (We know that we're the only
7093                                  * thread in COOLDOWN, so there is no race.)
7094                                  */
7095                                 void *activity = &state->dts_activity;
7096                                 dtrace_activity_t current = state->dts_activity;
7097 
7098                                 if (current == DTRACE_ACTIVITY_COOLDOWN)
7099                                         break;
7100 
7101                                 if (current != DTRACE_ACTIVITY_WARMUP)
7102                                         current = DTRACE_ACTIVITY_ACTIVE;
7103 
7104                                 if (dtrace_cas32(activity, current,
7105                                     DTRACE_ACTIVITY_DRAINING) != current) {
7106                                         *flags |= CPU_DTRACE_DROP;
7107                                         continue;
7108                                 }
7109 
7110                                 break;
7111                         }
7112 
7113                         default:
7114                                 ASSERT(0);
7115                         }
7116 
7117                         if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ||
7118                             dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) {
7119                                 uintptr_t end = valoffs + size;
7120 
7121                                 if (tracememsize != 0 &&
7122                                     valoffs + tracememsize < end) {
7123                                         end = valoffs + tracememsize;
7124                                         tracememsize = 0;
7125                                 }
7126 
7127                                 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF &&
7128                                     !dtrace_vcanload((void *)(uintptr_t)val,
7129                                     &dp->dtdo_rtype, &mstate, vstate))
7130                                         continue;
7131 
7132                                 dtrace_store_by_ref(dp, tomax, size, &valoffs,
7133                                     &val, end, act->dta_intuple,
7134                                     dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ?
7135                                     DIF_TF_BYREF: DIF_TF_BYUREF);
7136                                 continue;
7137                         }
7138 
7139                         switch (size) {
7140                         case 0:
7141                                 break;
7142 
7143                         case sizeof (uint8_t):
7144                                 DTRACE_STORE(uint8_t, tomax, valoffs, val);
7145                                 break;
7146                         case sizeof (uint16_t):
7147                                 DTRACE_STORE(uint16_t, tomax, valoffs, val);
7148                                 break;
7149                         case sizeof (uint32_t):
7150                                 DTRACE_STORE(uint32_t, tomax, valoffs, val);
7151                                 break;
7152                         case sizeof (uint64_t):
7153                                 DTRACE_STORE(uint64_t, tomax, valoffs, val);
7154                                 break;
7155                         default:
7156                                 /*
7157                                  * Any other size should have been returned by
7158                                  * reference, not by value.
7159                                  */
7160                                 ASSERT(0);
7161                                 break;
7162                         }
7163                 }
7164 
7165                 if (*flags & CPU_DTRACE_DROP)
7166                         continue;
7167 
7168                 if (*flags & CPU_DTRACE_FAULT) {
7169                         int ndx;
7170                         dtrace_action_t *err;
7171 
7172                         buf->dtb_errors++;
7173 
7174                         if (probe->dtpr_id == dtrace_probeid_error) {
7175                                 /*
7176                                  * There's nothing we can do -- we had an
7177                                  * error on the error probe.  We bump an
7178                                  * error counter to at least indicate that
7179                                  * this condition happened.
7180                                  */
7181                                 dtrace_error(&state->dts_dblerrors);
7182                                 continue;
7183                         }
7184 
7185                         if (vtime) {
7186                                 /*
7187                                  * Before recursing on dtrace_probe(), we
7188                                  * need to explicitly clear out our start
7189                                  * time to prevent it from being accumulated
7190                                  * into t_dtrace_vtime.
7191                                  */
7192                                 curthread->t_dtrace_start = 0;
7193                         }
7194 
7195                         /*
7196                          * Iterate over the actions to figure out which action
7197                          * we were processing when we experienced the error.
7198                          * Note that act points _past_ the faulting action; if
7199                          * act is ecb->dte_action, the fault was in the
7200                          * predicate, if it's ecb->dte_action->dta_next it's
7201                          * in action #1, and so on.
7202                          */
7203                         for (err = ecb->dte_action, ndx = 0;
7204                             err != act; err = err->dta_next, ndx++)
7205                                 continue;
7206 
7207                         dtrace_probe_error(state, ecb->dte_epid, ndx,
7208                             (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
7209                             mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
7210                             cpu_core[cpuid].cpuc_dtrace_illval);
7211 
7212                         continue;
7213                 }
7214 
7215                 if (!committed)
7216                         buf->dtb_offset = offs + ecb->dte_size;
7217         }
7218 
7219         end = dtrace_gethrtime();
7220         if (vtime)
7221                 curthread->t_dtrace_start = end;
7222 
7223         CPU->cpu_dtrace_nsec += end - now;
7224 
7225         dtrace_interrupt_enable(cookie);
7226 }
7227 
7228 /*
7229  * DTrace Probe Hashing Functions
7230  *
7231  * The functions in this section (and indeed, the functions in remaining
7232  * sections) are not _called_ from probe context.  (Any exceptions to this are
7233  * marked with a "Note:".)  Rather, they are called from elsewhere in the
7234  * DTrace framework to look-up probes in, add probes to and remove probes from
7235  * the DTrace probe hashes.  (Each probe is hashed by each element of the
7236  * probe tuple -- allowing for fast lookups, regardless of what was
7237  * specified.)
7238  */
7239 static uint_t
7240 dtrace_hash_str(char *p)
7241 {
7242         unsigned int g;
7243         uint_t hval = 0;
7244 
7245         while (*p) {
7246                 hval = (hval << 4) + *p++;
7247                 if ((g = (hval & 0xf0000000)) != 0)
7248                         hval ^= g >> 24;
7249                 hval &= ~g;
7250         }
7251         return (hval);
7252 }
7253 
7254 static dtrace_hash_t *
7255 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
7256 {
7257         dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
7258 
7259         hash->dth_stroffs = stroffs;
7260         hash->dth_nextoffs = nextoffs;
7261         hash->dth_prevoffs = prevoffs;
7262 
7263         hash->dth_size = 1;
7264         hash->dth_mask = hash->dth_size - 1;
7265 
7266         hash->dth_tab = kmem_zalloc(hash->dth_size *
7267             sizeof (dtrace_hashbucket_t *), KM_SLEEP);
7268 
7269         return (hash);
7270 }
7271 
7272 static void
7273 dtrace_hash_destroy(dtrace_hash_t *hash)
7274 {
7275 #ifdef DEBUG
7276         int i;
7277 
7278         for (i = 0; i < hash->dth_size; i++)
7279                 ASSERT(hash->dth_tab[i] == NULL);
7280 #endif
7281 
7282         kmem_free(hash->dth_tab,
7283             hash->dth_size * sizeof (dtrace_hashbucket_t *));
7284         kmem_free(hash, sizeof (dtrace_hash_t));
7285 }
7286 
7287 static void
7288 dtrace_hash_resize(dtrace_hash_t *hash)
7289 {
7290         int size = hash->dth_size, i, ndx;
7291         int new_size = hash->dth_size << 1;
7292         int new_mask = new_size - 1;
7293         dtrace_hashbucket_t **new_tab, *bucket, *next;
7294 
7295         ASSERT((new_size & new_mask) == 0);
7296 
7297         new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
7298 
7299         for (i = 0; i < size; i++) {
7300                 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
7301                         dtrace_probe_t *probe = bucket->dthb_chain;
7302 
7303                         ASSERT(probe != NULL);
7304                         ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
7305 
7306                         next = bucket->dthb_next;
7307                         bucket->dthb_next = new_tab[ndx];
7308                         new_tab[ndx] = bucket;
7309                 }
7310         }
7311 
7312         kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
7313         hash->dth_tab = new_tab;
7314         hash->dth_size = new_size;
7315         hash->dth_mask = new_mask;
7316 }
7317 
7318 static void
7319 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
7320 {
7321         int hashval = DTRACE_HASHSTR(hash, new);
7322         int ndx = hashval & hash->dth_mask;
7323         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7324         dtrace_probe_t **nextp, **prevp;
7325 
7326         for (; bucket != NULL; bucket = bucket->dthb_next) {
7327                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
7328                         goto add;
7329         }
7330 
7331         if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
7332                 dtrace_hash_resize(hash);
7333                 dtrace_hash_add(hash, new);
7334                 return;
7335         }
7336 
7337         bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
7338         bucket->dthb_next = hash->dth_tab[ndx];
7339         hash->dth_tab[ndx] = bucket;
7340         hash->dth_nbuckets++;
7341 
7342 add:
7343         nextp = DTRACE_HASHNEXT(hash, new);
7344         ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
7345         *nextp = bucket->dthb_chain;
7346 
7347         if (bucket->dthb_chain != NULL) {
7348                 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
7349                 ASSERT(*prevp == NULL);
7350                 *prevp = new;
7351         }
7352 
7353         bucket->dthb_chain = new;
7354         bucket->dthb_len++;
7355 }
7356 
7357 static dtrace_probe_t *
7358 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
7359 {
7360         int hashval = DTRACE_HASHSTR(hash, template);
7361         int ndx = hashval & hash->dth_mask;
7362         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7363 
7364         for (; bucket != NULL; bucket = bucket->dthb_next) {
7365                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7366                         return (bucket->dthb_chain);
7367         }
7368 
7369         return (NULL);
7370 }
7371 
7372 static int
7373 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
7374 {
7375         int hashval = DTRACE_HASHSTR(hash, template);
7376         int ndx = hashval & hash->dth_mask;
7377         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7378 
7379         for (; bucket != NULL; bucket = bucket->dthb_next) {
7380                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7381                         return (bucket->dthb_len);
7382         }
7383 
7384         return (NULL);
7385 }
7386 
7387 static void
7388 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
7389 {
7390         int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
7391         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7392 
7393         dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
7394         dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
7395 
7396         /*
7397          * Find the bucket that we're removing this probe from.
7398          */
7399         for (; bucket != NULL; bucket = bucket->dthb_next) {
7400                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
7401                         break;
7402         }
7403 
7404         ASSERT(bucket != NULL);
7405 
7406         if (*prevp == NULL) {
7407                 if (*nextp == NULL) {
7408                         /*
7409                          * The removed probe was the only probe on this
7410                          * bucket; we need to remove the bucket.
7411                          */
7412                         dtrace_hashbucket_t *b = hash->dth_tab[ndx];
7413 
7414                         ASSERT(bucket->dthb_chain == probe);
7415                         ASSERT(b != NULL);
7416 
7417                         if (b == bucket) {
7418                                 hash->dth_tab[ndx] = bucket->dthb_next;
7419                         } else {
7420                                 while (b->dthb_next != bucket)
7421                                         b = b->dthb_next;
7422                                 b->dthb_next = bucket->dthb_next;
7423                         }
7424 
7425                         ASSERT(hash->dth_nbuckets > 0);
7426                         hash->dth_nbuckets--;
7427                         kmem_free(bucket, sizeof (dtrace_hashbucket_t));
7428                         return;
7429                 }
7430 
7431                 bucket->dthb_chain = *nextp;
7432         } else {
7433                 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
7434         }
7435 
7436         if (*nextp != NULL)
7437                 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
7438 }
7439 
7440 /*
7441  * DTrace Utility Functions
7442  *
7443  * These are random utility functions that are _not_ called from probe context.
7444  */
7445 static int
7446 dtrace_badattr(const dtrace_attribute_t *a)
7447 {
7448         return (a->dtat_name > DTRACE_STABILITY_MAX ||
7449             a->dtat_data > DTRACE_STABILITY_MAX ||
7450             a->dtat_class > DTRACE_CLASS_MAX);
7451 }
7452 
7453 /*
7454  * Return a duplicate copy of a string.  If the specified string is NULL,
7455  * this function returns a zero-length string.
7456  */
7457 static char *
7458 dtrace_strdup(const char *str)
7459 {
7460         char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
7461 
7462         if (str != NULL)
7463                 (void) strcpy(new, str);
7464 
7465         return (new);
7466 }
7467 
7468 #define DTRACE_ISALPHA(c)       \
7469         (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
7470 
7471 static int
7472 dtrace_badname(const char *s)
7473 {
7474         char c;
7475 
7476         if (s == NULL || (c = *s++) == '\0')
7477                 return (0);
7478 
7479         if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
7480                 return (1);
7481 
7482         while ((c = *s++) != '\0') {
7483                 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
7484                     c != '-' && c != '_' && c != '.' && c != '`')
7485                         return (1);
7486         }
7487 
7488         return (0);
7489 }
7490 
7491 static void
7492 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
7493 {
7494         uint32_t priv;
7495 
7496         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
7497                 /*
7498                  * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
7499                  */
7500                 priv = DTRACE_PRIV_ALL;
7501         } else {
7502                 *uidp = crgetuid(cr);
7503                 *zoneidp = crgetzoneid(cr);
7504 
7505                 priv = 0;
7506                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
7507                         priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
7508                 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
7509                         priv |= DTRACE_PRIV_USER;
7510                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
7511                         priv |= DTRACE_PRIV_PROC;
7512                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
7513                         priv |= DTRACE_PRIV_OWNER;
7514                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
7515                         priv |= DTRACE_PRIV_ZONEOWNER;
7516         }
7517 
7518         *privp = priv;
7519 }
7520 
7521 #ifdef DTRACE_ERRDEBUG
7522 static void
7523 dtrace_errdebug(const char *str)
7524 {
7525         int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
7526         int occupied = 0;
7527 
7528         mutex_enter(&dtrace_errlock);
7529         dtrace_errlast = str;
7530         dtrace_errthread = curthread;
7531 
7532         while (occupied++ < DTRACE_ERRHASHSZ) {
7533                 if (dtrace_errhash[hval].dter_msg == str) {
7534                         dtrace_errhash[hval].dter_count++;
7535                         goto out;
7536                 }
7537 
7538                 if (dtrace_errhash[hval].dter_msg != NULL) {
7539                         hval = (hval + 1) % DTRACE_ERRHASHSZ;
7540                         continue;
7541                 }
7542 
7543                 dtrace_errhash[hval].dter_msg = str;
7544                 dtrace_errhash[hval].dter_count = 1;
7545                 goto out;
7546         }
7547 
7548         panic("dtrace: undersized error hash");
7549 out:
7550         mutex_exit(&dtrace_errlock);
7551 }
7552 #endif
7553 
7554 /*
7555  * DTrace Matching Functions
7556  *
7557  * These functions are used to match groups of probes, given some elements of
7558  * a probe tuple, or some globbed expressions for elements of a probe tuple.
7559  */
7560 static int
7561 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
7562     zoneid_t zoneid)
7563 {
7564         if (priv != DTRACE_PRIV_ALL) {
7565                 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
7566                 uint32_t match = priv & ppriv;
7567 
7568                 /*
7569                  * No PRIV_DTRACE_* privileges...
7570                  */
7571                 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
7572                     DTRACE_PRIV_KERNEL)) == 0)
7573                         return (0);
7574 
7575                 /*
7576                  * No matching bits, but there were bits to match...
7577                  */
7578                 if (match == 0 && ppriv != 0)
7579                         return (0);
7580 
7581                 /*
7582                  * Need to have permissions to the process, but don't...
7583                  */
7584                 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
7585                     uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
7586                         return (0);
7587                 }
7588 
7589                 /*
7590                  * Need to be in the same zone unless we possess the
7591                  * privilege to examine all zones.
7592                  */
7593                 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
7594                     zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
7595                         return (0);
7596                 }
7597         }
7598 
7599         return (1);
7600 }
7601 
7602 /*
7603  * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
7604  * consists of input pattern strings and an ops-vector to evaluate them.
7605  * This function returns >0 for match, 0 for no match, and <0 for error.
7606  */
7607 static int
7608 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
7609     uint32_t priv, uid_t uid, zoneid_t zoneid)
7610 {
7611         dtrace_provider_t *pvp = prp->dtpr_provider;
7612         int rv;
7613 
7614         if (pvp->dtpv_defunct)
7615                 return (0);
7616 
7617         if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
7618                 return (rv);
7619 
7620         if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
7621                 return (rv);
7622 
7623         if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
7624                 return (rv);
7625 
7626         if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
7627                 return (rv);
7628 
7629         if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
7630                 return (0);
7631 
7632         return (rv);
7633 }
7634 
7635 /*
7636  * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
7637  * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
7638  * libc's version, the kernel version only applies to 8-bit ASCII strings.
7639  * In addition, all of the recursion cases except for '*' matching have been
7640  * unwound.  For '*', we still implement recursive evaluation, but a depth
7641  * counter is maintained and matching is aborted if we recurse too deep.
7642  * The function returns 0 if no match, >0 if match, and <0 if recursion error.
7643  */
7644 static int
7645 dtrace_match_glob(const char *s, const char *p, int depth)
7646 {
7647         const char *olds;
7648         char s1, c;
7649         int gs;
7650 
7651         if (depth > DTRACE_PROBEKEY_MAXDEPTH)
7652                 return (-1);
7653 
7654         if (s == NULL)
7655                 s = ""; /* treat NULL as empty string */
7656 
7657 top:
7658         olds = s;
7659         s1 = *s++;
7660 
7661         if (p == NULL)
7662                 return (0);
7663 
7664         if ((c = *p++) == '\0')
7665                 return (s1 == '\0');
7666 
7667         switch (c) {
7668         case '[': {
7669                 int ok = 0, notflag = 0;
7670                 char lc = '\0';
7671 
7672                 if (s1 == '\0')
7673                         return (0);
7674 
7675                 if (*p == '!') {
7676                         notflag = 1;
7677                         p++;
7678                 }
7679 
7680                 if ((c = *p++) == '\0')
7681                         return (0);
7682 
7683                 do {
7684                         if (c == '-' && lc != '\0' && *p != ']') {
7685                                 if ((c = *p++) == '\0')
7686                                         return (0);
7687                                 if (c == '\\' && (c = *p++) == '\0')
7688                                         return (0);
7689 
7690                                 if (notflag) {
7691                                         if (s1 < lc || s1 > c)
7692                                                 ok++;
7693                                         else
7694                                                 return (0);
7695                                 } else if (lc <= s1 && s1 <= c)
7696                                         ok++;
7697 
7698                         } else if (c == '\\' && (c = *p++) == '\0')
7699                                 return (0);
7700 
7701                         lc = c; /* save left-hand 'c' for next iteration */
7702 
7703                         if (notflag) {
7704                                 if (s1 != c)
7705                                         ok++;
7706                                 else
7707                                         return (0);
7708                         } else if (s1 == c)
7709                                 ok++;
7710 
7711                         if ((c = *p++) == '\0')
7712                                 return (0);
7713 
7714                 } while (c != ']');
7715 
7716                 if (ok)
7717                         goto top;
7718 
7719                 return (0);
7720         }
7721 
7722         case '\\':
7723                 if ((c = *p++) == '\0')
7724                         return (0);
7725                 /*FALLTHRU*/
7726 
7727         default:
7728                 if (c != s1)
7729                         return (0);
7730                 /*FALLTHRU*/
7731 
7732         case '?':
7733                 if (s1 != '\0')
7734                         goto top;
7735                 return (0);
7736 
7737         case '*':
7738                 while (*p == '*')
7739                         p++; /* consecutive *'s are identical to a single one */
7740 
7741                 if (*p == '\0')
7742                         return (1);
7743 
7744                 for (s = olds; *s != '\0'; s++) {
7745                         if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
7746                                 return (gs);
7747                 }
7748 
7749                 return (0);
7750         }
7751 }
7752 
7753 /*ARGSUSED*/
7754 static int
7755 dtrace_match_string(const char *s, const char *p, int depth)
7756 {
7757         return (s != NULL && strcmp(s, p) == 0);
7758 }
7759 
7760 /*ARGSUSED*/
7761 static int
7762 dtrace_match_nul(const char *s, const char *p, int depth)
7763 {
7764         return (1); /* always match the empty pattern */
7765 }
7766 
7767 /*ARGSUSED*/
7768 static int
7769 dtrace_match_nonzero(const char *s, const char *p, int depth)
7770 {
7771         return (s != NULL && s[0] != '\0');
7772 }
7773 
7774 static int
7775 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
7776     zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
7777 {
7778         dtrace_probe_t template, *probe;
7779         dtrace_hash_t *hash = NULL;
7780         int len, rc, best = INT_MAX, nmatched = 0;
7781         dtrace_id_t i;
7782 
7783         ASSERT(MUTEX_HELD(&dtrace_lock));
7784 
7785         /*
7786          * If the probe ID is specified in the key, just lookup by ID and
7787          * invoke the match callback once if a matching probe is found.
7788          */
7789         if (pkp->dtpk_id != DTRACE_IDNONE) {
7790                 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
7791                     dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
7792                         if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
7793                                 return (DTRACE_MATCH_FAIL);
7794                         nmatched++;
7795                 }
7796                 return (nmatched);
7797         }
7798 
7799         template.dtpr_mod = (char *)pkp->dtpk_mod;
7800         template.dtpr_func = (char *)pkp->dtpk_func;
7801         template.dtpr_name = (char *)pkp->dtpk_name;
7802 
7803         /*
7804          * We want to find the most distinct of the module name, function
7805          * name, and name.  So for each one that is not a glob pattern or
7806          * empty string, we perform a lookup in the corresponding hash and
7807          * use the hash table with the fewest collisions to do our search.
7808          */
7809         if (pkp->dtpk_mmatch == &dtrace_match_string &&
7810             (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
7811                 best = len;
7812                 hash = dtrace_bymod;
7813         }
7814 
7815         if (pkp->dtpk_fmatch == &dtrace_match_string &&
7816             (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
7817                 best = len;
7818                 hash = dtrace_byfunc;
7819         }
7820 
7821         if (pkp->dtpk_nmatch == &dtrace_match_string &&
7822             (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
7823                 best = len;
7824                 hash = dtrace_byname;
7825         }
7826 
7827         /*
7828          * If we did not select a hash table, iterate over every probe and
7829          * invoke our callback for each one that matches our input probe key.
7830          */
7831         if (hash == NULL) {
7832                 for (i = 0; i < dtrace_nprobes; i++) {
7833                         if ((probe = dtrace_probes[i]) == NULL ||
7834                             dtrace_match_probe(probe, pkp, priv, uid,
7835                             zoneid) <= 0)
7836                                 continue;
7837 
7838                         nmatched++;
7839 
7840                         if ((rc = (*matched)(probe, arg)) !=
7841                             DTRACE_MATCH_NEXT) {
7842                                 if (rc == DTRACE_MATCH_FAIL)
7843                                         return (DTRACE_MATCH_FAIL);
7844                                 break;
7845                         }
7846                 }
7847 
7848                 return (nmatched);
7849         }
7850 
7851         /*
7852          * If we selected a hash table, iterate over each probe of the same key
7853          * name and invoke the callback for every probe that matches the other
7854          * attributes of our input probe key.
7855          */
7856         for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
7857             probe = *(DTRACE_HASHNEXT(hash, probe))) {
7858 
7859                 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
7860                         continue;
7861 
7862                 nmatched++;
7863 
7864                 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
7865                         if (rc == DTRACE_MATCH_FAIL)
7866                                 return (DTRACE_MATCH_FAIL);
7867                         break;
7868                 }
7869         }
7870 
7871         return (nmatched);
7872 }
7873 
7874 /*
7875  * Return the function pointer dtrace_probecmp() should use to compare the
7876  * specified pattern with a string.  For NULL or empty patterns, we select
7877  * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
7878  * For non-empty non-glob strings, we use dtrace_match_string().
7879  */
7880 static dtrace_probekey_f *
7881 dtrace_probekey_func(const char *p)
7882 {
7883         char c;
7884 
7885         if (p == NULL || *p == '\0')
7886                 return (&dtrace_match_nul);
7887 
7888         while ((c = *p++) != '\0') {
7889                 if (c == '[' || c == '?' || c == '*' || c == '\\')
7890                         return (&dtrace_match_glob);
7891         }
7892 
7893         return (&dtrace_match_string);
7894 }
7895 
7896 /*
7897  * Build a probe comparison key for use with dtrace_match_probe() from the
7898  * given probe description.  By convention, a null key only matches anchored
7899  * probes: if each field is the empty string, reset dtpk_fmatch to
7900  * dtrace_match_nonzero().
7901  */
7902 static void
7903 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
7904 {
7905         pkp->dtpk_prov = pdp->dtpd_provider;
7906         pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
7907 
7908         pkp->dtpk_mod = pdp->dtpd_mod;
7909         pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
7910 
7911         pkp->dtpk_func = pdp->dtpd_func;
7912         pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
7913 
7914         pkp->dtpk_name = pdp->dtpd_name;
7915         pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
7916 
7917         pkp->dtpk_id = pdp->dtpd_id;
7918 
7919         if (pkp->dtpk_id == DTRACE_IDNONE &&
7920             pkp->dtpk_pmatch == &dtrace_match_nul &&
7921             pkp->dtpk_mmatch == &dtrace_match_nul &&
7922             pkp->dtpk_fmatch == &dtrace_match_nul &&
7923             pkp->dtpk_nmatch == &dtrace_match_nul)
7924                 pkp->dtpk_fmatch = &dtrace_match_nonzero;
7925 }
7926 
7927 /*
7928  * DTrace Provider-to-Framework API Functions
7929  *
7930  * These functions implement much of the Provider-to-Framework API, as
7931  * described in <sys/dtrace.h>.  The parts of the API not in this section are
7932  * the functions in the API for probe management (found below), and
7933  * dtrace_probe() itself (found above).
7934  */
7935 
7936 /*
7937  * Register the calling provider with the DTrace framework.  This should
7938  * generally be called by DTrace providers in their attach(9E) entry point.
7939  */
7940 int
7941 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
7942     cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
7943 {
7944         dtrace_provider_t *provider;
7945 
7946         if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
7947                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7948                     "arguments", name ? name : "<NULL>");
7949                 return (EINVAL);
7950         }
7951 
7952         if (name[0] == '\0' || dtrace_badname(name)) {
7953                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7954                     "provider name", name);
7955                 return (EINVAL);
7956         }
7957 
7958         if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
7959             pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
7960             pops->dtps_destroy == NULL ||
7961             ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
7962                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7963                     "provider ops", name);
7964                 return (EINVAL);
7965         }
7966 
7967         if (dtrace_badattr(&pap->dtpa_provider) ||
7968             dtrace_badattr(&pap->dtpa_mod) ||
7969             dtrace_badattr(&pap->dtpa_func) ||
7970             dtrace_badattr(&pap->dtpa_name) ||
7971             dtrace_badattr(&pap->dtpa_args)) {
7972                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7973                     "provider attributes", name);
7974                 return (EINVAL);
7975         }
7976 
7977         if (priv & ~DTRACE_PRIV_ALL) {
7978                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7979                     "privilege attributes", name);
7980                 return (EINVAL);
7981         }
7982 
7983         if ((priv & DTRACE_PRIV_KERNEL) &&
7984             (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
7985             pops->dtps_mode == NULL) {
7986                 cmn_err(CE_WARN, "failed to register provider '%s': need "
7987                     "dtps_mode() op for given privilege attributes", name);
7988                 return (EINVAL);
7989         }
7990 
7991         provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
7992         provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
7993         (void) strcpy(provider->dtpv_name, name);
7994 
7995         provider->dtpv_attr = *pap;
7996         provider->dtpv_priv.dtpp_flags = priv;
7997         if (cr != NULL) {
7998                 provider->dtpv_priv.dtpp_uid = crgetuid(cr);
7999                 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
8000         }
8001         provider->dtpv_pops = *pops;
8002 
8003         if (pops->dtps_provide == NULL) {
8004                 ASSERT(pops->dtps_provide_module != NULL);
8005                 provider->dtpv_pops.dtps_provide =
8006                     (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
8007         }
8008 
8009         if (pops->dtps_provide_module == NULL) {
8010                 ASSERT(pops->dtps_provide != NULL);
8011                 provider->dtpv_pops.dtps_provide_module =
8012                     (void (*)(void *, struct modctl *))dtrace_nullop;
8013         }
8014 
8015         if (pops->dtps_suspend == NULL) {
8016                 ASSERT(pops->dtps_resume == NULL);
8017                 provider->dtpv_pops.dtps_suspend =
8018                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
8019                 provider->dtpv_pops.dtps_resume =
8020                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
8021         }
8022 
8023         provider->dtpv_arg = arg;
8024         *idp = (dtrace_provider_id_t)provider;
8025 
8026         if (pops == &dtrace_provider_ops) {
8027                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8028                 ASSERT(MUTEX_HELD(&dtrace_lock));
8029                 ASSERT(dtrace_anon.dta_enabling == NULL);
8030 
8031                 /*
8032                  * We make sure that the DTrace provider is at the head of
8033                  * the provider chain.
8034                  */
8035                 provider->dtpv_next = dtrace_provider;
8036                 dtrace_provider = provider;
8037                 return (0);
8038         }
8039 
8040         mutex_enter(&dtrace_provider_lock);
8041         mutex_enter(&dtrace_lock);
8042 
8043         /*
8044          * If there is at least one provider registered, we'll add this
8045          * provider after the first provider.
8046          */
8047         if (dtrace_provider != NULL) {
8048                 provider->dtpv_next = dtrace_provider->dtpv_next;
8049                 dtrace_provider->dtpv_next = provider;
8050         } else {
8051                 dtrace_provider = provider;
8052         }
8053 
8054         if (dtrace_retained != NULL) {
8055                 dtrace_enabling_provide(provider);
8056 
8057                 /*
8058                  * Now we need to call dtrace_enabling_matchall() -- which
8059                  * will acquire cpu_lock and dtrace_lock.  We therefore need
8060                  * to drop all of our locks before calling into it...
8061                  */
8062                 mutex_exit(&dtrace_lock);
8063                 mutex_exit(&dtrace_provider_lock);
8064                 dtrace_enabling_matchall();
8065 
8066                 return (0);
8067         }
8068 
8069         mutex_exit(&dtrace_lock);
8070         mutex_exit(&dtrace_provider_lock);
8071 
8072         return (0);
8073 }
8074 
8075 /*
8076  * Unregister the specified provider from the DTrace framework.  This should
8077  * generally be called by DTrace providers in their detach(9E) entry point.
8078  */
8079 int
8080 dtrace_unregister(dtrace_provider_id_t id)
8081 {
8082         dtrace_provider_t *old = (dtrace_provider_t *)id;
8083         dtrace_provider_t *prev = NULL;
8084         int i, self = 0, noreap = 0;
8085         dtrace_probe_t *probe, *first = NULL;
8086 
8087         if (old->dtpv_pops.dtps_enable ==
8088             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) {
8089                 /*
8090                  * If DTrace itself is the provider, we're called with locks
8091                  * already held.
8092                  */
8093                 ASSERT(old == dtrace_provider);
8094                 ASSERT(dtrace_devi != NULL);
8095                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8096                 ASSERT(MUTEX_HELD(&dtrace_lock));
8097                 self = 1;
8098 
8099                 if (dtrace_provider->dtpv_next != NULL) {
8100                         /*
8101                          * There's another provider here; return failure.
8102                          */
8103                         return (EBUSY);
8104                 }
8105         } else {
8106                 mutex_enter(&dtrace_provider_lock);
8107                 mutex_enter(&mod_lock);
8108                 mutex_enter(&dtrace_lock);
8109         }
8110 
8111         /*
8112          * If anyone has /dev/dtrace open, or if there are anonymous enabled
8113          * probes, we refuse to let providers slither away, unless this
8114          * provider has already been explicitly invalidated.
8115          */
8116         if (!old->dtpv_defunct &&
8117             (dtrace_opens || (dtrace_anon.dta_state != NULL &&
8118             dtrace_anon.dta_state->dts_necbs > 0))) {
8119                 if (!self) {
8120                         mutex_exit(&dtrace_lock);
8121                         mutex_exit(&mod_lock);
8122                         mutex_exit(&dtrace_provider_lock);
8123                 }
8124                 return (EBUSY);
8125         }
8126 
8127         /*
8128          * Attempt to destroy the probes associated with this provider.
8129          */
8130         for (i = 0; i < dtrace_nprobes; i++) {
8131                 if ((probe = dtrace_probes[i]) == NULL)
8132                         continue;
8133 
8134                 if (probe->dtpr_provider != old)
8135                         continue;
8136 
8137                 if (probe->dtpr_ecb == NULL)
8138                         continue;
8139 
8140                 /*
8141                  * If we are trying to unregister a defunct provider, and the
8142                  * provider was made defunct within the interval dictated by
8143                  * dtrace_unregister_defunct_reap, we'll (asynchronously)
8144                  * attempt to reap our enablings.  To denote that the provider
8145                  * should reattempt to unregister itself at some point in the
8146                  * future, we will return a differentiable error code (EAGAIN
8147                  * instead of EBUSY) in this case.
8148                  */
8149                 if (dtrace_gethrtime() - old->dtpv_defunct >
8150                     dtrace_unregister_defunct_reap)
8151                         noreap = 1;
8152 
8153                 if (!self) {
8154                         mutex_exit(&dtrace_lock);
8155                         mutex_exit(&mod_lock);
8156                         mutex_exit(&dtrace_provider_lock);
8157                 }
8158 
8159                 if (noreap)
8160                         return (EBUSY);
8161 
8162                 (void) taskq_dispatch(dtrace_taskq,
8163                     (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
8164 
8165                 return (EAGAIN);
8166         }
8167 
8168         /*
8169          * All of the probes for this provider are disabled; we can safely
8170          * remove all of them from their hash chains and from the probe array.
8171          */
8172         for (i = 0; i < dtrace_nprobes; i++) {
8173                 if ((probe = dtrace_probes[i]) == NULL)
8174                         continue;
8175 
8176                 if (probe->dtpr_provider != old)
8177                         continue;
8178 
8179                 dtrace_probes[i] = NULL;
8180 
8181                 dtrace_hash_remove(dtrace_bymod, probe);
8182                 dtrace_hash_remove(dtrace_byfunc, probe);
8183                 dtrace_hash_remove(dtrace_byname, probe);
8184 
8185                 if (first == NULL) {
8186                         first = probe;
8187                         probe->dtpr_nextmod = NULL;
8188                 } else {
8189                         probe->dtpr_nextmod = first;
8190                         first = probe;
8191                 }
8192         }
8193 
8194         /*
8195          * The provider's probes have been removed from the hash chains and
8196          * from the probe array.  Now issue a dtrace_sync() to be sure that
8197          * everyone has cleared out from any probe array processing.
8198          */
8199         dtrace_sync();
8200 
8201         for (probe = first; probe != NULL; probe = first) {
8202                 first = probe->dtpr_nextmod;
8203 
8204                 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
8205                     probe->dtpr_arg);
8206                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8207                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8208                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8209                 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
8210                 kmem_free(probe, sizeof (dtrace_probe_t));
8211         }
8212 
8213         if ((prev = dtrace_provider) == old) {
8214                 ASSERT(self || dtrace_devi == NULL);
8215                 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
8216                 dtrace_provider = old->dtpv_next;
8217         } else {
8218                 while (prev != NULL && prev->dtpv_next != old)
8219                         prev = prev->dtpv_next;
8220 
8221                 if (prev == NULL) {
8222                         panic("attempt to unregister non-existent "
8223                             "dtrace provider %p\n", (void *)id);
8224                 }
8225 
8226                 prev->dtpv_next = old->dtpv_next;
8227         }
8228 
8229         if (!self) {
8230                 mutex_exit(&dtrace_lock);
8231                 mutex_exit(&mod_lock);
8232                 mutex_exit(&dtrace_provider_lock);
8233         }
8234 
8235         kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
8236         kmem_free(old, sizeof (dtrace_provider_t));
8237 
8238         return (0);
8239 }
8240 
8241 /*
8242  * Invalidate the specified provider.  All subsequent probe lookups for the
8243  * specified provider will fail, but its probes will not be removed.
8244  */
8245 void
8246 dtrace_invalidate(dtrace_provider_id_t id)
8247 {
8248         dtrace_provider_t *pvp = (dtrace_provider_t *)id;
8249 
8250         ASSERT(pvp->dtpv_pops.dtps_enable !=
8251             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
8252 
8253         mutex_enter(&dtrace_provider_lock);
8254         mutex_enter(&dtrace_lock);
8255 
8256         pvp->dtpv_defunct = dtrace_gethrtime();
8257 
8258         mutex_exit(&dtrace_lock);
8259         mutex_exit(&dtrace_provider_lock);
8260 }
8261 
8262 /*
8263  * Indicate whether or not DTrace has attached.
8264  */
8265 int
8266 dtrace_attached(void)
8267 {
8268         /*
8269          * dtrace_provider will be non-NULL iff the DTrace driver has
8270          * attached.  (It's non-NULL because DTrace is always itself a
8271          * provider.)
8272          */
8273         return (dtrace_provider != NULL);
8274 }
8275 
8276 /*
8277  * Remove all the unenabled probes for the given provider.  This function is
8278  * not unlike dtrace_unregister(), except that it doesn't remove the provider
8279  * -- just as many of its associated probes as it can.
8280  */
8281 int
8282 dtrace_condense(dtrace_provider_id_t id)
8283 {
8284         dtrace_provider_t *prov = (dtrace_provider_t *)id;
8285         int i;
8286         dtrace_probe_t *probe;
8287 
8288         /*
8289          * Make sure this isn't the dtrace provider itself.
8290          */
8291         ASSERT(prov->dtpv_pops.dtps_enable !=
8292             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
8293 
8294         mutex_enter(&dtrace_provider_lock);
8295         mutex_enter(&dtrace_lock);
8296 
8297         /*
8298          * Attempt to destroy the probes associated with this provider.
8299          */
8300         for (i = 0; i < dtrace_nprobes; i++) {
8301                 if ((probe = dtrace_probes[i]) == NULL)
8302                         continue;
8303 
8304                 if (probe->dtpr_provider != prov)
8305                         continue;
8306 
8307                 if (probe->dtpr_ecb != NULL)
8308                         continue;
8309 
8310                 dtrace_probes[i] = NULL;
8311 
8312                 dtrace_hash_remove(dtrace_bymod, probe);
8313                 dtrace_hash_remove(dtrace_byfunc, probe);
8314                 dtrace_hash_remove(dtrace_byname, probe);
8315 
8316                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
8317                     probe->dtpr_arg);
8318                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8319                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8320                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8321                 kmem_free(probe, sizeof (dtrace_probe_t));
8322                 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
8323         }
8324 
8325         mutex_exit(&dtrace_lock);
8326         mutex_exit(&dtrace_provider_lock);
8327 
8328         return (0);
8329 }
8330 
8331 /*
8332  * DTrace Probe Management Functions
8333  *
8334  * The functions in this section perform the DTrace probe management,
8335  * including functions to create probes, look-up probes, and call into the
8336  * providers to request that probes be provided.  Some of these functions are
8337  * in the Provider-to-Framework API; these functions can be identified by the
8338  * fact that they are not declared "static".
8339  */
8340 
8341 /*
8342  * Create a probe with the specified module name, function name, and name.
8343  */
8344 dtrace_id_t
8345 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
8346     const char *func, const char *name, int aframes, void *arg)
8347 {
8348         dtrace_probe_t *probe, **probes;
8349         dtrace_provider_t *provider = (dtrace_provider_t *)prov;
8350         dtrace_id_t id;
8351 
8352         if (provider == dtrace_provider) {
8353                 ASSERT(MUTEX_HELD(&dtrace_lock));
8354         } else {
8355                 mutex_enter(&dtrace_lock);
8356         }
8357 
8358         id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
8359             VM_BESTFIT | VM_SLEEP);
8360         probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
8361 
8362         probe->dtpr_id = id;
8363         probe->dtpr_gen = dtrace_probegen++;
8364         probe->dtpr_mod = dtrace_strdup(mod);
8365         probe->dtpr_func = dtrace_strdup(func);
8366         probe->dtpr_name = dtrace_strdup(name);
8367         probe->dtpr_arg = arg;
8368         probe->dtpr_aframes = aframes;
8369         probe->dtpr_provider = provider;
8370 
8371         dtrace_hash_add(dtrace_bymod, probe);
8372         dtrace_hash_add(dtrace_byfunc, probe);
8373         dtrace_hash_add(dtrace_byname, probe);
8374 
8375         if (id - 1 >= dtrace_nprobes) {
8376                 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
8377                 size_t nsize = osize << 1;
8378 
8379                 if (nsize == 0) {
8380                         ASSERT(osize == 0);
8381                         ASSERT(dtrace_probes == NULL);
8382                         nsize = sizeof (dtrace_probe_t *);
8383                 }
8384 
8385                 probes = kmem_zalloc(nsize, KM_SLEEP);
8386 
8387                 if (dtrace_probes == NULL) {
8388                         ASSERT(osize == 0);
8389                         dtrace_probes = probes;
8390                         dtrace_nprobes = 1;
8391                 } else {
8392                         dtrace_probe_t **oprobes = dtrace_probes;
8393 
8394                         bcopy(oprobes, probes, osize);
8395                         dtrace_membar_producer();
8396                         dtrace_probes = probes;
8397 
8398                         dtrace_sync();
8399 
8400                         /*
8401                          * All CPUs are now seeing the new probes array; we can
8402                          * safely free the old array.
8403                          */
8404                         kmem_free(oprobes, osize);
8405                         dtrace_nprobes <<= 1;
8406                 }
8407 
8408                 ASSERT(id - 1 < dtrace_nprobes);
8409         }
8410 
8411         ASSERT(dtrace_probes[id - 1] == NULL);
8412         dtrace_probes[id - 1] = probe;
8413 
8414         if (provider != dtrace_provider)
8415                 mutex_exit(&dtrace_lock);
8416 
8417         return (id);
8418 }
8419 
8420 static dtrace_probe_t *
8421 dtrace_probe_lookup_id(dtrace_id_t id)
8422 {
8423         ASSERT(MUTEX_HELD(&dtrace_lock));
8424 
8425         if (id == 0 || id > dtrace_nprobes)
8426                 return (NULL);
8427 
8428         return (dtrace_probes[id - 1]);
8429 }
8430 
8431 static int
8432 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
8433 {
8434         *((dtrace_id_t *)arg) = probe->dtpr_id;
8435 
8436         return (DTRACE_MATCH_DONE);
8437 }
8438 
8439 /*
8440  * Look up a probe based on provider and one or more of module name, function
8441  * name and probe name.
8442  */
8443 dtrace_id_t
8444 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
8445     const char *func, const char *name)
8446 {
8447         dtrace_probekey_t pkey;
8448         dtrace_id_t id;
8449         int match;
8450 
8451         pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
8452         pkey.dtpk_pmatch = &dtrace_match_string;
8453         pkey.dtpk_mod = mod;
8454         pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
8455         pkey.dtpk_func = func;
8456         pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
8457         pkey.dtpk_name = name;
8458         pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
8459         pkey.dtpk_id = DTRACE_IDNONE;
8460 
8461         mutex_enter(&dtrace_lock);
8462         match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
8463             dtrace_probe_lookup_match, &id);
8464         mutex_exit(&dtrace_lock);
8465 
8466         ASSERT(match == 1 || match == 0);
8467         return (match ? id : 0);
8468 }
8469 
8470 /*
8471  * Returns the probe argument associated with the specified probe.
8472  */
8473 void *
8474 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
8475 {
8476         dtrace_probe_t *probe;
8477         void *rval = NULL;
8478 
8479         mutex_enter(&dtrace_lock);
8480 
8481         if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
8482             probe->dtpr_provider == (dtrace_provider_t *)id)
8483                 rval = probe->dtpr_arg;
8484 
8485         mutex_exit(&dtrace_lock);
8486 
8487         return (rval);
8488 }
8489 
8490 /*
8491  * Copy a probe into a probe description.
8492  */
8493 static void
8494 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
8495 {
8496         bzero(pdp, sizeof (dtrace_probedesc_t));
8497         pdp->dtpd_id = prp->dtpr_id;
8498 
8499         (void) strncpy(pdp->dtpd_provider,
8500             prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
8501 
8502         (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
8503         (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
8504         (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
8505 }
8506 
8507 /*
8508  * Called to indicate that a probe -- or probes -- should be provided by a
8509  * specfied provider.  If the specified description is NULL, the provider will
8510  * be told to provide all of its probes.  (This is done whenever a new
8511  * consumer comes along, or whenever a retained enabling is to be matched.) If
8512  * the specified description is non-NULL, the provider is given the
8513  * opportunity to dynamically provide the specified probe, allowing providers
8514  * to support the creation of probes on-the-fly.  (So-called _autocreated_
8515  * probes.)  If the provider is NULL, the operations will be applied to all
8516  * providers; if the provider is non-NULL the operations will only be applied
8517  * to the specified provider.  The dtrace_provider_lock must be held, and the
8518  * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
8519  * will need to grab the dtrace_lock when it reenters the framework through
8520  * dtrace_probe_lookup(), dtrace_probe_create(), etc.
8521  */
8522 static void
8523 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
8524 {
8525         struct modctl *ctl;
8526         int all = 0;
8527 
8528         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8529 
8530         if (prv == NULL) {
8531                 all = 1;
8532                 prv = dtrace_provider;
8533         }
8534 
8535         do {
8536                 /*
8537                  * First, call the blanket provide operation.
8538                  */
8539                 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
8540 
8541                 /*
8542                  * Now call the per-module provide operation.  We will grab
8543                  * mod_lock to prevent the list from being modified.  Note
8544                  * that this also prevents the mod_busy bits from changing.
8545                  * (mod_busy can only be changed with mod_lock held.)
8546                  */
8547                 mutex_enter(&mod_lock);
8548 
8549                 ctl = &modules;
8550                 do {
8551                         if (ctl->mod_busy || ctl->mod_mp == NULL)
8552                                 continue;
8553 
8554                         prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
8555 
8556                 } while ((ctl = ctl->mod_next) != &modules);
8557 
8558                 mutex_exit(&mod_lock);
8559         } while (all && (prv = prv->dtpv_next) != NULL);
8560 }
8561 
8562 /*
8563  * Iterate over each probe, and call the Framework-to-Provider API function
8564  * denoted by offs.
8565  */
8566 static void
8567 dtrace_probe_foreach(uintptr_t offs)
8568 {
8569         dtrace_provider_t *prov;
8570         void (*func)(void *, dtrace_id_t, void *);
8571         dtrace_probe_t *probe;
8572         dtrace_icookie_t cookie;
8573         int i;
8574 
8575         /*
8576          * We disable interrupts to walk through the probe array.  This is
8577          * safe -- the dtrace_sync() in dtrace_unregister() assures that we
8578          * won't see stale data.
8579          */
8580         cookie = dtrace_interrupt_disable();
8581 
8582         for (i = 0; i < dtrace_nprobes; i++) {
8583                 if ((probe = dtrace_probes[i]) == NULL)
8584                         continue;
8585 
8586                 if (probe->dtpr_ecb == NULL) {
8587                         /*
8588                          * This probe isn't enabled -- don't call the function.
8589                          */
8590                         continue;
8591                 }
8592 
8593                 prov = probe->dtpr_provider;
8594                 func = *((void(**)(void *, dtrace_id_t, void *))
8595                     ((uintptr_t)&prov->dtpv_pops + offs));
8596 
8597                 func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
8598         }
8599 
8600         dtrace_interrupt_enable(cookie);
8601 }
8602 
8603 static int
8604 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
8605 {
8606         dtrace_probekey_t pkey;
8607         uint32_t priv;
8608         uid_t uid;
8609         zoneid_t zoneid;
8610 
8611         ASSERT(MUTEX_HELD(&dtrace_lock));
8612         dtrace_ecb_create_cache = NULL;
8613 
8614         if (desc == NULL) {
8615                 /*
8616                  * If we're passed a NULL description, we're being asked to
8617                  * create an ECB with a NULL probe.
8618                  */
8619                 (void) dtrace_ecb_create_enable(NULL, enab);
8620                 return (0);
8621         }
8622 
8623         dtrace_probekey(desc, &pkey);
8624         dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
8625             &priv, &uid, &zoneid);
8626 
8627         return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
8628             enab));
8629 }
8630 
8631 /*
8632  * DTrace Helper Provider Functions
8633  */
8634 static void
8635 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
8636 {
8637         attr->dtat_name = DOF_ATTR_NAME(dofattr);
8638         attr->dtat_data = DOF_ATTR_DATA(dofattr);
8639         attr->dtat_class = DOF_ATTR_CLASS(dofattr);
8640 }
8641 
8642 static void
8643 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
8644     const dof_provider_t *dofprov, char *strtab)
8645 {
8646         hprov->dthpv_provname = strtab + dofprov->dofpv_name;
8647         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
8648             dofprov->dofpv_provattr);
8649         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
8650             dofprov->dofpv_modattr);
8651         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
8652             dofprov->dofpv_funcattr);
8653         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
8654             dofprov->dofpv_nameattr);
8655         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
8656             dofprov->dofpv_argsattr);
8657 }
8658 
8659 static void
8660 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8661 {
8662         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8663         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8664         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
8665         dof_provider_t *provider;
8666         dof_probe_t *probe;
8667         uint32_t *off, *enoff;
8668         uint8_t *arg;
8669         char *strtab;
8670         uint_t i, nprobes;
8671         dtrace_helper_provdesc_t dhpv;
8672         dtrace_helper_probedesc_t dhpb;
8673         dtrace_meta_t *meta = dtrace_meta_pid;
8674         dtrace_mops_t *mops = &meta->dtm_mops;
8675         void *parg;
8676 
8677         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8678         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8679             provider->dofpv_strtab * dof->dofh_secsize);
8680         prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8681             provider->dofpv_probes * dof->dofh_secsize);
8682         arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8683             provider->dofpv_prargs * dof->dofh_secsize);
8684         off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8685             provider->dofpv_proffs * dof->dofh_secsize);
8686 
8687         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8688         off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
8689         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
8690         enoff = NULL;
8691 
8692         /*
8693          * See dtrace_helper_provider_validate().
8694          */
8695         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
8696             provider->dofpv_prenoffs != DOF_SECT_NONE) {
8697                 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8698                     provider->dofpv_prenoffs * dof->dofh_secsize);
8699                 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
8700         }
8701 
8702         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
8703 
8704         /*
8705          * Create the provider.
8706          */
8707         dtrace_dofprov2hprov(&dhpv, provider, strtab);
8708 
8709         if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
8710                 return;
8711 
8712         meta->dtm_count++;
8713 
8714         /*
8715          * Create the probes.
8716          */
8717         for (i = 0; i < nprobes; i++) {
8718                 probe = (dof_probe_t *)(uintptr_t)(daddr +
8719                     prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
8720 
8721                 dhpb.dthpb_mod = dhp->dofhp_mod;
8722                 dhpb.dthpb_func = strtab + probe->dofpr_func;
8723                 dhpb.dthpb_name = strtab + probe->dofpr_name;
8724                 dhpb.dthpb_base = probe->dofpr_addr;
8725                 dhpb.dthpb_offs = off + probe->dofpr_offidx;
8726                 dhpb.dthpb_noffs = probe->dofpr_noffs;
8727                 if (enoff != NULL) {
8728                         dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
8729                         dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
8730                 } else {
8731                         dhpb.dthpb_enoffs = NULL;
8732                         dhpb.dthpb_nenoffs = 0;
8733                 }
8734                 dhpb.dthpb_args = arg + probe->dofpr_argidx;
8735                 dhpb.dthpb_nargc = probe->dofpr_nargc;
8736                 dhpb.dthpb_xargc = probe->dofpr_xargc;
8737                 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
8738                 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
8739 
8740                 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
8741         }
8742 }
8743 
8744 static void
8745 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
8746 {
8747         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8748         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8749         int i;
8750 
8751         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8752 
8753         for (i = 0; i < dof->dofh_secnum; i++) {
8754                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8755                     dof->dofh_secoff + i * dof->dofh_secsize);
8756 
8757                 if (sec->dofs_type != DOF_SECT_PROVIDER)
8758                         continue;
8759 
8760                 dtrace_helper_provide_one(dhp, sec, pid);
8761         }
8762 
8763         /*
8764          * We may have just created probes, so we must now rematch against
8765          * any retained enablings.  Note that this call will acquire both
8766          * cpu_lock and dtrace_lock; the fact that we are holding
8767          * dtrace_meta_lock now is what defines the ordering with respect to
8768          * these three locks.
8769          */
8770         dtrace_enabling_matchall();
8771 }
8772 
8773 static void
8774 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8775 {
8776         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8777         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8778         dof_sec_t *str_sec;
8779         dof_provider_t *provider;
8780         char *strtab;
8781         dtrace_helper_provdesc_t dhpv;
8782         dtrace_meta_t *meta = dtrace_meta_pid;
8783         dtrace_mops_t *mops = &meta->dtm_mops;
8784 
8785         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8786         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8787             provider->dofpv_strtab * dof->dofh_secsize);
8788 
8789         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8790 
8791         /*
8792          * Create the provider.
8793          */
8794         dtrace_dofprov2hprov(&dhpv, provider, strtab);
8795 
8796         mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
8797 
8798         meta->dtm_count--;
8799 }
8800 
8801 static void
8802 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
8803 {
8804         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8805         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8806         int i;
8807 
8808         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8809 
8810         for (i = 0; i < dof->dofh_secnum; i++) {
8811                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8812                     dof->dofh_secoff + i * dof->dofh_secsize);
8813 
8814                 if (sec->dofs_type != DOF_SECT_PROVIDER)
8815                         continue;
8816 
8817                 dtrace_helper_provider_remove_one(dhp, sec, pid);
8818         }
8819 }
8820 
8821 /*
8822  * DTrace Meta Provider-to-Framework API Functions
8823  *
8824  * These functions implement the Meta Provider-to-Framework API, as described
8825  * in <sys/dtrace.h>.
8826  */
8827 int
8828 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
8829     dtrace_meta_provider_id_t *idp)
8830 {
8831         dtrace_meta_t *meta;
8832         dtrace_helpers_t *help, *next;
8833         int i;
8834 
8835         *idp = DTRACE_METAPROVNONE;
8836 
8837         /*
8838          * We strictly don't need the name, but we hold onto it for
8839          * debuggability. All hail error queues!
8840          */
8841         if (name == NULL) {
8842                 cmn_err(CE_WARN, "failed to register meta-provider: "
8843                     "invalid name");
8844                 return (EINVAL);
8845         }
8846 
8847         if (mops == NULL ||
8848             mops->dtms_create_probe == NULL ||
8849             mops->dtms_provide_pid == NULL ||
8850             mops->dtms_remove_pid == NULL) {
8851                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8852                     "invalid ops", name);
8853                 return (EINVAL);
8854         }
8855 
8856         meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
8857         meta->dtm_mops = *mops;
8858         meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
8859         (void) strcpy(meta->dtm_name, name);
8860         meta->dtm_arg = arg;
8861 
8862         mutex_enter(&dtrace_meta_lock);
8863         mutex_enter(&dtrace_lock);
8864 
8865         if (dtrace_meta_pid != NULL) {
8866                 mutex_exit(&dtrace_lock);
8867                 mutex_exit(&dtrace_meta_lock);
8868                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8869                     "user-land meta-provider exists", name);
8870                 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
8871                 kmem_free(meta, sizeof (dtrace_meta_t));
8872                 return (EINVAL);
8873         }
8874 
8875         dtrace_meta_pid = meta;
8876         *idp = (dtrace_meta_provider_id_t)meta;
8877 
8878         /*
8879          * If there are providers and probes ready to go, pass them
8880          * off to the new meta provider now.
8881          */
8882 
8883         help = dtrace_deferred_pid;
8884         dtrace_deferred_pid = NULL;
8885 
8886         mutex_exit(&dtrace_lock);
8887 
8888         while (help != NULL) {
8889                 for (i = 0; i < help->dthps_nprovs; i++) {
8890                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
8891                             help->dthps_pid);
8892                 }
8893 
8894                 next = help->dthps_next;
8895                 help->dthps_next = NULL;
8896                 help->dthps_prev = NULL;
8897                 help->dthps_deferred = 0;
8898                 help = next;
8899         }
8900 
8901         mutex_exit(&dtrace_meta_lock);
8902 
8903         return (0);
8904 }
8905 
8906 int
8907 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
8908 {
8909         dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
8910 
8911         mutex_enter(&dtrace_meta_lock);
8912         mutex_enter(&dtrace_lock);
8913 
8914         if (old == dtrace_meta_pid) {
8915                 pp = &dtrace_meta_pid;
8916         } else {
8917                 panic("attempt to unregister non-existent "
8918                     "dtrace meta-provider %p\n", (void *)old);
8919         }
8920 
8921         if (old->dtm_count != 0) {
8922                 mutex_exit(&dtrace_lock);
8923                 mutex_exit(&dtrace_meta_lock);
8924                 return (EBUSY);
8925         }
8926 
8927         *pp = NULL;
8928 
8929         mutex_exit(&dtrace_lock);
8930         mutex_exit(&dtrace_meta_lock);
8931 
8932         kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
8933         kmem_free(old, sizeof (dtrace_meta_t));
8934 
8935         return (0);
8936 }
8937 
8938 
8939 /*
8940  * DTrace DIF Object Functions
8941  */
8942 static int
8943 dtrace_difo_err(uint_t pc, const char *format, ...)
8944 {
8945         if (dtrace_err_verbose) {
8946                 va_list alist;
8947 
8948                 (void) uprintf("dtrace DIF object error: [%u]: ", pc);
8949                 va_start(alist, format);
8950                 (void) vuprintf(format, alist);
8951                 va_end(alist);
8952         }
8953 
8954 #ifdef DTRACE_ERRDEBUG
8955         dtrace_errdebug(format);
8956 #endif
8957         return (1);
8958 }
8959 
8960 /*
8961  * Validate a DTrace DIF object by checking the IR instructions.  The following
8962  * rules are currently enforced by dtrace_difo_validate():
8963  *
8964  * 1. Each instruction must have a valid opcode
8965  * 2. Each register, string, variable, or subroutine reference must be valid
8966  * 3. No instruction can modify register %r0 (must be zero)
8967  * 4. All instruction reserved bits must be set to zero
8968  * 5. The last instruction must be a "ret" instruction
8969  * 6. All branch targets must reference a valid instruction _after_ the branch
8970  */
8971 static int
8972 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
8973     cred_t *cr)
8974 {
8975         int err = 0, i;
8976         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8977         int kcheckload;
8978         uint_t pc;
8979 
8980         kcheckload = cr == NULL ||
8981             (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
8982 
8983         dp->dtdo_destructive = 0;
8984 
8985         for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
8986                 dif_instr_t instr = dp->dtdo_buf[pc];
8987 
8988                 uint_t r1 = DIF_INSTR_R1(instr);
8989                 uint_t r2 = DIF_INSTR_R2(instr);
8990                 uint_t rd = DIF_INSTR_RD(instr);
8991                 uint_t rs = DIF_INSTR_RS(instr);
8992                 uint_t label = DIF_INSTR_LABEL(instr);
8993                 uint_t v = DIF_INSTR_VAR(instr);
8994                 uint_t subr = DIF_INSTR_SUBR(instr);
8995                 uint_t type = DIF_INSTR_TYPE(instr);
8996                 uint_t op = DIF_INSTR_OP(instr);
8997 
8998                 switch (op) {
8999                 case DIF_OP_OR:
9000                 case DIF_OP_XOR:
9001                 case DIF_OP_AND:
9002                 case DIF_OP_SLL:
9003                 case DIF_OP_SRL:
9004                 case DIF_OP_SRA:
9005                 case DIF_OP_SUB:
9006                 case DIF_OP_ADD:
9007                 case DIF_OP_MUL:
9008                 case DIF_OP_SDIV:
9009                 case DIF_OP_UDIV:
9010                 case DIF_OP_SREM:
9011                 case DIF_OP_UREM:
9012                 case DIF_OP_COPYS:
9013                         if (r1 >= nregs)
9014                                 err += efunc(pc, "invalid register %u\n", r1);
9015                         if (r2 >= nregs)
9016                                 err += efunc(pc, "invalid register %u\n", r2);
9017                         if (rd >= nregs)
9018                                 err += efunc(pc, "invalid register %u\n", rd);
9019                         if (rd == 0)
9020                                 err += efunc(pc, "cannot write to %r0\n");
9021                         break;
9022                 case DIF_OP_NOT:
9023                 case DIF_OP_MOV:
9024                 case DIF_OP_ALLOCS:
9025                         if (r1 >= nregs)
9026                                 err += efunc(pc, "invalid register %u\n", r1);
9027                         if (r2 != 0)
9028                                 err += efunc(pc, "non-zero reserved bits\n");
9029                         if (rd >= nregs)
9030                                 err += efunc(pc, "invalid register %u\n", rd);
9031                         if (rd == 0)
9032                                 err += efunc(pc, "cannot write to %r0\n");
9033                         break;
9034                 case DIF_OP_LDSB:
9035                 case DIF_OP_LDSH:
9036                 case DIF_OP_LDSW:
9037                 case DIF_OP_LDUB:
9038                 case DIF_OP_LDUH:
9039                 case DIF_OP_LDUW:
9040                 case DIF_OP_LDX:
9041                         if (r1 >= nregs)
9042                                 err += efunc(pc, "invalid register %u\n", r1);
9043                         if (r2 != 0)
9044                                 err += efunc(pc, "non-zero reserved bits\n");
9045                         if (rd >= nregs)
9046                                 err += efunc(pc, "invalid register %u\n", rd);
9047                         if (rd == 0)
9048                                 err += efunc(pc, "cannot write to %r0\n");
9049                         if (kcheckload)
9050                                 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
9051                                     DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
9052                         break;
9053                 case DIF_OP_RLDSB:
9054                 case DIF_OP_RLDSH:
9055                 case DIF_OP_RLDSW:
9056                 case DIF_OP_RLDUB:
9057                 case DIF_OP_RLDUH:
9058                 case DIF_OP_RLDUW:
9059                 case DIF_OP_RLDX:
9060                         if (r1 >= nregs)
9061                                 err += efunc(pc, "invalid register %u\n", r1);
9062                         if (r2 != 0)
9063                                 err += efunc(pc, "non-zero reserved bits\n");
9064                         if (rd >= nregs)
9065                                 err += efunc(pc, "invalid register %u\n", rd);
9066                         if (rd == 0)
9067                                 err += efunc(pc, "cannot write to %r0\n");
9068                         break;
9069                 case DIF_OP_ULDSB:
9070                 case DIF_OP_ULDSH:
9071                 case DIF_OP_ULDSW:
9072                 case DIF_OP_ULDUB:
9073                 case DIF_OP_ULDUH:
9074                 case DIF_OP_ULDUW:
9075                 case DIF_OP_ULDX:
9076                         if (r1 >= nregs)
9077                                 err += efunc(pc, "invalid register %u\n", r1);
9078                         if (r2 != 0)
9079                                 err += efunc(pc, "non-zero reserved bits\n");
9080                         if (rd >= nregs)
9081                                 err += efunc(pc, "invalid register %u\n", rd);
9082                         if (rd == 0)
9083                                 err += efunc(pc, "cannot write to %r0\n");
9084                         break;
9085                 case DIF_OP_STB:
9086                 case DIF_OP_STH:
9087                 case DIF_OP_STW:
9088                 case DIF_OP_STX:
9089                         if (r1 >= nregs)
9090                                 err += efunc(pc, "invalid register %u\n", r1);
9091                         if (r2 != 0)
9092                                 err += efunc(pc, "non-zero reserved bits\n");
9093                         if (rd >= nregs)
9094                                 err += efunc(pc, "invalid register %u\n", rd);
9095                         if (rd == 0)
9096                                 err += efunc(pc, "cannot write to 0 address\n");
9097                         break;
9098                 case DIF_OP_CMP:
9099                 case DIF_OP_SCMP:
9100                         if (r1 >= nregs)
9101                                 err += efunc(pc, "invalid register %u\n", r1);
9102                         if (r2 >= nregs)
9103                                 err += efunc(pc, "invalid register %u\n", r2);
9104                         if (rd != 0)
9105                                 err += efunc(pc, "non-zero reserved bits\n");
9106                         break;
9107                 case DIF_OP_TST:
9108                         if (r1 >= nregs)
9109                                 err += efunc(pc, "invalid register %u\n", r1);
9110                         if (r2 != 0 || rd != 0)
9111                                 err += efunc(pc, "non-zero reserved bits\n");
9112                         break;
9113                 case DIF_OP_BA:
9114                 case DIF_OP_BE:
9115                 case DIF_OP_BNE:
9116                 case DIF_OP_BG:
9117                 case DIF_OP_BGU:
9118                 case DIF_OP_BGE:
9119                 case DIF_OP_BGEU:
9120                 case DIF_OP_BL:
9121                 case DIF_OP_BLU:
9122                 case DIF_OP_BLE:
9123                 case DIF_OP_BLEU:
9124                         if (label >= dp->dtdo_len) {
9125                                 err += efunc(pc, "invalid branch target %u\n",
9126                                     label);
9127                         }
9128                         if (label <= pc) {
9129                                 err += efunc(pc, "backward branch to %u\n",
9130                                     label);
9131                         }
9132                         break;
9133                 case DIF_OP_RET:
9134                         if (r1 != 0 || r2 != 0)
9135                                 err += efunc(pc, "non-zero reserved bits\n");
9136                         if (rd >= nregs)
9137                                 err += efunc(pc, "invalid register %u\n", rd);
9138                         break;
9139                 case DIF_OP_NOP:
9140                 case DIF_OP_POPTS:
9141                 case DIF_OP_FLUSHTS:
9142                         if (r1 != 0 || r2 != 0 || rd != 0)
9143                                 err += efunc(pc, "non-zero reserved bits\n");
9144                         break;
9145                 case DIF_OP_SETX:
9146                         if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
9147                                 err += efunc(pc, "invalid integer ref %u\n",
9148                                     DIF_INSTR_INTEGER(instr));
9149                         }
9150                         if (rd >= nregs)
9151                                 err += efunc(pc, "invalid register %u\n", rd);
9152                         if (rd == 0)
9153                                 err += efunc(pc, "cannot write to %r0\n");
9154                         break;
9155                 case DIF_OP_SETS:
9156                         if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
9157                                 err += efunc(pc, "invalid string ref %u\n",
9158                                     DIF_INSTR_STRING(instr));
9159                         }
9160                         if (rd >= nregs)
9161                                 err += efunc(pc, "invalid register %u\n", rd);
9162                         if (rd == 0)
9163                                 err += efunc(pc, "cannot write to %r0\n");
9164                         break;
9165                 case DIF_OP_LDGA:
9166                 case DIF_OP_LDTA:
9167                         if (r1 > DIF_VAR_ARRAY_MAX)
9168                                 err += efunc(pc, "invalid array %u\n", r1);
9169                         if (r2 >= nregs)
9170                                 err += efunc(pc, "invalid register %u\n", r2);
9171                         if (rd >= nregs)
9172                                 err += efunc(pc, "invalid register %u\n", rd);
9173                         if (rd == 0)
9174                                 err += efunc(pc, "cannot write to %r0\n");
9175                         break;
9176                 case DIF_OP_LDGS:
9177                 case DIF_OP_LDTS:
9178                 case DIF_OP_LDLS:
9179                 case DIF_OP_LDGAA:
9180                 case DIF_OP_LDTAA:
9181                         if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
9182                                 err += efunc(pc, "invalid variable %u\n", v);
9183                         if (rd >= nregs)
9184                                 err += efunc(pc, "invalid register %u\n", rd);
9185                         if (rd == 0)
9186                                 err += efunc(pc, "cannot write to %r0\n");
9187                         break;
9188                 case DIF_OP_STGS:
9189                 case DIF_OP_STTS:
9190                 case DIF_OP_STLS:
9191                 case DIF_OP_STGAA:
9192                 case DIF_OP_STTAA:
9193                         if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
9194                                 err += efunc(pc, "invalid variable %u\n", v);
9195                         if (rs >= nregs)
9196                                 err += efunc(pc, "invalid register %u\n", rd);
9197                         break;
9198                 case DIF_OP_CALL:
9199                         if (subr > DIF_SUBR_MAX)
9200                                 err += efunc(pc, "invalid subr %u\n", subr);
9201                         if (rd >= nregs)
9202                                 err += efunc(pc, "invalid register %u\n", rd);
9203                         if (rd == 0)
9204                                 err += efunc(pc, "cannot write to %r0\n");
9205 
9206                         if (subr == DIF_SUBR_COPYOUT ||
9207                             subr == DIF_SUBR_COPYOUTSTR) {
9208                                 dp->dtdo_destructive = 1;
9209                         }
9210 
9211                         if (subr == DIF_SUBR_GETF) {
9212                                 /*
9213                                  * If we have a getf() we need to record that
9214                                  * in our state.  Note that our state can be
9215                                  * NULL if this is a helper -- but in that
9216                                  * case, the call to getf() is itself illegal,
9217                                  * and will be caught (slightly later) when
9218                                  * the helper is validated.
9219                                  */
9220                                 if (vstate->dtvs_state != NULL)
9221                                         vstate->dtvs_state->dts_getf++;
9222                         }
9223 
9224                         break;
9225                 case DIF_OP_PUSHTR:
9226                         if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
9227                                 err += efunc(pc, "invalid ref type %u\n", type);
9228                         if (r2 >= nregs)
9229                                 err += efunc(pc, "invalid register %u\n", r2);
9230                         if (rs >= nregs)
9231                                 err += efunc(pc, "invalid register %u\n", rs);
9232                         break;
9233                 case DIF_OP_PUSHTV:
9234                         if (type != DIF_TYPE_CTF)
9235                                 err += efunc(pc, "invalid val type %u\n", type);
9236                         if (r2 >= nregs)
9237                                 err += efunc(pc, "invalid register %u\n", r2);
9238                         if (rs >= nregs)
9239                                 err += efunc(pc, "invalid register %u\n", rs);
9240                         break;
9241                 default:
9242                         err += efunc(pc, "invalid opcode %u\n",
9243                             DIF_INSTR_OP(instr));
9244                 }
9245         }
9246 
9247         if (dp->dtdo_len != 0 &&
9248             DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
9249                 err += efunc(dp->dtdo_len - 1,
9250                     "expected 'ret' as last DIF instruction\n");
9251         }
9252 
9253         if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) {
9254                 /*
9255                  * If we're not returning by reference, the size must be either
9256                  * 0 or the size of one of the base types.
9257                  */
9258                 switch (dp->dtdo_rtype.dtdt_size) {
9259                 case 0:
9260                 case sizeof (uint8_t):
9261                 case sizeof (uint16_t):
9262                 case sizeof (uint32_t):
9263                 case sizeof (uint64_t):
9264                         break;
9265 
9266                 default:
9267                         err += efunc(dp->dtdo_len - 1, "bad return size\n");
9268                 }
9269         }
9270 
9271         for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
9272                 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
9273                 dtrace_diftype_t *vt, *et;
9274                 uint_t id, ndx;
9275 
9276                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
9277                     v->dtdv_scope != DIFV_SCOPE_THREAD &&
9278                     v->dtdv_scope != DIFV_SCOPE_LOCAL) {
9279                         err += efunc(i, "unrecognized variable scope %d\n",
9280                             v->dtdv_scope);
9281                         break;
9282                 }
9283 
9284                 if (v->dtdv_kind != DIFV_KIND_ARRAY &&
9285                     v->dtdv_kind != DIFV_KIND_SCALAR) {
9286                         err += efunc(i, "unrecognized variable type %d\n",
9287                             v->dtdv_kind);
9288                         break;
9289                 }
9290 
9291                 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
9292                         err += efunc(i, "%d exceeds variable id limit\n", id);
9293                         break;
9294                 }
9295 
9296                 if (id < DIF_VAR_OTHER_UBASE)
9297                         continue;
9298 
9299                 /*
9300                  * For user-defined variables, we need to check that this
9301                  * definition is identical to any previous definition that we
9302                  * encountered.
9303                  */
9304                 ndx = id - DIF_VAR_OTHER_UBASE;
9305 
9306                 switch (v->dtdv_scope) {
9307                 case DIFV_SCOPE_GLOBAL:
9308                         if (ndx < vstate->dtvs_nglobals) {
9309                                 dtrace_statvar_t *svar;
9310 
9311                                 if ((svar = vstate->dtvs_globals[ndx]) != NULL)
9312                                         existing = &svar->dtsv_var;
9313                         }
9314 
9315                         break;
9316 
9317                 case DIFV_SCOPE_THREAD:
9318                         if (ndx < vstate->dtvs_ntlocals)
9319                                 existing = &vstate->dtvs_tlocals[ndx];
9320                         break;
9321 
9322                 case DIFV_SCOPE_LOCAL:
9323                         if (ndx < vstate->dtvs_nlocals) {
9324                                 dtrace_statvar_t *svar;
9325 
9326                                 if ((svar = vstate->dtvs_locals[ndx]) != NULL)
9327                                         existing = &svar->dtsv_var;
9328                         }
9329 
9330                         break;
9331                 }
9332 
9333                 vt = &v->dtdv_type;
9334 
9335                 if (vt->dtdt_flags & DIF_TF_BYREF) {
9336                         if (vt->dtdt_size == 0) {
9337                                 err += efunc(i, "zero-sized variable\n");
9338                                 break;
9339                         }
9340 
9341                         if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
9342                             vt->dtdt_size > dtrace_global_maxsize) {
9343                                 err += efunc(i, "oversized by-ref global\n");
9344                                 break;
9345                         }
9346                 }
9347 
9348                 if (existing == NULL || existing->dtdv_id == 0)
9349                         continue;
9350 
9351                 ASSERT(existing->dtdv_id == v->dtdv_id);
9352                 ASSERT(existing->dtdv_scope == v->dtdv_scope);
9353 
9354                 if (existing->dtdv_kind != v->dtdv_kind)
9355                         err += efunc(i, "%d changed variable kind\n", id);
9356 
9357                 et = &existing->dtdv_type;
9358 
9359                 if (vt->dtdt_flags != et->dtdt_flags) {
9360                         err += efunc(i, "%d changed variable type flags\n", id);
9361                         break;
9362                 }
9363 
9364                 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
9365                         err += efunc(i, "%d changed variable type size\n", id);
9366                         break;
9367                 }
9368         }
9369 
9370         return (err);
9371 }
9372 
9373 /*
9374  * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
9375  * are much more constrained than normal DIFOs.  Specifically, they may
9376  * not:
9377  *
9378  * 1. Make calls to subroutines other than copyin(), copyinstr() or
9379  *    miscellaneous string routines
9380  * 2. Access DTrace variables other than the args[] array, and the
9381  *    curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
9382  * 3. Have thread-local variables.
9383  * 4. Have dynamic variables.
9384  */
9385 static int
9386 dtrace_difo_validate_helper(dtrace_difo_t *dp)
9387 {
9388         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
9389         int err = 0;
9390         uint_t pc;
9391 
9392         for (pc = 0; pc < dp->dtdo_len; pc++) {
9393                 dif_instr_t instr = dp->dtdo_buf[pc];
9394 
9395                 uint_t v = DIF_INSTR_VAR(instr);
9396                 uint_t subr = DIF_INSTR_SUBR(instr);
9397                 uint_t op = DIF_INSTR_OP(instr);
9398 
9399                 switch (op) {
9400                 case DIF_OP_OR:
9401                 case DIF_OP_XOR:
9402                 case DIF_OP_AND:
9403                 case DIF_OP_SLL:
9404                 case DIF_OP_SRL:
9405                 case DIF_OP_SRA:
9406                 case DIF_OP_SUB:
9407                 case DIF_OP_ADD:
9408                 case DIF_OP_MUL:
9409                 case DIF_OP_SDIV:
9410                 case DIF_OP_UDIV:
9411                 case DIF_OP_SREM:
9412                 case DIF_OP_UREM:
9413                 case DIF_OP_COPYS:
9414                 case DIF_OP_NOT:
9415                 case DIF_OP_MOV:
9416                 case DIF_OP_RLDSB:
9417                 case DIF_OP_RLDSH:
9418                 case DIF_OP_RLDSW:
9419                 case DIF_OP_RLDUB:
9420                 case DIF_OP_RLDUH:
9421                 case DIF_OP_RLDUW:
9422                 case DIF_OP_RLDX:
9423                 case DIF_OP_ULDSB:
9424                 case DIF_OP_ULDSH:
9425                 case DIF_OP_ULDSW:
9426                 case DIF_OP_ULDUB:
9427                 case DIF_OP_ULDUH:
9428                 case DIF_OP_ULDUW:
9429                 case DIF_OP_ULDX:
9430                 case DIF_OP_STB:
9431                 case DIF_OP_STH:
9432                 case DIF_OP_STW:
9433                 case DIF_OP_STX:
9434                 case DIF_OP_ALLOCS:
9435                 case DIF_OP_CMP:
9436                 case DIF_OP_SCMP:
9437                 case DIF_OP_TST:
9438                 case DIF_OP_BA:
9439                 case DIF_OP_BE:
9440                 case DIF_OP_BNE:
9441                 case DIF_OP_BG:
9442                 case DIF_OP_BGU:
9443                 case DIF_OP_BGE:
9444                 case DIF_OP_BGEU:
9445                 case DIF_OP_BL:
9446                 case DIF_OP_BLU:
9447                 case DIF_OP_BLE:
9448                 case DIF_OP_BLEU:
9449                 case DIF_OP_RET:
9450                 case DIF_OP_NOP:
9451                 case DIF_OP_POPTS:
9452                 case DIF_OP_FLUSHTS:
9453                 case DIF_OP_SETX:
9454                 case DIF_OP_SETS:
9455                 case DIF_OP_LDGA:
9456                 case DIF_OP_LDLS:
9457                 case DIF_OP_STGS:
9458                 case DIF_OP_STLS:
9459                 case DIF_OP_PUSHTR:
9460                 case DIF_OP_PUSHTV:
9461                         break;
9462 
9463                 case DIF_OP_LDGS:
9464                         if (v >= DIF_VAR_OTHER_UBASE)
9465                                 break;
9466 
9467                         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
9468                                 break;
9469 
9470                         if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
9471                             v == DIF_VAR_PPID || v == DIF_VAR_TID ||
9472                             v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
9473                             v == DIF_VAR_UID || v == DIF_VAR_GID)
9474                                 break;
9475 
9476                         err += efunc(pc, "illegal variable %u\n", v);
9477                         break;
9478 
9479                 case DIF_OP_LDTA:
9480                 case DIF_OP_LDTS:
9481                 case DIF_OP_LDGAA:
9482                 case DIF_OP_LDTAA:
9483                         err += efunc(pc, "illegal dynamic variable load\n");
9484                         break;
9485 
9486                 case DIF_OP_STTS:
9487                 case DIF_OP_STGAA:
9488                 case DIF_OP_STTAA:
9489                         err += efunc(pc, "illegal dynamic variable store\n");
9490                         break;
9491 
9492                 case DIF_OP_CALL:
9493                         if (subr == DIF_SUBR_ALLOCA ||
9494                             subr == DIF_SUBR_BCOPY ||
9495                             subr == DIF_SUBR_COPYIN ||
9496                             subr == DIF_SUBR_COPYINTO ||
9497                             subr == DIF_SUBR_COPYINSTR ||
9498                             subr == DIF_SUBR_INDEX ||
9499                             subr == DIF_SUBR_INET_NTOA ||
9500                             subr == DIF_SUBR_INET_NTOA6 ||
9501                             subr == DIF_SUBR_INET_NTOP ||
9502                             subr == DIF_SUBR_JSON ||
9503                             subr == DIF_SUBR_LLTOSTR ||
9504                             subr == DIF_SUBR_STRTOLL ||
9505                             subr == DIF_SUBR_RINDEX ||
9506                             subr == DIF_SUBR_STRCHR ||
9507                             subr == DIF_SUBR_STRJOIN ||
9508                             subr == DIF_SUBR_STRRCHR ||
9509                             subr == DIF_SUBR_STRSTR ||
9510                             subr == DIF_SUBR_HTONS ||
9511                             subr == DIF_SUBR_HTONL ||
9512                             subr == DIF_SUBR_HTONLL ||
9513                             subr == DIF_SUBR_NTOHS ||
9514                             subr == DIF_SUBR_NTOHL ||
9515                             subr == DIF_SUBR_NTOHLL)
9516                                 break;
9517 
9518                         err += efunc(pc, "invalid subr %u\n", subr);
9519                         break;
9520 
9521                 default:
9522                         err += efunc(pc, "invalid opcode %u\n",
9523                             DIF_INSTR_OP(instr));
9524                 }
9525         }
9526 
9527         return (err);
9528 }
9529 
9530 /*
9531  * Returns 1 if the expression in the DIF object can be cached on a per-thread
9532  * basis; 0 if not.
9533  */
9534 static int
9535 dtrace_difo_cacheable(dtrace_difo_t *dp)
9536 {
9537         int i;
9538 
9539         if (dp == NULL)
9540                 return (0);
9541 
9542         for (i = 0; i < dp->dtdo_varlen; i++) {
9543                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9544 
9545                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
9546                         continue;
9547 
9548                 switch (v->dtdv_id) {
9549                 case DIF_VAR_CURTHREAD:
9550                 case DIF_VAR_PID:
9551                 case DIF_VAR_TID:
9552                 case DIF_VAR_EXECNAME:
9553                 case DIF_VAR_ZONENAME:
9554                         break;
9555 
9556                 default:
9557                         return (0);
9558                 }
9559         }
9560 
9561         /*
9562          * This DIF object may be cacheable.  Now we need to look for any
9563          * array loading instructions, any memory loading instructions, or
9564          * any stores to thread-local variables.
9565          */
9566         for (i = 0; i < dp->dtdo_len; i++) {
9567                 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
9568 
9569                 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
9570                     (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
9571                     (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
9572                     op == DIF_OP_LDGA || op == DIF_OP_STTS)
9573                         return (0);
9574         }
9575 
9576         return (1);
9577 }
9578 
9579 static void
9580 dtrace_difo_hold(dtrace_difo_t *dp)
9581 {
9582         int i;
9583 
9584         ASSERT(MUTEX_HELD(&dtrace_lock));
9585 
9586         dp->dtdo_refcnt++;
9587         ASSERT(dp->dtdo_refcnt != 0);
9588 
9589         /*
9590          * We need to check this DIF object for references to the variable
9591          * DIF_VAR_VTIMESTAMP.
9592          */
9593         for (i = 0; i < dp->dtdo_varlen; i++) {
9594                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9595 
9596                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9597                         continue;
9598 
9599                 if (dtrace_vtime_references++ == 0)
9600                         dtrace_vtime_enable();
9601         }
9602 }
9603 
9604 /*
9605  * This routine calculates the dynamic variable chunksize for a given DIF
9606  * object.  The calculation is not fool-proof, and can probably be tricked by
9607  * malicious DIF -- but it works for all compiler-generated DIF.  Because this
9608  * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
9609  * if a dynamic variable size exceeds the chunksize.
9610  */
9611 static void
9612 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9613 {
9614         uint64_t sval;
9615         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
9616         const dif_instr_t *text = dp->dtdo_buf;
9617         uint_t pc, srd = 0;
9618         uint_t ttop = 0;
9619         size_t size, ksize;
9620         uint_t id, i;
9621 
9622         for (pc = 0; pc < dp->dtdo_len; pc++) {
9623                 dif_instr_t instr = text[pc];
9624                 uint_t op = DIF_INSTR_OP(instr);
9625                 uint_t rd = DIF_INSTR_RD(instr);
9626                 uint_t r1 = DIF_INSTR_R1(instr);
9627                 uint_t nkeys = 0;
9628                 uchar_t scope;
9629 
9630                 dtrace_key_t *key = tupregs;
9631 
9632                 switch (op) {
9633                 case DIF_OP_SETX:
9634                         sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
9635                         srd = rd;
9636                         continue;
9637 
9638                 case DIF_OP_STTS:
9639                         key = &tupregs[DIF_DTR_NREGS];
9640                         key[0].dttk_size = 0;
9641                         key[1].dttk_size = 0;
9642                         nkeys = 2;
9643                         scope = DIFV_SCOPE_THREAD;
9644                         break;
9645 
9646                 case DIF_OP_STGAA:
9647                 case DIF_OP_STTAA:
9648                         nkeys = ttop;
9649 
9650                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
9651                                 key[nkeys++].dttk_size = 0;
9652 
9653                         key[nkeys++].dttk_size = 0;
9654 
9655                         if (op == DIF_OP_STTAA) {
9656                                 scope = DIFV_SCOPE_THREAD;
9657                         } else {
9658                                 scope = DIFV_SCOPE_GLOBAL;
9659                         }
9660 
9661                         break;
9662 
9663                 case DIF_OP_PUSHTR:
9664                         if (ttop == DIF_DTR_NREGS)
9665                                 return;
9666 
9667                         if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
9668                                 /*
9669                                  * If the register for the size of the "pushtr"
9670                                  * is %r0 (or the value is 0) and the type is
9671                                  * a string, we'll use the system-wide default
9672                                  * string size.
9673                                  */
9674                                 tupregs[ttop++].dttk_size =
9675                                     dtrace_strsize_default;
9676                         } else {
9677                                 if (srd == 0)
9678                                         return;
9679 
9680                                 tupregs[ttop++].dttk_size = sval;
9681                         }
9682 
9683                         break;
9684 
9685                 case DIF_OP_PUSHTV:
9686                         if (ttop == DIF_DTR_NREGS)
9687                                 return;
9688 
9689                         tupregs[ttop++].dttk_size = 0;
9690                         break;
9691 
9692                 case DIF_OP_FLUSHTS:
9693                         ttop = 0;
9694                         break;
9695 
9696                 case DIF_OP_POPTS:
9697                         if (ttop != 0)
9698                                 ttop--;
9699                         break;
9700                 }
9701 
9702                 sval = 0;
9703                 srd = 0;
9704 
9705                 if (nkeys == 0)
9706                         continue;
9707 
9708                 /*
9709                  * We have a dynamic variable allocation; calculate its size.
9710                  */
9711                 for (ksize = 0, i = 0; i < nkeys; i++)
9712                         ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
9713 
9714                 size = sizeof (dtrace_dynvar_t);
9715                 size += sizeof (dtrace_key_t) * (nkeys - 1);
9716                 size += ksize;
9717 
9718                 /*
9719                  * Now we need to determine the size of the stored data.
9720                  */
9721                 id = DIF_INSTR_VAR(instr);
9722 
9723                 for (i = 0; i < dp->dtdo_varlen; i++) {
9724                         dtrace_difv_t *v = &dp->dtdo_vartab[i];
9725 
9726                         if (v->dtdv_id == id && v->dtdv_scope == scope) {
9727                                 size += v->dtdv_type.dtdt_size;
9728                                 break;
9729                         }
9730                 }
9731 
9732                 if (i == dp->dtdo_varlen)
9733                         return;
9734 
9735                 /*
9736                  * We have the size.  If this is larger than the chunk size
9737                  * for our dynamic variable state, reset the chunk size.
9738                  */
9739                 size = P2ROUNDUP(size, sizeof (uint64_t));
9740 
9741                 if (size > vstate->dtvs_dynvars.dtds_chunksize)
9742                         vstate->dtvs_dynvars.dtds_chunksize = size;
9743         }
9744 }
9745 
9746 static void
9747 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9748 {
9749         int i, oldsvars, osz, nsz, otlocals, ntlocals;
9750         uint_t id;
9751 
9752         ASSERT(MUTEX_HELD(&dtrace_lock));
9753         ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
9754 
9755         for (i = 0; i < dp->dtdo_varlen; i++) {
9756                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9757                 dtrace_statvar_t *svar, ***svarp;
9758                 size_t dsize = 0;
9759                 uint8_t scope = v->dtdv_scope;
9760                 int *np;
9761 
9762                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9763                         continue;
9764 
9765                 id -= DIF_VAR_OTHER_UBASE;
9766 
9767                 switch (scope) {
9768                 case DIFV_SCOPE_THREAD:
9769                         while (id >= (otlocals = vstate->dtvs_ntlocals)) {
9770                                 dtrace_difv_t *tlocals;
9771 
9772                                 if ((ntlocals = (otlocals << 1)) == 0)
9773                                         ntlocals = 1;
9774 
9775                                 osz = otlocals * sizeof (dtrace_difv_t);
9776                                 nsz = ntlocals * sizeof (dtrace_difv_t);
9777 
9778                                 tlocals = kmem_zalloc(nsz, KM_SLEEP);
9779 
9780                                 if (osz != 0) {
9781                                         bcopy(vstate->dtvs_tlocals,
9782                                             tlocals, osz);
9783                                         kmem_free(vstate->dtvs_tlocals, osz);
9784                                 }
9785 
9786                                 vstate->dtvs_tlocals = tlocals;
9787                                 vstate->dtvs_ntlocals = ntlocals;
9788                         }
9789 
9790                         vstate->dtvs_tlocals[id] = *v;
9791                         continue;
9792 
9793                 case DIFV_SCOPE_LOCAL:
9794                         np = &vstate->dtvs_nlocals;
9795                         svarp = &vstate->dtvs_locals;
9796 
9797                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9798                                 dsize = NCPU * (v->dtdv_type.dtdt_size +
9799                                     sizeof (uint64_t));
9800                         else
9801                                 dsize = NCPU * sizeof (uint64_t);
9802 
9803                         break;
9804 
9805                 case DIFV_SCOPE_GLOBAL:
9806                         np = &vstate->dtvs_nglobals;
9807                         svarp = &vstate->dtvs_globals;
9808 
9809                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9810                                 dsize = v->dtdv_type.dtdt_size +
9811                                     sizeof (uint64_t);
9812 
9813                         break;
9814 
9815                 default:
9816                         ASSERT(0);
9817                 }
9818 
9819                 while (id >= (oldsvars = *np)) {
9820                         dtrace_statvar_t **statics;
9821                         int newsvars, oldsize, newsize;
9822 
9823                         if ((newsvars = (oldsvars << 1)) == 0)
9824                                 newsvars = 1;
9825 
9826                         oldsize = oldsvars * sizeof (dtrace_statvar_t *);
9827                         newsize = newsvars * sizeof (dtrace_statvar_t *);
9828 
9829                         statics = kmem_zalloc(newsize, KM_SLEEP);
9830 
9831                         if (oldsize != 0) {
9832                                 bcopy(*svarp, statics, oldsize);
9833                                 kmem_free(*svarp, oldsize);
9834                         }
9835 
9836                         *svarp = statics;
9837                         *np = newsvars;
9838                 }
9839 
9840                 if ((svar = (*svarp)[id]) == NULL) {
9841                         svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
9842                         svar->dtsv_var = *v;
9843 
9844                         if ((svar->dtsv_size = dsize) != 0) {
9845                                 svar->dtsv_data = (uint64_t)(uintptr_t)
9846                                     kmem_zalloc(dsize, KM_SLEEP);
9847                         }
9848 
9849                         (*svarp)[id] = svar;
9850                 }
9851 
9852                 svar->dtsv_refcnt++;
9853         }
9854 
9855         dtrace_difo_chunksize(dp, vstate);
9856         dtrace_difo_hold(dp);
9857 }
9858 
9859 static dtrace_difo_t *
9860 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9861 {
9862         dtrace_difo_t *new;
9863         size_t sz;
9864 
9865         ASSERT(dp->dtdo_buf != NULL);
9866         ASSERT(dp->dtdo_refcnt != 0);
9867 
9868         new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
9869 
9870         ASSERT(dp->dtdo_buf != NULL);
9871         sz = dp->dtdo_len * sizeof (dif_instr_t);
9872         new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
9873         bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
9874         new->dtdo_len = dp->dtdo_len;
9875 
9876         if (dp->dtdo_strtab != NULL) {
9877                 ASSERT(dp->dtdo_strlen != 0);
9878                 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
9879                 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
9880                 new->dtdo_strlen = dp->dtdo_strlen;
9881         }
9882 
9883         if (dp->dtdo_inttab != NULL) {
9884                 ASSERT(dp->dtdo_intlen != 0);
9885                 sz = dp->dtdo_intlen * sizeof (uint64_t);
9886                 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
9887                 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
9888                 new->dtdo_intlen = dp->dtdo_intlen;
9889         }
9890 
9891         if (dp->dtdo_vartab != NULL) {
9892                 ASSERT(dp->dtdo_varlen != 0);
9893                 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
9894                 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
9895                 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
9896                 new->dtdo_varlen = dp->dtdo_varlen;
9897         }
9898 
9899         dtrace_difo_init(new, vstate);
9900         return (new);
9901 }
9902 
9903 static void
9904 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9905 {
9906         int i;
9907 
9908         ASSERT(dp->dtdo_refcnt == 0);
9909 
9910         for (i = 0; i < dp->dtdo_varlen; i++) {
9911                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9912                 dtrace_statvar_t *svar, **svarp;
9913                 uint_t id;
9914                 uint8_t scope = v->dtdv_scope;
9915                 int *np;
9916 
9917                 switch (scope) {
9918                 case DIFV_SCOPE_THREAD:
9919                         continue;
9920 
9921                 case DIFV_SCOPE_LOCAL:
9922                         np = &vstate->dtvs_nlocals;
9923                         svarp = vstate->dtvs_locals;
9924                         break;
9925 
9926                 case DIFV_SCOPE_GLOBAL:
9927                         np = &vstate->dtvs_nglobals;
9928                         svarp = vstate->dtvs_globals;
9929                         break;
9930 
9931                 default:
9932                         ASSERT(0);
9933                 }
9934 
9935                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9936                         continue;
9937 
9938                 id -= DIF_VAR_OTHER_UBASE;
9939                 ASSERT(id < *np);
9940 
9941                 svar = svarp[id];
9942                 ASSERT(svar != NULL);
9943                 ASSERT(svar->dtsv_refcnt > 0);
9944 
9945                 if (--svar->dtsv_refcnt > 0)
9946                         continue;
9947 
9948                 if (svar->dtsv_size != 0) {
9949                         ASSERT(svar->dtsv_data != NULL);
9950                         kmem_free((void *)(uintptr_t)svar->dtsv_data,
9951                             svar->dtsv_size);
9952                 }
9953 
9954                 kmem_free(svar, sizeof (dtrace_statvar_t));
9955                 svarp[id] = NULL;
9956         }
9957 
9958         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
9959         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
9960         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
9961         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
9962 
9963         kmem_free(dp, sizeof (dtrace_difo_t));
9964 }
9965 
9966 static void
9967 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9968 {
9969         int i;
9970 
9971         ASSERT(MUTEX_HELD(&dtrace_lock));
9972         ASSERT(dp->dtdo_refcnt != 0);
9973 
9974         for (i = 0; i < dp->dtdo_varlen; i++) {
9975                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9976 
9977                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9978                         continue;
9979 
9980                 ASSERT(dtrace_vtime_references > 0);
9981                 if (--dtrace_vtime_references == 0)
9982                         dtrace_vtime_disable();
9983         }
9984 
9985         if (--dp->dtdo_refcnt == 0)
9986                 dtrace_difo_destroy(dp, vstate);
9987 }
9988 
9989 /*
9990  * DTrace Format Functions
9991  */
9992 static uint16_t
9993 dtrace_format_add(dtrace_state_t *state, char *str)
9994 {
9995         char *fmt, **new;
9996         uint16_t ndx, len = strlen(str) + 1;
9997 
9998         fmt = kmem_zalloc(len, KM_SLEEP);
9999         bcopy(str, fmt, len);
10000 
10001         for (ndx = 0; ndx < state->dts_nformats; ndx++) {
10002                 if (state->dts_formats[ndx] == NULL) {
10003                         state->dts_formats[ndx] = fmt;
10004                         return (ndx + 1);
10005                 }
10006         }
10007 
10008         if (state->dts_nformats == USHRT_MAX) {
10009                 /*
10010                  * This is only likely if a denial-of-service attack is being
10011                  * attempted.  As such, it's okay to fail silently here.
10012                  */
10013                 kmem_free(fmt, len);
10014                 return (0);
10015         }
10016 
10017         /*
10018          * For simplicity, we always resize the formats array to be exactly the
10019          * number of formats.
10020          */
10021         ndx = state->dts_nformats++;
10022         new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
10023 
10024         if (state->dts_formats != NULL) {
10025                 ASSERT(ndx != 0);
10026                 bcopy(state->dts_formats, new, ndx * sizeof (char *));
10027                 kmem_free(state->dts_formats, ndx * sizeof (char *));
10028         }
10029 
10030         state->dts_formats = new;
10031         state->dts_formats[ndx] = fmt;
10032 
10033         return (ndx + 1);
10034 }
10035 
10036 static void
10037 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
10038 {
10039         char *fmt;
10040 
10041         ASSERT(state->dts_formats != NULL);
10042         ASSERT(format <= state->dts_nformats);
10043         ASSERT(state->dts_formats[format - 1] != NULL);
10044 
10045         fmt = state->dts_formats[format - 1];
10046         kmem_free(fmt, strlen(fmt) + 1);
10047         state->dts_formats[format - 1] = NULL;
10048 }
10049 
10050 static void
10051 dtrace_format_destroy(dtrace_state_t *state)
10052 {
10053         int i;
10054 
10055         if (state->dts_nformats == 0) {
10056                 ASSERT(state->dts_formats == NULL);
10057                 return;
10058         }
10059 
10060         ASSERT(state->dts_formats != NULL);
10061 
10062         for (i = 0; i < state->dts_nformats; i++) {
10063                 char *fmt = state->dts_formats[i];
10064 
10065                 if (fmt == NULL)
10066                         continue;
10067 
10068                 kmem_free(fmt, strlen(fmt) + 1);
10069         }
10070 
10071         kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
10072         state->dts_nformats = 0;
10073         state->dts_formats = NULL;
10074 }
10075 
10076 /*
10077  * DTrace Predicate Functions
10078  */
10079 static dtrace_predicate_t *
10080 dtrace_predicate_create(dtrace_difo_t *dp)
10081 {
10082         dtrace_predicate_t *pred;
10083 
10084         ASSERT(MUTEX_HELD(&dtrace_lock));
10085         ASSERT(dp->dtdo_refcnt != 0);
10086 
10087         pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
10088         pred->dtp_difo = dp;
10089         pred->dtp_refcnt = 1;
10090 
10091         if (!dtrace_difo_cacheable(dp))
10092                 return (pred);
10093 
10094         if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
10095                 /*
10096                  * This is only theoretically possible -- we have had 2^32
10097                  * cacheable predicates on this machine.  We cannot allow any
10098                  * more predicates to become cacheable:  as unlikely as it is,
10099                  * there may be a thread caching a (now stale) predicate cache
10100                  * ID. (N.B.: the temptation is being successfully resisted to
10101                  * have this cmn_err() "Holy shit -- we executed this code!")
10102                  */
10103                 return (pred);
10104         }
10105 
10106         pred->dtp_cacheid = dtrace_predcache_id++;
10107 
10108         return (pred);
10109 }
10110 
10111 static void
10112 dtrace_predicate_hold(dtrace_predicate_t *pred)
10113 {
10114         ASSERT(MUTEX_HELD(&dtrace_lock));
10115         ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
10116         ASSERT(pred->dtp_refcnt > 0);
10117 
10118         pred->dtp_refcnt++;
10119 }
10120 
10121 static void
10122 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
10123 {
10124         dtrace_difo_t *dp = pred->dtp_difo;
10125 
10126         ASSERT(MUTEX_HELD(&dtrace_lock));
10127         ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
10128         ASSERT(pred->dtp_refcnt > 0);
10129 
10130         if (--pred->dtp_refcnt == 0) {
10131                 dtrace_difo_release(pred->dtp_difo, vstate);
10132                 kmem_free(pred, sizeof (dtrace_predicate_t));
10133         }
10134 }
10135 
10136 /*
10137  * DTrace Action Description Functions
10138  */
10139 static dtrace_actdesc_t *
10140 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
10141     uint64_t uarg, uint64_t arg)
10142 {
10143         dtrace_actdesc_t *act;
10144 
10145         ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
10146             arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
10147 
10148         act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
10149         act->dtad_kind = kind;
10150         act->dtad_ntuple = ntuple;
10151         act->dtad_uarg = uarg;
10152         act->dtad_arg = arg;
10153         act->dtad_refcnt = 1;
10154 
10155         return (act);
10156 }
10157 
10158 static void
10159 dtrace_actdesc_hold(dtrace_actdesc_t *act)
10160 {
10161         ASSERT(act->dtad_refcnt >= 1);
10162         act->dtad_refcnt++;
10163 }
10164 
10165 static void
10166 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
10167 {
10168         dtrace_actkind_t kind = act->dtad_kind;
10169         dtrace_difo_t *dp;
10170 
10171         ASSERT(act->dtad_refcnt >= 1);
10172 
10173         if (--act->dtad_refcnt != 0)
10174                 return;
10175 
10176         if ((dp = act->dtad_difo) != NULL)
10177                 dtrace_difo_release(dp, vstate);
10178 
10179         if (DTRACEACT_ISPRINTFLIKE(kind)) {
10180                 char *str = (char *)(uintptr_t)act->dtad_arg;
10181 
10182                 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
10183                     (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
10184 
10185                 if (str != NULL)
10186                         kmem_free(str, strlen(str) + 1);
10187         }
10188 
10189         kmem_free(act, sizeof (dtrace_actdesc_t));
10190 }
10191 
10192 /*
10193  * DTrace ECB Functions
10194  */
10195 static dtrace_ecb_t *
10196 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
10197 {
10198         dtrace_ecb_t *ecb;
10199         dtrace_epid_t epid;
10200 
10201         ASSERT(MUTEX_HELD(&dtrace_lock));
10202 
10203         ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
10204         ecb->dte_predicate = NULL;
10205         ecb->dte_probe = probe;
10206 
10207         /*
10208          * The default size is the size of the default action: recording
10209          * the header.
10210          */
10211         ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
10212         ecb->dte_alignment = sizeof (dtrace_epid_t);
10213 
10214         epid = state->dts_epid++;
10215 
10216         if (epid - 1 >= state->dts_necbs) {
10217                 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
10218                 int necbs = state->dts_necbs << 1;
10219 
10220                 ASSERT(epid == state->dts_necbs + 1);
10221 
10222                 if (necbs == 0) {
10223                         ASSERT(oecbs == NULL);
10224                         necbs = 1;
10225                 }
10226 
10227                 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
10228 
10229                 if (oecbs != NULL)
10230                         bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
10231 
10232                 dtrace_membar_producer();
10233                 state->dts_ecbs = ecbs;
10234 
10235                 if (oecbs != NULL) {
10236                         /*
10237                          * If this state is active, we must dtrace_sync()
10238                          * before we can free the old dts_ecbs array:  we're
10239                          * coming in hot, and there may be active ring
10240                          * buffer processing (which indexes into the dts_ecbs
10241                          * array) on another CPU.
10242                          */
10243                         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
10244                                 dtrace_sync();
10245 
10246                         kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
10247                 }
10248 
10249                 dtrace_membar_producer();
10250                 state->dts_necbs = necbs;
10251         }
10252 
10253         ecb->dte_state = state;
10254 
10255         ASSERT(state->dts_ecbs[epid - 1] == NULL);
10256         dtrace_membar_producer();
10257         state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
10258 
10259         return (ecb);
10260 }
10261 
10262 static int
10263 dtrace_ecb_enable(dtrace_ecb_t *ecb)
10264 {
10265         dtrace_probe_t *probe = ecb->dte_probe;
10266 
10267         ASSERT(MUTEX_HELD(&cpu_lock));
10268         ASSERT(MUTEX_HELD(&dtrace_lock));
10269         ASSERT(ecb->dte_next == NULL);
10270 
10271         if (probe == NULL) {
10272                 /*
10273                  * This is the NULL probe -- there's nothing to do.
10274                  */
10275                 return (0);
10276         }
10277 
10278         if (probe->dtpr_ecb == NULL) {
10279                 dtrace_provider_t *prov = probe->dtpr_provider;
10280 
10281                 /*
10282                  * We're the first ECB on this probe.
10283                  */
10284                 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
10285 
10286                 if (ecb->dte_predicate != NULL)
10287                         probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
10288 
10289                 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
10290                     probe->dtpr_id, probe->dtpr_arg));
10291         } else {
10292                 /*
10293                  * This probe is already active.  Swing the last pointer to
10294                  * point to the new ECB, and issue a dtrace_sync() to assure
10295                  * that all CPUs have seen the change.
10296                  */
10297                 ASSERT(probe->dtpr_ecb_last != NULL);
10298                 probe->dtpr_ecb_last->dte_next = ecb;
10299                 probe->dtpr_ecb_last = ecb;
10300                 probe->dtpr_predcache = 0;
10301 
10302                 dtrace_sync();
10303                 return (0);
10304         }
10305 }
10306 
10307 static void
10308 dtrace_ecb_resize(dtrace_ecb_t *ecb)
10309 {
10310         dtrace_action_t *act;
10311         uint32_t curneeded = UINT32_MAX;
10312         uint32_t aggbase = UINT32_MAX;
10313 
10314         /*
10315          * If we record anything, we always record the dtrace_rechdr_t.  (And
10316          * we always record it first.)
10317          */
10318         ecb->dte_size = sizeof (dtrace_rechdr_t);
10319         ecb->dte_alignment = sizeof (dtrace_epid_t);
10320 
10321         for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
10322                 dtrace_recdesc_t *rec = &act->dta_rec;
10323                 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
10324 
10325                 ecb->dte_alignment = MAX(ecb->dte_alignment,
10326                     rec->dtrd_alignment);
10327 
10328                 if (DTRACEACT_ISAGG(act->dta_kind)) {
10329                         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10330 
10331                         ASSERT(rec->dtrd_size != 0);
10332                         ASSERT(agg->dtag_first != NULL);
10333                         ASSERT(act->dta_prev->dta_intuple);
10334                         ASSERT(aggbase != UINT32_MAX);
10335                         ASSERT(curneeded != UINT32_MAX);
10336 
10337                         agg->dtag_base = aggbase;
10338 
10339                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10340                         rec->dtrd_offset = curneeded;
10341                         curneeded += rec->dtrd_size;
10342                         ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
10343 
10344                         aggbase = UINT32_MAX;
10345                         curneeded = UINT32_MAX;
10346                 } else if (act->dta_intuple) {
10347                         if (curneeded == UINT32_MAX) {
10348                                 /*
10349                                  * This is the first record in a tuple.  Align
10350                                  * curneeded to be at offset 4 in an 8-byte
10351                                  * aligned block.
10352                                  */
10353                                 ASSERT(act->dta_prev == NULL ||
10354                                     !act->dta_prev->dta_intuple);
10355                                 ASSERT3U(aggbase, ==, UINT32_MAX);
10356                                 curneeded = P2PHASEUP(ecb->dte_size,
10357                                     sizeof (uint64_t), sizeof (dtrace_aggid_t));
10358 
10359                                 aggbase = curneeded - sizeof (dtrace_aggid_t);
10360                                 ASSERT(IS_P2ALIGNED(aggbase,
10361                                     sizeof (uint64_t)));
10362                         }
10363                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10364                         rec->dtrd_offset = curneeded;
10365                         curneeded += rec->dtrd_size;
10366                 } else {
10367                         /* tuples must be followed by an aggregation */
10368                         ASSERT(act->dta_prev == NULL ||
10369                             !act->dta_prev->dta_intuple);
10370 
10371                         ecb->dte_size = P2ROUNDUP(ecb->dte_size,
10372                             rec->dtrd_alignment);
10373                         rec->dtrd_offset = ecb->dte_size;
10374                         ecb->dte_size += rec->dtrd_size;
10375                         ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
10376                 }
10377         }
10378 
10379         if ((act = ecb->dte_action) != NULL &&
10380             !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
10381             ecb->dte_size == sizeof (dtrace_rechdr_t)) {
10382                 /*
10383                  * If the size is still sizeof (dtrace_rechdr_t), then all
10384                  * actions store no data; set the size to 0.
10385                  */
10386                 ecb->dte_size = 0;
10387         }
10388 
10389         ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
10390         ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
10391         ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
10392             ecb->dte_needed);
10393 }
10394 
10395 static dtrace_action_t *
10396 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10397 {
10398         dtrace_aggregation_t *agg;
10399         size_t size = sizeof (uint64_t);
10400         int ntuple = desc->dtad_ntuple;
10401         dtrace_action_t *act;
10402         dtrace_recdesc_t *frec;
10403         dtrace_aggid_t aggid;
10404         dtrace_state_t *state = ecb->dte_state;
10405 
10406         agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
10407         agg->dtag_ecb = ecb;
10408 
10409         ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
10410 
10411         switch (desc->dtad_kind) {
10412         case DTRACEAGG_MIN:
10413                 agg->dtag_initial = INT64_MAX;
10414                 agg->dtag_aggregate = dtrace_aggregate_min;
10415                 break;
10416 
10417         case DTRACEAGG_MAX:
10418                 agg->dtag_initial = INT64_MIN;
10419                 agg->dtag_aggregate = dtrace_aggregate_max;
10420                 break;
10421 
10422         case DTRACEAGG_COUNT:
10423                 agg->dtag_aggregate = dtrace_aggregate_count;
10424                 break;
10425 
10426         case DTRACEAGG_QUANTIZE:
10427                 agg->dtag_aggregate = dtrace_aggregate_quantize;
10428                 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
10429                     sizeof (uint64_t);
10430                 break;
10431 
10432         case DTRACEAGG_LQUANTIZE: {
10433                 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
10434                 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
10435 
10436                 agg->dtag_initial = desc->dtad_arg;
10437                 agg->dtag_aggregate = dtrace_aggregate_lquantize;
10438 
10439                 if (step == 0 || levels == 0)
10440                         goto err;
10441 
10442                 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
10443                 break;
10444         }
10445 
10446         case DTRACEAGG_LLQUANTIZE: {
10447                 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
10448                 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
10449                 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
10450                 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
10451                 int64_t v;
10452 
10453                 agg->dtag_initial = desc->dtad_arg;
10454                 agg->dtag_aggregate = dtrace_aggregate_llquantize;
10455 
10456                 if (factor < 2 || low >= high || nsteps < factor)
10457                         goto err;
10458 
10459                 /*
10460                  * Now check that the number of steps evenly divides a power
10461                  * of the factor.  (This assures both integer bucket size and
10462                  * linearity within each magnitude.)
10463                  */
10464                 for (v = factor; v < nsteps; v *= factor)
10465                         continue;
10466 
10467                 if ((v % nsteps) || (nsteps % factor))
10468                         goto err;
10469 
10470                 size = (dtrace_aggregate_llquantize_bucket(factor,
10471                     low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
10472                 break;
10473         }
10474 
10475         case DTRACEAGG_AVG:
10476                 agg->dtag_aggregate = dtrace_aggregate_avg;
10477                 size = sizeof (uint64_t) * 2;
10478                 break;
10479 
10480         case DTRACEAGG_STDDEV:
10481                 agg->dtag_aggregate = dtrace_aggregate_stddev;
10482                 size = sizeof (uint64_t) * 4;
10483                 break;
10484 
10485         case DTRACEAGG_SUM:
10486                 agg->dtag_aggregate = dtrace_aggregate_sum;
10487                 break;
10488 
10489         default:
10490                 goto err;
10491         }
10492 
10493         agg->dtag_action.dta_rec.dtrd_size = size;
10494 
10495         if (ntuple == 0)
10496                 goto err;
10497 
10498         /*
10499          * We must make sure that we have enough actions for the n-tuple.
10500          */
10501         for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
10502                 if (DTRACEACT_ISAGG(act->dta_kind))
10503                         break;
10504 
10505                 if (--ntuple == 0) {
10506                         /*
10507                          * This is the action with which our n-tuple begins.
10508                          */
10509                         agg->dtag_first = act;
10510                         goto success;
10511                 }
10512         }
10513 
10514         /*
10515          * This n-tuple is short by ntuple elements.  Return failure.
10516          */
10517         ASSERT(ntuple != 0);
10518 err:
10519         kmem_free(agg, sizeof (dtrace_aggregation_t));
10520         return (NULL);
10521 
10522 success:
10523         /*
10524          * If the last action in the tuple has a size of zero, it's actually
10525          * an expression argument for the aggregating action.
10526          */
10527         ASSERT(ecb->dte_action_last != NULL);
10528         act = ecb->dte_action_last;
10529 
10530         if (act->dta_kind == DTRACEACT_DIFEXPR) {
10531                 ASSERT(act->dta_difo != NULL);
10532 
10533                 if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
10534                         agg->dtag_hasarg = 1;
10535         }
10536 
10537         /*
10538          * We need to allocate an id for this aggregation.
10539          */
10540         aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
10541             VM_BESTFIT | VM_SLEEP);
10542 
10543         if (aggid - 1 >= state->dts_naggregations) {
10544                 dtrace_aggregation_t **oaggs = state->dts_aggregations;
10545                 dtrace_aggregation_t **aggs;
10546                 int naggs = state->dts_naggregations << 1;
10547                 int onaggs = state->dts_naggregations;
10548 
10549                 ASSERT(aggid == state->dts_naggregations + 1);
10550 
10551                 if (naggs == 0) {
10552                         ASSERT(oaggs == NULL);
10553                         naggs = 1;
10554                 }
10555 
10556                 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
10557 
10558                 if (oaggs != NULL) {
10559                         bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
10560                         kmem_free(oaggs, onaggs * sizeof (*aggs));
10561                 }
10562 
10563                 state->dts_aggregations = aggs;
10564                 state->dts_naggregations = naggs;
10565         }
10566 
10567         ASSERT(state->dts_aggregations[aggid - 1] == NULL);
10568         state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
10569 
10570         frec = &agg->dtag_first->dta_rec;
10571         if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
10572                 frec->dtrd_alignment = sizeof (dtrace_aggid_t);
10573 
10574         for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
10575                 ASSERT(!act->dta_intuple);
10576                 act->dta_intuple = 1;
10577         }
10578 
10579         return (&agg->dtag_action);
10580 }
10581 
10582 static void
10583 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
10584 {
10585         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10586         dtrace_state_t *state = ecb->dte_state;
10587         dtrace_aggid_t aggid = agg->dtag_id;
10588 
10589         ASSERT(DTRACEACT_ISAGG(act->dta_kind));
10590         vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
10591 
10592         ASSERT(state->dts_aggregations[aggid - 1] == agg);
10593         state->dts_aggregations[aggid - 1] = NULL;
10594 
10595         kmem_free(agg, sizeof (dtrace_aggregation_t));
10596 }
10597 
10598 static int
10599 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10600 {
10601         dtrace_action_t *action, *last;
10602         dtrace_difo_t *dp = desc->dtad_difo;
10603         uint32_t size = 0, align = sizeof (uint8_t), mask;
10604         uint16_t format = 0;
10605         dtrace_recdesc_t *rec;
10606         dtrace_state_t *state = ecb->dte_state;
10607         dtrace_optval_t *opt = state->dts_options, nframes, strsize;
10608         uint64_t arg = desc->dtad_arg;
10609 
10610         ASSERT(MUTEX_HELD(&dtrace_lock));
10611         ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
10612 
10613         if (DTRACEACT_ISAGG(desc->dtad_kind)) {
10614                 /*
10615                  * If this is an aggregating action, there must be neither
10616                  * a speculate nor a commit on the action chain.
10617                  */
10618                 dtrace_action_t *act;
10619 
10620                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
10621                         if (act->dta_kind == DTRACEACT_COMMIT)
10622                                 return (EINVAL);
10623 
10624                         if (act->dta_kind == DTRACEACT_SPECULATE)
10625                                 return (EINVAL);
10626                 }
10627 
10628                 action = dtrace_ecb_aggregation_create(ecb, desc);
10629 
10630                 if (action == NULL)
10631                         return (EINVAL);
10632         } else {
10633                 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
10634                     (desc->dtad_kind == DTRACEACT_DIFEXPR &&
10635                     dp != NULL && dp->dtdo_destructive)) {
10636                         state->dts_destructive = 1;
10637                 }
10638 
10639                 switch (desc->dtad_kind) {
10640                 case DTRACEACT_PRINTF:
10641                 case DTRACEACT_PRINTA:
10642                 case DTRACEACT_SYSTEM:
10643                 case DTRACEACT_FREOPEN:
10644                 case DTRACEACT_DIFEXPR:
10645                         /*
10646                          * We know that our arg is a string -- turn it into a
10647                          * format.
10648                          */
10649                         if (arg == NULL) {
10650                                 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
10651                                     desc->dtad_kind == DTRACEACT_DIFEXPR);
10652                                 format = 0;
10653                         } else {
10654                                 ASSERT(arg != NULL);
10655                                 ASSERT(arg > KERNELBASE);
10656                                 format = dtrace_format_add(state,
10657                                     (char *)(uintptr_t)arg);
10658                         }
10659 
10660                         /*FALLTHROUGH*/
10661                 case DTRACEACT_LIBACT:
10662                 case DTRACEACT_TRACEMEM:
10663                 case DTRACEACT_TRACEMEM_DYNSIZE:
10664                         if (dp == NULL)
10665                                 return (EINVAL);
10666 
10667                         if ((size = dp->dtdo_rtype.dtdt_size) != 0)
10668                                 break;
10669 
10670                         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
10671                                 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10672                                         return (EINVAL);
10673 
10674                                 size = opt[DTRACEOPT_STRSIZE];
10675                         }
10676 
10677                         break;
10678 
10679                 case DTRACEACT_STACK:
10680                         if ((nframes = arg) == 0) {
10681                                 nframes = opt[DTRACEOPT_STACKFRAMES];
10682                                 ASSERT(nframes > 0);
10683                                 arg = nframes;
10684                         }
10685 
10686                         size = nframes * sizeof (pc_t);
10687                         break;
10688 
10689                 case DTRACEACT_JSTACK:
10690                         if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
10691                                 strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
10692 
10693                         if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
10694                                 nframes = opt[DTRACEOPT_JSTACKFRAMES];
10695 
10696                         arg = DTRACE_USTACK_ARG(nframes, strsize);
10697 
10698                         /*FALLTHROUGH*/
10699                 case DTRACEACT_USTACK:
10700                         if (desc->dtad_kind != DTRACEACT_JSTACK &&
10701                             (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
10702                                 strsize = DTRACE_USTACK_STRSIZE(arg);
10703                                 nframes = opt[DTRACEOPT_USTACKFRAMES];
10704                                 ASSERT(nframes > 0);
10705                                 arg = DTRACE_USTACK_ARG(nframes, strsize);
10706                         }
10707 
10708                         /*
10709                          * Save a slot for the pid.
10710                          */
10711                         size = (nframes + 1) * sizeof (uint64_t);
10712                         size += DTRACE_USTACK_STRSIZE(arg);
10713                         size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
10714 
10715                         break;
10716 
10717                 case DTRACEACT_SYM:
10718                 case DTRACEACT_MOD:
10719                         if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
10720                             sizeof (uint64_t)) ||
10721                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10722                                 return (EINVAL);
10723                         break;
10724 
10725                 case DTRACEACT_USYM:
10726                 case DTRACEACT_UMOD:
10727                 case DTRACEACT_UADDR:
10728                         if (dp == NULL ||
10729                             (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
10730                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10731                                 return (EINVAL);
10732 
10733                         /*
10734                          * We have a slot for the pid, plus a slot for the
10735                          * argument.  To keep things simple (aligned with
10736                          * bitness-neutral sizing), we store each as a 64-bit
10737                          * quantity.
10738                          */
10739                         size = 2 * sizeof (uint64_t);
10740                         break;
10741 
10742                 case DTRACEACT_STOP:
10743                 case DTRACEACT_BREAKPOINT:
10744                 case DTRACEACT_PANIC:
10745                         break;
10746 
10747                 case DTRACEACT_CHILL:
10748                 case DTRACEACT_DISCARD:
10749                 case DTRACEACT_RAISE:
10750                         if (dp == NULL)
10751                                 return (EINVAL);
10752                         break;
10753 
10754                 case DTRACEACT_EXIT:
10755                         if (dp == NULL ||
10756                             (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
10757                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
10758                                 return (EINVAL);
10759                         break;
10760 
10761                 case DTRACEACT_SPECULATE:
10762                         if (ecb->dte_size > sizeof (dtrace_rechdr_t))
10763                                 return (EINVAL);
10764 
10765                         if (dp == NULL)
10766                                 return (EINVAL);
10767 
10768                         state->dts_speculates = 1;
10769                         break;
10770 
10771                 case DTRACEACT_COMMIT: {
10772                         dtrace_action_t *act = ecb->dte_action;
10773 
10774                         for (; act != NULL; act = act->dta_next) {
10775                                 if (act->dta_kind == DTRACEACT_COMMIT)
10776                                         return (EINVAL);
10777                         }
10778 
10779                         if (dp == NULL)
10780                                 return (EINVAL);
10781                         break;
10782                 }
10783 
10784                 default:
10785                         return (EINVAL);
10786                 }
10787 
10788                 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
10789                         /*
10790                          * If this is a data-storing action or a speculate,
10791                          * we must be sure that there isn't a commit on the
10792                          * action chain.
10793                          */
10794                         dtrace_action_t *act = ecb->dte_action;
10795 
10796                         for (; act != NULL; act = act->dta_next) {
10797                                 if (act->dta_kind == DTRACEACT_COMMIT)
10798                                         return (EINVAL);
10799                         }
10800                 }
10801 
10802                 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
10803                 action->dta_rec.dtrd_size = size;
10804         }
10805 
10806         action->dta_refcnt = 1;
10807         rec = &action->dta_rec;
10808         size = rec->dtrd_size;
10809 
10810         for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
10811                 if (!(size & mask)) {
10812                         align = mask + 1;
10813                         break;
10814                 }
10815         }
10816 
10817         action->dta_kind = desc->dtad_kind;
10818 
10819         if ((action->dta_difo = dp) != NULL)
10820                 dtrace_difo_hold(dp);
10821 
10822         rec->dtrd_action = action->dta_kind;
10823         rec->dtrd_arg = arg;
10824         rec->dtrd_uarg = desc->dtad_uarg;
10825         rec->dtrd_alignment = (uint16_t)align;
10826         rec->dtrd_format = format;
10827 
10828         if ((last = ecb->dte_action_last) != NULL) {
10829                 ASSERT(ecb->dte_action != NULL);
10830                 action->dta_prev = last;
10831                 last->dta_next = action;
10832         } else {
10833                 ASSERT(ecb->dte_action == NULL);
10834                 ecb->dte_action = action;
10835         }
10836 
10837         ecb->dte_action_last = action;
10838 
10839         return (0);
10840 }
10841 
10842 static void
10843 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
10844 {
10845         dtrace_action_t *act = ecb->dte_action, *next;
10846         dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
10847         dtrace_difo_t *dp;
10848         uint16_t format;
10849 
10850         if (act != NULL && act->dta_refcnt > 1) {
10851                 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
10852                 act->dta_refcnt--;
10853         } else {
10854                 for (; act != NULL; act = next) {
10855                         next = act->dta_next;
10856                         ASSERT(next != NULL || act == ecb->dte_action_last);
10857                         ASSERT(act->dta_refcnt == 1);
10858 
10859                         if ((format = act->dta_rec.dtrd_format) != 0)
10860                                 dtrace_format_remove(ecb->dte_state, format);
10861 
10862                         if ((dp = act->dta_difo) != NULL)
10863                                 dtrace_difo_release(dp, vstate);
10864 
10865                         if (DTRACEACT_ISAGG(act->dta_kind)) {
10866                                 dtrace_ecb_aggregation_destroy(ecb, act);
10867                         } else {
10868                                 kmem_free(act, sizeof (dtrace_action_t));
10869                         }
10870                 }
10871         }
10872 
10873         ecb->dte_action = NULL;
10874         ecb->dte_action_last = NULL;
10875         ecb->dte_size = 0;
10876 }
10877 
10878 static void
10879 dtrace_ecb_disable(dtrace_ecb_t *ecb)
10880 {
10881         /*
10882          * We disable the ECB by removing it from its probe.
10883          */
10884         dtrace_ecb_t *pecb, *prev = NULL;
10885         dtrace_probe_t *probe = ecb->dte_probe;
10886 
10887         ASSERT(MUTEX_HELD(&dtrace_lock));
10888 
10889         if (probe == NULL) {
10890                 /*
10891                  * This is the NULL probe; there is nothing to disable.
10892                  */
10893                 return;
10894         }
10895 
10896         for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
10897                 if (pecb == ecb)
10898                         break;
10899                 prev = pecb;
10900         }
10901 
10902         ASSERT(pecb != NULL);
10903 
10904         if (prev == NULL) {
10905                 probe->dtpr_ecb = ecb->dte_next;
10906         } else {
10907                 prev->dte_next = ecb->dte_next;
10908         }
10909 
10910         if (ecb == probe->dtpr_ecb_last) {
10911                 ASSERT(ecb->dte_next == NULL);
10912                 probe->dtpr_ecb_last = prev;
10913         }
10914 
10915         /*
10916          * The ECB has been disconnected from the probe; now sync to assure
10917          * that all CPUs have seen the change before returning.
10918          */
10919         dtrace_sync();
10920 
10921         if (probe->dtpr_ecb == NULL) {
10922                 /*
10923                  * That was the last ECB on the probe; clear the predicate
10924                  * cache ID for the probe, disable it and sync one more time
10925                  * to assure that we'll never hit it again.
10926                  */
10927                 dtrace_provider_t *prov = probe->dtpr_provider;
10928 
10929                 ASSERT(ecb->dte_next == NULL);
10930                 ASSERT(probe->dtpr_ecb_last == NULL);
10931                 probe->dtpr_predcache = DTRACE_CACHEIDNONE;
10932                 prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
10933                     probe->dtpr_id, probe->dtpr_arg);
10934                 dtrace_sync();
10935         } else {
10936                 /*
10937                  * There is at least one ECB remaining on the probe.  If there
10938                  * is _exactly_ one, set the probe's predicate cache ID to be
10939                  * the predicate cache ID of the remaining ECB.
10940                  */
10941                 ASSERT(probe->dtpr_ecb_last != NULL);
10942                 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
10943 
10944                 if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
10945                         dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
10946 
10947                         ASSERT(probe->dtpr_ecb->dte_next == NULL);
10948 
10949                         if (p != NULL)
10950                                 probe->dtpr_predcache = p->dtp_cacheid;
10951                 }
10952 
10953                 ecb->dte_next = NULL;
10954         }
10955 }
10956 
10957 static void
10958 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
10959 {
10960         dtrace_state_t *state = ecb->dte_state;
10961         dtrace_vstate_t *vstate = &state->dts_vstate;
10962         dtrace_predicate_t *pred;
10963         dtrace_epid_t epid = ecb->dte_epid;
10964 
10965         ASSERT(MUTEX_HELD(&dtrace_lock));
10966         ASSERT(ecb->dte_next == NULL);
10967         ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
10968 
10969         if ((pred = ecb->dte_predicate) != NULL)
10970                 dtrace_predicate_release(pred, vstate);
10971 
10972         dtrace_ecb_action_remove(ecb);
10973 
10974         ASSERT(state->dts_ecbs[epid - 1] == ecb);
10975         state->dts_ecbs[epid - 1] = NULL;
10976 
10977         kmem_free(ecb, sizeof (dtrace_ecb_t));
10978 }
10979 
10980 static dtrace_ecb_t *
10981 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
10982     dtrace_enabling_t *enab)
10983 {
10984         dtrace_ecb_t *ecb;
10985         dtrace_predicate_t *pred;
10986         dtrace_actdesc_t *act;
10987         dtrace_provider_t *prov;
10988         dtrace_ecbdesc_t *desc = enab->dten_current;
10989 
10990         ASSERT(MUTEX_HELD(&dtrace_lock));
10991         ASSERT(state != NULL);
10992 
10993         ecb = dtrace_ecb_add(state, probe);
10994         ecb->dte_uarg = desc->dted_uarg;
10995 
10996         if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
10997                 dtrace_predicate_hold(pred);
10998                 ecb->dte_predicate = pred;
10999         }
11000 
11001         if (probe != NULL) {
11002                 /*
11003                  * If the provider shows more leg than the consumer is old
11004                  * enough to see, we need to enable the appropriate implicit
11005                  * predicate bits to prevent the ecb from activating at
11006                  * revealing times.
11007                  *
11008                  * Providers specifying DTRACE_PRIV_USER at register time
11009                  * are stating that they need the /proc-style privilege
11010                  * model to be enforced, and this is what DTRACE_COND_OWNER
11011                  * and DTRACE_COND_ZONEOWNER will then do at probe time.
11012                  */
11013                 prov = probe->dtpr_provider;
11014                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
11015                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
11016                         ecb->dte_cond |= DTRACE_COND_OWNER;
11017 
11018                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
11019                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
11020                         ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
11021 
11022                 /*
11023                  * If the provider shows us kernel innards and the user
11024                  * is lacking sufficient privilege, enable the
11025                  * DTRACE_COND_USERMODE implicit predicate.
11026                  */
11027                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
11028                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
11029                         ecb->dte_cond |= DTRACE_COND_USERMODE;
11030         }
11031 
11032         if (dtrace_ecb_create_cache != NULL) {
11033                 /*
11034                  * If we have a cached ecb, we'll use its action list instead
11035                  * of creating our own (saving both time and space).
11036                  */
11037                 dtrace_ecb_t *cached = dtrace_ecb_create_cache;
11038                 dtrace_action_t *act = cached->dte_action;
11039 
11040                 if (act != NULL) {
11041                         ASSERT(act->dta_refcnt > 0);
11042                         act->dta_refcnt++;
11043                         ecb->dte_action = act;
11044                         ecb->dte_action_last = cached->dte_action_last;
11045                         ecb->dte_needed = cached->dte_needed;
11046                         ecb->dte_size = cached->dte_size;
11047                         ecb->dte_alignment = cached->dte_alignment;
11048                 }
11049 
11050                 return (ecb);
11051         }
11052 
11053         for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
11054                 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
11055                         dtrace_ecb_destroy(ecb);
11056                         return (NULL);
11057                 }
11058         }
11059 
11060         dtrace_ecb_resize(ecb);
11061 
11062         return (dtrace_ecb_create_cache = ecb);
11063 }
11064 
11065 static int
11066 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
11067 {
11068         dtrace_ecb_t *ecb;
11069         dtrace_enabling_t *enab = arg;
11070         dtrace_state_t *state = enab->dten_vstate->dtvs_state;
11071 
11072         ASSERT(state != NULL);
11073 
11074         if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
11075                 /*
11076                  * This probe was created in a generation for which this
11077                  * enabling has previously created ECBs; we don't want to
11078                  * enable it again, so just kick out.
11079                  */
11080                 return (DTRACE_MATCH_NEXT);
11081         }
11082 
11083         if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
11084                 return (DTRACE_MATCH_DONE);
11085 
11086         if (dtrace_ecb_enable(ecb) < 0)
11087                 return (DTRACE_MATCH_FAIL);
11088 
11089         return (DTRACE_MATCH_NEXT);
11090 }
11091 
11092 static dtrace_ecb_t *
11093 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
11094 {
11095         dtrace_ecb_t *ecb;
11096 
11097         ASSERT(MUTEX_HELD(&dtrace_lock));
11098 
11099         if (id == 0 || id > state->dts_necbs)
11100                 return (NULL);
11101 
11102         ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
11103         ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
11104 
11105         return (state->dts_ecbs[id - 1]);
11106 }
11107 
11108 static dtrace_aggregation_t *
11109 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
11110 {
11111         dtrace_aggregation_t *agg;
11112 
11113         ASSERT(MUTEX_HELD(&dtrace_lock));
11114 
11115         if (id == 0 || id > state->dts_naggregations)
11116                 return (NULL);
11117 
11118         ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
11119         ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
11120             agg->dtag_id == id);
11121 
11122         return (state->dts_aggregations[id - 1]);
11123 }
11124 
11125 /*
11126  * DTrace Buffer Functions
11127  *
11128  * The following functions manipulate DTrace buffers.  Most of these functions
11129  * are called in the context of establishing or processing consumer state;
11130  * exceptions are explicitly noted.
11131  */
11132 
11133 /*
11134  * Note:  called from cross call context.  This function switches the two
11135  * buffers on a given CPU.  The atomicity of this operation is assured by
11136  * disabling interrupts while the actual switch takes place; the disabling of
11137  * interrupts serializes the execution with any execution of dtrace_probe() on
11138  * the same CPU.
11139  */
11140 static void
11141 dtrace_buffer_switch(dtrace_buffer_t *buf)
11142 {
11143         caddr_t tomax = buf->dtb_tomax;
11144         caddr_t xamot = buf->dtb_xamot;
11145         dtrace_icookie_t cookie;
11146         hrtime_t now;
11147 
11148         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
11149         ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
11150 
11151         cookie = dtrace_interrupt_disable();
11152         now = dtrace_gethrtime();
11153         buf->dtb_tomax = xamot;
11154         buf->dtb_xamot = tomax;
11155         buf->dtb_xamot_drops = buf->dtb_drops;
11156         buf->dtb_xamot_offset = buf->dtb_offset;
11157         buf->dtb_xamot_errors = buf->dtb_errors;
11158         buf->dtb_xamot_flags = buf->dtb_flags;
11159         buf->dtb_offset = 0;
11160         buf->dtb_drops = 0;
11161         buf->dtb_errors = 0;
11162         buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
11163         buf->dtb_interval = now - buf->dtb_switched;
11164         buf->dtb_switched = now;
11165         dtrace_interrupt_enable(cookie);
11166 }
11167 
11168 /*
11169  * Note:  called from cross call context.  This function activates a buffer
11170  * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
11171  * is guaranteed by the disabling of interrupts.
11172  */
11173 static void
11174 dtrace_buffer_activate(dtrace_state_t *state)
11175 {
11176         dtrace_buffer_t *buf;
11177         dtrace_icookie_t cookie = dtrace_interrupt_disable();
11178 
11179         buf = &state->dts_buffer[CPU->cpu_id];
11180 
11181         if (buf->dtb_tomax != NULL) {
11182                 /*
11183                  * We might like to assert that the buffer is marked inactive,
11184                  * but this isn't necessarily true:  the buffer for the CPU
11185                  * that processes the BEGIN probe has its buffer activated
11186                  * manually.  In this case, we take the (harmless) action
11187                  * re-clearing the bit INACTIVE bit.
11188                  */
11189                 buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
11190         }
11191 
11192         dtrace_interrupt_enable(cookie);
11193 }
11194 
11195 static int
11196 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
11197     processorid_t cpu, int *factor)
11198 {
11199         cpu_t *cp;
11200         dtrace_buffer_t *buf;
11201         int allocated = 0, desired = 0;
11202 
11203         ASSERT(MUTEX_HELD(&cpu_lock));
11204         ASSERT(MUTEX_HELD(&dtrace_lock));
11205 
11206         *factor = 1;
11207 
11208         if (size > dtrace_nonroot_maxsize &&
11209             !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
11210                 return (EFBIG);
11211 
11212         cp = cpu_list;
11213 
11214         do {
11215                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11216                         continue;
11217 
11218                 buf = &bufs[cp->cpu_id];
11219 
11220                 /*
11221                  * If there is already a buffer allocated for this CPU, it
11222                  * is only possible that this is a DR event.  In this case,
11223                  * the buffer size must match our specified size.
11224                  */
11225                 if (buf->dtb_tomax != NULL) {
11226                         ASSERT(buf->dtb_size == size);
11227                         continue;
11228                 }
11229 
11230                 ASSERT(buf->dtb_xamot == NULL);
11231 
11232                 if ((buf->dtb_tomax = kmem_zalloc(size,
11233                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11234                         goto err;
11235 
11236                 buf->dtb_size = size;
11237                 buf->dtb_flags = flags;
11238                 buf->dtb_offset = 0;
11239                 buf->dtb_drops = 0;
11240 
11241                 if (flags & DTRACEBUF_NOSWITCH)
11242                         continue;
11243 
11244                 if ((buf->dtb_xamot = kmem_zalloc(size,
11245                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11246                         goto err;
11247         } while ((cp = cp->cpu_next) != cpu_list);
11248 
11249         return (0);
11250 
11251 err:
11252         cp = cpu_list;
11253 
11254         do {
11255                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11256                         continue;
11257 
11258                 buf = &bufs[cp->cpu_id];
11259                 desired += 2;
11260 
11261                 if (buf->dtb_xamot != NULL) {
11262                         ASSERT(buf->dtb_tomax != NULL);
11263                         ASSERT(buf->dtb_size == size);
11264                         kmem_free(buf->dtb_xamot, size);
11265                         allocated++;
11266                 }
11267 
11268                 if (buf->dtb_tomax != NULL) {
11269                         ASSERT(buf->dtb_size == size);
11270                         kmem_free(buf->dtb_tomax, size);
11271                         allocated++;
11272                 }
11273 
11274                 buf->dtb_tomax = NULL;
11275                 buf->dtb_xamot = NULL;
11276                 buf->dtb_size = 0;
11277         } while ((cp = cp->cpu_next) != cpu_list);
11278 
11279         *factor = desired / (allocated > 0 ? allocated : 1);
11280 
11281         return (ENOMEM);
11282 }
11283 
11284 /*
11285  * Note:  called from probe context.  This function just increments the drop
11286  * count on a buffer.  It has been made a function to allow for the
11287  * possibility of understanding the source of mysterious drop counts.  (A
11288  * problem for which one may be particularly disappointed that DTrace cannot
11289  * be used to understand DTrace.)
11290  */
11291 static void
11292 dtrace_buffer_drop(dtrace_buffer_t *buf)
11293 {
11294         buf->dtb_drops++;
11295 }
11296 
11297 /*
11298  * Note:  called from probe context.  This function is called to reserve space
11299  * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
11300  * mstate.  Returns the new offset in the buffer, or a negative value if an
11301  * error has occurred.
11302  */
11303 static intptr_t
11304 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
11305     dtrace_state_t *state, dtrace_mstate_t *mstate)
11306 {
11307         intptr_t offs = buf->dtb_offset, soffs;
11308         intptr_t woffs;
11309         caddr_t tomax;
11310         size_t total;
11311 
11312         if (buf->dtb_flags & DTRACEBUF_INACTIVE)
11313                 return (-1);
11314 
11315         if ((tomax = buf->dtb_tomax) == NULL) {
11316                 dtrace_buffer_drop(buf);
11317                 return (-1);
11318         }
11319 
11320         if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
11321                 while (offs & (align - 1)) {
11322                         /*
11323                          * Assert that our alignment is off by a number which
11324                          * is itself sizeof (uint32_t) aligned.
11325                          */
11326                         ASSERT(!((align - (offs & (align - 1))) &
11327                             (sizeof (uint32_t) - 1)));
11328                         DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11329                         offs += sizeof (uint32_t);
11330                 }
11331 
11332                 if ((soffs = offs + needed) > buf->dtb_size) {
11333                         dtrace_buffer_drop(buf);
11334                         return (-1);
11335                 }
11336 
11337                 if (mstate == NULL)
11338                         return (offs);
11339 
11340                 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
11341                 mstate->dtms_scratch_size = buf->dtb_size - soffs;
11342                 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11343 
11344                 return (offs);
11345         }
11346 
11347         if (buf->dtb_flags & DTRACEBUF_FILL) {
11348                 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
11349                     (buf->dtb_flags & DTRACEBUF_FULL))
11350                         return (-1);
11351                 goto out;
11352         }
11353 
11354         total = needed + (offs & (align - 1));
11355 
11356         /*
11357          * For a ring buffer, life is quite a bit more complicated.  Before
11358          * we can store any padding, we need to adjust our wrapping offset.
11359          * (If we've never before wrapped or we're not about to, no adjustment
11360          * is required.)
11361          */
11362         if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
11363             offs + total > buf->dtb_size) {
11364                 woffs = buf->dtb_xamot_offset;
11365 
11366                 if (offs + total > buf->dtb_size) {
11367                         /*
11368                          * We can't fit in the end of the buffer.  First, a
11369                          * sanity check that we can fit in the buffer at all.
11370                          */
11371                         if (total > buf->dtb_size) {
11372                                 dtrace_buffer_drop(buf);
11373                                 return (-1);
11374                         }
11375 
11376                         /*
11377                          * We're going to be storing at the top of the buffer,
11378                          * so now we need to deal with the wrapped offset.  We
11379                          * only reset our wrapped offset to 0 if it is
11380                          * currently greater than the current offset.  If it
11381                          * is less than the current offset, it is because a
11382                          * previous allocation induced a wrap -- but the
11383                          * allocation didn't subsequently take the space due
11384                          * to an error or false predicate evaluation.  In this
11385                          * case, we'll just leave the wrapped offset alone: if
11386                          * the wrapped offset hasn't been advanced far enough
11387                          * for this allocation, it will be adjusted in the
11388                          * lower loop.
11389                          */
11390                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
11391                                 if (woffs >= offs)
11392                                         woffs = 0;
11393                         } else {
11394                                 woffs = 0;
11395                         }
11396 
11397                         /*
11398                          * Now we know that we're going to be storing to the
11399                          * top of the buffer and that there is room for us
11400                          * there.  We need to clear the buffer from the current
11401                          * offset to the end (there may be old gunk there).
11402                          */
11403                         while (offs < buf->dtb_size)
11404                                 tomax[offs++] = 0;
11405 
11406                         /*
11407                          * We need to set our offset to zero.  And because we
11408                          * are wrapping, we need to set the bit indicating as
11409                          * much.  We can also adjust our needed space back
11410                          * down to the space required by the ECB -- we know
11411                          * that the top of the buffer is aligned.
11412                          */
11413                         offs = 0;
11414                         total = needed;
11415                         buf->dtb_flags |= DTRACEBUF_WRAPPED;
11416                 } else {
11417                         /*
11418                          * There is room for us in the buffer, so we simply
11419                          * need to check the wrapped offset.
11420                          */
11421                         if (woffs < offs) {
11422                                 /*
11423                                  * The wrapped offset is less than the offset.
11424                                  * This can happen if we allocated buffer space
11425                                  * that induced a wrap, but then we didn't
11426                                  * subsequently take the space due to an error
11427                                  * or false predicate evaluation.  This is
11428                                  * okay; we know that _this_ allocation isn't
11429                                  * going to induce a wrap.  We still can't
11430                                  * reset the wrapped offset to be zero,
11431                                  * however: the space may have been trashed in
11432                                  * the previous failed probe attempt.  But at
11433                                  * least the wrapped offset doesn't need to
11434                                  * be adjusted at all...
11435                                  */
11436                                 goto out;
11437                         }
11438                 }
11439 
11440                 while (offs + total > woffs) {
11441                         dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
11442                         size_t size;
11443 
11444                         if (epid == DTRACE_EPIDNONE) {
11445                                 size = sizeof (uint32_t);
11446                         } else {
11447                                 ASSERT3U(epid, <=, state->dts_necbs);
11448                                 ASSERT(state->dts_ecbs[epid - 1] != NULL);
11449 
11450                                 size = state->dts_ecbs[epid - 1]->dte_size;
11451                         }
11452 
11453                         ASSERT(woffs + size <= buf->dtb_size);
11454                         ASSERT(size != 0);
11455 
11456                         if (woffs + size == buf->dtb_size) {
11457                                 /*
11458                                  * We've reached the end of the buffer; we want
11459                                  * to set the wrapped offset to 0 and break
11460                                  * out.  However, if the offs is 0, then we're
11461                                  * in a strange edge-condition:  the amount of
11462                                  * space that we want to reserve plus the size
11463                                  * of the record that we're overwriting is
11464                                  * greater than the size of the buffer.  This
11465                                  * is problematic because if we reserve the
11466                                  * space but subsequently don't consume it (due
11467                                  * to a failed predicate or error) the wrapped
11468                                  * offset will be 0 -- yet the EPID at offset 0
11469                                  * will not be committed.  This situation is
11470                                  * relatively easy to deal with:  if we're in
11471                                  * this case, the buffer is indistinguishable
11472                                  * from one that hasn't wrapped; we need only
11473                                  * finish the job by clearing the wrapped bit,
11474                                  * explicitly setting the offset to be 0, and
11475                                  * zero'ing out the old data in the buffer.
11476                                  */
11477                                 if (offs == 0) {
11478                                         buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
11479                                         buf->dtb_offset = 0;
11480                                         woffs = total;
11481 
11482                                         while (woffs < buf->dtb_size)
11483                                                 tomax[woffs++] = 0;
11484                                 }
11485 
11486                                 woffs = 0;
11487                                 break;
11488                         }
11489 
11490                         woffs += size;
11491                 }
11492 
11493                 /*
11494                  * We have a wrapped offset.  It may be that the wrapped offset
11495                  * has become zero -- that's okay.
11496                  */
11497                 buf->dtb_xamot_offset = woffs;
11498         }
11499 
11500 out:
11501         /*
11502          * Now we can plow the buffer with any necessary padding.
11503          */
11504         while (offs & (align - 1)) {
11505                 /*
11506                  * Assert that our alignment is off by a number which
11507                  * is itself sizeof (uint32_t) aligned.
11508                  */
11509                 ASSERT(!((align - (offs & (align - 1))) &
11510                     (sizeof (uint32_t) - 1)));
11511                 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11512                 offs += sizeof (uint32_t);
11513         }
11514 
11515         if (buf->dtb_flags & DTRACEBUF_FILL) {
11516                 if (offs + needed > buf->dtb_size - state->dts_reserve) {
11517                         buf->dtb_flags |= DTRACEBUF_FULL;
11518                         return (-1);
11519                 }
11520         }
11521 
11522         if (mstate == NULL)
11523                 return (offs);
11524 
11525         /*
11526          * For ring buffers and fill buffers, the scratch space is always
11527          * the inactive buffer.
11528          */
11529         mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
11530         mstate->dtms_scratch_size = buf->dtb_size;
11531         mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11532 
11533         return (offs);
11534 }
11535 
11536 static void
11537 dtrace_buffer_polish(dtrace_buffer_t *buf)
11538 {
11539         ASSERT(buf->dtb_flags & DTRACEBUF_RING);
11540         ASSERT(MUTEX_HELD(&dtrace_lock));
11541 
11542         if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
11543                 return;
11544 
11545         /*
11546          * We need to polish the ring buffer.  There are three cases:
11547          *
11548          * - The first (and presumably most common) is that there is no gap
11549          *   between the buffer offset and the wrapped offset.  In this case,
11550          *   there is nothing in the buffer that isn't valid data; we can
11551          *   mark the buffer as polished and return.
11552          *
11553          * - The second (less common than the first but still more common
11554          *   than the third) is that there is a gap between the buffer offset
11555          *   and the wrapped offset, and the wrapped offset is larger than the
11556          *   buffer offset.  This can happen because of an alignment issue, or
11557          *   can happen because of a call to dtrace_buffer_reserve() that
11558          *   didn't subsequently consume the buffer space.  In this case,
11559          *   we need to zero the data from the buffer offset to the wrapped
11560          *   offset.
11561          *
11562          * - The third (and least common) is that there is a gap between the
11563          *   buffer offset and the wrapped offset, but the wrapped offset is
11564          *   _less_ than the buffer offset.  This can only happen because a
11565          *   call to dtrace_buffer_reserve() induced a wrap, but the space
11566          *   was not subsequently consumed.  In this case, we need to zero the
11567          *   space from the offset to the end of the buffer _and_ from the
11568          *   top of the buffer to the wrapped offset.
11569          */
11570         if (buf->dtb_offset < buf->dtb_xamot_offset) {
11571                 bzero(buf->dtb_tomax + buf->dtb_offset,
11572                     buf->dtb_xamot_offset - buf->dtb_offset);
11573         }
11574 
11575         if (buf->dtb_offset > buf->dtb_xamot_offset) {
11576                 bzero(buf->dtb_tomax + buf->dtb_offset,
11577                     buf->dtb_size - buf->dtb_offset);
11578                 bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
11579         }
11580 }
11581 
11582 /*
11583  * This routine determines if data generated at the specified time has likely
11584  * been entirely consumed at user-level.  This routine is called to determine
11585  * if an ECB on a defunct probe (but for an active enabling) can be safely
11586  * disabled and destroyed.
11587  */
11588 static int
11589 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
11590 {
11591         int i;
11592 
11593         for (i = 0; i < NCPU; i++) {
11594                 dtrace_buffer_t *buf = &bufs[i];
11595 
11596                 if (buf->dtb_size == 0)
11597                         continue;
11598 
11599                 if (buf->dtb_flags & DTRACEBUF_RING)
11600                         return (0);
11601 
11602                 if (!buf->dtb_switched && buf->dtb_offset != 0)
11603                         return (0);
11604 
11605                 if (buf->dtb_switched - buf->dtb_interval < when)
11606                         return (0);
11607         }
11608 
11609         return (1);
11610 }
11611 
11612 static void
11613 dtrace_buffer_free(dtrace_buffer_t *bufs)
11614 {
11615         int i;
11616 
11617         for (i = 0; i < NCPU; i++) {
11618                 dtrace_buffer_t *buf = &bufs[i];
11619 
11620                 if (buf->dtb_tomax == NULL) {
11621                         ASSERT(buf->dtb_xamot == NULL);
11622                         ASSERT(buf->dtb_size == 0);
11623                         continue;
11624                 }
11625 
11626                 if (buf->dtb_xamot != NULL) {
11627                         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
11628                         kmem_free(buf->dtb_xamot, buf->dtb_size);
11629                 }
11630 
11631                 kmem_free(buf->dtb_tomax, buf->dtb_size);
11632                 buf->dtb_size = 0;
11633                 buf->dtb_tomax = NULL;
11634                 buf->dtb_xamot = NULL;
11635         }
11636 }
11637 
11638 /*
11639  * DTrace Enabling Functions
11640  */
11641 static dtrace_enabling_t *
11642 dtrace_enabling_create(dtrace_vstate_t *vstate)
11643 {
11644         dtrace_enabling_t *enab;
11645 
11646         enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
11647         enab->dten_vstate = vstate;
11648 
11649         return (enab);
11650 }
11651 
11652 static void
11653 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
11654 {
11655         dtrace_ecbdesc_t **ndesc;
11656         size_t osize, nsize;
11657 
11658         /*
11659          * We can't add to enablings after we've enabled them, or after we've
11660          * retained them.
11661          */
11662         ASSERT(enab->dten_probegen == 0);
11663         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11664 
11665         if (enab->dten_ndesc < enab->dten_maxdesc) {
11666                 enab->dten_desc[enab->dten_ndesc++] = ecb;
11667                 return;
11668         }
11669 
11670         osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
11671 
11672         if (enab->dten_maxdesc == 0) {
11673                 enab->dten_maxdesc = 1;
11674         } else {
11675                 enab->dten_maxdesc <<= 1;
11676         }
11677 
11678         ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
11679 
11680         nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
11681         ndesc = kmem_zalloc(nsize, KM_SLEEP);
11682         bcopy(enab->dten_desc, ndesc, osize);
11683         kmem_free(enab->dten_desc, osize);
11684 
11685         enab->dten_desc = ndesc;
11686         enab->dten_desc[enab->dten_ndesc++] = ecb;
11687 }
11688 
11689 static void
11690 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
11691     dtrace_probedesc_t *pd)
11692 {
11693         dtrace_ecbdesc_t *new;
11694         dtrace_predicate_t *pred;
11695         dtrace_actdesc_t *act;
11696 
11697         /*
11698          * We're going to create a new ECB description that matches the
11699          * specified ECB in every way, but has the specified probe description.
11700          */
11701         new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
11702 
11703         if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
11704                 dtrace_predicate_hold(pred);
11705 
11706         for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
11707                 dtrace_actdesc_hold(act);
11708 
11709         new->dted_action = ecb->dted_action;
11710         new->dted_pred = ecb->dted_pred;
11711         new->dted_probe = *pd;
11712         new->dted_uarg = ecb->dted_uarg;
11713 
11714         dtrace_enabling_add(enab, new);
11715 }
11716 
11717 static void
11718 dtrace_enabling_dump(dtrace_enabling_t *enab)
11719 {
11720         int i;
11721 
11722         for (i = 0; i < enab->dten_ndesc; i++) {
11723                 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
11724 
11725                 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
11726                     desc->dtpd_provider, desc->dtpd_mod,
11727                     desc->dtpd_func, desc->dtpd_name);
11728         }
11729 }
11730 
11731 static void
11732 dtrace_enabling_destroy(dtrace_enabling_t *enab)
11733 {
11734         int i;
11735         dtrace_ecbdesc_t *ep;
11736         dtrace_vstate_t *vstate = enab->dten_vstate;
11737 
11738         ASSERT(MUTEX_HELD(&dtrace_lock));
11739 
11740         for (i = 0; i < enab->dten_ndesc; i++) {
11741                 dtrace_actdesc_t *act, *next;
11742                 dtrace_predicate_t *pred;
11743 
11744                 ep = enab->dten_desc[i];
11745 
11746                 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
11747                         dtrace_predicate_release(pred, vstate);
11748 
11749                 for (act = ep->dted_action; act != NULL; act = next) {
11750                         next = act->dtad_next;
11751                         dtrace_actdesc_release(act, vstate);
11752                 }
11753 
11754                 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
11755         }
11756 
11757         kmem_free(enab->dten_desc,
11758             enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
11759 
11760         /*
11761          * If this was a retained enabling, decrement the dts_nretained count
11762          * and take it off of the dtrace_retained list.
11763          */
11764         if (enab->dten_prev != NULL || enab->dten_next != NULL ||
11765             dtrace_retained == enab) {
11766                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11767                 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
11768                 enab->dten_vstate->dtvs_state->dts_nretained--;
11769                 dtrace_retained_gen++;
11770         }
11771 
11772         if (enab->dten_prev == NULL) {
11773                 if (dtrace_retained == enab) {
11774                         dtrace_retained = enab->dten_next;
11775 
11776                         if (dtrace_retained != NULL)
11777                                 dtrace_retained->dten_prev = NULL;
11778                 }
11779         } else {
11780                 ASSERT(enab != dtrace_retained);
11781                 ASSERT(dtrace_retained != NULL);
11782                 enab->dten_prev->dten_next = enab->dten_next;
11783         }
11784 
11785         if (enab->dten_next != NULL) {
11786                 ASSERT(dtrace_retained != NULL);
11787                 enab->dten_next->dten_prev = enab->dten_prev;
11788         }
11789 
11790         kmem_free(enab, sizeof (dtrace_enabling_t));
11791 }
11792 
11793 static int
11794 dtrace_enabling_retain(dtrace_enabling_t *enab)
11795 {
11796         dtrace_state_t *state;
11797 
11798         ASSERT(MUTEX_HELD(&dtrace_lock));
11799         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11800         ASSERT(enab->dten_vstate != NULL);
11801 
11802         state = enab->dten_vstate->dtvs_state;
11803         ASSERT(state != NULL);
11804 
11805         /*
11806          * We only allow each state to retain dtrace_retain_max enablings.
11807          */
11808         if (state->dts_nretained >= dtrace_retain_max)
11809                 return (ENOSPC);
11810 
11811         state->dts_nretained++;
11812         dtrace_retained_gen++;
11813 
11814         if (dtrace_retained == NULL) {
11815                 dtrace_retained = enab;
11816                 return (0);
11817         }
11818 
11819         enab->dten_next = dtrace_retained;
11820         dtrace_retained->dten_prev = enab;
11821         dtrace_retained = enab;
11822 
11823         return (0);
11824 }
11825 
11826 static int
11827 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
11828     dtrace_probedesc_t *create)
11829 {
11830         dtrace_enabling_t *new, *enab;
11831         int found = 0, err = ENOENT;
11832 
11833         ASSERT(MUTEX_HELD(&dtrace_lock));
11834         ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
11835         ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
11836         ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
11837         ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
11838 
11839         new = dtrace_enabling_create(&state->dts_vstate);
11840 
11841         /*
11842          * Iterate over all retained enablings, looking for enablings that
11843          * match the specified state.
11844          */
11845         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11846                 int i;
11847 
11848                 /*
11849                  * dtvs_state can only be NULL for helper enablings -- and
11850                  * helper enablings can't be retained.
11851                  */
11852                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11853 
11854                 if (enab->dten_vstate->dtvs_state != state)
11855                         continue;
11856 
11857                 /*
11858                  * Now iterate over each probe description; we're looking for
11859                  * an exact match to the specified probe description.
11860                  */
11861                 for (i = 0; i < enab->dten_ndesc; i++) {
11862                         dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11863                         dtrace_probedesc_t *pd = &ep->dted_probe;
11864 
11865                         if (strcmp(pd->dtpd_provider, match->dtpd_provider))
11866                                 continue;
11867 
11868                         if (strcmp(pd->dtpd_mod, match->dtpd_mod))
11869                                 continue;
11870 
11871                         if (strcmp(pd->dtpd_func, match->dtpd_func))
11872                                 continue;
11873 
11874                         if (strcmp(pd->dtpd_name, match->dtpd_name))
11875                                 continue;
11876 
11877                         /*
11878                          * We have a winning probe!  Add it to our growing
11879                          * enabling.
11880                          */
11881                         found = 1;
11882                         dtrace_enabling_addlike(new, ep, create);
11883                 }
11884         }
11885 
11886         if (!found || (err = dtrace_enabling_retain(new)) != 0) {
11887                 dtrace_enabling_destroy(new);
11888                 return (err);
11889         }
11890 
11891         return (0);
11892 }
11893 
11894 static void
11895 dtrace_enabling_retract(dtrace_state_t *state)
11896 {
11897         dtrace_enabling_t *enab, *next;
11898 
11899         ASSERT(MUTEX_HELD(&dtrace_lock));
11900 
11901         /*
11902          * Iterate over all retained enablings, destroy the enablings retained
11903          * for the specified state.
11904          */
11905         for (enab = dtrace_retained; enab != NULL; enab = next) {
11906                 next = enab->dten_next;
11907 
11908                 /*
11909                  * dtvs_state can only be NULL for helper enablings -- and
11910                  * helper enablings can't be retained.
11911                  */
11912                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11913 
11914                 if (enab->dten_vstate->dtvs_state == state) {
11915                         ASSERT(state->dts_nretained > 0);
11916                         dtrace_enabling_destroy(enab);
11917                 }
11918         }
11919 
11920         ASSERT(state->dts_nretained == 0);
11921 }
11922 
11923 static int
11924 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
11925 {
11926         int i = 0;
11927         int total_matched = 0, matched = 0;
11928 
11929         ASSERT(MUTEX_HELD(&cpu_lock));
11930         ASSERT(MUTEX_HELD(&dtrace_lock));
11931 
11932         for (i = 0; i < enab->dten_ndesc; i++) {
11933                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11934 
11935                 enab->dten_current = ep;
11936                 enab->dten_error = 0;
11937 
11938                 /*
11939                  * If a provider failed to enable a probe then get out and
11940                  * let the consumer know we failed.
11941                  */
11942                 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
11943                         return (EBUSY);
11944 
11945                 total_matched += matched;
11946 
11947                 if (enab->dten_error != 0) {
11948                         /*
11949                          * If we get an error half-way through enabling the
11950                          * probes, we kick out -- perhaps with some number of
11951                          * them enabled.  Leaving enabled probes enabled may
11952                          * be slightly confusing for user-level, but we expect
11953                          * that no one will attempt to actually drive on in
11954                          * the face of such errors.  If this is an anonymous
11955                          * enabling (indicated with a NULL nmatched pointer),
11956                          * we cmn_err() a message.  We aren't expecting to
11957                          * get such an error -- such as it can exist at all,
11958                          * it would be a result of corrupted DOF in the driver
11959                          * properties.
11960                          */
11961                         if (nmatched == NULL) {
11962                                 cmn_err(CE_WARN, "dtrace_enabling_match() "
11963                                     "error on %p: %d", (void *)ep,
11964                                     enab->dten_error);
11965                         }
11966 
11967                         return (enab->dten_error);
11968                 }
11969         }
11970 
11971         enab->dten_probegen = dtrace_probegen;
11972         if (nmatched != NULL)
11973                 *nmatched = total_matched;
11974 
11975         return (0);
11976 }
11977 
11978 static void
11979 dtrace_enabling_matchall(void)
11980 {
11981         dtrace_enabling_t *enab;
11982 
11983         mutex_enter(&cpu_lock);
11984         mutex_enter(&dtrace_lock);
11985 
11986         /*
11987          * Iterate over all retained enablings to see if any probes match
11988          * against them.  We only perform this operation on enablings for which
11989          * we have sufficient permissions by virtue of being in the global zone
11990          * or in the same zone as the DTrace client.  Because we can be called
11991          * after dtrace_detach() has been called, we cannot assert that there
11992          * are retained enablings.  We can safely load from dtrace_retained,
11993          * however:  the taskq_destroy() at the end of dtrace_detach() will
11994          * block pending our completion.
11995          */
11996         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11997                 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred;
11998                 cred_t *cr = dcr->dcr_cred;
11999                 zoneid_t zone = cr != NULL ? crgetzoneid(cr) : 0;
12000 
12001                 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL &&
12002                     (zone == GLOBAL_ZONEID || getzoneid() == zone)))
12003                         (void) dtrace_enabling_match(enab, NULL);
12004         }
12005 
12006         mutex_exit(&dtrace_lock);
12007         mutex_exit(&cpu_lock);
12008 }
12009 
12010 /*
12011  * If an enabling is to be enabled without having matched probes (that is, if
12012  * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
12013  * enabling must be _primed_ by creating an ECB for every ECB description.
12014  * This must be done to assure that we know the number of speculations, the
12015  * number of aggregations, the minimum buffer size needed, etc. before we
12016  * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
12017  * enabling any probes, we create ECBs for every ECB decription, but with a
12018  * NULL probe -- which is exactly what this function does.
12019  */
12020 static void
12021 dtrace_enabling_prime(dtrace_state_t *state)
12022 {
12023         dtrace_enabling_t *enab;
12024         int i;
12025 
12026         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
12027                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
12028 
12029                 if (enab->dten_vstate->dtvs_state != state)
12030                         continue;
12031 
12032                 /*
12033                  * We don't want to prime an enabling more than once, lest
12034                  * we allow a malicious user to induce resource exhaustion.
12035                  * (The ECBs that result from priming an enabling aren't
12036                  * leaked -- but they also aren't deallocated until the
12037                  * consumer state is destroyed.)
12038                  */
12039                 if (enab->dten_primed)
12040                         continue;
12041 
12042                 for (i = 0; i < enab->dten_ndesc; i++) {
12043                         enab->dten_current = enab->dten_desc[i];
12044                         (void) dtrace_probe_enable(NULL, enab);
12045                 }
12046 
12047                 enab->dten_primed = 1;
12048         }
12049 }
12050 
12051 /*
12052  * Called to indicate that probes should be provided due to retained
12053  * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
12054  * must take an initial lap through the enabling calling the dtps_provide()
12055  * entry point explicitly to allow for autocreated probes.
12056  */
12057 static void
12058 dtrace_enabling_provide(dtrace_provider_t *prv)
12059 {
12060         int i, all = 0;
12061         dtrace_probedesc_t desc;
12062         dtrace_genid_t gen;
12063 
12064         ASSERT(MUTEX_HELD(&dtrace_lock));
12065         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
12066 
12067         if (prv == NULL) {
12068                 all = 1;
12069                 prv = dtrace_provider;
12070         }
12071 
12072         do {
12073                 dtrace_enabling_t *enab;
12074                 void *parg = prv->dtpv_arg;
12075 
12076 retry:
12077                 gen = dtrace_retained_gen;
12078                 for (enab = dtrace_retained; enab != NULL;
12079                     enab = enab->dten_next) {
12080                         for (i = 0; i < enab->dten_ndesc; i++) {
12081                                 desc = enab->dten_desc[i]->dted_probe;
12082                                 mutex_exit(&dtrace_lock);
12083                                 prv->dtpv_pops.dtps_provide(parg, &desc);
12084                                 mutex_enter(&dtrace_lock);
12085                                 /*
12086                                  * Process the retained enablings again if
12087                                  * they have changed while we weren't holding
12088                                  * dtrace_lock.
12089                                  */
12090                                 if (gen != dtrace_retained_gen)
12091                                         goto retry;
12092                         }
12093                 }
12094         } while (all && (prv = prv->dtpv_next) != NULL);
12095 
12096         mutex_exit(&dtrace_lock);
12097         dtrace_probe_provide(NULL, all ? NULL : prv);
12098         mutex_enter(&dtrace_lock);
12099 }
12100 
12101 /*
12102  * Called to reap ECBs that are attached to probes from defunct providers.
12103  */
12104 static void
12105 dtrace_enabling_reap(void)
12106 {
12107         dtrace_provider_t *prov;
12108         dtrace_probe_t *probe;
12109         dtrace_ecb_t *ecb;
12110         hrtime_t when;
12111         int i;
12112 
12113         mutex_enter(&cpu_lock);
12114         mutex_enter(&dtrace_lock);
12115 
12116         for (i = 0; i < dtrace_nprobes; i++) {
12117                 if ((probe = dtrace_probes[i]) == NULL)
12118                         continue;
12119 
12120                 if (probe->dtpr_ecb == NULL)
12121                         continue;
12122 
12123                 prov = probe->dtpr_provider;
12124 
12125                 if ((when = prov->dtpv_defunct) == 0)
12126                         continue;
12127 
12128                 /*
12129                  * We have ECBs on a defunct provider:  we want to reap these
12130                  * ECBs to allow the provider to unregister.  The destruction
12131                  * of these ECBs must be done carefully:  if we destroy the ECB
12132                  * and the consumer later wishes to consume an EPID that
12133                  * corresponds to the destroyed ECB (and if the EPID metadata
12134                  * has not been previously consumed), the consumer will abort
12135                  * processing on the unknown EPID.  To reduce (but not, sadly,
12136                  * eliminate) the possibility of this, we will only destroy an
12137                  * ECB for a defunct provider if, for the state that
12138                  * corresponds to the ECB:
12139                  *
12140                  *  (a) There is no speculative tracing (which can effectively
12141                  *      cache an EPID for an arbitrary amount of time).
12142                  *
12143                  *  (b) The principal buffers have been switched twice since the
12144                  *      provider became defunct.
12145                  *
12146                  *  (c) The aggregation buffers are of zero size or have been
12147                  *      switched twice since the provider became defunct.
12148                  *
12149                  * We use dts_speculates to determine (a) and call a function
12150                  * (dtrace_buffer_consumed()) to determine (b) and (c).  Note
12151                  * that as soon as we've been unable to destroy one of the ECBs
12152                  * associated with the probe, we quit trying -- reaping is only
12153                  * fruitful in as much as we can destroy all ECBs associated
12154                  * with the defunct provider's probes.
12155                  */
12156                 while ((ecb = probe->dtpr_ecb) != NULL) {
12157                         dtrace_state_t *state = ecb->dte_state;
12158                         dtrace_buffer_t *buf = state->dts_buffer;
12159                         dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
12160 
12161                         if (state->dts_speculates)
12162                                 break;
12163 
12164                         if (!dtrace_buffer_consumed(buf, when))
12165                                 break;
12166 
12167                         if (!dtrace_buffer_consumed(aggbuf, when))
12168                                 break;
12169 
12170                         dtrace_ecb_disable(ecb);
12171                         ASSERT(probe->dtpr_ecb != ecb);
12172                         dtrace_ecb_destroy(ecb);
12173                 }
12174         }
12175 
12176         mutex_exit(&dtrace_lock);
12177         mutex_exit(&cpu_lock);
12178 }
12179 
12180 /*
12181  * DTrace DOF Functions
12182  */
12183 /*ARGSUSED*/
12184 static void
12185 dtrace_dof_error(dof_hdr_t *dof, const char *str)
12186 {
12187         if (dtrace_err_verbose)
12188                 cmn_err(CE_WARN, "failed to process DOF: %s", str);
12189 
12190 #ifdef DTRACE_ERRDEBUG
12191         dtrace_errdebug(str);
12192 #endif
12193 }
12194 
12195 /*
12196  * Create DOF out of a currently enabled state.  Right now, we only create
12197  * DOF containing the run-time options -- but this could be expanded to create
12198  * complete DOF representing the enabled state.
12199  */
12200 static dof_hdr_t *
12201 dtrace_dof_create(dtrace_state_t *state)
12202 {
12203         dof_hdr_t *dof;
12204         dof_sec_t *sec;
12205         dof_optdesc_t *opt;
12206         int i, len = sizeof (dof_hdr_t) +
12207             roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
12208             sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12209 
12210         ASSERT(MUTEX_HELD(&dtrace_lock));
12211 
12212         dof = kmem_zalloc(len, KM_SLEEP);
12213         dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
12214         dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
12215         dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
12216         dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
12217 
12218         dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
12219         dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
12220         dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
12221         dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
12222         dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
12223         dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
12224 
12225         dof->dofh_flags = 0;
12226         dof->dofh_hdrsize = sizeof (dof_hdr_t);
12227         dof->dofh_secsize = sizeof (dof_sec_t);
12228         dof->dofh_secnum = 1;        /* only DOF_SECT_OPTDESC */
12229         dof->dofh_secoff = sizeof (dof_hdr_t);
12230         dof->dofh_loadsz = len;
12231         dof->dofh_filesz = len;
12232         dof->dofh_pad = 0;
12233 
12234         /*
12235          * Fill in the option section header...
12236          */
12237         sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
12238         sec->dofs_type = DOF_SECT_OPTDESC;
12239         sec->dofs_align = sizeof (uint64_t);
12240         sec->dofs_flags = DOF_SECF_LOAD;
12241         sec->dofs_entsize = sizeof (dof_optdesc_t);
12242 
12243         opt = (dof_optdesc_t *)((uintptr_t)sec +
12244             roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
12245 
12246         sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
12247         sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12248 
12249         for (i = 0; i < DTRACEOPT_MAX; i++) {
12250                 opt[i].dofo_option = i;
12251                 opt[i].dofo_strtab = DOF_SECIDX_NONE;
12252                 opt[i].dofo_value = state->dts_options[i];
12253         }
12254 
12255         return (dof);
12256 }
12257 
12258 static dof_hdr_t *
12259 dtrace_dof_copyin(uintptr_t uarg, int *errp)
12260 {
12261         dof_hdr_t hdr, *dof;
12262 
12263         ASSERT(!MUTEX_HELD(&dtrace_lock));
12264 
12265         /*
12266          * First, we're going to copyin() the sizeof (dof_hdr_t).
12267          */
12268         if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
12269                 dtrace_dof_error(NULL, "failed to copyin DOF header");
12270                 *errp = EFAULT;
12271                 return (NULL);
12272         }
12273 
12274         /*
12275          * Now we'll allocate the entire DOF and copy it in -- provided
12276          * that the length isn't outrageous.
12277          */
12278         if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
12279                 dtrace_dof_error(&hdr, "load size exceeds maximum");
12280                 *errp = E2BIG;
12281                 return (NULL);
12282         }
12283 
12284         if (hdr.dofh_loadsz < sizeof (hdr)) {
12285                 dtrace_dof_error(&hdr, "invalid load size");
12286                 *errp = EINVAL;
12287                 return (NULL);
12288         }
12289 
12290         dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
12291 
12292         if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
12293             dof->dofh_loadsz != hdr.dofh_loadsz) {
12294                 kmem_free(dof, hdr.dofh_loadsz);
12295                 *errp = EFAULT;
12296                 return (NULL);
12297         }
12298 
12299         return (dof);
12300 }
12301 
12302 static dof_hdr_t *
12303 dtrace_dof_property(const char *name)
12304 {
12305         uchar_t *buf;
12306         uint64_t loadsz;
12307         unsigned int len, i;
12308         dof_hdr_t *dof;
12309 
12310         /*
12311          * Unfortunately, array of values in .conf files are always (and
12312          * only) interpreted to be integer arrays.  We must read our DOF
12313          * as an integer array, and then squeeze it into a byte array.
12314          */
12315         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
12316             (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
12317                 return (NULL);
12318 
12319         for (i = 0; i < len; i++)
12320                 buf[i] = (uchar_t)(((int *)buf)[i]);
12321 
12322         if (len < sizeof (dof_hdr_t)) {
12323                 ddi_prop_free(buf);
12324                 dtrace_dof_error(NULL, "truncated header");
12325                 return (NULL);
12326         }
12327 
12328         if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
12329                 ddi_prop_free(buf);
12330                 dtrace_dof_error(NULL, "truncated DOF");
12331                 return (NULL);
12332         }
12333 
12334         if (loadsz >= dtrace_dof_maxsize) {
12335                 ddi_prop_free(buf);
12336                 dtrace_dof_error(NULL, "oversized DOF");
12337                 return (NULL);
12338         }
12339 
12340         dof = kmem_alloc(loadsz, KM_SLEEP);
12341         bcopy(buf, dof, loadsz);
12342         ddi_prop_free(buf);
12343 
12344         return (dof);
12345 }
12346 
12347 static void
12348 dtrace_dof_destroy(dof_hdr_t *dof)
12349 {
12350         kmem_free(dof, dof->dofh_loadsz);
12351 }
12352 
12353 /*
12354  * Return the dof_sec_t pointer corresponding to a given section index.  If the
12355  * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
12356  * a type other than DOF_SECT_NONE is specified, the header is checked against
12357  * this type and NULL is returned if the types do not match.
12358  */
12359 static dof_sec_t *
12360 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
12361 {
12362         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
12363             ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
12364 
12365         if (i >= dof->dofh_secnum) {
12366                 dtrace_dof_error(dof, "referenced section index is invalid");
12367                 return (NULL);
12368         }
12369 
12370         if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
12371                 dtrace_dof_error(dof, "referenced section is not loadable");
12372                 return (NULL);
12373         }
12374 
12375         if (type != DOF_SECT_NONE && type != sec->dofs_type) {
12376                 dtrace_dof_error(dof, "referenced section is the wrong type");
12377                 return (NULL);
12378         }
12379 
12380         return (sec);
12381 }
12382 
12383 static dtrace_probedesc_t *
12384 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
12385 {
12386         dof_probedesc_t *probe;
12387         dof_sec_t *strtab;
12388         uintptr_t daddr = (uintptr_t)dof;
12389         uintptr_t str;
12390         size_t size;
12391 
12392         if (sec->dofs_type != DOF_SECT_PROBEDESC) {
12393                 dtrace_dof_error(dof, "invalid probe section");
12394                 return (NULL);
12395         }
12396 
12397         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12398                 dtrace_dof_error(dof, "bad alignment in probe description");
12399                 return (NULL);
12400         }
12401 
12402         if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
12403                 dtrace_dof_error(dof, "truncated probe description");
12404                 return (NULL);
12405         }
12406 
12407         probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
12408         strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
12409 
12410         if (strtab == NULL)
12411                 return (NULL);
12412 
12413         str = daddr + strtab->dofs_offset;
12414         size = strtab->dofs_size;
12415 
12416         if (probe->dofp_provider >= strtab->dofs_size) {
12417                 dtrace_dof_error(dof, "corrupt probe provider");
12418                 return (NULL);
12419         }
12420 
12421         (void) strncpy(desc->dtpd_provider,
12422             (char *)(str + probe->dofp_provider),
12423             MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
12424 
12425         if (probe->dofp_mod >= strtab->dofs_size) {
12426                 dtrace_dof_error(dof, "corrupt probe module");
12427                 return (NULL);
12428         }
12429 
12430         (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
12431             MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
12432 
12433         if (probe->dofp_func >= strtab->dofs_size) {
12434                 dtrace_dof_error(dof, "corrupt probe function");
12435                 return (NULL);
12436         }
12437 
12438         (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
12439             MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
12440 
12441         if (probe->dofp_name >= strtab->dofs_size) {
12442                 dtrace_dof_error(dof, "corrupt probe name");
12443                 return (NULL);
12444         }
12445 
12446         (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
12447             MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
12448 
12449         return (desc);
12450 }
12451 
12452 static dtrace_difo_t *
12453 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12454     cred_t *cr)
12455 {
12456         dtrace_difo_t *dp;
12457         size_t ttl = 0;
12458         dof_difohdr_t *dofd;
12459         uintptr_t daddr = (uintptr_t)dof;
12460         size_t max = dtrace_difo_maxsize;
12461         int i, l, n;
12462 
12463         static const struct {
12464                 int section;
12465                 int bufoffs;
12466                 int lenoffs;
12467                 int entsize;
12468                 int align;
12469                 const char *msg;
12470         } difo[] = {
12471                 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
12472                 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
12473                 sizeof (dif_instr_t), "multiple DIF sections" },
12474 
12475                 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
12476                 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
12477                 sizeof (uint64_t), "multiple integer tables" },
12478 
12479                 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
12480                 offsetof(dtrace_difo_t, dtdo_strlen), 0,
12481                 sizeof (char), "multiple string tables" },
12482 
12483                 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
12484                 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
12485                 sizeof (uint_t), "multiple variable tables" },
12486 
12487                 { DOF_SECT_NONE, 0, 0, 0, NULL }
12488         };
12489 
12490         if (sec->dofs_type != DOF_SECT_DIFOHDR) {
12491                 dtrace_dof_error(dof, "invalid DIFO header section");
12492                 return (NULL);
12493         }
12494 
12495         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12496                 dtrace_dof_error(dof, "bad alignment in DIFO header");
12497                 return (NULL);
12498         }
12499 
12500         if (sec->dofs_size < sizeof (dof_difohdr_t) ||
12501             sec->dofs_size % sizeof (dof_secidx_t)) {
12502                 dtrace_dof_error(dof, "bad size in DIFO header");
12503                 return (NULL);
12504         }
12505 
12506         dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12507         n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
12508 
12509         dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
12510         dp->dtdo_rtype = dofd->dofd_rtype;
12511 
12512         for (l = 0; l < n; l++) {
12513                 dof_sec_t *subsec;
12514                 void **bufp;
12515                 uint32_t *lenp;
12516 
12517                 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
12518                     dofd->dofd_links[l])) == NULL)
12519                         goto err; /* invalid section link */
12520 
12521                 if (ttl + subsec->dofs_size > max) {
12522                         dtrace_dof_error(dof, "exceeds maximum size");
12523                         goto err;
12524                 }
12525 
12526                 ttl += subsec->dofs_size;
12527 
12528                 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
12529                         if (subsec->dofs_type != difo[i].section)
12530                                 continue;
12531 
12532                         if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
12533                                 dtrace_dof_error(dof, "section not loaded");
12534                                 goto err;
12535                         }
12536 
12537                         if (subsec->dofs_align != difo[i].align) {
12538                                 dtrace_dof_error(dof, "bad alignment");
12539                                 goto err;
12540                         }
12541 
12542                         bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
12543                         lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
12544 
12545                         if (*bufp != NULL) {
12546                                 dtrace_dof_error(dof, difo[i].msg);
12547                                 goto err;
12548                         }
12549 
12550                         if (difo[i].entsize != subsec->dofs_entsize) {
12551                                 dtrace_dof_error(dof, "entry size mismatch");
12552                                 goto err;
12553                         }
12554 
12555                         if (subsec->dofs_entsize != 0 &&
12556                             (subsec->dofs_size % subsec->dofs_entsize) != 0) {
12557                                 dtrace_dof_error(dof, "corrupt entry size");
12558                                 goto err;
12559                         }
12560 
12561                         *lenp = subsec->dofs_size;
12562                         *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
12563                         bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
12564                             *bufp, subsec->dofs_size);
12565 
12566                         if (subsec->dofs_entsize != 0)
12567                                 *lenp /= subsec->dofs_entsize;
12568 
12569                         break;
12570                 }
12571 
12572                 /*
12573                  * If we encounter a loadable DIFO sub-section that is not
12574                  * known to us, assume this is a broken program and fail.
12575                  */
12576                 if (difo[i].section == DOF_SECT_NONE &&
12577                     (subsec->dofs_flags & DOF_SECF_LOAD)) {
12578                         dtrace_dof_error(dof, "unrecognized DIFO subsection");
12579                         goto err;
12580                 }
12581         }
12582 
12583         if (dp->dtdo_buf == NULL) {
12584                 /*
12585                  * We can't have a DIF object without DIF text.
12586                  */
12587                 dtrace_dof_error(dof, "missing DIF text");
12588                 goto err;
12589         }
12590 
12591         /*
12592          * Before we validate the DIF object, run through the variable table
12593          * looking for the strings -- if any of their size are under, we'll set
12594          * their size to be the system-wide default string size.  Note that
12595          * this should _not_ happen if the "strsize" option has been set --
12596          * in this case, the compiler should have set the size to reflect the
12597          * setting of the option.
12598          */
12599         for (i = 0; i < dp->dtdo_varlen; i++) {
12600                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
12601                 dtrace_diftype_t *t = &v->dtdv_type;
12602 
12603                 if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
12604                         continue;
12605 
12606                 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
12607                         t->dtdt_size = dtrace_strsize_default;
12608         }
12609 
12610         if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
12611                 goto err;
12612 
12613         dtrace_difo_init(dp, vstate);
12614         return (dp);
12615 
12616 err:
12617         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
12618         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
12619         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
12620         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
12621 
12622         kmem_free(dp, sizeof (dtrace_difo_t));
12623         return (NULL);
12624 }
12625 
12626 static dtrace_predicate_t *
12627 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12628     cred_t *cr)
12629 {
12630         dtrace_difo_t *dp;
12631 
12632         if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
12633                 return (NULL);
12634 
12635         return (dtrace_predicate_create(dp));
12636 }
12637 
12638 static dtrace_actdesc_t *
12639 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12640     cred_t *cr)
12641 {
12642         dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
12643         dof_actdesc_t *desc;
12644         dof_sec_t *difosec;
12645         size_t offs;
12646         uintptr_t daddr = (uintptr_t)dof;
12647         uint64_t arg;
12648         dtrace_actkind_t kind;
12649 
12650         if (sec->dofs_type != DOF_SECT_ACTDESC) {
12651                 dtrace_dof_error(dof, "invalid action section");
12652                 return (NULL);
12653         }
12654 
12655         if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
12656                 dtrace_dof_error(dof, "truncated action description");
12657                 return (NULL);
12658         }
12659 
12660         if (sec->dofs_align != sizeof (uint64_t)) {
12661                 dtrace_dof_error(dof, "bad alignment in action description");
12662                 return (NULL);
12663         }
12664 
12665         if (sec->dofs_size < sec->dofs_entsize) {
12666                 dtrace_dof_error(dof, "section entry size exceeds total size");
12667                 return (NULL);
12668         }
12669 
12670         if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
12671                 dtrace_dof_error(dof, "bad entry size in action description");
12672                 return (NULL);
12673         }
12674 
12675         if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
12676                 dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
12677                 return (NULL);
12678         }
12679 
12680         for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
12681                 desc = (dof_actdesc_t *)(daddr +
12682                     (uintptr_t)sec->dofs_offset + offs);
12683                 kind = (dtrace_actkind_t)desc->dofa_kind;
12684 
12685                 if ((DTRACEACT_ISPRINTFLIKE(kind) &&
12686                     (kind != DTRACEACT_PRINTA ||
12687                     desc->dofa_strtab != DOF_SECIDX_NONE)) ||
12688                     (kind == DTRACEACT_DIFEXPR &&
12689                     desc->dofa_strtab != DOF_SECIDX_NONE)) {
12690                         dof_sec_t *strtab;
12691                         char *str, *fmt;
12692                         uint64_t i;
12693 
12694                         /*
12695                          * The argument to these actions is an index into the
12696                          * DOF string table.  For printf()-like actions, this
12697                          * is the format string.  For print(), this is the
12698                          * CTF type of the expression result.
12699                          */
12700                         if ((strtab = dtrace_dof_sect(dof,
12701                             DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
12702                                 goto err;
12703 
12704                         str = (char *)((uintptr_t)dof +
12705                             (uintptr_t)strtab->dofs_offset);
12706 
12707                         for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
12708                                 if (str[i] == '\0')
12709                                         break;
12710                         }
12711 
12712                         if (i >= strtab->dofs_size) {
12713                                 dtrace_dof_error(dof, "bogus format string");
12714                                 goto err;
12715                         }
12716 
12717                         if (i == desc->dofa_arg) {
12718                                 dtrace_dof_error(dof, "empty format string");
12719                                 goto err;
12720                         }
12721 
12722                         i -= desc->dofa_arg;
12723                         fmt = kmem_alloc(i + 1, KM_SLEEP);
12724                         bcopy(&str[desc->dofa_arg], fmt, i + 1);
12725                         arg = (uint64_t)(uintptr_t)fmt;
12726                 } else {
12727                         if (kind == DTRACEACT_PRINTA) {
12728                                 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
12729                                 arg = 0;
12730                         } else {
12731                                 arg = desc->dofa_arg;
12732                         }
12733                 }
12734 
12735                 act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
12736                     desc->dofa_uarg, arg);
12737 
12738                 if (last != NULL) {
12739                         last->dtad_next = act;
12740                 } else {
12741                         first = act;
12742                 }
12743 
12744                 last = act;
12745 
12746                 if (desc->dofa_difo == DOF_SECIDX_NONE)
12747                         continue;
12748 
12749                 if ((difosec = dtrace_dof_sect(dof,
12750                     DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
12751                         goto err;
12752 
12753                 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
12754 
12755                 if (act->dtad_difo == NULL)
12756                         goto err;
12757         }
12758 
12759         ASSERT(first != NULL);
12760         return (first);
12761 
12762 err:
12763         for (act = first; act != NULL; act = next) {
12764                 next = act->dtad_next;
12765                 dtrace_actdesc_release(act, vstate);
12766         }
12767 
12768         return (NULL);
12769 }
12770 
12771 static dtrace_ecbdesc_t *
12772 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12773     cred_t *cr)
12774 {
12775         dtrace_ecbdesc_t *ep;
12776         dof_ecbdesc_t *ecb;
12777         dtrace_probedesc_t *desc;
12778         dtrace_predicate_t *pred = NULL;
12779 
12780         if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
12781                 dtrace_dof_error(dof, "truncated ECB description");
12782                 return (NULL);
12783         }
12784 
12785         if (sec->dofs_align != sizeof (uint64_t)) {
12786                 dtrace_dof_error(dof, "bad alignment in ECB description");
12787                 return (NULL);
12788         }
12789 
12790         ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
12791         sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
12792 
12793         if (sec == NULL)
12794                 return (NULL);
12795 
12796         ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
12797         ep->dted_uarg = ecb->dofe_uarg;
12798         desc = &ep->dted_probe;
12799 
12800         if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
12801                 goto err;
12802 
12803         if (ecb->dofe_pred != DOF_SECIDX_NONE) {
12804                 if ((sec = dtrace_dof_sect(dof,
12805                     DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
12806                         goto err;
12807 
12808                 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
12809                         goto err;
12810 
12811                 ep->dted_pred.dtpdd_predicate = pred;
12812         }
12813 
12814         if (ecb->dofe_actions != DOF_SECIDX_NONE) {
12815                 if ((sec = dtrace_dof_sect(dof,
12816                     DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
12817                         goto err;
12818 
12819                 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
12820 
12821                 if (ep->dted_action == NULL)
12822                         goto err;
12823         }
12824 
12825         return (ep);
12826 
12827 err:
12828         if (pred != NULL)
12829                 dtrace_predicate_release(pred, vstate);
12830         kmem_free(ep, sizeof (dtrace_ecbdesc_t));
12831         return (NULL);
12832 }
12833 
12834 /*
12835  * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
12836  * specified DOF.  At present, this amounts to simply adding 'ubase' to the
12837  * site of any user SETX relocations to account for load object base address.
12838  * In the future, if we need other relocations, this function can be extended.
12839  */
12840 static int
12841 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
12842 {
12843         uintptr_t daddr = (uintptr_t)dof;
12844         dof_relohdr_t *dofr =
12845             (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12846         dof_sec_t *ss, *rs, *ts;
12847         dof_relodesc_t *r;
12848         uint_t i, n;
12849 
12850         if (sec->dofs_size < sizeof (dof_relohdr_t) ||
12851             sec->dofs_align != sizeof (dof_secidx_t)) {
12852                 dtrace_dof_error(dof, "invalid relocation header");
12853                 return (-1);
12854         }
12855 
12856         ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
12857         rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
12858         ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
12859 
12860         if (ss == NULL || rs == NULL || ts == NULL)
12861                 return (-1); /* dtrace_dof_error() has been called already */
12862 
12863         if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
12864             rs->dofs_align != sizeof (uint64_t)) {
12865                 dtrace_dof_error(dof, "invalid relocation section");
12866                 return (-1);
12867         }
12868 
12869         r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
12870         n = rs->dofs_size / rs->dofs_entsize;
12871 
12872         for (i = 0; i < n; i++) {
12873                 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
12874 
12875                 switch (r->dofr_type) {
12876                 case DOF_RELO_NONE:
12877                         break;
12878                 case DOF_RELO_SETX:
12879                         if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
12880                             sizeof (uint64_t) > ts->dofs_size) {
12881                                 dtrace_dof_error(dof, "bad relocation offset");
12882                                 return (-1);
12883                         }
12884 
12885                         if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
12886                                 dtrace_dof_error(dof, "misaligned setx relo");
12887                                 return (-1);
12888                         }
12889 
12890                         *(uint64_t *)taddr += ubase;
12891                         break;
12892                 default:
12893                         dtrace_dof_error(dof, "invalid relocation type");
12894                         return (-1);
12895                 }
12896 
12897                 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
12898         }
12899 
12900         return (0);
12901 }
12902 
12903 /*
12904  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
12905  * header:  it should be at the front of a memory region that is at least
12906  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
12907  * size.  It need not be validated in any other way.
12908  */
12909 static int
12910 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
12911     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
12912 {
12913         uint64_t len = dof->dofh_loadsz, seclen;
12914         uintptr_t daddr = (uintptr_t)dof;
12915         dtrace_ecbdesc_t *ep;
12916         dtrace_enabling_t *enab;
12917         uint_t i;
12918 
12919         ASSERT(MUTEX_HELD(&dtrace_lock));
12920         ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
12921 
12922         /*
12923          * Check the DOF header identification bytes.  In addition to checking
12924          * valid settings, we also verify that unused bits/bytes are zeroed so
12925          * we can use them later without fear of regressing existing binaries.
12926          */
12927         if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
12928             DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
12929                 dtrace_dof_error(dof, "DOF magic string mismatch");
12930                 return (-1);
12931         }
12932 
12933         if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
12934             dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
12935                 dtrace_dof_error(dof, "DOF has invalid data model");
12936                 return (-1);
12937         }
12938 
12939         if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
12940                 dtrace_dof_error(dof, "DOF encoding mismatch");
12941                 return (-1);
12942         }
12943 
12944         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
12945             dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
12946                 dtrace_dof_error(dof, "DOF version mismatch");
12947                 return (-1);
12948         }
12949 
12950         if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
12951                 dtrace_dof_error(dof, "DOF uses unsupported instruction set");
12952                 return (-1);
12953         }
12954 
12955         if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
12956                 dtrace_dof_error(dof, "DOF uses too many integer registers");
12957                 return (-1);
12958         }
12959 
12960         if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
12961                 dtrace_dof_error(dof, "DOF uses too many tuple registers");
12962                 return (-1);
12963         }
12964 
12965         for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
12966                 if (dof->dofh_ident[i] != 0) {
12967                         dtrace_dof_error(dof, "DOF has invalid ident byte set");
12968                         return (-1);
12969                 }
12970         }
12971 
12972         if (dof->dofh_flags & ~DOF_FL_VALID) {
12973                 dtrace_dof_error(dof, "DOF has invalid flag bits set");
12974                 return (-1);
12975         }
12976 
12977         if (dof->dofh_secsize == 0) {
12978                 dtrace_dof_error(dof, "zero section header size");
12979                 return (-1);
12980         }
12981 
12982         /*
12983          * Check that the section headers don't exceed the amount of DOF
12984          * data.  Note that we cast the section size and number of sections
12985          * to uint64_t's to prevent possible overflow in the multiplication.
12986          */
12987         seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
12988 
12989         if (dof->dofh_secoff > len || seclen > len ||
12990             dof->dofh_secoff + seclen > len) {
12991                 dtrace_dof_error(dof, "truncated section headers");
12992                 return (-1);
12993         }
12994 
12995         if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
12996                 dtrace_dof_error(dof, "misaligned section headers");
12997                 return (-1);
12998         }
12999 
13000         if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
13001                 dtrace_dof_error(dof, "misaligned section size");
13002                 return (-1);
13003         }
13004 
13005         /*
13006          * Take an initial pass through the section headers to be sure that
13007          * the headers don't have stray offsets.  If the 'noprobes' flag is
13008          * set, do not permit sections relating to providers, probes, or args.
13009          */
13010         for (i = 0; i < dof->dofh_secnum; i++) {
13011                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13012                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13013 
13014                 if (noprobes) {
13015                         switch (sec->dofs_type) {
13016                         case DOF_SECT_PROVIDER:
13017                         case DOF_SECT_PROBES:
13018                         case DOF_SECT_PRARGS:
13019                         case DOF_SECT_PROFFS:
13020                                 dtrace_dof_error(dof, "illegal sections "
13021                                     "for enabling");
13022                                 return (-1);
13023                         }
13024                 }
13025 
13026                 if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
13027                     !(sec->dofs_flags & DOF_SECF_LOAD)) {
13028                         dtrace_dof_error(dof, "loadable section with load "
13029                             "flag unset");
13030                         return (-1);
13031                 }
13032 
13033                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
13034                         continue; /* just ignore non-loadable sections */
13035 
13036                 if (sec->dofs_align & (sec->dofs_align - 1)) {
13037                         dtrace_dof_error(dof, "bad section alignment");
13038                         return (-1);
13039                 }
13040 
13041                 if (sec->dofs_offset & (sec->dofs_align - 1)) {
13042                         dtrace_dof_error(dof, "misaligned section");
13043                         return (-1);
13044                 }
13045 
13046                 if (sec->dofs_offset > len || sec->dofs_size > len ||
13047                     sec->dofs_offset + sec->dofs_size > len) {
13048                         dtrace_dof_error(dof, "corrupt section header");
13049                         return (-1);
13050                 }
13051 
13052                 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
13053                     sec->dofs_offset + sec->dofs_size - 1) != '\0') {
13054                         dtrace_dof_error(dof, "non-terminating string table");
13055                         return (-1);
13056                 }
13057         }
13058 
13059         /*
13060          * Take a second pass through the sections and locate and perform any
13061          * relocations that are present.  We do this after the first pass to
13062          * be sure that all sections have had their headers validated.
13063          */
13064         for (i = 0; i < dof->dofh_secnum; i++) {
13065                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13066                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13067 
13068                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
13069                         continue; /* skip sections that are not loadable */
13070 
13071                 switch (sec->dofs_type) {
13072                 case DOF_SECT_URELHDR:
13073                         if (dtrace_dof_relocate(dof, sec, ubase) != 0)
13074                                 return (-1);
13075                         break;
13076                 }
13077         }
13078 
13079         if ((enab = *enabp) == NULL)
13080                 enab = *enabp = dtrace_enabling_create(vstate);
13081 
13082         for (i = 0; i < dof->dofh_secnum; i++) {
13083                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13084                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13085 
13086                 if (sec->dofs_type != DOF_SECT_ECBDESC)
13087                         continue;
13088 
13089                 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
13090                         dtrace_enabling_destroy(enab);
13091                         *enabp = NULL;
13092                         return (-1);
13093                 }
13094 
13095                 dtrace_enabling_add(enab, ep);
13096         }
13097 
13098         return (0);
13099 }
13100 
13101 /*
13102  * Process DOF for any options.  This routine assumes that the DOF has been
13103  * at least processed by dtrace_dof_slurp().
13104  */
13105 static int
13106 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
13107 {
13108         int i, rval;
13109         uint32_t entsize;
13110         size_t offs;
13111         dof_optdesc_t *desc;
13112 
13113         for (i = 0; i < dof->dofh_secnum; i++) {
13114                 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
13115                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13116 
13117                 if (sec->dofs_type != DOF_SECT_OPTDESC)
13118                         continue;
13119 
13120                 if (sec->dofs_align != sizeof (uint64_t)) {
13121                         dtrace_dof_error(dof, "bad alignment in "
13122                             "option description");
13123                         return (EINVAL);
13124                 }
13125 
13126                 if ((entsize = sec->dofs_entsize) == 0) {
13127                         dtrace_dof_error(dof, "zeroed option entry size");
13128                         return (EINVAL);
13129                 }
13130 
13131                 if (entsize < sizeof (dof_optdesc_t)) {
13132                         dtrace_dof_error(dof, "bad option entry size");
13133                         return (EINVAL);
13134                 }
13135 
13136                 for (offs = 0; offs < sec->dofs_size; offs += entsize) {
13137                         desc = (dof_optdesc_t *)((uintptr_t)dof +
13138                             (uintptr_t)sec->dofs_offset + offs);
13139 
13140                         if (desc->dofo_strtab != DOF_SECIDX_NONE) {
13141                                 dtrace_dof_error(dof, "non-zero option string");
13142                                 return (EINVAL);
13143                         }
13144 
13145                         if (desc->dofo_value == DTRACEOPT_UNSET) {
13146                                 dtrace_dof_error(dof, "unset option");
13147                                 return (EINVAL);
13148                         }
13149 
13150                         if ((rval = dtrace_state_option(state,
13151                             desc->dofo_option, desc->dofo_value)) != 0) {
13152                                 dtrace_dof_error(dof, "rejected option");
13153                                 return (rval);
13154                         }
13155                 }
13156         }
13157 
13158         return (0);
13159 }
13160 
13161 /*
13162  * DTrace Consumer State Functions
13163  */
13164 int
13165 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
13166 {
13167         size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
13168         void *base;
13169         uintptr_t limit;
13170         dtrace_dynvar_t *dvar, *next, *start;
13171         int i;
13172 
13173         ASSERT(MUTEX_HELD(&dtrace_lock));
13174         ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
13175 
13176         bzero(dstate, sizeof (dtrace_dstate_t));
13177 
13178         if ((dstate->dtds_chunksize = chunksize) == 0)
13179                 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
13180 
13181         if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
13182                 size = min;
13183 
13184         if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
13185                 return (ENOMEM);
13186 
13187         dstate->dtds_size = size;
13188         dstate->dtds_base = base;
13189         dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
13190         bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
13191 
13192         hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
13193 
13194         if (hashsize != 1 && (hashsize & 1))
13195                 hashsize--;
13196 
13197         dstate->dtds_hashsize = hashsize;
13198         dstate->dtds_hash = dstate->dtds_base;
13199 
13200         /*
13201          * Set all of our hash buckets to point to the single sink, and (if
13202          * it hasn't already been set), set the sink's hash value to be the
13203          * sink sentinel value.  The sink is needed for dynamic variable
13204          * lookups to know that they have iterated over an entire, valid hash
13205          * chain.
13206          */
13207         for (i = 0; i < hashsize; i++)
13208                 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
13209 
13210         if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
13211                 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
13212 
13213         /*
13214          * Determine number of active CPUs.  Divide free list evenly among
13215          * active CPUs.
13216          */
13217         start = (dtrace_dynvar_t *)
13218             ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
13219         limit = (uintptr_t)base + size;
13220 
13221         maxper = (limit - (uintptr_t)start) / NCPU;
13222         maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
13223 
13224         for (i = 0; i < NCPU; i++) {
13225                 dstate->dtds_percpu[i].dtdsc_free = dvar = start;
13226 
13227                 /*
13228                  * If we don't even have enough chunks to make it once through
13229                  * NCPUs, we're just going to allocate everything to the first
13230                  * CPU.  And if we're on the last CPU, we're going to allocate
13231                  * whatever is left over.  In either case, we set the limit to
13232                  * be the limit of the dynamic variable space.
13233                  */
13234                 if (maxper == 0 || i == NCPU - 1) {
13235                         limit = (uintptr_t)base + size;
13236                         start = NULL;
13237                 } else {
13238                         limit = (uintptr_t)start + maxper;
13239                         start = (dtrace_dynvar_t *)limit;
13240                 }
13241 
13242                 ASSERT(limit <= (uintptr_t)base + size);
13243 
13244                 for (;;) {
13245                         next = (dtrace_dynvar_t *)((uintptr_t)dvar +
13246                             dstate->dtds_chunksize);
13247 
13248                         if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
13249                                 break;
13250 
13251                         dvar->dtdv_next = next;
13252                         dvar = next;
13253                 }
13254 
13255                 if (maxper == 0)
13256                         break;
13257         }
13258 
13259         return (0);
13260 }
13261 
13262 void
13263 dtrace_dstate_fini(dtrace_dstate_t *dstate)
13264 {
13265         ASSERT(MUTEX_HELD(&cpu_lock));
13266 
13267         if (dstate->dtds_base == NULL)
13268                 return;
13269 
13270         kmem_free(dstate->dtds_base, dstate->dtds_size);
13271         kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
13272 }
13273 
13274 static void
13275 dtrace_vstate_fini(dtrace_vstate_t *vstate)
13276 {
13277         /*
13278          * Logical XOR, where are you?
13279          */
13280         ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
13281 
13282         if (vstate->dtvs_nglobals > 0) {
13283                 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
13284                     sizeof (dtrace_statvar_t *));
13285         }
13286 
13287         if (vstate->dtvs_ntlocals > 0) {
13288                 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
13289                     sizeof (dtrace_difv_t));
13290         }
13291 
13292         ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
13293 
13294         if (vstate->dtvs_nlocals > 0) {
13295                 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
13296                     sizeof (dtrace_statvar_t *));
13297         }
13298 }
13299 
13300 static void
13301 dtrace_state_clean(dtrace_state_t *state)
13302 {
13303         if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
13304                 return;
13305 
13306         dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
13307         dtrace_speculation_clean(state);
13308 }
13309 
13310 static void
13311 dtrace_state_deadman(dtrace_state_t *state)
13312 {
13313         hrtime_t now;
13314 
13315         dtrace_sync();
13316 
13317         now = dtrace_gethrtime();
13318 
13319         if (state != dtrace_anon.dta_state &&
13320             now - state->dts_laststatus >= dtrace_deadman_user)
13321                 return;
13322 
13323         /*
13324          * We must be sure that dts_alive never appears to be less than the
13325          * value upon entry to dtrace_state_deadman(), and because we lack a
13326          * dtrace_cas64(), we cannot store to it atomically.  We thus instead
13327          * store INT64_MAX to it, followed by a memory barrier, followed by
13328          * the new value.  This assures that dts_alive never appears to be
13329          * less than its true value, regardless of the order in which the
13330          * stores to the underlying storage are issued.
13331          */
13332         state->dts_alive = INT64_MAX;
13333         dtrace_membar_producer();
13334         state->dts_alive = now;
13335 }
13336 
13337 dtrace_state_t *
13338 dtrace_state_create(dev_t *devp, cred_t *cr)
13339 {
13340         minor_t minor;
13341         major_t major;
13342         char c[30];
13343         dtrace_state_t *state;
13344         dtrace_optval_t *opt;
13345         int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
13346 
13347         ASSERT(MUTEX_HELD(&dtrace_lock));
13348         ASSERT(MUTEX_HELD(&cpu_lock));
13349 
13350         minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
13351             VM_BESTFIT | VM_SLEEP);
13352 
13353         if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
13354                 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
13355                 return (NULL);
13356         }
13357 
13358         state = ddi_get_soft_state(dtrace_softstate, minor);
13359         state->dts_epid = DTRACE_EPIDNONE + 1;
13360 
13361         (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
13362         state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
13363             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
13364 
13365         if (devp != NULL) {
13366                 major = getemajor(*devp);
13367         } else {
13368                 major = ddi_driver_major(dtrace_devi);
13369         }
13370 
13371         state->dts_dev = makedevice(major, minor);
13372 
13373         if (devp != NULL)
13374                 *devp = state->dts_dev;
13375 
13376         /*
13377          * We allocate NCPU buffers.  On the one hand, this can be quite
13378          * a bit of memory per instance (nearly 36K on a Starcat).  On the
13379          * other hand, it saves an additional memory reference in the probe
13380          * path.
13381          */
13382         state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
13383         state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
13384         state->dts_cleaner = CYCLIC_NONE;
13385         state->dts_deadman = CYCLIC_NONE;
13386         state->dts_vstate.dtvs_state = state;
13387 
13388         for (i = 0; i < DTRACEOPT_MAX; i++)
13389                 state->dts_options[i] = DTRACEOPT_UNSET;
13390 
13391         /*
13392          * Set the default options.
13393          */
13394         opt = state->dts_options;
13395         opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
13396         opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
13397         opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
13398         opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
13399         opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
13400         opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
13401         opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
13402         opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
13403         opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
13404         opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
13405         opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
13406         opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
13407         opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
13408         opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
13409 
13410         state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
13411 
13412         /*
13413          * Depending on the user credentials, we set flag bits which alter probe
13414          * visibility or the amount of destructiveness allowed.  In the case of
13415          * actual anonymous tracing, or the possession of all privileges, all of
13416          * the normal checks are bypassed.
13417          */
13418         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
13419                 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
13420                 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
13421         } else {
13422                 /*
13423                  * Set up the credentials for this instantiation.  We take a
13424                  * hold on the credential to prevent it from disappearing on
13425                  * us; this in turn prevents the zone_t referenced by this
13426                  * credential from disappearing.  This means that we can
13427                  * examine the credential and the zone from probe context.
13428                  */
13429                 crhold(cr);
13430                 state->dts_cred.dcr_cred = cr;
13431 
13432                 /*
13433                  * CRA_PROC means "we have *some* privilege for dtrace" and
13434                  * unlocks the use of variables like pid, zonename, etc.
13435                  */
13436                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
13437                     PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13438                         state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
13439                 }
13440 
13441                 /*
13442                  * dtrace_user allows use of syscall and profile providers.
13443                  * If the user also has proc_owner and/or proc_zone, we
13444                  * extend the scope to include additional visibility and
13445                  * destructive power.
13446                  */
13447                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
13448                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
13449                                 state->dts_cred.dcr_visible |=
13450                                     DTRACE_CRV_ALLPROC;
13451 
13452                                 state->dts_cred.dcr_action |=
13453                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13454                         }
13455 
13456                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
13457                                 state->dts_cred.dcr_visible |=
13458                                     DTRACE_CRV_ALLZONE;
13459 
13460                                 state->dts_cred.dcr_action |=
13461                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13462                         }
13463 
13464                         /*
13465                          * If we have all privs in whatever zone this is,
13466                          * we can do destructive things to processes which
13467                          * have altered credentials.
13468                          */
13469                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13470                             cr->cr_zone->zone_privset)) {
13471                                 state->dts_cred.dcr_action |=
13472                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13473                         }
13474                 }
13475 
13476                 /*
13477                  * Holding the dtrace_kernel privilege also implies that
13478                  * the user has the dtrace_user privilege from a visibility
13479                  * perspective.  But without further privileges, some
13480                  * destructive actions are not available.
13481                  */
13482                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
13483                         /*
13484                          * Make all probes in all zones visible.  However,
13485                          * this doesn't mean that all actions become available
13486                          * to all zones.
13487                          */
13488                         state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
13489                             DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
13490 
13491                         state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
13492                             DTRACE_CRA_PROC;
13493                         /*
13494                          * Holding proc_owner means that destructive actions
13495                          * for *this* zone are allowed.
13496                          */
13497                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13498                                 state->dts_cred.dcr_action |=
13499                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13500 
13501                         /*
13502                          * Holding proc_zone means that destructive actions
13503                          * for this user/group ID in all zones is allowed.
13504                          */
13505                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13506                                 state->dts_cred.dcr_action |=
13507                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13508 
13509                         /*
13510                          * If we have all privs in whatever zone this is,
13511                          * we can do destructive things to processes which
13512                          * have altered credentials.
13513                          */
13514                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13515                             cr->cr_zone->zone_privset)) {
13516                                 state->dts_cred.dcr_action |=
13517                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13518                         }
13519                 }
13520 
13521                 /*
13522                  * Holding the dtrace_proc privilege gives control over fasttrap
13523                  * and pid providers.  We need to grant wider destructive
13524                  * privileges in the event that the user has proc_owner and/or
13525                  * proc_zone.
13526                  */
13527                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13528                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13529                                 state->dts_cred.dcr_action |=
13530                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13531 
13532                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13533                                 state->dts_cred.dcr_action |=
13534                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13535                 }
13536         }
13537 
13538         return (state);
13539 }
13540 
13541 static int
13542 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
13543 {
13544         dtrace_optval_t *opt = state->dts_options, size;
13545         processorid_t cpu;
13546         int flags = 0, rval, factor, divisor = 1;
13547 
13548         ASSERT(MUTEX_HELD(&dtrace_lock));
13549         ASSERT(MUTEX_HELD(&cpu_lock));
13550         ASSERT(which < DTRACEOPT_MAX);
13551         ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
13552             (state == dtrace_anon.dta_state &&
13553             state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
13554 
13555         if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
13556                 return (0);
13557 
13558         if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
13559                 cpu = opt[DTRACEOPT_CPU];
13560 
13561         if (which == DTRACEOPT_SPECSIZE)
13562                 flags |= DTRACEBUF_NOSWITCH;
13563 
13564         if (which == DTRACEOPT_BUFSIZE) {
13565                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
13566                         flags |= DTRACEBUF_RING;
13567 
13568                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
13569                         flags |= DTRACEBUF_FILL;
13570 
13571                 if (state != dtrace_anon.dta_state ||
13572                     state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
13573                         flags |= DTRACEBUF_INACTIVE;
13574         }
13575 
13576         for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
13577                 /*
13578                  * The size must be 8-byte aligned.  If the size is not 8-byte
13579                  * aligned, drop it down by the difference.
13580                  */
13581                 if (size & (sizeof (uint64_t) - 1))
13582                         size -= size & (sizeof (uint64_t) - 1);
13583 
13584                 if (size < state->dts_reserve) {
13585                         /*
13586                          * Buffers always must be large enough to accommodate
13587                          * their prereserved space.  We return E2BIG instead
13588                          * of ENOMEM in this case to allow for user-level
13589                          * software to differentiate the cases.
13590                          */
13591                         return (E2BIG);
13592                 }
13593 
13594                 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
13595 
13596                 if (rval != ENOMEM) {
13597                         opt[which] = size;
13598                         return (rval);
13599                 }
13600 
13601                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13602                         return (rval);
13603 
13604                 for (divisor = 2; divisor < factor; divisor <<= 1)
13605                         continue;
13606         }
13607 
13608         return (ENOMEM);
13609 }
13610 
13611 static int
13612 dtrace_state_buffers(dtrace_state_t *state)
13613 {
13614         dtrace_speculation_t *spec = state->dts_speculations;
13615         int rval, i;
13616 
13617         if ((rval = dtrace_state_buffer(state, state->dts_buffer,
13618             DTRACEOPT_BUFSIZE)) != 0)
13619                 return (rval);
13620 
13621         if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
13622             DTRACEOPT_AGGSIZE)) != 0)
13623                 return (rval);
13624 
13625         for (i = 0; i < state->dts_nspeculations; i++) {
13626                 if ((rval = dtrace_state_buffer(state,
13627                     spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
13628                         return (rval);
13629         }
13630 
13631         return (0);
13632 }
13633 
13634 static void
13635 dtrace_state_prereserve(dtrace_state_t *state)
13636 {
13637         dtrace_ecb_t *ecb;
13638         dtrace_probe_t *probe;
13639 
13640         state->dts_reserve = 0;
13641 
13642         if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
13643                 return;
13644 
13645         /*
13646          * If our buffer policy is a "fill" buffer policy, we need to set the
13647          * prereserved space to be the space required by the END probes.
13648          */
13649         probe = dtrace_probes[dtrace_probeid_end - 1];
13650         ASSERT(probe != NULL);
13651 
13652         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
13653                 if (ecb->dte_state != state)
13654                         continue;
13655 
13656                 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
13657         }
13658 }
13659 
13660 static int
13661 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
13662 {
13663         dtrace_optval_t *opt = state->dts_options, sz, nspec;
13664         dtrace_speculation_t *spec;
13665         dtrace_buffer_t *buf;
13666         cyc_handler_t hdlr;
13667         cyc_time_t when;
13668         int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
13669         dtrace_icookie_t cookie;
13670 
13671         mutex_enter(&cpu_lock);
13672         mutex_enter(&dtrace_lock);
13673 
13674         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
13675                 rval = EBUSY;
13676                 goto out;
13677         }
13678 
13679         /*
13680          * Before we can perform any checks, we must prime all of the
13681          * retained enablings that correspond to this state.
13682          */
13683         dtrace_enabling_prime(state);
13684 
13685         if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
13686                 rval = EACCES;
13687                 goto out;
13688         }
13689 
13690         dtrace_state_prereserve(state);
13691 
13692         /*
13693          * Now we want to do is try to allocate our speculations.
13694          * We do not automatically resize the number of speculations; if
13695          * this fails, we will fail the operation.
13696          */
13697         nspec = opt[DTRACEOPT_NSPEC];
13698         ASSERT(nspec != DTRACEOPT_UNSET);
13699 
13700         if (nspec > INT_MAX) {
13701                 rval = ENOMEM;
13702                 goto out;
13703         }
13704 
13705         spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
13706             KM_NOSLEEP | KM_NORMALPRI);
13707 
13708         if (spec == NULL) {
13709                 rval = ENOMEM;
13710                 goto out;
13711         }
13712 
13713         state->dts_speculations = spec;
13714         state->dts_nspeculations = (int)nspec;
13715 
13716         for (i = 0; i < nspec; i++) {
13717                 if ((buf = kmem_zalloc(bufsize,
13718                     KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
13719                         rval = ENOMEM;
13720                         goto err;
13721                 }
13722 
13723                 spec[i].dtsp_buffer = buf;
13724         }
13725 
13726         if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
13727                 if (dtrace_anon.dta_state == NULL) {
13728                         rval = ENOENT;
13729                         goto out;
13730                 }
13731 
13732                 if (state->dts_necbs != 0) {
13733                         rval = EALREADY;
13734                         goto out;
13735                 }
13736 
13737                 state->dts_anon = dtrace_anon_grab();
13738                 ASSERT(state->dts_anon != NULL);
13739                 state = state->dts_anon;
13740 
13741                 /*
13742                  * We want "grabanon" to be set in the grabbed state, so we'll
13743                  * copy that option value from the grabbing state into the
13744                  * grabbed state.
13745                  */
13746                 state->dts_options[DTRACEOPT_GRABANON] =
13747                     opt[DTRACEOPT_GRABANON];
13748 
13749                 *cpu = dtrace_anon.dta_beganon;
13750 
13751                 /*
13752                  * If the anonymous state is active (as it almost certainly
13753                  * is if the anonymous enabling ultimately matched anything),
13754                  * we don't allow any further option processing -- but we
13755                  * don't return failure.
13756                  */
13757                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
13758                         goto out;
13759         }
13760 
13761         if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
13762             opt[DTRACEOPT_AGGSIZE] != 0) {
13763                 if (state->dts_aggregations == NULL) {
13764                         /*
13765                          * We're not going to create an aggregation buffer
13766                          * because we don't have any ECBs that contain
13767                          * aggregations -- set this option to 0.
13768                          */
13769                         opt[DTRACEOPT_AGGSIZE] = 0;
13770                 } else {
13771                         /*
13772                          * If we have an aggregation buffer, we must also have
13773                          * a buffer to use as scratch.
13774                          */
13775                         if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
13776                             opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
13777                                 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
13778                         }
13779                 }
13780         }
13781 
13782         if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
13783             opt[DTRACEOPT_SPECSIZE] != 0) {
13784                 if (!state->dts_speculates) {
13785                         /*
13786                          * We're not going to create speculation buffers
13787                          * because we don't have any ECBs that actually
13788                          * speculate -- set the speculation size to 0.
13789                          */
13790                         opt[DTRACEOPT_SPECSIZE] = 0;
13791                 }
13792         }
13793 
13794         /*
13795          * The bare minimum size for any buffer that we're actually going to
13796          * do anything to is sizeof (uint64_t).
13797          */
13798         sz = sizeof (uint64_t);
13799 
13800         if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
13801             (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
13802             (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
13803                 /*
13804                  * A buffer size has been explicitly set to 0 (or to a size
13805                  * that will be adjusted to 0) and we need the space -- we
13806                  * need to return failure.  We return ENOSPC to differentiate
13807                  * it from failing to allocate a buffer due to failure to meet
13808                  * the reserve (for which we return E2BIG).
13809                  */
13810                 rval = ENOSPC;
13811                 goto out;
13812         }
13813 
13814         if ((rval = dtrace_state_buffers(state)) != 0)
13815                 goto err;
13816 
13817         if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
13818                 sz = dtrace_dstate_defsize;
13819 
13820         do {
13821                 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
13822 
13823                 if (rval == 0)
13824                         break;
13825 
13826                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13827                         goto err;
13828         } while (sz >>= 1);
13829 
13830         opt[DTRACEOPT_DYNVARSIZE] = sz;
13831 
13832         if (rval != 0)
13833                 goto err;
13834 
13835         if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
13836                 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
13837 
13838         if (opt[DTRACEOPT_CLEANRATE] == 0)
13839                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13840 
13841         if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
13842                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
13843 
13844         if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
13845                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13846 
13847         hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
13848         hdlr.cyh_arg = state;
13849         hdlr.cyh_level = CY_LOW_LEVEL;
13850 
13851         when.cyt_when = 0;
13852         when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
13853 
13854         state->dts_cleaner = cyclic_add(&hdlr, &when);
13855 
13856         hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
13857         hdlr.cyh_arg = state;
13858         hdlr.cyh_level = CY_LOW_LEVEL;
13859 
13860         when.cyt_when = 0;
13861         when.cyt_interval = dtrace_deadman_interval;
13862 
13863         state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
13864         state->dts_deadman = cyclic_add(&hdlr, &when);
13865 
13866         state->dts_activity = DTRACE_ACTIVITY_WARMUP;
13867 
13868         if (state->dts_getf != 0 &&
13869             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
13870                 /*
13871                  * We don't have kernel privs but we have at least one call
13872                  * to getf(); we need to bump our zone's count, and (if
13873                  * this is the first enabling to have an unprivileged call
13874                  * to getf()) we need to hook into closef().
13875                  */
13876                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++;
13877 
13878                 if (dtrace_getf++ == 0) {
13879                         ASSERT(dtrace_closef == NULL);
13880                         dtrace_closef = dtrace_getf_barrier;
13881                 }
13882         }
13883 
13884         /*
13885          * Now it's time to actually fire the BEGIN probe.  We need to disable
13886          * interrupts here both to record the CPU on which we fired the BEGIN
13887          * probe (the data from this CPU will be processed first at user
13888          * level) and to manually activate the buffer for this CPU.
13889          */
13890         cookie = dtrace_interrupt_disable();
13891         *cpu = CPU->cpu_id;
13892         ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
13893         state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
13894 
13895         dtrace_probe(dtrace_probeid_begin,
13896             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13897         dtrace_interrupt_enable(cookie);
13898         /*
13899          * We may have had an exit action from a BEGIN probe; only change our
13900          * state to ACTIVE if we're still in WARMUP.
13901          */
13902         ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
13903             state->dts_activity == DTRACE_ACTIVITY_DRAINING);
13904 
13905         if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
13906                 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
13907 
13908         /*
13909          * Regardless of whether or not now we're in ACTIVE or DRAINING, we
13910          * want each CPU to transition its principal buffer out of the
13911          * INACTIVE state.  Doing this assures that no CPU will suddenly begin
13912          * processing an ECB halfway down a probe's ECB chain; all CPUs will
13913          * atomically transition from processing none of a state's ECBs to
13914          * processing all of them.
13915          */
13916         dtrace_xcall(DTRACE_CPUALL,
13917             (dtrace_xcall_t)dtrace_buffer_activate, state);
13918         goto out;
13919 
13920 err:
13921         dtrace_buffer_free(state->dts_buffer);
13922         dtrace_buffer_free(state->dts_aggbuffer);
13923 
13924         if ((nspec = state->dts_nspeculations) == 0) {
13925                 ASSERT(state->dts_speculations == NULL);
13926                 goto out;
13927         }
13928 
13929         spec = state->dts_speculations;
13930         ASSERT(spec != NULL);
13931 
13932         for (i = 0; i < state->dts_nspeculations; i++) {
13933                 if ((buf = spec[i].dtsp_buffer) == NULL)
13934                         break;
13935 
13936                 dtrace_buffer_free(buf);
13937                 kmem_free(buf, bufsize);
13938         }
13939 
13940         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13941         state->dts_nspeculations = 0;
13942         state->dts_speculations = NULL;
13943 
13944 out:
13945         mutex_exit(&dtrace_lock);
13946         mutex_exit(&cpu_lock);
13947 
13948         return (rval);
13949 }
13950 
13951 static int
13952 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
13953 {
13954         dtrace_icookie_t cookie;
13955 
13956         ASSERT(MUTEX_HELD(&dtrace_lock));
13957 
13958         if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
13959             state->dts_activity != DTRACE_ACTIVITY_DRAINING)
13960                 return (EINVAL);
13961 
13962         /*
13963          * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
13964          * to be sure that every CPU has seen it.  See below for the details
13965          * on why this is done.
13966          */
13967         state->dts_activity = DTRACE_ACTIVITY_DRAINING;
13968         dtrace_sync();
13969 
13970         /*
13971          * By this point, it is impossible for any CPU to be still processing
13972          * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
13973          * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
13974          * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
13975          * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
13976          * iff we're in the END probe.
13977          */
13978         state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
13979         dtrace_sync();
13980         ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
13981 
13982         /*
13983          * Finally, we can release the reserve and call the END probe.  We
13984          * disable interrupts across calling the END probe to allow us to
13985          * return the CPU on which we actually called the END probe.  This
13986          * allows user-land to be sure that this CPU's principal buffer is
13987          * processed last.
13988          */
13989         state->dts_reserve = 0;
13990 
13991         cookie = dtrace_interrupt_disable();
13992         *cpu = CPU->cpu_id;
13993         dtrace_probe(dtrace_probeid_end,
13994             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13995         dtrace_interrupt_enable(cookie);
13996 
13997         state->dts_activity = DTRACE_ACTIVITY_STOPPED;
13998         dtrace_sync();
13999 
14000         if (state->dts_getf != 0 &&
14001             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
14002                 /*
14003                  * We don't have kernel privs but we have at least one call
14004                  * to getf(); we need to lower our zone's count, and (if
14005                  * this is the last enabling to have an unprivileged call
14006                  * to getf()) we need to clear the closef() hook.
14007                  */
14008                 ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0);
14009                 ASSERT(dtrace_closef == dtrace_getf_barrier);
14010                 ASSERT(dtrace_getf > 0);
14011 
14012                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--;
14013 
14014                 if (--dtrace_getf == 0)
14015                         dtrace_closef = NULL;
14016         }
14017 
14018         return (0);
14019 }
14020 
14021 static int
14022 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
14023     dtrace_optval_t val)
14024 {
14025         ASSERT(MUTEX_HELD(&dtrace_lock));
14026 
14027         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
14028                 return (EBUSY);
14029 
14030         if (option >= DTRACEOPT_MAX)
14031                 return (EINVAL);
14032 
14033         if (option != DTRACEOPT_CPU && val < 0)
14034                 return (EINVAL);
14035 
14036         switch (option) {
14037         case DTRACEOPT_DESTRUCTIVE:
14038                 if (dtrace_destructive_disallow)
14039                         return (EACCES);
14040 
14041                 state->dts_cred.dcr_destructive = 1;
14042                 break;
14043 
14044         case DTRACEOPT_BUFSIZE:
14045         case DTRACEOPT_DYNVARSIZE:
14046         case DTRACEOPT_AGGSIZE:
14047         case DTRACEOPT_SPECSIZE:
14048         case DTRACEOPT_STRSIZE:
14049                 if (val < 0)
14050                         return (EINVAL);
14051 
14052                 if (val >= LONG_MAX) {
14053                         /*
14054                          * If this is an otherwise negative value, set it to
14055                          * the highest multiple of 128m less than LONG_MAX.
14056                          * Technically, we're adjusting the size without
14057                          * regard to the buffer resizing policy, but in fact,
14058                          * this has no effect -- if we set the buffer size to
14059                          * ~LONG_MAX and the buffer policy is ultimately set to
14060                          * be "manual", the buffer allocation is guaranteed to
14061                          * fail, if only because the allocation requires two
14062                          * buffers.  (We set the the size to the highest
14063                          * multiple of 128m because it ensures that the size
14064                          * will remain a multiple of a megabyte when
14065                          * repeatedly halved -- all the way down to 15m.)
14066                          */
14067                         val = LONG_MAX - (1 << 27) + 1;
14068                 }
14069         }
14070 
14071         state->dts_options[option] = val;
14072 
14073         return (0);
14074 }
14075 
14076 static void
14077 dtrace_state_destroy(dtrace_state_t *state)
14078 {
14079         dtrace_ecb_t *ecb;
14080         dtrace_vstate_t *vstate = &state->dts_vstate;
14081         minor_t minor = getminor(state->dts_dev);
14082         int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
14083         dtrace_speculation_t *spec = state->dts_speculations;
14084         int nspec = state->dts_nspeculations;
14085         uint32_t match;
14086 
14087         ASSERT(MUTEX_HELD(&dtrace_lock));
14088         ASSERT(MUTEX_HELD(&cpu_lock));
14089 
14090         /*
14091          * First, retract any retained enablings for this state.
14092          */
14093         dtrace_enabling_retract(state);
14094         ASSERT(state->dts_nretained == 0);
14095 
14096         if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
14097             state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
14098                 /*
14099                  * We have managed to come into dtrace_state_destroy() on a
14100                  * hot enabling -- almost certainly because of a disorderly
14101                  * shutdown of a consumer.  (That is, a consumer that is
14102                  * exiting without having called dtrace_stop().) In this case,
14103                  * we're going to set our activity to be KILLED, and then
14104                  * issue a sync to be sure that everyone is out of probe
14105                  * context before we start blowing away ECBs.
14106                  */
14107                 state->dts_activity = DTRACE_ACTIVITY_KILLED;
14108                 dtrace_sync();
14109         }
14110 
14111         /*
14112          * Release the credential hold we took in dtrace_state_create().
14113          */
14114         if (state->dts_cred.dcr_cred != NULL)
14115                 crfree(state->dts_cred.dcr_cred);
14116 
14117         /*
14118          * Now we can safely disable and destroy any enabled probes.  Because
14119          * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
14120          * (especially if they're all enabled), we take two passes through the
14121          * ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes, and
14122          * in the second we disable whatever is left over.
14123          */
14124         for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
14125                 for (i = 0; i < state->dts_necbs; i++) {
14126                         if ((ecb = state->dts_ecbs[i]) == NULL)
14127                                 continue;
14128 
14129                         if (match && ecb->dte_probe != NULL) {
14130                                 dtrace_probe_t *probe = ecb->dte_probe;
14131                                 dtrace_provider_t *prov = probe->dtpr_provider;
14132 
14133                                 if (!(prov->dtpv_priv.dtpp_flags & match))
14134                                         continue;
14135                         }
14136 
14137                         dtrace_ecb_disable(ecb);
14138                         dtrace_ecb_destroy(ecb);
14139                 }
14140 
14141                 if (!match)
14142                         break;
14143         }
14144 
14145         /*
14146          * Before we free the buffers, perform one more sync to assure that
14147          * every CPU is out of probe context.
14148          */
14149         dtrace_sync();
14150 
14151         dtrace_buffer_free(state->dts_buffer);
14152         dtrace_buffer_free(state->dts_aggbuffer);
14153 
14154         for (i = 0; i < nspec; i++)
14155                 dtrace_buffer_free(spec[i].dtsp_buffer);
14156 
14157         if (state->dts_cleaner != CYCLIC_NONE)
14158                 cyclic_remove(state->dts_cleaner);
14159 
14160         if (state->dts_deadman != CYCLIC_NONE)
14161                 cyclic_remove(state->dts_deadman);
14162 
14163         dtrace_dstate_fini(&vstate->dtvs_dynvars);
14164         dtrace_vstate_fini(vstate);
14165         kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
14166 
14167         if (state->dts_aggregations != NULL) {
14168 #ifdef DEBUG
14169                 for (i = 0; i < state->dts_naggregations; i++)
14170                         ASSERT(state->dts_aggregations[i] == NULL);
14171 #endif
14172                 ASSERT(state->dts_naggregations > 0);
14173                 kmem_free(state->dts_aggregations,
14174                     state->dts_naggregations * sizeof (dtrace_aggregation_t *));
14175         }
14176 
14177         kmem_free(state->dts_buffer, bufsize);
14178         kmem_free(state->dts_aggbuffer, bufsize);
14179 
14180         for (i = 0; i < nspec; i++)
14181                 kmem_free(spec[i].dtsp_buffer, bufsize);
14182 
14183         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
14184 
14185         dtrace_format_destroy(state);
14186 
14187         vmem_destroy(state->dts_aggid_arena);
14188         ddi_soft_state_free(dtrace_softstate, minor);
14189         vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
14190 }
14191 
14192 /*
14193  * DTrace Anonymous Enabling Functions
14194  */
14195 static dtrace_state_t *
14196 dtrace_anon_grab(void)
14197 {
14198         dtrace_state_t *state;
14199 
14200         ASSERT(MUTEX_HELD(&dtrace_lock));
14201 
14202         if ((state = dtrace_anon.dta_state) == NULL) {
14203                 ASSERT(dtrace_anon.dta_enabling == NULL);
14204                 return (NULL);
14205         }
14206 
14207         ASSERT(dtrace_anon.dta_enabling != NULL);
14208         ASSERT(dtrace_retained != NULL);
14209 
14210         dtrace_enabling_destroy(dtrace_anon.dta_enabling);
14211         dtrace_anon.dta_enabling = NULL;
14212         dtrace_anon.dta_state = NULL;
14213 
14214         return (state);
14215 }
14216 
14217 static void
14218 dtrace_anon_property(void)
14219 {
14220         int i, rv;
14221         dtrace_state_t *state;
14222         dof_hdr_t *dof;
14223         char c[32];             /* enough for "dof-data-" + digits */
14224 
14225         ASSERT(MUTEX_HELD(&dtrace_lock));
14226         ASSERT(MUTEX_HELD(&cpu_lock));
14227 
14228         for (i = 0; ; i++) {
14229                 (void) snprintf(c, sizeof (c), "dof-data-%d", i);
14230 
14231                 dtrace_err_verbose = 1;
14232 
14233                 if ((dof = dtrace_dof_property(c)) == NULL) {
14234                         dtrace_err_verbose = 0;
14235                         break;
14236                 }
14237 
14238                 /*
14239                  * We want to create anonymous state, so we need to transition
14240                  * the kernel debugger to indicate that DTrace is active.  If
14241                  * this fails (e.g. because the debugger has modified text in
14242                  * some way), we won't continue with the processing.
14243                  */
14244                 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
14245                         cmn_err(CE_NOTE, "kernel debugger active; anonymous "
14246                             "enabling ignored.");
14247                         dtrace_dof_destroy(dof);
14248                         break;
14249                 }
14250 
14251                 /*
14252                  * If we haven't allocated an anonymous state, we'll do so now.
14253                  */
14254                 if ((state = dtrace_anon.dta_state) == NULL) {
14255                         state = dtrace_state_create(NULL, NULL);
14256                         dtrace_anon.dta_state = state;
14257 
14258                         if (state == NULL) {
14259                                 /*
14260                                  * This basically shouldn't happen:  the only
14261                                  * failure mode from dtrace_state_create() is a
14262                                  * failure of ddi_soft_state_zalloc() that
14263                                  * itself should never happen.  Still, the
14264                                  * interface allows for a failure mode, and
14265                                  * we want to fail as gracefully as possible:
14266                                  * we'll emit an error message and cease
14267                                  * processing anonymous state in this case.
14268                                  */
14269                                 cmn_err(CE_WARN, "failed to create "
14270                                     "anonymous state");
14271                                 dtrace_dof_destroy(dof);
14272                                 break;
14273                         }
14274                 }
14275 
14276                 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
14277                     &dtrace_anon.dta_enabling, 0, B_TRUE);
14278 
14279                 if (rv == 0)
14280                         rv = dtrace_dof_options(dof, state);
14281 
14282                 dtrace_err_verbose = 0;
14283                 dtrace_dof_destroy(dof);
14284 
14285                 if (rv != 0) {
14286                         /*
14287                          * This is malformed DOF; chuck any anonymous state
14288                          * that we created.
14289                          */
14290                         ASSERT(dtrace_anon.dta_enabling == NULL);
14291                         dtrace_state_destroy(state);
14292                         dtrace_anon.dta_state = NULL;
14293                         break;
14294                 }
14295 
14296                 ASSERT(dtrace_anon.dta_enabling != NULL);
14297         }
14298 
14299         if (dtrace_anon.dta_enabling != NULL) {
14300                 int rval;
14301 
14302                 /*
14303                  * dtrace_enabling_retain() can only fail because we are
14304                  * trying to retain more enablings than are allowed -- but
14305                  * we only have one anonymous enabling, and we are guaranteed
14306                  * to be allowed at least one retained enabling; we assert
14307                  * that dtrace_enabling_retain() returns success.
14308                  */
14309                 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
14310                 ASSERT(rval == 0);
14311 
14312                 dtrace_enabling_dump(dtrace_anon.dta_enabling);
14313         }
14314 }
14315 
14316 /*
14317  * DTrace Helper Functions
14318  */
14319 static void
14320 dtrace_helper_trace(dtrace_helper_action_t *helper,
14321     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
14322 {
14323         uint32_t size, next, nnext, i;
14324         dtrace_helptrace_t *ent;
14325         uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14326 
14327         if (!dtrace_helptrace_enabled)
14328                 return;
14329 
14330         ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
14331 
14332         /*
14333          * What would a tracing framework be without its own tracing
14334          * framework?  (Well, a hell of a lot simpler, for starters...)
14335          */
14336         size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
14337             sizeof (uint64_t) - sizeof (uint64_t);
14338 
14339         /*
14340          * Iterate until we can allocate a slot in the trace buffer.
14341          */
14342         do {
14343                 next = dtrace_helptrace_next;
14344 
14345                 if (next + size < dtrace_helptrace_bufsize) {
14346                         nnext = next + size;
14347                 } else {
14348                         nnext = size;
14349                 }
14350         } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
14351 
14352         /*
14353          * We have our slot; fill it in.
14354          */
14355         if (nnext == size)
14356                 next = 0;
14357 
14358         ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
14359         ent->dtht_helper = helper;
14360         ent->dtht_where = where;
14361         ent->dtht_nlocals = vstate->dtvs_nlocals;
14362 
14363         ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
14364             mstate->dtms_fltoffs : -1;
14365         ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
14366         ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
14367 
14368         for (i = 0; i < vstate->dtvs_nlocals; i++) {
14369                 dtrace_statvar_t *svar;
14370 
14371                 if ((svar = vstate->dtvs_locals[i]) == NULL)
14372                         continue;
14373 
14374                 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
14375                 ent->dtht_locals[i] =
14376                     ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
14377         }
14378 }
14379 
14380 static uint64_t
14381 dtrace_helper(int which, dtrace_mstate_t *mstate,
14382     dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
14383 {
14384         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14385         uint64_t sarg0 = mstate->dtms_arg[0];
14386         uint64_t sarg1 = mstate->dtms_arg[1];
14387         uint64_t rval;
14388         dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
14389         dtrace_helper_action_t *helper;
14390         dtrace_vstate_t *vstate;
14391         dtrace_difo_t *pred;
14392         int i, trace = dtrace_helptrace_enabled;
14393 
14394         ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
14395 
14396         if (helpers == NULL)
14397                 return (0);
14398 
14399         if ((helper = helpers->dthps_actions[which]) == NULL)
14400                 return (0);
14401 
14402         vstate = &helpers->dthps_vstate;
14403         mstate->dtms_arg[0] = arg0;
14404         mstate->dtms_arg[1] = arg1;
14405 
14406         /*
14407          * Now iterate over each helper.  If its predicate evaluates to 'true',
14408          * we'll call the corresponding actions.  Note that the below calls
14409          * to dtrace_dif_emulate() may set faults in machine state.  This is
14410          * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
14411          * the stored DIF offset with its own (which is the desired behavior).
14412          * Also, note the calls to dtrace_dif_emulate() may allocate scratch
14413          * from machine state; this is okay, too.
14414          */
14415         for (; helper != NULL; helper = helper->dtha_next) {
14416                 if ((pred = helper->dtha_predicate) != NULL) {
14417                         if (trace)
14418                                 dtrace_helper_trace(helper, mstate, vstate, 0);
14419 
14420                         if (!dtrace_dif_emulate(pred, mstate, vstate, state))
14421                                 goto next;
14422 
14423                         if (*flags & CPU_DTRACE_FAULT)
14424                                 goto err;
14425                 }
14426 
14427                 for (i = 0; i < helper->dtha_nactions; i++) {
14428                         if (trace)
14429                                 dtrace_helper_trace(helper,
14430                                     mstate, vstate, i + 1);
14431 
14432                         rval = dtrace_dif_emulate(helper->dtha_actions[i],
14433                             mstate, vstate, state);
14434 
14435                         if (*flags & CPU_DTRACE_FAULT)
14436                                 goto err;
14437                 }
14438 
14439 next:
14440                 if (trace)
14441                         dtrace_helper_trace(helper, mstate, vstate,
14442                             DTRACE_HELPTRACE_NEXT);
14443         }
14444 
14445         if (trace)
14446                 dtrace_helper_trace(helper, mstate, vstate,
14447                     DTRACE_HELPTRACE_DONE);
14448 
14449         /*
14450          * Restore the arg0 that we saved upon entry.
14451          */
14452         mstate->dtms_arg[0] = sarg0;
14453         mstate->dtms_arg[1] = sarg1;
14454 
14455         return (rval);
14456 
14457 err:
14458         if (trace)
14459                 dtrace_helper_trace(helper, mstate, vstate,
14460                     DTRACE_HELPTRACE_ERR);
14461 
14462         /*
14463          * Restore the arg0 that we saved upon entry.
14464          */
14465         mstate->dtms_arg[0] = sarg0;
14466         mstate->dtms_arg[1] = sarg1;
14467 
14468         return (NULL);
14469 }
14470 
14471 static void
14472 dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
14473     dtrace_vstate_t *vstate)
14474 {
14475         int i;
14476 
14477         if (helper->dtha_predicate != NULL)
14478                 dtrace_difo_release(helper->dtha_predicate, vstate);
14479 
14480         for (i = 0; i < helper->dtha_nactions; i++) {
14481                 ASSERT(helper->dtha_actions[i] != NULL);
14482                 dtrace_difo_release(helper->dtha_actions[i], vstate);
14483         }
14484 
14485         kmem_free(helper->dtha_actions,
14486             helper->dtha_nactions * sizeof (dtrace_difo_t *));
14487         kmem_free(helper, sizeof (dtrace_helper_action_t));
14488 }
14489 
14490 static int
14491 dtrace_helper_destroygen(int gen)
14492 {
14493         proc_t *p = curproc;
14494         dtrace_helpers_t *help = p->p_dtrace_helpers;
14495         dtrace_vstate_t *vstate;
14496         int i;
14497 
14498         ASSERT(MUTEX_HELD(&dtrace_lock));
14499 
14500         if (help == NULL || gen > help->dthps_generation)
14501                 return (EINVAL);
14502 
14503         vstate = &help->dthps_vstate;
14504 
14505         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14506                 dtrace_helper_action_t *last = NULL, *h, *next;
14507 
14508                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
14509                         next = h->dtha_next;
14510 
14511                         if (h->dtha_generation == gen) {
14512                                 if (last != NULL) {
14513                                         last->dtha_next = next;
14514                                 } else {
14515                                         help->dthps_actions[i] = next;
14516                                 }
14517 
14518                                 dtrace_helper_action_destroy(h, vstate);
14519                         } else {
14520                                 last = h;
14521                         }
14522                 }
14523         }
14524 
14525         /*
14526          * Interate until we've cleared out all helper providers with the
14527          * given generation number.
14528          */
14529         for (;;) {
14530                 dtrace_helper_provider_t *prov;
14531 
14532                 /*
14533                  * Look for a helper provider with the right generation. We
14534                  * have to start back at the beginning of the list each time
14535                  * because we drop dtrace_lock. It's unlikely that we'll make
14536                  * more than two passes.
14537                  */
14538                 for (i = 0; i < help->dthps_nprovs; i++) {
14539                         prov = help->dthps_provs[i];
14540 
14541                         if (prov->dthp_generation == gen)
14542                                 break;
14543                 }
14544 
14545                 /*
14546                  * If there were no matches, we're done.
14547                  */
14548                 if (i == help->dthps_nprovs)
14549                         break;
14550 
14551                 /*
14552                  * Move the last helper provider into this slot.
14553                  */
14554                 help->dthps_nprovs--;
14555                 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
14556                 help->dthps_provs[help->dthps_nprovs] = NULL;
14557 
14558                 mutex_exit(&dtrace_lock);
14559 
14560                 /*
14561                  * If we have a meta provider, remove this helper provider.
14562                  */
14563                 mutex_enter(&dtrace_meta_lock);
14564                 if (dtrace_meta_pid != NULL) {
14565                         ASSERT(dtrace_deferred_pid == NULL);
14566                         dtrace_helper_provider_remove(&prov->dthp_prov,
14567                             p->p_pid);
14568                 }
14569                 mutex_exit(&dtrace_meta_lock);
14570 
14571                 dtrace_helper_provider_destroy(prov);
14572 
14573                 mutex_enter(&dtrace_lock);
14574         }
14575 
14576         return (0);
14577 }
14578 
14579 static int
14580 dtrace_helper_validate(dtrace_helper_action_t *helper)
14581 {
14582         int err = 0, i;
14583         dtrace_difo_t *dp;
14584 
14585         if ((dp = helper->dtha_predicate) != NULL)
14586                 err += dtrace_difo_validate_helper(dp);
14587 
14588         for (i = 0; i < helper->dtha_nactions; i++)
14589                 err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
14590 
14591         return (err == 0);
14592 }
14593 
14594 static int
14595 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
14596 {
14597         dtrace_helpers_t *help;
14598         dtrace_helper_action_t *helper, *last;
14599         dtrace_actdesc_t *act;
14600         dtrace_vstate_t *vstate;
14601         dtrace_predicate_t *pred;
14602         int count = 0, nactions = 0, i;
14603 
14604         if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
14605                 return (EINVAL);
14606 
14607         help = curproc->p_dtrace_helpers;
14608         last = help->dthps_actions[which];
14609         vstate = &help->dthps_vstate;
14610 
14611         for (count = 0; last != NULL; last = last->dtha_next) {
14612                 count++;
14613                 if (last->dtha_next == NULL)
14614                         break;
14615         }
14616 
14617         /*
14618          * If we already have dtrace_helper_actions_max helper actions for this
14619          * helper action type, we'll refuse to add a new one.
14620          */
14621         if (count >= dtrace_helper_actions_max)
14622                 return (ENOSPC);
14623 
14624         helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
14625         helper->dtha_generation = help->dthps_generation;
14626 
14627         if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
14628                 ASSERT(pred->dtp_difo != NULL);
14629                 dtrace_difo_hold(pred->dtp_difo);
14630                 helper->dtha_predicate = pred->dtp_difo;
14631         }
14632 
14633         for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
14634                 if (act->dtad_kind != DTRACEACT_DIFEXPR)
14635                         goto err;
14636 
14637                 if (act->dtad_difo == NULL)
14638                         goto err;
14639 
14640                 nactions++;
14641         }
14642 
14643         helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
14644             (helper->dtha_nactions = nactions), KM_SLEEP);
14645 
14646         for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
14647                 dtrace_difo_hold(act->dtad_difo);
14648                 helper->dtha_actions[i++] = act->dtad_difo;
14649         }
14650 
14651         if (!dtrace_helper_validate(helper))
14652                 goto err;
14653 
14654         if (last == NULL) {
14655                 help->dthps_actions[which] = helper;
14656         } else {
14657                 last->dtha_next = helper;
14658         }
14659 
14660         if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
14661                 dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
14662                 dtrace_helptrace_next = 0;
14663         }
14664 
14665         return (0);
14666 err:
14667         dtrace_helper_action_destroy(helper, vstate);
14668         return (EINVAL);
14669 }
14670 
14671 static void
14672 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
14673     dof_helper_t *dofhp)
14674 {
14675         ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
14676 
14677         mutex_enter(&dtrace_meta_lock);
14678         mutex_enter(&dtrace_lock);
14679 
14680         if (!dtrace_attached() || dtrace_meta_pid == NULL) {
14681                 /*
14682                  * If the dtrace module is loaded but not attached, or if
14683                  * there aren't isn't a meta provider registered to deal with
14684                  * these provider descriptions, we need to postpone creating
14685                  * the actual providers until later.
14686                  */
14687 
14688                 if (help->dthps_next == NULL && help->dthps_prev == NULL &&
14689                     dtrace_deferred_pid != help) {
14690                         help->dthps_deferred = 1;
14691                         help->dthps_pid = p->p_pid;
14692                         help->dthps_next = dtrace_deferred_pid;
14693                         help->dthps_prev = NULL;
14694                         if (dtrace_deferred_pid != NULL)
14695                                 dtrace_deferred_pid->dthps_prev = help;
14696                         dtrace_deferred_pid = help;
14697                 }
14698 
14699                 mutex_exit(&dtrace_lock);
14700 
14701         } else if (dofhp != NULL) {
14702                 /*
14703                  * If the dtrace module is loaded and we have a particular
14704                  * helper provider description, pass that off to the
14705                  * meta provider.
14706                  */
14707 
14708                 mutex_exit(&dtrace_lock);
14709 
14710                 dtrace_helper_provide(dofhp, p->p_pid);
14711 
14712         } else {
14713                 /*
14714                  * Otherwise, just pass all the helper provider descriptions
14715                  * off to the meta provider.
14716                  */
14717 
14718                 int i;
14719                 mutex_exit(&dtrace_lock);
14720 
14721                 for (i = 0; i < help->dthps_nprovs; i++) {
14722                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
14723                             p->p_pid);
14724                 }
14725         }
14726 
14727         mutex_exit(&dtrace_meta_lock);
14728 }
14729 
14730 static int
14731 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
14732 {
14733         dtrace_helpers_t *help;
14734         dtrace_helper_provider_t *hprov, **tmp_provs;
14735         uint_t tmp_maxprovs, i;
14736 
14737         ASSERT(MUTEX_HELD(&dtrace_lock));
14738 
14739         help = curproc->p_dtrace_helpers;
14740         ASSERT(help != NULL);
14741 
14742         /*
14743          * If we already have dtrace_helper_providers_max helper providers,
14744          * we're refuse to add a new one.
14745          */
14746         if (help->dthps_nprovs >= dtrace_helper_providers_max)
14747                 return (ENOSPC);
14748 
14749         /*
14750          * Check to make sure this isn't a duplicate.
14751          */
14752         for (i = 0; i < help->dthps_nprovs; i++) {
14753                 if (dofhp->dofhp_dof ==
14754                     help->dthps_provs[i]->dthp_prov.dofhp_dof)
14755                         return (EALREADY);
14756         }
14757 
14758         hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
14759         hprov->dthp_prov = *dofhp;
14760         hprov->dthp_ref = 1;
14761         hprov->dthp_generation = gen;
14762 
14763         /*
14764          * Allocate a bigger table for helper providers if it's already full.
14765          */
14766         if (help->dthps_maxprovs == help->dthps_nprovs) {
14767                 tmp_maxprovs = help->dthps_maxprovs;
14768                 tmp_provs = help->dthps_provs;
14769 
14770                 if (help->dthps_maxprovs == 0)
14771                         help->dthps_maxprovs = 2;
14772                 else
14773                         help->dthps_maxprovs *= 2;
14774                 if (help->dthps_maxprovs > dtrace_helper_providers_max)
14775                         help->dthps_maxprovs = dtrace_helper_providers_max;
14776 
14777                 ASSERT(tmp_maxprovs < help->dthps_maxprovs);
14778 
14779                 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
14780                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
14781 
14782                 if (tmp_provs != NULL) {
14783                         bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
14784                             sizeof (dtrace_helper_provider_t *));
14785                         kmem_free(tmp_provs, tmp_maxprovs *
14786                             sizeof (dtrace_helper_provider_t *));
14787                 }
14788         }
14789 
14790         help->dthps_provs[help->dthps_nprovs] = hprov;
14791         help->dthps_nprovs++;
14792 
14793         return (0);
14794 }
14795 
14796 static void
14797 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
14798 {
14799         mutex_enter(&dtrace_lock);
14800 
14801         if (--hprov->dthp_ref == 0) {
14802                 dof_hdr_t *dof;
14803                 mutex_exit(&dtrace_lock);
14804                 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
14805                 dtrace_dof_destroy(dof);
14806                 kmem_free(hprov, sizeof (dtrace_helper_provider_t));
14807         } else {
14808                 mutex_exit(&dtrace_lock);
14809         }
14810 }
14811 
14812 static int
14813 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
14814 {
14815         uintptr_t daddr = (uintptr_t)dof;
14816         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
14817         dof_provider_t *provider;
14818         dof_probe_t *probe;
14819         uint8_t *arg;
14820         char *strtab, *typestr;
14821         dof_stridx_t typeidx;
14822         size_t typesz;
14823         uint_t nprobes, j, k;
14824 
14825         ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
14826 
14827         if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
14828                 dtrace_dof_error(dof, "misaligned section offset");
14829                 return (-1);
14830         }
14831 
14832         /*
14833          * The section needs to be large enough to contain the DOF provider
14834          * structure appropriate for the given version.
14835          */
14836         if (sec->dofs_size <
14837             ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
14838             offsetof(dof_provider_t, dofpv_prenoffs) :
14839             sizeof (dof_provider_t))) {
14840                 dtrace_dof_error(dof, "provider section too small");
14841                 return (-1);
14842         }
14843 
14844         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
14845         str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
14846         prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
14847         arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
14848         off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
14849 
14850         if (str_sec == NULL || prb_sec == NULL ||
14851             arg_sec == NULL || off_sec == NULL)
14852                 return (-1);
14853 
14854         enoff_sec = NULL;
14855 
14856         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
14857             provider->dofpv_prenoffs != DOF_SECT_NONE &&
14858             (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
14859             provider->dofpv_prenoffs)) == NULL)
14860                 return (-1);
14861 
14862         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
14863 
14864         if (provider->dofpv_name >= str_sec->dofs_size ||
14865             strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
14866                 dtrace_dof_error(dof, "invalid provider name");
14867                 return (-1);
14868         }
14869 
14870         if (prb_sec->dofs_entsize == 0 ||
14871             prb_sec->dofs_entsize > prb_sec->dofs_size) {
14872                 dtrace_dof_error(dof, "invalid entry size");
14873                 return (-1);
14874         }
14875 
14876         if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
14877                 dtrace_dof_error(dof, "misaligned entry size");
14878                 return (-1);
14879         }
14880 
14881         if (off_sec->dofs_entsize != sizeof (uint32_t)) {
14882                 dtrace_dof_error(dof, "invalid entry size");
14883                 return (-1);
14884         }
14885 
14886         if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
14887                 dtrace_dof_error(dof, "misaligned section offset");
14888                 return (-1);
14889         }
14890 
14891         if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
14892                 dtrace_dof_error(dof, "invalid entry size");
14893                 return (-1);
14894         }
14895 
14896         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
14897 
14898         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
14899 
14900         /*
14901          * Take a pass through the probes to check for errors.
14902          */
14903         for (j = 0; j < nprobes; j++) {
14904                 probe = (dof_probe_t *)(uintptr_t)(daddr +
14905                     prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
14906 
14907                 if (probe->dofpr_func >= str_sec->dofs_size) {
14908                         dtrace_dof_error(dof, "invalid function name");
14909                         return (-1);
14910                 }
14911 
14912                 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
14913                         dtrace_dof_error(dof, "function name too long");
14914                         return (-1);
14915                 }
14916 
14917                 if (probe->dofpr_name >= str_sec->dofs_size ||
14918                     strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
14919                         dtrace_dof_error(dof, "invalid probe name");
14920                         return (-1);
14921                 }
14922 
14923                 /*
14924                  * The offset count must not wrap the index, and the offsets
14925                  * must also not overflow the section's data.
14926                  */
14927                 if (probe->dofpr_offidx + probe->dofpr_noffs <
14928                     probe->dofpr_offidx ||
14929                     (probe->dofpr_offidx + probe->dofpr_noffs) *
14930                     off_sec->dofs_entsize > off_sec->dofs_size) {
14931                         dtrace_dof_error(dof, "invalid probe offset");
14932                         return (-1);
14933                 }
14934 
14935                 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
14936                         /*
14937                          * If there's no is-enabled offset section, make sure
14938                          * there aren't any is-enabled offsets. Otherwise
14939                          * perform the same checks as for probe offsets
14940                          * (immediately above).
14941                          */
14942                         if (enoff_sec == NULL) {
14943                                 if (probe->dofpr_enoffidx != 0 ||
14944                                     probe->dofpr_nenoffs != 0) {
14945                                         dtrace_dof_error(dof, "is-enabled "
14946                                             "offsets with null section");
14947                                         return (-1);
14948                                 }
14949                         } else if (probe->dofpr_enoffidx +
14950                             probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
14951                             (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
14952                             enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
14953                                 dtrace_dof_error(dof, "invalid is-enabled "
14954                                     "offset");
14955                                 return (-1);
14956                         }
14957 
14958                         if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
14959                                 dtrace_dof_error(dof, "zero probe and "
14960                                     "is-enabled offsets");
14961                                 return (-1);
14962                         }
14963                 } else if (probe->dofpr_noffs == 0) {
14964                         dtrace_dof_error(dof, "zero probe offsets");
14965                         return (-1);
14966                 }
14967 
14968                 if (probe->dofpr_argidx + probe->dofpr_xargc <
14969                     probe->dofpr_argidx ||
14970                     (probe->dofpr_argidx + probe->dofpr_xargc) *
14971                     arg_sec->dofs_entsize > arg_sec->dofs_size) {
14972                         dtrace_dof_error(dof, "invalid args");
14973                         return (-1);
14974                 }
14975 
14976                 typeidx = probe->dofpr_nargv;
14977                 typestr = strtab + probe->dofpr_nargv;
14978                 for (k = 0; k < probe->dofpr_nargc; k++) {
14979                         if (typeidx >= str_sec->dofs_size) {
14980                                 dtrace_dof_error(dof, "bad "
14981                                     "native argument type");
14982                                 return (-1);
14983                         }
14984 
14985                         typesz = strlen(typestr) + 1;
14986                         if (typesz > DTRACE_ARGTYPELEN) {
14987                                 dtrace_dof_error(dof, "native "
14988                                     "argument type too long");
14989                                 return (-1);
14990                         }
14991                         typeidx += typesz;
14992                         typestr += typesz;
14993                 }
14994 
14995                 typeidx = probe->dofpr_xargv;
14996                 typestr = strtab + probe->dofpr_xargv;
14997                 for (k = 0; k < probe->dofpr_xargc; k++) {
14998                         if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
14999                                 dtrace_dof_error(dof, "bad "
15000                                     "native argument index");
15001                                 return (-1);
15002                         }
15003 
15004                         if (typeidx >= str_sec->dofs_size) {
15005                                 dtrace_dof_error(dof, "bad "
15006                                     "translated argument type");
15007                                 return (-1);
15008                         }
15009 
15010                         typesz = strlen(typestr) + 1;
15011                         if (typesz > DTRACE_ARGTYPELEN) {
15012                                 dtrace_dof_error(dof, "translated argument "
15013                                     "type too long");
15014                                 return (-1);
15015                         }
15016 
15017                         typeidx += typesz;
15018                         typestr += typesz;
15019                 }
15020         }
15021 
15022         return (0);
15023 }
15024 
15025 static int
15026 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
15027 {
15028         dtrace_helpers_t *help;
15029         dtrace_vstate_t *vstate;
15030         dtrace_enabling_t *enab = NULL;
15031         int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
15032         uintptr_t daddr = (uintptr_t)dof;
15033 
15034         ASSERT(MUTEX_HELD(&dtrace_lock));
15035 
15036         if ((help = curproc->p_dtrace_helpers) == NULL)
15037                 help = dtrace_helpers_create(curproc);
15038 
15039         vstate = &help->dthps_vstate;
15040 
15041         if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
15042             dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
15043                 dtrace_dof_destroy(dof);
15044                 return (rv);
15045         }
15046 
15047         /*
15048          * Look for helper providers and validate their descriptions.
15049          */
15050         if (dhp != NULL) {
15051                 for (i = 0; i < dof->dofh_secnum; i++) {
15052                         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
15053                             dof->dofh_secoff + i * dof->dofh_secsize);
15054 
15055                         if (sec->dofs_type != DOF_SECT_PROVIDER)
15056                                 continue;
15057 
15058                         if (dtrace_helper_provider_validate(dof, sec) != 0) {
15059                                 dtrace_enabling_destroy(enab);
15060                                 dtrace_dof_destroy(dof);
15061                                 return (-1);
15062                         }
15063 
15064                         nprovs++;
15065                 }
15066         }
15067 
15068         /*
15069          * Now we need to walk through the ECB descriptions in the enabling.
15070          */
15071         for (i = 0; i < enab->dten_ndesc; i++) {
15072                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
15073                 dtrace_probedesc_t *desc = &ep->dted_probe;
15074 
15075                 if (strcmp(desc->dtpd_provider, "dtrace") != 0)
15076                         continue;
15077 
15078                 if (strcmp(desc->dtpd_mod, "helper") != 0)
15079                         continue;
15080 
15081                 if (strcmp(desc->dtpd_func, "ustack") != 0)
15082                         continue;
15083 
15084                 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
15085                     ep)) != 0) {
15086                         /*
15087                          * Adding this helper action failed -- we are now going
15088                          * to rip out the entire generation and return failure.
15089                          */
15090                         (void) dtrace_helper_destroygen(help->dthps_generation);
15091                         dtrace_enabling_destroy(enab);
15092                         dtrace_dof_destroy(dof);
15093                         return (-1);
15094                 }
15095 
15096                 nhelpers++;
15097         }
15098 
15099         if (nhelpers < enab->dten_ndesc)
15100                 dtrace_dof_error(dof, "unmatched helpers");
15101 
15102         gen = help->dthps_generation++;
15103         dtrace_enabling_destroy(enab);
15104 
15105         if (dhp != NULL && nprovs > 0) {
15106                 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
15107                 if (dtrace_helper_provider_add(dhp, gen) == 0) {
15108                         mutex_exit(&dtrace_lock);
15109                         dtrace_helper_provider_register(curproc, help, dhp);
15110                         mutex_enter(&dtrace_lock);
15111 
15112                         destroy = 0;
15113                 }
15114         }
15115 
15116         if (destroy)
15117                 dtrace_dof_destroy(dof);
15118 
15119         return (gen);
15120 }
15121 
15122 static dtrace_helpers_t *
15123 dtrace_helpers_create(proc_t *p)
15124 {
15125         dtrace_helpers_t *help;
15126 
15127         ASSERT(MUTEX_HELD(&dtrace_lock));
15128         ASSERT(p->p_dtrace_helpers == NULL);
15129 
15130         help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
15131         help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
15132             DTRACE_NHELPER_ACTIONS, KM_SLEEP);
15133 
15134         p->p_dtrace_helpers = help;
15135         dtrace_helpers++;
15136 
15137         return (help);
15138 }
15139 
15140 static void
15141 dtrace_helpers_destroy(void)
15142 {
15143         dtrace_helpers_t *help;
15144         dtrace_vstate_t *vstate;
15145         proc_t *p = curproc;
15146         int i;
15147 
15148         mutex_enter(&dtrace_lock);
15149 
15150         ASSERT(p->p_dtrace_helpers != NULL);
15151         ASSERT(dtrace_helpers > 0);
15152 
15153         help = p->p_dtrace_helpers;
15154         vstate = &help->dthps_vstate;
15155 
15156         /*
15157          * We're now going to lose the help from this process.
15158          */
15159         p->p_dtrace_helpers = NULL;
15160         dtrace_sync();
15161 
15162         /*
15163          * Destory the helper actions.
15164          */
15165         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15166                 dtrace_helper_action_t *h, *next;
15167 
15168                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
15169                         next = h->dtha_next;
15170                         dtrace_helper_action_destroy(h, vstate);
15171                         h = next;
15172                 }
15173         }
15174 
15175         mutex_exit(&dtrace_lock);
15176 
15177         /*
15178          * Destroy the helper providers.
15179          */
15180         if (help->dthps_maxprovs > 0) {
15181                 mutex_enter(&dtrace_meta_lock);
15182                 if (dtrace_meta_pid != NULL) {
15183                         ASSERT(dtrace_deferred_pid == NULL);
15184 
15185                         for (i = 0; i < help->dthps_nprovs; i++) {
15186                                 dtrace_helper_provider_remove(
15187                                     &help->dthps_provs[i]->dthp_prov, p->p_pid);
15188                         }
15189                 } else {
15190                         mutex_enter(&dtrace_lock);
15191                         ASSERT(help->dthps_deferred == 0 ||
15192                             help->dthps_next != NULL ||
15193                             help->dthps_prev != NULL ||
15194                             help == dtrace_deferred_pid);
15195 
15196                         /*
15197                          * Remove the helper from the deferred list.
15198                          */
15199                         if (help->dthps_next != NULL)
15200                                 help->dthps_next->dthps_prev = help->dthps_prev;
15201                         if (help->dthps_prev != NULL)
15202                                 help->dthps_prev->dthps_next = help->dthps_next;
15203                         if (dtrace_deferred_pid == help) {
15204                                 dtrace_deferred_pid = help->dthps_next;
15205                                 ASSERT(help->dthps_prev == NULL);
15206                         }
15207 
15208                         mutex_exit(&dtrace_lock);
15209                 }
15210 
15211                 mutex_exit(&dtrace_meta_lock);
15212 
15213                 for (i = 0; i < help->dthps_nprovs; i++) {
15214                         dtrace_helper_provider_destroy(help->dthps_provs[i]);
15215                 }
15216 
15217                 kmem_free(help->dthps_provs, help->dthps_maxprovs *
15218                     sizeof (dtrace_helper_provider_t *));
15219         }
15220 
15221         mutex_enter(&dtrace_lock);
15222 
15223         dtrace_vstate_fini(&help->dthps_vstate);
15224         kmem_free(help->dthps_actions,
15225             sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
15226         kmem_free(help, sizeof (dtrace_helpers_t));
15227 
15228         --dtrace_helpers;
15229         mutex_exit(&dtrace_lock);
15230 }
15231 
15232 static void
15233 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
15234 {
15235         dtrace_helpers_t *help, *newhelp;
15236         dtrace_helper_action_t *helper, *new, *last;
15237         dtrace_difo_t *dp;
15238         dtrace_vstate_t *vstate;
15239         int i, j, sz, hasprovs = 0;
15240 
15241         mutex_enter(&dtrace_lock);
15242         ASSERT(from->p_dtrace_helpers != NULL);
15243         ASSERT(dtrace_helpers > 0);
15244 
15245         help = from->p_dtrace_helpers;
15246         newhelp = dtrace_helpers_create(to);
15247         ASSERT(to->p_dtrace_helpers != NULL);
15248 
15249         newhelp->dthps_generation = help->dthps_generation;
15250         vstate = &newhelp->dthps_vstate;
15251 
15252         /*
15253          * Duplicate the helper actions.
15254          */
15255         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15256                 if ((helper = help->dthps_actions[i]) == NULL)
15257                         continue;
15258 
15259                 for (last = NULL; helper != NULL; helper = helper->dtha_next) {
15260                         new = kmem_zalloc(sizeof (dtrace_helper_action_t),
15261                             KM_SLEEP);
15262                         new->dtha_generation = helper->dtha_generation;
15263 
15264                         if ((dp = helper->dtha_predicate) != NULL) {
15265                                 dp = dtrace_difo_duplicate(dp, vstate);
15266                                 new->dtha_predicate = dp;
15267                         }
15268 
15269                         new->dtha_nactions = helper->dtha_nactions;
15270                         sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
15271                         new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
15272 
15273                         for (j = 0; j < new->dtha_nactions; j++) {
15274                                 dtrace_difo_t *dp = helper->dtha_actions[j];
15275 
15276                                 ASSERT(dp != NULL);
15277                                 dp = dtrace_difo_duplicate(dp, vstate);
15278                                 new->dtha_actions[j] = dp;
15279                         }
15280 
15281                         if (last != NULL) {
15282                                 last->dtha_next = new;
15283                         } else {
15284                                 newhelp->dthps_actions[i] = new;
15285                         }
15286 
15287                         last = new;
15288                 }
15289         }
15290 
15291         /*
15292          * Duplicate the helper providers and register them with the
15293          * DTrace framework.
15294          */
15295         if (help->dthps_nprovs > 0) {
15296                 newhelp->dthps_nprovs = help->dthps_nprovs;
15297                 newhelp->dthps_maxprovs = help->dthps_nprovs;
15298                 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
15299                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
15300                 for (i = 0; i < newhelp->dthps_nprovs; i++) {
15301                         newhelp->dthps_provs[i] = help->dthps_provs[i];
15302                         newhelp->dthps_provs[i]->dthp_ref++;
15303                 }
15304 
15305                 hasprovs = 1;
15306         }
15307 
15308         mutex_exit(&dtrace_lock);
15309 
15310         if (hasprovs)
15311                 dtrace_helper_provider_register(to, newhelp, NULL);
15312 }
15313 
15314 /*
15315  * DTrace Hook Functions
15316  */
15317 static void
15318 dtrace_module_loaded(struct modctl *ctl)
15319 {
15320         dtrace_provider_t *prv;
15321 
15322         mutex_enter(&dtrace_provider_lock);
15323         mutex_enter(&mod_lock);
15324 
15325         ASSERT(ctl->mod_busy);
15326 
15327         /*
15328          * We're going to call each providers per-module provide operation
15329          * specifying only this module.
15330          */
15331         for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
15332                 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
15333 
15334         mutex_exit(&mod_lock);
15335         mutex_exit(&dtrace_provider_lock);
15336 
15337         /*
15338          * If we have any retained enablings, we need to match against them.
15339          * Enabling probes requires that cpu_lock be held, and we cannot hold
15340          * cpu_lock here -- it is legal for cpu_lock to be held when loading a
15341          * module.  (In particular, this happens when loading scheduling
15342          * classes.)  So if we have any retained enablings, we need to dispatch
15343          * our task queue to do the match for us.
15344          */
15345         mutex_enter(&dtrace_lock);
15346 
15347         if (dtrace_retained == NULL) {
15348                 mutex_exit(&dtrace_lock);
15349                 return;
15350         }
15351 
15352         (void) taskq_dispatch(dtrace_taskq,
15353             (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
15354 
15355         mutex_exit(&dtrace_lock);
15356 
15357         /*
15358          * And now, for a little heuristic sleaze:  in general, we want to
15359          * match modules as soon as they load.  However, we cannot guarantee
15360          * this, because it would lead us to the lock ordering violation
15361          * outlined above.  The common case, of course, is that cpu_lock is
15362          * _not_ held -- so we delay here for a clock tick, hoping that that's
15363          * long enough for the task queue to do its work.  If it's not, it's
15364          * not a serious problem -- it just means that the module that we
15365          * just loaded may not be immediately instrumentable.
15366          */
15367         delay(1);
15368 }
15369 
15370 static void
15371 dtrace_module_unloaded(struct modctl *ctl)
15372 {
15373         dtrace_probe_t template, *probe, *first, *next;
15374         dtrace_provider_t *prov;
15375 
15376         template.dtpr_mod = ctl->mod_modname;
15377 
15378         mutex_enter(&dtrace_provider_lock);
15379         mutex_enter(&mod_lock);
15380         mutex_enter(&dtrace_lock);
15381 
15382         if (dtrace_bymod == NULL) {
15383                 /*
15384                  * The DTrace module is loaded (obviously) but not attached;
15385                  * we don't have any work to do.
15386                  */
15387                 mutex_exit(&dtrace_provider_lock);
15388                 mutex_exit(&mod_lock);
15389                 mutex_exit(&dtrace_lock);
15390                 return;
15391         }
15392 
15393         for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
15394             probe != NULL; probe = probe->dtpr_nextmod) {
15395                 if (probe->dtpr_ecb != NULL) {
15396                         mutex_exit(&dtrace_provider_lock);
15397                         mutex_exit(&mod_lock);
15398                         mutex_exit(&dtrace_lock);
15399 
15400                         /*
15401                          * This shouldn't _actually_ be possible -- we're
15402                          * unloading a module that has an enabled probe in it.
15403                          * (It's normally up to the provider to make sure that
15404                          * this can't happen.)  However, because dtps_enable()
15405                          * doesn't have a failure mode, there can be an
15406                          * enable/unload race.  Upshot:  we don't want to
15407                          * assert, but we're not going to disable the
15408                          * probe, either.
15409                          */
15410                         if (dtrace_err_verbose) {
15411                                 cmn_err(CE_WARN, "unloaded module '%s' had "
15412                                     "enabled probes", ctl->mod_modname);
15413                         }
15414 
15415                         return;
15416                 }
15417         }
15418 
15419         probe = first;
15420 
15421         for (first = NULL; probe != NULL; probe = next) {
15422                 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
15423 
15424                 dtrace_probes[probe->dtpr_id - 1] = NULL;
15425 
15426                 next = probe->dtpr_nextmod;
15427                 dtrace_hash_remove(dtrace_bymod, probe);
15428                 dtrace_hash_remove(dtrace_byfunc, probe);
15429                 dtrace_hash_remove(dtrace_byname, probe);
15430 
15431                 if (first == NULL) {
15432                         first = probe;
15433                         probe->dtpr_nextmod = NULL;
15434                 } else {
15435                         probe->dtpr_nextmod = first;
15436                         first = probe;
15437                 }
15438         }
15439 
15440         /*
15441          * We've removed all of the module's probes from the hash chains and
15442          * from the probe array.  Now issue a dtrace_sync() to be sure that
15443          * everyone has cleared out from any probe array processing.
15444          */
15445         dtrace_sync();
15446 
15447         for (probe = first; probe != NULL; probe = first) {
15448                 first = probe->dtpr_nextmod;
15449                 prov = probe->dtpr_provider;
15450                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
15451                     probe->dtpr_arg);
15452                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
15453                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
15454                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
15455                 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
15456                 kmem_free(probe, sizeof (dtrace_probe_t));
15457         }
15458 
15459         mutex_exit(&dtrace_lock);
15460         mutex_exit(&mod_lock);
15461         mutex_exit(&dtrace_provider_lock);
15462 }
15463 
15464 void
15465 dtrace_suspend(void)
15466 {
15467         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
15468 }
15469 
15470 void
15471 dtrace_resume(void)
15472 {
15473         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
15474 }
15475 
15476 static int
15477 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
15478 {
15479         ASSERT(MUTEX_HELD(&cpu_lock));
15480         mutex_enter(&dtrace_lock);
15481 
15482         switch (what) {
15483         case CPU_CONFIG: {
15484                 dtrace_state_t *state;
15485                 dtrace_optval_t *opt, rs, c;
15486 
15487                 /*
15488                  * For now, we only allocate a new buffer for anonymous state.
15489                  */
15490                 if ((state = dtrace_anon.dta_state) == NULL)
15491                         break;
15492 
15493                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
15494                         break;
15495 
15496                 opt = state->dts_options;
15497                 c = opt[DTRACEOPT_CPU];
15498 
15499                 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
15500                         break;
15501 
15502                 /*
15503                  * Regardless of what the actual policy is, we're going to
15504                  * temporarily set our resize policy to be manual.  We're
15505                  * also going to temporarily set our CPU option to denote
15506                  * the newly configured CPU.
15507                  */
15508                 rs = opt[DTRACEOPT_BUFRESIZE];
15509                 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
15510                 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
15511 
15512                 (void) dtrace_state_buffers(state);
15513 
15514                 opt[DTRACEOPT_BUFRESIZE] = rs;
15515                 opt[DTRACEOPT_CPU] = c;
15516 
15517                 break;
15518         }
15519 
15520         case CPU_UNCONFIG:
15521                 /*
15522                  * We don't free the buffer in the CPU_UNCONFIG case.  (The
15523                  * buffer will be freed when the consumer exits.)
15524                  */
15525                 break;
15526 
15527         default:
15528                 break;
15529         }
15530 
15531         mutex_exit(&dtrace_lock);
15532         return (0);
15533 }
15534 
15535 static void
15536 dtrace_cpu_setup_initial(processorid_t cpu)
15537 {
15538         (void) dtrace_cpu_setup(CPU_CONFIG, cpu);
15539 }
15540 
15541 static void
15542 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
15543 {
15544         if (dtrace_toxranges >= dtrace_toxranges_max) {
15545                 int osize, nsize;
15546                 dtrace_toxrange_t *range;
15547 
15548                 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15549 
15550                 if (osize == 0) {
15551                         ASSERT(dtrace_toxrange == NULL);
15552                         ASSERT(dtrace_toxranges_max == 0);
15553                         dtrace_toxranges_max = 1;
15554                 } else {
15555                         dtrace_toxranges_max <<= 1;
15556                 }
15557 
15558                 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15559                 range = kmem_zalloc(nsize, KM_SLEEP);
15560 
15561                 if (dtrace_toxrange != NULL) {
15562                         ASSERT(osize != 0);
15563                         bcopy(dtrace_toxrange, range, osize);
15564                         kmem_free(dtrace_toxrange, osize);
15565                 }
15566 
15567                 dtrace_toxrange = range;
15568         }
15569 
15570         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
15571         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
15572 
15573         dtrace_toxrange[dtrace_toxranges].dtt_base = base;
15574         dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
15575         dtrace_toxranges++;
15576 }
15577 
15578 static void
15579 dtrace_getf_barrier()
15580 {
15581         /*
15582          * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
15583          * that contain calls to getf(), this routine will be called on every
15584          * closef() before either the underlying vnode is released or the
15585          * file_t itself is freed.  By the time we are here, it is essential
15586          * that the file_t can no longer be accessed from a call to getf()
15587          * in probe context -- that assures that a dtrace_sync() can be used
15588          * to clear out any enablings referring to the old structures.
15589          */
15590         if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 ||
15591             kcred->cr_zone->zone_dtrace_getf != 0)
15592                 dtrace_sync();
15593 }
15594 
15595 /*
15596  * DTrace Driver Cookbook Functions
15597  */
15598 /*ARGSUSED*/
15599 static int
15600 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
15601 {
15602         dtrace_provider_id_t id;
15603         dtrace_state_t *state = NULL;
15604         dtrace_enabling_t *enab;
15605 
15606         mutex_enter(&cpu_lock);
15607         mutex_enter(&dtrace_provider_lock);
15608         mutex_enter(&dtrace_lock);
15609 
15610         if (ddi_soft_state_init(&dtrace_softstate,
15611             sizeof (dtrace_state_t), 0) != 0) {
15612                 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
15613                 mutex_exit(&cpu_lock);
15614                 mutex_exit(&dtrace_provider_lock);
15615                 mutex_exit(&dtrace_lock);
15616                 return (DDI_FAILURE);
15617         }
15618 
15619         if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
15620             DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
15621             ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
15622             DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
15623                 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
15624                 ddi_remove_minor_node(devi, NULL);
15625                 ddi_soft_state_fini(&dtrace_softstate);
15626                 mutex_exit(&cpu_lock);
15627                 mutex_exit(&dtrace_provider_lock);
15628                 mutex_exit(&dtrace_lock);
15629                 return (DDI_FAILURE);
15630         }
15631 
15632         ddi_report_dev(devi);
15633         dtrace_devi = devi;
15634 
15635         dtrace_modload = dtrace_module_loaded;
15636         dtrace_modunload = dtrace_module_unloaded;
15637         dtrace_cpu_init = dtrace_cpu_setup_initial;
15638         dtrace_helpers_cleanup = dtrace_helpers_destroy;
15639         dtrace_helpers_fork = dtrace_helpers_duplicate;
15640         dtrace_cpustart_init = dtrace_suspend;
15641         dtrace_cpustart_fini = dtrace_resume;
15642         dtrace_debugger_init = dtrace_suspend;
15643         dtrace_debugger_fini = dtrace_resume;
15644 
15645         register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
15646 
15647         ASSERT(MUTEX_HELD(&cpu_lock));
15648 
15649         dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
15650             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
15651         dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
15652             UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
15653             VM_SLEEP | VMC_IDENTIFIER);
15654         dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
15655             1, INT_MAX, 0);
15656 
15657         dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
15658             sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
15659             NULL, NULL, NULL, NULL, NULL, 0);
15660 
15661         ASSERT(MUTEX_HELD(&cpu_lock));
15662         dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
15663             offsetof(dtrace_probe_t, dtpr_nextmod),
15664             offsetof(dtrace_probe_t, dtpr_prevmod));
15665 
15666         dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
15667             offsetof(dtrace_probe_t, dtpr_nextfunc),
15668             offsetof(dtrace_probe_t, dtpr_prevfunc));
15669 
15670         dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
15671             offsetof(dtrace_probe_t, dtpr_nextname),
15672             offsetof(dtrace_probe_t, dtpr_prevname));
15673 
15674         if (dtrace_retain_max < 1) {
15675                 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
15676                     "setting to 1", dtrace_retain_max);
15677                 dtrace_retain_max = 1;
15678         }
15679 
15680         /*
15681          * Now discover our toxic ranges.
15682          */
15683         dtrace_toxic_ranges(dtrace_toxrange_add);
15684 
15685         /*
15686          * Before we register ourselves as a provider to our own framework,
15687          * we would like to assert that dtrace_provider is NULL -- but that's
15688          * not true if we were loaded as a dependency of a DTrace provider.
15689          * Once we've registered, we can assert that dtrace_provider is our
15690          * pseudo provider.
15691          */
15692         (void) dtrace_register("dtrace", &dtrace_provider_attr,
15693             DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
15694 
15695         ASSERT(dtrace_provider != NULL);
15696         ASSERT((dtrace_provider_id_t)dtrace_provider == id);
15697 
15698         dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
15699             dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
15700         dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
15701             dtrace_provider, NULL, NULL, "END", 0, NULL);
15702         dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
15703             dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
15704 
15705         dtrace_anon_property();
15706         mutex_exit(&cpu_lock);
15707 
15708         /*
15709          * If DTrace helper tracing is enabled, we need to allocate the
15710          * trace buffer and initialize the values.
15711          */
15712         if (dtrace_helptrace_enabled) {
15713                 ASSERT(dtrace_helptrace_buffer == NULL);
15714                 dtrace_helptrace_buffer =
15715                     kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
15716                 dtrace_helptrace_next = 0;
15717         }
15718 
15719         /*
15720          * If there are already providers, we must ask them to provide their
15721          * probes, and then match any anonymous enabling against them.  Note
15722          * that there should be no other retained enablings at this time:
15723          * the only retained enablings at this time should be the anonymous
15724          * enabling.
15725          */
15726         if (dtrace_anon.dta_enabling != NULL) {
15727                 ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
15728 
15729                 dtrace_enabling_provide(NULL);
15730                 state = dtrace_anon.dta_state;
15731 
15732                 /*
15733                  * We couldn't hold cpu_lock across the above call to
15734                  * dtrace_enabling_provide(), but we must hold it to actually
15735                  * enable the probes.  We have to drop all of our locks, pick
15736                  * up cpu_lock, and regain our locks before matching the
15737                  * retained anonymous enabling.
15738                  */
15739                 mutex_exit(&dtrace_lock);
15740                 mutex_exit(&dtrace_provider_lock);
15741 
15742                 mutex_enter(&cpu_lock);
15743                 mutex_enter(&dtrace_provider_lock);
15744                 mutex_enter(&dtrace_lock);
15745 
15746                 if ((enab = dtrace_anon.dta_enabling) != NULL)
15747                         (void) dtrace_enabling_match(enab, NULL);
15748 
15749                 mutex_exit(&cpu_lock);
15750         }
15751 
15752         mutex_exit(&dtrace_lock);
15753         mutex_exit(&dtrace_provider_lock);
15754 
15755         if (state != NULL) {
15756                 /*
15757                  * If we created any anonymous state, set it going now.
15758                  */
15759                 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
15760         }
15761 
15762         return (DDI_SUCCESS);
15763 }
15764 
15765 /*ARGSUSED*/
15766 static int
15767 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
15768 {
15769         dtrace_state_t *state;
15770         uint32_t priv;
15771         uid_t uid;
15772         zoneid_t zoneid;
15773 
15774         if (getminor(*devp) == DTRACEMNRN_HELPER)
15775                 return (0);
15776 
15777         /*
15778          * If this wasn't an open with the "helper" minor, then it must be
15779          * the "dtrace" minor.
15780          */
15781         if (getminor(*devp) != DTRACEMNRN_DTRACE)
15782                 return (ENXIO);
15783 
15784         /*
15785          * If no DTRACE_PRIV_* bits are set in the credential, then the
15786          * caller lacks sufficient permission to do anything with DTrace.
15787          */
15788         dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
15789         if (priv == DTRACE_PRIV_NONE)
15790                 return (EACCES);
15791 
15792         /*
15793          * Ask all providers to provide all their probes.
15794          */
15795         mutex_enter(&dtrace_provider_lock);
15796         dtrace_probe_provide(NULL, NULL);
15797         mutex_exit(&dtrace_provider_lock);
15798 
15799         mutex_enter(&cpu_lock);
15800         mutex_enter(&dtrace_lock);
15801         dtrace_opens++;
15802         dtrace_membar_producer();
15803 
15804         /*
15805          * If the kernel debugger is active (that is, if the kernel debugger
15806          * modified text in some way), we won't allow the open.
15807          */
15808         if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
15809                 dtrace_opens--;
15810                 mutex_exit(&cpu_lock);
15811                 mutex_exit(&dtrace_lock);
15812                 return (EBUSY);
15813         }
15814 
15815         state = dtrace_state_create(devp, cred_p);
15816         mutex_exit(&cpu_lock);
15817 
15818         if (state == NULL) {
15819                 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15820                         (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15821                 mutex_exit(&dtrace_lock);
15822                 return (EAGAIN);
15823         }
15824 
15825         mutex_exit(&dtrace_lock);
15826 
15827         return (0);
15828 }
15829 
15830 /*ARGSUSED*/
15831 static int
15832 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
15833 {
15834         minor_t minor = getminor(dev);
15835         dtrace_state_t *state;
15836 
15837         if (minor == DTRACEMNRN_HELPER)
15838                 return (0);
15839 
15840         state = ddi_get_soft_state(dtrace_softstate, minor);
15841 
15842         mutex_enter(&cpu_lock);
15843         mutex_enter(&dtrace_lock);
15844 
15845         if (state->dts_anon) {
15846                 /*
15847                  * There is anonymous state. Destroy that first.
15848                  */
15849                 ASSERT(dtrace_anon.dta_state == NULL);
15850                 dtrace_state_destroy(state->dts_anon);
15851         }
15852 
15853         dtrace_state_destroy(state);
15854         ASSERT(dtrace_opens > 0);
15855 
15856         /*
15857          * Only relinquish control of the kernel debugger interface when there
15858          * are no consumers and no anonymous enablings.
15859          */
15860         if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15861                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15862 
15863         mutex_exit(&dtrace_lock);
15864         mutex_exit(&cpu_lock);
15865 
15866         return (0);
15867 }
15868 
15869 /*ARGSUSED*/
15870 static int
15871 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
15872 {
15873         int rval;
15874         dof_helper_t help, *dhp = NULL;
15875 
15876         switch (cmd) {
15877         case DTRACEHIOC_ADDDOF:
15878                 if (copyin((void *)arg, &help, sizeof (help)) != 0) {
15879                         dtrace_dof_error(NULL, "failed to copyin DOF helper");
15880                         return (EFAULT);
15881                 }
15882 
15883                 dhp = &help;
15884                 arg = (intptr_t)help.dofhp_dof;
15885                 /*FALLTHROUGH*/
15886 
15887         case DTRACEHIOC_ADD: {
15888                 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
15889 
15890                 if (dof == NULL)
15891                         return (rval);
15892 
15893                 mutex_enter(&dtrace_lock);
15894 
15895                 /*
15896                  * dtrace_helper_slurp() takes responsibility for the dof --
15897                  * it may free it now or it may save it and free it later.
15898                  */
15899                 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
15900                         *rv = rval;
15901                         rval = 0;
15902                 } else {
15903                         rval = EINVAL;
15904                 }
15905 
15906                 mutex_exit(&dtrace_lock);
15907                 return (rval);
15908         }
15909 
15910         case DTRACEHIOC_REMOVE: {
15911                 mutex_enter(&dtrace_lock);
15912                 rval = dtrace_helper_destroygen(arg);
15913                 mutex_exit(&dtrace_lock);
15914 
15915                 return (rval);
15916         }
15917 
15918         default:
15919                 break;
15920         }
15921 
15922         return (ENOTTY);
15923 }
15924 
15925 /*ARGSUSED*/
15926 static int
15927 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
15928 {
15929         minor_t minor = getminor(dev);
15930         dtrace_state_t *state;
15931         int rval;
15932 
15933         if (minor == DTRACEMNRN_HELPER)
15934                 return (dtrace_ioctl_helper(cmd, arg, rv));
15935 
15936         state = ddi_get_soft_state(dtrace_softstate, minor);
15937 
15938         if (state->dts_anon) {
15939                 ASSERT(dtrace_anon.dta_state == NULL);
15940                 state = state->dts_anon;
15941         }
15942 
15943         switch (cmd) {
15944         case DTRACEIOC_PROVIDER: {
15945                 dtrace_providerdesc_t pvd;
15946                 dtrace_provider_t *pvp;
15947 
15948                 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
15949                         return (EFAULT);
15950 
15951                 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
15952                 mutex_enter(&dtrace_provider_lock);
15953 
15954                 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
15955                         if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
15956                                 break;
15957                 }
15958 
15959                 mutex_exit(&dtrace_provider_lock);
15960 
15961                 if (pvp == NULL)
15962                         return (ESRCH);
15963 
15964                 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
15965                 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
15966                 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
15967                         return (EFAULT);
15968 
15969                 return (0);
15970         }
15971 
15972         case DTRACEIOC_EPROBE: {
15973                 dtrace_eprobedesc_t epdesc;
15974                 dtrace_ecb_t *ecb;
15975                 dtrace_action_t *act;
15976                 void *buf;
15977                 size_t size;
15978                 uintptr_t dest;
15979                 int nrecs;
15980 
15981                 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
15982                         return (EFAULT);
15983 
15984                 mutex_enter(&dtrace_lock);
15985 
15986                 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
15987                         mutex_exit(&dtrace_lock);
15988                         return (EINVAL);
15989                 }
15990 
15991                 if (ecb->dte_probe == NULL) {
15992                         mutex_exit(&dtrace_lock);
15993                         return (EINVAL);
15994                 }
15995 
15996                 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
15997                 epdesc.dtepd_uarg = ecb->dte_uarg;
15998                 epdesc.dtepd_size = ecb->dte_size;
15999 
16000                 nrecs = epdesc.dtepd_nrecs;
16001                 epdesc.dtepd_nrecs = 0;
16002                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
16003                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
16004                                 continue;
16005 
16006                         epdesc.dtepd_nrecs++;
16007                 }
16008 
16009                 /*
16010                  * Now that we have the size, we need to allocate a temporary
16011                  * buffer in which to store the complete description.  We need
16012                  * the temporary buffer to be able to drop dtrace_lock()
16013                  * across the copyout(), below.
16014                  */
16015                 size = sizeof (dtrace_eprobedesc_t) +
16016                     (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
16017 
16018                 buf = kmem_alloc(size, KM_SLEEP);
16019                 dest = (uintptr_t)buf;
16020 
16021                 bcopy(&epdesc, (void *)dest, sizeof (epdesc));
16022                 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
16023 
16024                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
16025                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
16026                                 continue;
16027 
16028                         if (nrecs-- == 0)
16029                                 break;
16030 
16031                         bcopy(&act->dta_rec, (void *)dest,
16032                             sizeof (dtrace_recdesc_t));
16033                         dest += sizeof (dtrace_recdesc_t);
16034                 }
16035 
16036                 mutex_exit(&dtrace_lock);
16037 
16038                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
16039                         kmem_free(buf, size);
16040                         return (EFAULT);
16041                 }
16042 
16043                 kmem_free(buf, size);
16044                 return (0);
16045         }
16046 
16047         case DTRACEIOC_AGGDESC: {
16048                 dtrace_aggdesc_t aggdesc;
16049                 dtrace_action_t *act;
16050                 dtrace_aggregation_t *agg;
16051                 int nrecs;
16052                 uint32_t offs;
16053                 dtrace_recdesc_t *lrec;
16054                 void *buf;
16055                 size_t size;
16056                 uintptr_t dest;
16057 
16058                 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
16059                         return (EFAULT);
16060 
16061                 mutex_enter(&dtrace_lock);
16062 
16063                 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
16064                         mutex_exit(&dtrace_lock);
16065                         return (EINVAL);
16066                 }
16067 
16068                 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
16069 
16070                 nrecs = aggdesc.dtagd_nrecs;
16071                 aggdesc.dtagd_nrecs = 0;
16072 
16073                 offs = agg->dtag_base;
16074                 lrec = &agg->dtag_action.dta_rec;
16075                 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
16076 
16077                 for (act = agg->dtag_first; ; act = act->dta_next) {
16078                         ASSERT(act->dta_intuple ||
16079                             DTRACEACT_ISAGG(act->dta_kind));
16080 
16081                         /*
16082                          * If this action has a record size of zero, it
16083                          * denotes an argument to the aggregating action.
16084                          * Because the presence of this record doesn't (or
16085                          * shouldn't) affect the way the data is interpreted,
16086                          * we don't copy it out to save user-level the
16087                          * confusion of dealing with a zero-length record.
16088                          */
16089                         if (act->dta_rec.dtrd_size == 0) {
16090                                 ASSERT(agg->dtag_hasarg);
16091                                 continue;
16092                         }
16093 
16094                         aggdesc.dtagd_nrecs++;
16095 
16096                         if (act == &agg->dtag_action)
16097                                 break;
16098                 }
16099 
16100                 /*
16101                  * Now that we have the size, we need to allocate a temporary
16102                  * buffer in which to store the complete description.  We need
16103                  * the temporary buffer to be able to drop dtrace_lock()
16104                  * across the copyout(), below.
16105                  */
16106                 size = sizeof (dtrace_aggdesc_t) +
16107                     (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
16108 
16109                 buf = kmem_alloc(size, KM_SLEEP);
16110                 dest = (uintptr_t)buf;
16111 
16112                 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
16113                 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
16114 
16115                 for (act = agg->dtag_first; ; act = act->dta_next) {
16116                         dtrace_recdesc_t rec = act->dta_rec;
16117 
16118                         /*
16119                          * See the comment in the above loop for why we pass
16120                          * over zero-length records.
16121                          */
16122                         if (rec.dtrd_size == 0) {
16123                                 ASSERT(agg->dtag_hasarg);
16124                                 continue;
16125                         }
16126 
16127                         if (nrecs-- == 0)
16128                                 break;
16129 
16130                         rec.dtrd_offset -= offs;
16131                         bcopy(&rec, (void *)dest, sizeof (rec));
16132                         dest += sizeof (dtrace_recdesc_t);
16133 
16134                         if (act == &agg->dtag_action)
16135                                 break;
16136                 }
16137 
16138                 mutex_exit(&dtrace_lock);
16139 
16140                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
16141                         kmem_free(buf, size);
16142                         return (EFAULT);
16143                 }
16144 
16145                 kmem_free(buf, size);
16146                 return (0);
16147         }
16148 
16149         case DTRACEIOC_ENABLE: {
16150                 dof_hdr_t *dof;
16151                 dtrace_enabling_t *enab = NULL;
16152                 dtrace_vstate_t *vstate;
16153                 int err = 0;
16154 
16155                 *rv = 0;
16156 
16157                 /*
16158                  * If a NULL argument has been passed, we take this as our
16159                  * cue to reevaluate our enablings.
16160                  */
16161                 if (arg == NULL) {
16162                         dtrace_enabling_matchall();
16163 
16164                         return (0);
16165                 }
16166 
16167                 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
16168                         return (rval);
16169 
16170                 mutex_enter(&cpu_lock);
16171                 mutex_enter(&dtrace_lock);
16172                 vstate = &state->dts_vstate;
16173 
16174                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
16175                         mutex_exit(&dtrace_lock);
16176                         mutex_exit(&cpu_lock);
16177                         dtrace_dof_destroy(dof);
16178                         return (EBUSY);
16179                 }
16180 
16181                 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
16182                         mutex_exit(&dtrace_lock);
16183                         mutex_exit(&cpu_lock);
16184                         dtrace_dof_destroy(dof);
16185                         return (EINVAL);
16186                 }
16187 
16188                 if ((rval = dtrace_dof_options(dof, state)) != 0) {
16189                         dtrace_enabling_destroy(enab);
16190                         mutex_exit(&dtrace_lock);
16191                         mutex_exit(&cpu_lock);
16192                         dtrace_dof_destroy(dof);
16193                         return (rval);
16194                 }
16195 
16196                 if ((err = dtrace_enabling_match(enab, rv)) == 0) {
16197                         err = dtrace_enabling_retain(enab);
16198                 } else {
16199                         dtrace_enabling_destroy(enab);
16200                 }
16201 
16202                 mutex_exit(&cpu_lock);
16203                 mutex_exit(&dtrace_lock);
16204                 dtrace_dof_destroy(dof);
16205 
16206                 return (err);
16207         }
16208 
16209         case DTRACEIOC_REPLICATE: {
16210                 dtrace_repldesc_t desc;
16211                 dtrace_probedesc_t *match = &desc.dtrpd_match;
16212                 dtrace_probedesc_t *create = &desc.dtrpd_create;
16213                 int err;
16214 
16215                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16216                         return (EFAULT);
16217 
16218                 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16219                 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16220                 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16221                 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16222 
16223                 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16224                 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16225                 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16226                 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16227 
16228                 mutex_enter(&dtrace_lock);
16229                 err = dtrace_enabling_replicate(state, match, create);
16230                 mutex_exit(&dtrace_lock);
16231 
16232                 return (err);
16233         }
16234 
16235         case DTRACEIOC_PROBEMATCH:
16236         case DTRACEIOC_PROBES: {
16237                 dtrace_probe_t *probe = NULL;
16238                 dtrace_probedesc_t desc;
16239                 dtrace_probekey_t pkey;
16240                 dtrace_id_t i;
16241                 int m = 0;
16242                 uint32_t priv;
16243                 uid_t uid;
16244                 zoneid_t zoneid;
16245 
16246                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16247                         return (EFAULT);
16248 
16249                 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16250                 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16251                 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16252                 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16253 
16254                 /*
16255                  * Before we attempt to match this probe, we want to give
16256                  * all providers the opportunity to provide it.
16257                  */
16258                 if (desc.dtpd_id == DTRACE_IDNONE) {
16259                         mutex_enter(&dtrace_provider_lock);
16260                         dtrace_probe_provide(&desc, NULL);
16261                         mutex_exit(&dtrace_provider_lock);
16262                         desc.dtpd_id++;
16263                 }
16264 
16265                 if (cmd == DTRACEIOC_PROBEMATCH)  {
16266                         dtrace_probekey(&desc, &pkey);
16267                         pkey.dtpk_id = DTRACE_IDNONE;
16268                 }
16269 
16270                 dtrace_cred2priv(cr, &priv, &uid, &zoneid);
16271 
16272                 mutex_enter(&dtrace_lock);
16273 
16274                 if (cmd == DTRACEIOC_PROBEMATCH) {
16275                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16276                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16277                                     (m = dtrace_match_probe(probe, &pkey,
16278                                     priv, uid, zoneid)) != 0)
16279                                         break;
16280                         }
16281 
16282                         if (m < 0) {
16283                                 mutex_exit(&dtrace_lock);
16284                                 return (EINVAL);
16285                         }
16286 
16287                 } else {
16288                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16289                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16290                                     dtrace_match_priv(probe, priv, uid, zoneid))
16291                                         break;
16292                         }
16293                 }
16294 
16295                 if (probe == NULL) {
16296                         mutex_exit(&dtrace_lock);
16297                         return (ESRCH);
16298                 }
16299 
16300                 dtrace_probe_description(probe, &desc);
16301                 mutex_exit(&dtrace_lock);
16302 
16303                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16304                         return (EFAULT);
16305 
16306                 return (0);
16307         }
16308 
16309         case DTRACEIOC_PROBEARG: {
16310                 dtrace_argdesc_t desc;
16311                 dtrace_probe_t *probe;
16312                 dtrace_provider_t *prov;
16313 
16314                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16315                         return (EFAULT);
16316 
16317                 if (desc.dtargd_id == DTRACE_IDNONE)
16318                         return (EINVAL);
16319 
16320                 if (desc.dtargd_ndx == DTRACE_ARGNONE)
16321                         return (EINVAL);
16322 
16323                 mutex_enter(&dtrace_provider_lock);
16324                 mutex_enter(&mod_lock);
16325                 mutex_enter(&dtrace_lock);
16326 
16327                 if (desc.dtargd_id > dtrace_nprobes) {
16328                         mutex_exit(&dtrace_lock);
16329                         mutex_exit(&mod_lock);
16330                         mutex_exit(&dtrace_provider_lock);
16331                         return (EINVAL);
16332                 }
16333 
16334                 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
16335                         mutex_exit(&dtrace_lock);
16336                         mutex_exit(&mod_lock);
16337                         mutex_exit(&dtrace_provider_lock);
16338                         return (EINVAL);
16339                 }
16340 
16341                 mutex_exit(&dtrace_lock);
16342 
16343                 prov = probe->dtpr_provider;
16344 
16345                 if (prov->dtpv_pops.dtps_getargdesc == NULL) {
16346                         /*
16347                          * There isn't any typed information for this probe.
16348                          * Set the argument number to DTRACE_ARGNONE.
16349                          */
16350                         desc.dtargd_ndx = DTRACE_ARGNONE;
16351                 } else {
16352                         desc.dtargd_native[0] = '\0';
16353                         desc.dtargd_xlate[0] = '\0';
16354                         desc.dtargd_mapping = desc.dtargd_ndx;
16355 
16356                         prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
16357                             probe->dtpr_id, probe->dtpr_arg, &desc);
16358                 }
16359 
16360                 mutex_exit(&mod_lock);
16361                 mutex_exit(&dtrace_provider_lock);
16362 
16363                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16364                         return (EFAULT);
16365 
16366                 return (0);
16367         }
16368 
16369         case DTRACEIOC_GO: {
16370                 processorid_t cpuid;
16371                 rval = dtrace_state_go(state, &cpuid);
16372 
16373                 if (rval != 0)
16374                         return (rval);
16375 
16376                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16377                         return (EFAULT);
16378 
16379                 return (0);
16380         }
16381 
16382         case DTRACEIOC_STOP: {
16383                 processorid_t cpuid;
16384 
16385                 mutex_enter(&dtrace_lock);
16386                 rval = dtrace_state_stop(state, &cpuid);
16387                 mutex_exit(&dtrace_lock);
16388 
16389                 if (rval != 0)
16390                         return (rval);
16391 
16392                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16393                         return (EFAULT);
16394 
16395                 return (0);
16396         }
16397 
16398         case DTRACEIOC_DOFGET: {
16399                 dof_hdr_t hdr, *dof;
16400                 uint64_t len;
16401 
16402                 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
16403                         return (EFAULT);
16404 
16405                 mutex_enter(&dtrace_lock);
16406                 dof = dtrace_dof_create(state);
16407                 mutex_exit(&dtrace_lock);
16408 
16409                 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
16410                 rval = copyout(dof, (void *)arg, len);
16411                 dtrace_dof_destroy(dof);
16412 
16413                 return (rval == 0 ? 0 : EFAULT);
16414         }
16415 
16416         case DTRACEIOC_AGGSNAP:
16417         case DTRACEIOC_BUFSNAP: {
16418                 dtrace_bufdesc_t desc;
16419                 caddr_t cached;
16420                 dtrace_buffer_t *buf;
16421 
16422                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16423                         return (EFAULT);
16424 
16425                 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
16426                         return (EINVAL);
16427 
16428                 mutex_enter(&dtrace_lock);
16429 
16430                 if (cmd == DTRACEIOC_BUFSNAP) {
16431                         buf = &state->dts_buffer[desc.dtbd_cpu];
16432                 } else {
16433                         buf = &state->dts_aggbuffer[desc.dtbd_cpu];
16434                 }
16435 
16436                 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
16437                         size_t sz = buf->dtb_offset;
16438 
16439                         if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
16440                                 mutex_exit(&dtrace_lock);
16441                                 return (EBUSY);
16442                         }
16443 
16444                         /*
16445                          * If this buffer has already been consumed, we're
16446                          * going to indicate that there's nothing left here
16447                          * to consume.
16448                          */
16449                         if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
16450                                 mutex_exit(&dtrace_lock);
16451 
16452                                 desc.dtbd_size = 0;
16453                                 desc.dtbd_drops = 0;
16454                                 desc.dtbd_errors = 0;
16455                                 desc.dtbd_oldest = 0;
16456                                 sz = sizeof (desc);
16457 
16458                                 if (copyout(&desc, (void *)arg, sz) != 0)
16459                                         return (EFAULT);
16460 
16461                                 return (0);
16462                         }
16463 
16464                         /*
16465                          * If this is a ring buffer that has wrapped, we want
16466                          * to copy the whole thing out.
16467                          */
16468                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
16469                                 dtrace_buffer_polish(buf);
16470                                 sz = buf->dtb_size;
16471                         }
16472 
16473                         if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
16474                                 mutex_exit(&dtrace_lock);
16475                                 return (EFAULT);
16476                         }
16477 
16478                         desc.dtbd_size = sz;
16479                         desc.dtbd_drops = buf->dtb_drops;
16480                         desc.dtbd_errors = buf->dtb_errors;
16481                         desc.dtbd_oldest = buf->dtb_xamot_offset;
16482                         desc.dtbd_timestamp = dtrace_gethrtime();
16483 
16484                         mutex_exit(&dtrace_lock);
16485 
16486                         if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16487                                 return (EFAULT);
16488 
16489                         buf->dtb_flags |= DTRACEBUF_CONSUMED;
16490 
16491                         return (0);
16492                 }
16493 
16494                 if (buf->dtb_tomax == NULL) {
16495                         ASSERT(buf->dtb_xamot == NULL);
16496                         mutex_exit(&dtrace_lock);
16497                         return (ENOENT);
16498                 }
16499 
16500                 cached = buf->dtb_tomax;
16501                 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
16502 
16503                 dtrace_xcall(desc.dtbd_cpu,
16504                     (dtrace_xcall_t)dtrace_buffer_switch, buf);
16505 
16506                 state->dts_errors += buf->dtb_xamot_errors;
16507 
16508                 /*
16509                  * If the buffers did not actually switch, then the cross call
16510                  * did not take place -- presumably because the given CPU is
16511                  * not in the ready set.  If this is the case, we'll return
16512                  * ENOENT.
16513                  */
16514                 if (buf->dtb_tomax == cached) {
16515                         ASSERT(buf->dtb_xamot != cached);
16516                         mutex_exit(&dtrace_lock);
16517                         return (ENOENT);
16518                 }
16519 
16520                 ASSERT(cached == buf->dtb_xamot);
16521 
16522                 /*
16523                  * We have our snapshot; now copy it out.
16524                  */
16525                 if (copyout(buf->dtb_xamot, desc.dtbd_data,
16526                     buf->dtb_xamot_offset) != 0) {
16527                         mutex_exit(&dtrace_lock);
16528                         return (EFAULT);
16529                 }
16530 
16531                 desc.dtbd_size = buf->dtb_xamot_offset;
16532                 desc.dtbd_drops = buf->dtb_xamot_drops;
16533                 desc.dtbd_errors = buf->dtb_xamot_errors;
16534                 desc.dtbd_oldest = 0;
16535                 desc.dtbd_timestamp = buf->dtb_switched;
16536 
16537                 mutex_exit(&dtrace_lock);
16538 
16539                 /*
16540                  * Finally, copy out the buffer description.
16541                  */
16542                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16543                         return (EFAULT);
16544 
16545                 return (0);
16546         }
16547 
16548         case DTRACEIOC_CONF: {
16549                 dtrace_conf_t conf;
16550 
16551                 bzero(&conf, sizeof (conf));
16552                 conf.dtc_difversion = DIF_VERSION;
16553                 conf.dtc_difintregs = DIF_DIR_NREGS;
16554                 conf.dtc_diftupregs = DIF_DTR_NREGS;
16555                 conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
16556 
16557                 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
16558                         return (EFAULT);
16559 
16560                 return (0);
16561         }
16562 
16563         case DTRACEIOC_STATUS: {
16564                 dtrace_status_t stat;
16565                 dtrace_dstate_t *dstate;
16566                 int i, j;
16567                 uint64_t nerrs;
16568 
16569                 /*
16570                  * See the comment in dtrace_state_deadman() for the reason
16571                  * for setting dts_laststatus to INT64_MAX before setting
16572                  * it to the correct value.
16573                  */
16574                 state->dts_laststatus = INT64_MAX;
16575                 dtrace_membar_producer();
16576                 state->dts_laststatus = dtrace_gethrtime();
16577 
16578                 bzero(&stat, sizeof (stat));
16579 
16580                 mutex_enter(&dtrace_lock);
16581 
16582                 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
16583                         mutex_exit(&dtrace_lock);
16584                         return (ENOENT);
16585                 }
16586 
16587                 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
16588                         stat.dtst_exiting = 1;
16589 
16590                 nerrs = state->dts_errors;
16591                 dstate = &state->dts_vstate.dtvs_dynvars;
16592 
16593                 for (i = 0; i < NCPU; i++) {
16594                         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
16595 
16596                         stat.dtst_dyndrops += dcpu->dtdsc_drops;
16597                         stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
16598                         stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
16599 
16600                         if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
16601                                 stat.dtst_filled++;
16602 
16603                         nerrs += state->dts_buffer[i].dtb_errors;
16604 
16605                         for (j = 0; j < state->dts_nspeculations; j++) {
16606                                 dtrace_speculation_t *spec;
16607                                 dtrace_buffer_t *buf;
16608 
16609                                 spec = &state->dts_speculations[j];
16610                                 buf = &spec->dtsp_buffer[i];
16611                                 stat.dtst_specdrops += buf->dtb_xamot_drops;
16612                         }
16613                 }
16614 
16615                 stat.dtst_specdrops_busy = state->dts_speculations_busy;
16616                 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
16617                 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
16618                 stat.dtst_dblerrors = state->dts_dblerrors;
16619                 stat.dtst_killed =
16620                     (state->dts_activity == DTRACE_ACTIVITY_KILLED);
16621                 stat.dtst_errors = nerrs;
16622 
16623                 mutex_exit(&dtrace_lock);
16624 
16625                 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
16626                         return (EFAULT);
16627 
16628                 return (0);
16629         }
16630 
16631         case DTRACEIOC_FORMAT: {
16632                 dtrace_fmtdesc_t fmt;
16633                 char *str;
16634                 int len;
16635 
16636                 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
16637                         return (EFAULT);
16638 
16639                 mutex_enter(&dtrace_lock);
16640 
16641                 if (fmt.dtfd_format == 0 ||
16642                     fmt.dtfd_format > state->dts_nformats) {
16643                         mutex_exit(&dtrace_lock);
16644                         return (EINVAL);
16645                 }
16646 
16647                 /*
16648                  * Format strings are allocated contiguously and they are
16649                  * never freed; if a format index is less than the number
16650                  * of formats, we can assert that the format map is non-NULL
16651                  * and that the format for the specified index is non-NULL.
16652                  */
16653                 ASSERT(state->dts_formats != NULL);
16654                 str = state->dts_formats[fmt.dtfd_format - 1];
16655                 ASSERT(str != NULL);
16656 
16657                 len = strlen(str) + 1;
16658 
16659                 if (len > fmt.dtfd_length) {
16660                         fmt.dtfd_length = len;
16661 
16662                         if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
16663                                 mutex_exit(&dtrace_lock);
16664                                 return (EINVAL);
16665                         }
16666                 } else {
16667                         if (copyout(str, fmt.dtfd_string, len) != 0) {
16668                                 mutex_exit(&dtrace_lock);
16669                                 return (EINVAL);
16670                         }
16671                 }
16672 
16673                 mutex_exit(&dtrace_lock);
16674                 return (0);
16675         }
16676 
16677         default:
16678                 break;
16679         }
16680 
16681         return (ENOTTY);
16682 }
16683 
16684 /*ARGSUSED*/
16685 static int
16686 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
16687 {
16688         dtrace_state_t *state;
16689 
16690         switch (cmd) {
16691         case DDI_DETACH:
16692                 break;
16693 
16694         case DDI_SUSPEND:
16695                 return (DDI_SUCCESS);
16696 
16697         default:
16698                 return (DDI_FAILURE);
16699         }
16700 
16701         mutex_enter(&cpu_lock);
16702         mutex_enter(&dtrace_provider_lock);
16703         mutex_enter(&dtrace_lock);
16704 
16705         ASSERT(dtrace_opens == 0);
16706 
16707         if (dtrace_helpers > 0) {
16708                 mutex_exit(&dtrace_provider_lock);
16709                 mutex_exit(&dtrace_lock);
16710                 mutex_exit(&cpu_lock);
16711                 return (DDI_FAILURE);
16712         }
16713 
16714         if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
16715                 mutex_exit(&dtrace_provider_lock);
16716                 mutex_exit(&dtrace_lock);
16717                 mutex_exit(&cpu_lock);
16718                 return (DDI_FAILURE);
16719         }
16720 
16721         dtrace_provider = NULL;
16722 
16723         if ((state = dtrace_anon_grab()) != NULL) {
16724                 /*
16725                  * If there were ECBs on this state, the provider should
16726                  * have not been allowed to detach; assert that there is
16727                  * none.
16728                  */
16729                 ASSERT(state->dts_necbs == 0);
16730                 dtrace_state_destroy(state);
16731 
16732                 /*
16733                  * If we're being detached with anonymous state, we need to
16734                  * indicate to the kernel debugger that DTrace is now inactive.
16735                  */
16736                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
16737         }
16738 
16739         bzero(&dtrace_anon, sizeof (dtrace_anon_t));
16740         unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
16741         dtrace_cpu_init = NULL;
16742         dtrace_helpers_cleanup = NULL;
16743         dtrace_helpers_fork = NULL;
16744         dtrace_cpustart_init = NULL;
16745         dtrace_cpustart_fini = NULL;
16746         dtrace_debugger_init = NULL;
16747         dtrace_debugger_fini = NULL;
16748         dtrace_modload = NULL;
16749         dtrace_modunload = NULL;
16750 
16751         ASSERT(dtrace_getf == 0);
16752         ASSERT(dtrace_closef == NULL);
16753 
16754         mutex_exit(&cpu_lock);
16755 
16756         if (dtrace_helptrace_enabled) {
16757                 kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
16758                 dtrace_helptrace_buffer = NULL;
16759         }
16760 
16761         kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
16762         dtrace_probes = NULL;
16763         dtrace_nprobes = 0;
16764 
16765         dtrace_hash_destroy(dtrace_bymod);
16766         dtrace_hash_destroy(dtrace_byfunc);
16767         dtrace_hash_destroy(dtrace_byname);
16768         dtrace_bymod = NULL;
16769         dtrace_byfunc = NULL;
16770         dtrace_byname = NULL;
16771 
16772         kmem_cache_destroy(dtrace_state_cache);
16773         vmem_destroy(dtrace_minor);
16774         vmem_destroy(dtrace_arena);
16775 
16776         if (dtrace_toxrange != NULL) {
16777                 kmem_free(dtrace_toxrange,
16778                     dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
16779                 dtrace_toxrange = NULL;
16780                 dtrace_toxranges = 0;
16781                 dtrace_toxranges_max = 0;
16782         }
16783 
16784         ddi_remove_minor_node(dtrace_devi, NULL);
16785         dtrace_devi = NULL;
16786 
16787         ddi_soft_state_fini(&dtrace_softstate);
16788 
16789         ASSERT(dtrace_vtime_references == 0);
16790         ASSERT(dtrace_opens == 0);
16791         ASSERT(dtrace_retained == NULL);
16792 
16793         mutex_exit(&dtrace_lock);
16794         mutex_exit(&dtrace_provider_lock);
16795 
16796         /*
16797          * We don't destroy the task queue until after we have dropped our
16798          * locks (taskq_destroy() may block on running tasks).  To prevent
16799          * attempting to do work after we have effectively detached but before
16800          * the task queue has been destroyed, all tasks dispatched via the
16801          * task queue must check that DTrace is still attached before
16802          * performing any operation.
16803          */
16804         taskq_destroy(dtrace_taskq);
16805         dtrace_taskq = NULL;
16806 
16807         return (DDI_SUCCESS);
16808 }
16809 
16810 /*ARGSUSED*/
16811 static int
16812 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
16813 {
16814         int error;
16815 
16816         switch (infocmd) {
16817         case DDI_INFO_DEVT2DEVINFO:
16818                 *result = (void *)dtrace_devi;
16819                 error = DDI_SUCCESS;
16820                 break;
16821         case DDI_INFO_DEVT2INSTANCE:
16822                 *result = (void *)0;
16823                 error = DDI_SUCCESS;
16824                 break;
16825         default:
16826                 error = DDI_FAILURE;
16827         }
16828         return (error);
16829 }
16830 
16831 static struct cb_ops dtrace_cb_ops = {
16832         dtrace_open,            /* open */
16833         dtrace_close,           /* close */
16834         nulldev,                /* strategy */
16835         nulldev,                /* print */
16836         nodev,                  /* dump */
16837         nodev,                  /* read */
16838         nodev,                  /* write */
16839         dtrace_ioctl,           /* ioctl */
16840         nodev,                  /* devmap */
16841         nodev,                  /* mmap */
16842         nodev,                  /* segmap */
16843         nochpoll,               /* poll */
16844         ddi_prop_op,            /* cb_prop_op */
16845         0,                      /* streamtab  */
16846         D_NEW | D_MP            /* Driver compatibility flag */
16847 };
16848 
16849 static struct dev_ops dtrace_ops = {
16850         DEVO_REV,               /* devo_rev */
16851         0,                      /* refcnt */
16852         dtrace_info,            /* get_dev_info */
16853         nulldev,                /* identify */
16854         nulldev,                /* probe */
16855         dtrace_attach,          /* attach */
16856         dtrace_detach,          /* detach */
16857         nodev,                  /* reset */
16858         &dtrace_cb_ops,             /* driver operations */
16859         NULL,                   /* bus operations */
16860         nodev,                  /* dev power */
16861         ddi_quiesce_not_needed,         /* quiesce */
16862 };
16863 
16864 static struct modldrv modldrv = {
16865         &mod_driverops,             /* module type (this is a pseudo driver) */
16866         "Dynamic Tracing",      /* name of module */
16867         &dtrace_ops,                /* driver ops */
16868 };
16869 
16870 static struct modlinkage modlinkage = {
16871         MODREV_1,
16872         (void *)&modldrv,
16873         NULL
16874 };
16875 
16876 int
16877 _init(void)
16878 {
16879         return (mod_install(&modlinkage));
16880 }
16881 
16882 int
16883 _info(struct modinfo *modinfop)
16884 {
16885         return (mod_info(&modlinkage, modinfop));
16886 }
16887 
16888 int
16889 _fini(void)
16890 {
16891         return (mod_remove(&modlinkage));
16892 }