1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2019 Joyent, Inc.
  25  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  26  */
  27 
  28 /*
  29  * DTrace - Dynamic Tracing for Solaris
  30  *
  31  * This is the implementation of the Solaris Dynamic Tracing framework
  32  * (DTrace).  The user-visible interface to DTrace is described at length in
  33  * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
  34  * library, the in-kernel DTrace framework, and the DTrace providers are
  35  * described in the block comments in the <sys/dtrace.h> header file.  The
  36  * internal architecture of DTrace is described in the block comments in the
  37  * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
  38  * implementation very much assume mastery of all of these sources; if one has
  39  * an unanswered question about the implementation, one should consult them
  40  * first.
  41  *
  42  * The functions here are ordered roughly as follows:
  43  *
  44  *   - Probe context functions
  45  *   - Probe hashing functions
  46  *   - Non-probe context utility functions
  47  *   - Matching functions
  48  *   - Provider-to-Framework API functions
  49  *   - Probe management functions
  50  *   - DIF object functions
  51  *   - Format functions
  52  *   - Predicate functions
  53  *   - ECB functions
  54  *   - Buffer functions
  55  *   - Enabling functions
  56  *   - DOF functions
  57  *   - Anonymous enabling functions
  58  *   - Consumer state functions
  59  *   - Helper functions
  60  *   - Hook functions
  61  *   - Driver cookbook functions
  62  *
  63  * Each group of functions begins with a block comment labelled the "DTrace
  64  * [Group] Functions", allowing one to find each block by searching forward
  65  * on capital-f functions.
  66  */
  67 #include <sys/errno.h>
  68 #include <sys/stat.h>
  69 #include <sys/modctl.h>
  70 #include <sys/conf.h>
  71 #include <sys/systm.h>
  72 #include <sys/ddi.h>
  73 #include <sys/sunddi.h>
  74 #include <sys/cpuvar.h>
  75 #include <sys/kmem.h>
  76 #include <sys/strsubr.h>
  77 #include <sys/sysmacros.h>
  78 #include <sys/dtrace_impl.h>
  79 #include <sys/atomic.h>
  80 #include <sys/cmn_err.h>
  81 #include <sys/mutex_impl.h>
  82 #include <sys/rwlock_impl.h>
  83 #include <sys/ctf_api.h>
  84 #include <sys/panic.h>
  85 #include <sys/priv_impl.h>
  86 #include <sys/policy.h>
  87 #include <sys/cred_impl.h>
  88 #include <sys/procfs_isa.h>
  89 #include <sys/taskq.h>
  90 #include <sys/mkdev.h>
  91 #include <sys/kdi.h>
  92 #include <sys/zone.h>
  93 #include <sys/socket.h>
  94 #include <netinet/in.h>
  95 #include "strtolctype.h"
  96 
  97 /*
  98  * DTrace Tunable Variables
  99  *
 100  * The following variables may be tuned by adding a line to /etc/system that
 101  * includes both the name of the DTrace module ("dtrace") and the name of the
 102  * variable.  For example:
 103  *
 104  *   set dtrace:dtrace_destructive_disallow = 1
 105  *
 106  * In general, the only variables that one should be tuning this way are those
 107  * that affect system-wide DTrace behavior, and for which the default behavior
 108  * is undesirable.  Most of these variables are tunable on a per-consumer
 109  * basis using DTrace options, and need not be tuned on a system-wide basis.
 110  * When tuning these variables, avoid pathological values; while some attempt
 111  * is made to verify the integrity of these variables, they are not considered
 112  * part of the supported interface to DTrace, and they are therefore not
 113  * checked comprehensively.  Further, these variables should not be tuned
 114  * dynamically via "mdb -kw" or other means; they should only be tuned via
 115  * /etc/system.
 116  */
 117 int             dtrace_destructive_disallow = 0;
 118 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 119 size_t          dtrace_difo_maxsize = (256 * 1024);
 120 dtrace_optval_t dtrace_dof_maxsize = (8 * 1024 * 1024);
 121 size_t          dtrace_statvar_maxsize = (16 * 1024);
 122 size_t          dtrace_actions_max = (16 * 1024);
 123 size_t          dtrace_retain_max = 1024;
 124 dtrace_optval_t dtrace_helper_actions_max = 1024;
 125 dtrace_optval_t dtrace_helper_providers_max = 32;
 126 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
 127 size_t          dtrace_strsize_default = 256;
 128 dtrace_optval_t dtrace_cleanrate_default = 9900990;             /* 101 hz */
 129 dtrace_optval_t dtrace_cleanrate_min = 200000;                  /* 5000 hz */
 130 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;  /* 1/minute */
 131 dtrace_optval_t dtrace_aggrate_default = NANOSEC;               /* 1 hz */
 132 dtrace_optval_t dtrace_statusrate_default = NANOSEC;            /* 1 hz */
 133 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;  /* 6/minute */
 134 dtrace_optval_t dtrace_switchrate_default = NANOSEC;            /* 1 hz */
 135 dtrace_optval_t dtrace_nspec_default = 1;
 136 dtrace_optval_t dtrace_specsize_default = 32 * 1024;
 137 dtrace_optval_t dtrace_stackframes_default = 20;
 138 dtrace_optval_t dtrace_ustackframes_default = 20;
 139 dtrace_optval_t dtrace_jstackframes_default = 50;
 140 dtrace_optval_t dtrace_jstackstrsize_default = 512;
 141 int             dtrace_msgdsize_max = 128;
 142 hrtime_t        dtrace_chill_max = MSEC2NSEC(500);              /* 500 ms */
 143 hrtime_t        dtrace_chill_interval = NANOSEC;                /* 1000 ms */
 144 int             dtrace_devdepth_max = 32;
 145 int             dtrace_err_verbose;
 146 hrtime_t        dtrace_deadman_interval = NANOSEC;
 147 hrtime_t        dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
 148 hrtime_t        dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
 149 hrtime_t        dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
 150 
 151 /*
 152  * DTrace External Variables
 153  *
 154  * As dtrace(7D) is a kernel module, any DTrace variables are obviously
 155  * available to DTrace consumers via the backtick (`) syntax.  One of these,
 156  * dtrace_zero, is made deliberately so:  it is provided as a source of
 157  * well-known, zero-filled memory.  While this variable is not documented,
 158  * it is used by some translators as an implementation detail.
 159  */
 160 const char      dtrace_zero[256] = { 0 };       /* zero-filled memory */
 161 
 162 /*
 163  * DTrace Internal Variables
 164  */
 165 static dev_info_t       *dtrace_devi;           /* device info */
 166 static vmem_t           *dtrace_arena;          /* probe ID arena */
 167 static vmem_t           *dtrace_minor;          /* minor number arena */
 168 static taskq_t          *dtrace_taskq;          /* task queue */
 169 static dtrace_probe_t   **dtrace_probes;        /* array of all probes */
 170 static int              dtrace_nprobes;         /* number of probes */
 171 static dtrace_provider_t *dtrace_provider;      /* provider list */
 172 static dtrace_meta_t    *dtrace_meta_pid;       /* user-land meta provider */
 173 static int              dtrace_opens;           /* number of opens */
 174 static int              dtrace_helpers;         /* number of helpers */
 175 static int              dtrace_getf;            /* number of unpriv getf()s */
 176 static void             *dtrace_softstate;      /* softstate pointer */
 177 static dtrace_hash_t    *dtrace_bymod;          /* probes hashed by module */
 178 static dtrace_hash_t    *dtrace_byfunc;         /* probes hashed by function */
 179 static dtrace_hash_t    *dtrace_byname;         /* probes hashed by name */
 180 static dtrace_toxrange_t *dtrace_toxrange;      /* toxic range array */
 181 static int              dtrace_toxranges;       /* number of toxic ranges */
 182 static int              dtrace_toxranges_max;   /* size of toxic range array */
 183 static dtrace_anon_t    dtrace_anon;            /* anonymous enabling */
 184 static kmem_cache_t     *dtrace_state_cache;    /* cache for dynamic state */
 185 static uint64_t         dtrace_vtime_references; /* number of vtimestamp refs */
 186 static kthread_t        *dtrace_panicked;       /* panicking thread */
 187 static dtrace_ecb_t     *dtrace_ecb_create_cache; /* cached created ECB */
 188 static dtrace_genid_t   dtrace_probegen;        /* current probe generation */
 189 static dtrace_helpers_t *dtrace_deferred_pid;   /* deferred helper list */
 190 static dtrace_enabling_t *dtrace_retained;      /* list of retained enablings */
 191 static dtrace_genid_t   dtrace_retained_gen;    /* current retained enab gen */
 192 static dtrace_dynvar_t  dtrace_dynhash_sink;    /* end of dynamic hash chains */
 193 static int              dtrace_dynvar_failclean; /* dynvars failed to clean */
 194 
 195 /*
 196  * DTrace Locking
 197  * DTrace is protected by three (relatively coarse-grained) locks:
 198  *
 199  * (1) dtrace_lock is required to manipulate essentially any DTrace state,
 200  *     including enabling state, probes, ECBs, consumer state, helper state,
 201  *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
 202  *     probe context is lock-free -- synchronization is handled via the
 203  *     dtrace_sync() cross call mechanism.
 204  *
 205  * (2) dtrace_provider_lock is required when manipulating provider state, or
 206  *     when provider state must be held constant.
 207  *
 208  * (3) dtrace_meta_lock is required when manipulating meta provider state, or
 209  *     when meta provider state must be held constant.
 210  *
 211  * The lock ordering between these three locks is dtrace_meta_lock before
 212  * dtrace_provider_lock before dtrace_lock.  (In particular, there are
 213  * several places where dtrace_provider_lock is held by the framework as it
 214  * calls into the providers -- which then call back into the framework,
 215  * grabbing dtrace_lock.)
 216  *
 217  * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
 218  * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
 219  * role as a coarse-grained lock; it is acquired before both of these locks.
 220  * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
 221  * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
 222  * mod_lock is similar with respect to dtrace_provider_lock in that it must be
 223  * acquired _between_ dtrace_provider_lock and dtrace_lock.
 224  */
 225 static kmutex_t         dtrace_lock;            /* probe state lock */
 226 static kmutex_t         dtrace_provider_lock;   /* provider state lock */
 227 static kmutex_t         dtrace_meta_lock;       /* meta-provider state lock */
 228 
 229 /*
 230  * DTrace Provider Variables
 231  *
 232  * These are the variables relating to DTrace as a provider (that is, the
 233  * provider of the BEGIN, END, and ERROR probes).
 234  */
 235 static dtrace_pattr_t   dtrace_provider_attr = {
 236 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 237 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 238 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 239 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 240 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 241 };
 242 
 243 static void
 244 dtrace_nullop_provide(void *arg __unused,
 245     const dtrace_probedesc_t *spec __unused)
 246 {
 247 }
 248 
 249 static void
 250 dtrace_nullop_module(void *arg __unused, struct modctl *mp __unused)
 251 {
 252 }
 253 
 254 static void
 255 dtrace_nullop(void *arg __unused, dtrace_id_t id __unused, void *parg __unused)
 256 {
 257 }
 258 
 259 static int
 260 dtrace_enable_nullop(void *arg __unused, dtrace_id_t id __unused,
 261     void *parg __unused)
 262 {
 263         return (0);
 264 }
 265 
 266 static dtrace_pops_t    dtrace_provider_ops = {
 267         .dtps_provide = dtrace_nullop_provide,
 268         .dtps_provide_module = dtrace_nullop_module,
 269         .dtps_enable = dtrace_enable_nullop,
 270         .dtps_disable = dtrace_nullop,
 271         .dtps_suspend = dtrace_nullop,
 272         .dtps_resume = dtrace_nullop,
 273         .dtps_getargdesc = NULL,
 274         .dtps_getargval = NULL,
 275         .dtps_mode = NULL,
 276         .dtps_destroy = dtrace_nullop
 277 };
 278 
 279 static dtrace_id_t      dtrace_probeid_begin;   /* special BEGIN probe */
 280 static dtrace_id_t      dtrace_probeid_end;     /* special END probe */
 281 dtrace_id_t             dtrace_probeid_error;   /* special ERROR probe */
 282 
 283 /*
 284  * DTrace Helper Tracing Variables
 285  *
 286  * These variables should be set dynamically to enable helper tracing.  The
 287  * only variables that should be set are dtrace_helptrace_enable (which should
 288  * be set to a non-zero value to allocate helper tracing buffers on the next
 289  * open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a
 290  * non-zero value to deallocate helper tracing buffers on the next close of
 291  * /dev/dtrace).  When (and only when) helper tracing is disabled, the
 292  * buffer size may also be set via dtrace_helptrace_bufsize.
 293  */
 294 int                     dtrace_helptrace_enable = 0;
 295 int                     dtrace_helptrace_disable = 0;
 296 int                     dtrace_helptrace_bufsize = 16 * 1024 * 1024;
 297 uint32_t                dtrace_helptrace_nlocals;
 298 static dtrace_helptrace_t *dtrace_helptrace_buffer;
 299 static uint32_t         dtrace_helptrace_next = 0;
 300 static int              dtrace_helptrace_wrapped = 0;
 301 
 302 /*
 303  * DTrace Error Hashing
 304  *
 305  * On DEBUG kernels, DTrace will track the errors that has seen in a hash
 306  * table.  This is very useful for checking coverage of tests that are
 307  * expected to induce DIF or DOF processing errors, and may be useful for
 308  * debugging problems in the DIF code generator or in DOF generation .  The
 309  * error hash may be examined with the ::dtrace_errhash MDB dcmd.
 310  */
 311 #ifdef DEBUG
 312 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
 313 static const char *dtrace_errlast;
 314 static kthread_t *dtrace_errthread;
 315 static kmutex_t dtrace_errlock;
 316 #endif
 317 
 318 /*
 319  * DTrace Macros and Constants
 320  *
 321  * These are various macros that are useful in various spots in the
 322  * implementation, along with a few random constants that have no meaning
 323  * outside of the implementation.  There is no real structure to this cpp
 324  * mishmash -- but is there ever?
 325  */
 326 #define DTRACE_HASHSTR(hash, probe)     \
 327         dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
 328 
 329 #define DTRACE_HASHNEXT(hash, probe)    \
 330         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
 331 
 332 #define DTRACE_HASHPREV(hash, probe)    \
 333         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
 334 
 335 #define DTRACE_HASHEQ(hash, lhs, rhs)   \
 336         (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
 337             *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
 338 
 339 #define DTRACE_AGGHASHSIZE_SLEW         17
 340 
 341 #define DTRACE_V4MAPPED_OFFSET          (sizeof (uint32_t) * 3)
 342 
 343 /*
 344  * The key for a thread-local variable consists of the lower 61 bits of the
 345  * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
 346  * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
 347  * equal to a variable identifier.  This is necessary (but not sufficient) to
 348  * assure that global associative arrays never collide with thread-local
 349  * variables.  To guarantee that they cannot collide, we must also define the
 350  * order for keying dynamic variables.  That order is:
 351  *
 352  *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
 353  *
 354  * Because the variable-key and the tls-key are in orthogonal spaces, there is
 355  * no way for a global variable key signature to match a thread-local key
 356  * signature.
 357  */
 358 #define DTRACE_TLS_THRKEY(where) { \
 359         uint_t intr = 0; \
 360         uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
 361         for (; actv; actv >>= 1) \
 362                 intr++; \
 363         ASSERT(intr < (1 << 3)); \
 364         (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
 365             (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
 366 }
 367 
 368 #define DT_BSWAP_8(x)   ((x) & 0xff)
 369 #define DT_BSWAP_16(x)  ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
 370 #define DT_BSWAP_32(x)  ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
 371 #define DT_BSWAP_64(x)  ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
 372 
 373 #define DT_MASK_LO 0x00000000FFFFFFFFULL
 374 
 375 #define DTRACE_STORE(type, tomax, offset, what) \
 376         *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
 377 
 378 #ifndef __x86
 379 #define DTRACE_ALIGNCHECK(addr, size, flags)                            \
 380         if (addr & (size - 1)) {                                    \
 381                 *flags |= CPU_DTRACE_BADALIGN;                          \
 382                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 383                 return (0);                                             \
 384         }
 385 #else
 386 #define DTRACE_ALIGNCHECK(addr, size, flags)
 387 #endif
 388 
 389 /*
 390  * Test whether a range of memory starting at testaddr of size testsz falls
 391  * within the range of memory described by addr, sz.  We take care to avoid
 392  * problems with overflow and underflow of the unsigned quantities, and
 393  * disallow all negative sizes.  Ranges of size 0 are allowed.
 394  */
 395 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
 396         ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
 397         (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
 398         (testaddr) + (testsz) >= (testaddr))
 399 
 400 #define DTRACE_RANGE_REMAIN(remp, addr, baseaddr, basesz)               \
 401 do {                                                                    \
 402         if ((remp) != NULL) {                                           \
 403                 *(remp) = (uintptr_t)(baseaddr) + (basesz) - (addr);    \
 404         }                                                               \
 405 _NOTE(CONSTCOND) } while (0)
 406 
 407 
 408 /*
 409  * Test whether alloc_sz bytes will fit in the scratch region.  We isolate
 410  * alloc_sz on the righthand side of the comparison in order to avoid overflow
 411  * or underflow in the comparison with it.  This is simpler than the INRANGE
 412  * check above, because we know that the dtms_scratch_ptr is valid in the
 413  * range.  Allocations of size zero are allowed.
 414  */
 415 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
 416         ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
 417         (mstate)->dtms_scratch_ptr >= (alloc_sz))
 418 
 419 #define DTRACE_LOADFUNC(bits)                                           \
 420 /*CSTYLED*/                                                             \
 421 uint##bits##_t                                                          \
 422 dtrace_load##bits(uintptr_t addr)                                       \
 423 {                                                                       \
 424         size_t size = bits / NBBY;                                      \
 425         /*CSTYLED*/                                                     \
 426         uint##bits##_t rval;                                            \
 427         int i;                                                          \
 428         volatile uint16_t *flags = (volatile uint16_t *)                \
 429             &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;                    \
 430                                                                         \
 431         DTRACE_ALIGNCHECK(addr, size, flags);                           \
 432                                                                         \
 433         for (i = 0; i < dtrace_toxranges; i++) {                     \
 434                 if (addr >= dtrace_toxrange[i].dtt_limit)            \
 435                         continue;                                       \
 436                                                                         \
 437                 if (addr + size <= dtrace_toxrange[i].dtt_base)              \
 438                         continue;                                       \
 439                                                                         \
 440                 /*                                                      \
 441                  * This address falls within a toxic region; return 0.  \
 442                  */                                                     \
 443                 *flags |= CPU_DTRACE_BADADDR;                           \
 444                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 445                 return (0);                                             \
 446         }                                                               \
 447                                                                         \
 448         *flags |= CPU_DTRACE_NOFAULT;                                   \
 449         /*CSTYLED*/                                                     \
 450         rval = *((volatile uint##bits##_t *)addr);                      \
 451         *flags &= ~CPU_DTRACE_NOFAULT;                                      \
 452                                                                         \
 453         return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0);           \
 454 }
 455 
 456 #ifdef _LP64
 457 #define dtrace_loadptr  dtrace_load64
 458 #else
 459 #define dtrace_loadptr  dtrace_load32
 460 #endif
 461 
 462 #define DTRACE_DYNHASH_FREE     0
 463 #define DTRACE_DYNHASH_SINK     1
 464 #define DTRACE_DYNHASH_VALID    2
 465 
 466 #define DTRACE_MATCH_FAIL       -1
 467 #define DTRACE_MATCH_NEXT       0
 468 #define DTRACE_MATCH_DONE       1
 469 #define DTRACE_ANCHORED(probe)  ((probe)->dtpr_func[0] != '\0')
 470 #define DTRACE_STATE_ALIGN      64
 471 
 472 #define DTRACE_FLAGS2FLT(flags)                                         \
 473         (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :               \
 474         ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :            \
 475         ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :                \
 476         ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :            \
 477         ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :            \
 478         ((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :             \
 479         ((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :             \
 480         ((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :   \
 481         ((flags) & CPU_DTRACE_BADSTACK) ?  DTRACEFLT_BADSTACK :             \
 482         DTRACEFLT_UNKNOWN)
 483 
 484 #define DTRACEACT_ISSTRING(act)                                         \
 485         ((act)->dta_kind == DTRACEACT_DIFEXPR &&                     \
 486         (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
 487 
 488 static size_t dtrace_strlen(const char *, size_t);
 489 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
 490 static void dtrace_enabling_provide(dtrace_provider_t *);
 491 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
 492 static void dtrace_enabling_matchall(void);
 493 static void dtrace_enabling_reap(void);
 494 static dtrace_state_t *dtrace_anon_grab(void);
 495 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
 496     dtrace_state_t *, uint64_t, uint64_t);
 497 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
 498 static void dtrace_buffer_drop(dtrace_buffer_t *);
 499 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
 500 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
 501     dtrace_state_t *, dtrace_mstate_t *);
 502 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
 503     dtrace_optval_t);
 504 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
 505 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
 506 static int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *);
 507 static void dtrace_getf_barrier(void);
 508 static int dtrace_canload_remains(uint64_t, size_t, size_t *,
 509     dtrace_mstate_t *, dtrace_vstate_t *);
 510 static int dtrace_canstore_remains(uint64_t, size_t, size_t *,
 511     dtrace_mstate_t *, dtrace_vstate_t *);
 512 
 513 /*
 514  * DTrace Probe Context Functions
 515  *
 516  * These functions are called from probe context.  Because probe context is
 517  * any context in which C may be called, arbitrarily locks may be held,
 518  * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
 519  * As a result, functions called from probe context may only call other DTrace
 520  * support functions -- they may not interact at all with the system at large.
 521  * (Note that the ASSERT macro is made probe-context safe by redefining it in
 522  * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
 523  * loads are to be performed from probe context, they _must_ be in terms of
 524  * the safe dtrace_load*() variants.
 525  *
 526  * Some functions in this block are not actually called from probe context;
 527  * for these functions, there will be a comment above the function reading
 528  * "Note:  not called from probe context."
 529  */
 530 void
 531 dtrace_panic(const char *format, ...)
 532 {
 533         va_list alist;
 534 
 535         va_start(alist, format);
 536         dtrace_vpanic(format, alist);
 537         va_end(alist);
 538 }
 539 
 540 int
 541 dtrace_assfail(const char *a, const char *f, int l)
 542 {
 543         dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
 544 
 545         /*
 546          * We just need something here that even the most clever compiler
 547          * cannot optimize away.
 548          */
 549         return (a[(uintptr_t)f]);
 550 }
 551 
 552 /*
 553  * Atomically increment a specified error counter from probe context.
 554  */
 555 static void
 556 dtrace_error(uint32_t *counter)
 557 {
 558         /*
 559          * Most counters stored to in probe context are per-CPU counters.
 560          * However, there are some error conditions that are sufficiently
 561          * arcane that they don't merit per-CPU storage.  If these counters
 562          * are incremented concurrently on different CPUs, scalability will be
 563          * adversely affected -- but we don't expect them to be white-hot in a
 564          * correctly constructed enabling...
 565          */
 566         uint32_t oval, nval;
 567 
 568         do {
 569                 oval = *counter;
 570 
 571                 if ((nval = oval + 1) == 0) {
 572                         /*
 573                          * If the counter would wrap, set it to 1 -- assuring
 574                          * that the counter is never zero when we have seen
 575                          * errors.  (The counter must be 32-bits because we
 576                          * aren't guaranteed a 64-bit compare&swap operation.)
 577                          * To save this code both the infamy of being fingered
 578                          * by a priggish news story and the indignity of being
 579                          * the target of a neo-puritan witch trial, we're
 580                          * carefully avoiding any colorful description of the
 581                          * likelihood of this condition -- but suffice it to
 582                          * say that it is only slightly more likely than the
 583                          * overflow of predicate cache IDs, as discussed in
 584                          * dtrace_predicate_create().
 585                          */
 586                         nval = 1;
 587                 }
 588         } while (dtrace_cas32(counter, oval, nval) != oval);
 589 }
 590 
 591 /*
 592  * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
 593  * uint8_t, a uint16_t, a uint32_t and a uint64_t.
 594  */
 595 /* BEGIN CSTYLED */
 596 DTRACE_LOADFUNC(8)
 597 DTRACE_LOADFUNC(16)
 598 DTRACE_LOADFUNC(32)
 599 DTRACE_LOADFUNC(64)
 600 /* END CSTYLED */
 601 
 602 static int
 603 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
 604 {
 605         if (dest < mstate->dtms_scratch_base)
 606                 return (0);
 607 
 608         if (dest + size < dest)
 609                 return (0);
 610 
 611         if (dest + size > mstate->dtms_scratch_ptr)
 612                 return (0);
 613 
 614         return (1);
 615 }
 616 
 617 static int
 618 dtrace_canstore_statvar(uint64_t addr, size_t sz, size_t *remain,
 619     dtrace_statvar_t **svars, int nsvars)
 620 {
 621         int i;
 622         size_t maxglobalsize, maxlocalsize;
 623 
 624         if (nsvars == 0)
 625                 return (0);
 626 
 627         maxglobalsize = dtrace_statvar_maxsize + sizeof (uint64_t);
 628         maxlocalsize = maxglobalsize * NCPU;
 629 
 630         for (i = 0; i < nsvars; i++) {
 631                 dtrace_statvar_t *svar = svars[i];
 632                 uint8_t scope;
 633                 size_t size;
 634 
 635                 if (svar == NULL || (size = svar->dtsv_size) == 0)
 636                         continue;
 637 
 638                 scope = svar->dtsv_var.dtdv_scope;
 639 
 640                 /*
 641                  * We verify that our size is valid in the spirit of providing
 642                  * defense in depth:  we want to prevent attackers from using
 643                  * DTrace to escalate an orthogonal kernel heap corruption bug
 644                  * into the ability to store to arbitrary locations in memory.
 645                  */
 646                 VERIFY((scope == DIFV_SCOPE_GLOBAL && size <= maxglobalsize) ||
 647                     (scope == DIFV_SCOPE_LOCAL && size <= maxlocalsize));
 648 
 649                 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data,
 650                     svar->dtsv_size)) {
 651                         DTRACE_RANGE_REMAIN(remain, addr, svar->dtsv_data,
 652                             svar->dtsv_size);
 653                         return (1);
 654                 }
 655         }
 656 
 657         return (0);
 658 }
 659 
 660 /*
 661  * Check to see if the address is within a memory region to which a store may
 662  * be issued.  This includes the DTrace scratch areas, and any DTrace variable
 663  * region.  The caller of dtrace_canstore() is responsible for performing any
 664  * alignment checks that are needed before stores are actually executed.
 665  */
 666 static int
 667 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 668     dtrace_vstate_t *vstate)
 669 {
 670         return (dtrace_canstore_remains(addr, sz, NULL, mstate, vstate));
 671 }
 672 
 673 /*
 674  * Implementation of dtrace_canstore which communicates the upper bound of the
 675  * allowed memory region.
 676  */
 677 static int
 678 dtrace_canstore_remains(uint64_t addr, size_t sz, size_t *remain,
 679     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 680 {
 681         /*
 682          * First, check to see if the address is in scratch space...
 683          */
 684         if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
 685             mstate->dtms_scratch_size)) {
 686                 DTRACE_RANGE_REMAIN(remain, addr, mstate->dtms_scratch_base,
 687                     mstate->dtms_scratch_size);
 688                 return (1);
 689         }
 690 
 691         /*
 692          * Now check to see if it's a dynamic variable.  This check will pick
 693          * up both thread-local variables and any global dynamically-allocated
 694          * variables.
 695          */
 696         if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
 697             vstate->dtvs_dynvars.dtds_size)) {
 698                 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
 699                 uintptr_t base = (uintptr_t)dstate->dtds_base +
 700                     (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
 701                 uintptr_t chunkoffs;
 702                 dtrace_dynvar_t *dvar;
 703 
 704                 /*
 705                  * Before we assume that we can store here, we need to make
 706                  * sure that it isn't in our metadata -- storing to our
 707                  * dynamic variable metadata would corrupt our state.  For
 708                  * the range to not include any dynamic variable metadata,
 709                  * it must:
 710                  *
 711                  *      (1) Start above the hash table that is at the base of
 712                  *      the dynamic variable space
 713                  *
 714                  *      (2) Have a starting chunk offset that is beyond the
 715                  *      dtrace_dynvar_t that is at the base of every chunk
 716                  *
 717                  *      (3) Not span a chunk boundary
 718                  *
 719                  *      (4) Not be in the tuple space of a dynamic variable
 720                  *
 721                  */
 722                 if (addr < base)
 723                         return (0);
 724 
 725                 chunkoffs = (addr - base) % dstate->dtds_chunksize;
 726 
 727                 if (chunkoffs < sizeof (dtrace_dynvar_t))
 728                         return (0);
 729 
 730                 if (chunkoffs + sz > dstate->dtds_chunksize)
 731                         return (0);
 732 
 733                 dvar = (dtrace_dynvar_t *)((uintptr_t)addr - chunkoffs);
 734 
 735                 if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE)
 736                         return (0);
 737 
 738                 if (chunkoffs < sizeof (dtrace_dynvar_t) +
 739                     ((dvar->dtdv_tuple.dtt_nkeys - 1) * sizeof (dtrace_key_t)))
 740                         return (0);
 741 
 742                 DTRACE_RANGE_REMAIN(remain, addr, dvar, dstate->dtds_chunksize);
 743                 return (1);
 744         }
 745 
 746         /*
 747          * Finally, check the static local and global variables.  These checks
 748          * take the longest, so we perform them last.
 749          */
 750         if (dtrace_canstore_statvar(addr, sz, remain,
 751             vstate->dtvs_locals, vstate->dtvs_nlocals))
 752                 return (1);
 753 
 754         if (dtrace_canstore_statvar(addr, sz, remain,
 755             vstate->dtvs_globals, vstate->dtvs_nglobals))
 756                 return (1);
 757 
 758         return (0);
 759 }
 760 
 761 
 762 /*
 763  * Convenience routine to check to see if the address is within a memory
 764  * region in which a load may be issued given the user's privilege level;
 765  * if not, it sets the appropriate error flags and loads 'addr' into the
 766  * illegal value slot.
 767  *
 768  * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
 769  * appropriate memory access protection.
 770  */
 771 static int
 772 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 773     dtrace_vstate_t *vstate)
 774 {
 775         return (dtrace_canload_remains(addr, sz, NULL, mstate, vstate));
 776 }
 777 
 778 /*
 779  * Implementation of dtrace_canload which communicates the upper bound of the
 780  * allowed memory region.
 781  */
 782 static int
 783 dtrace_canload_remains(uint64_t addr, size_t sz, size_t *remain,
 784     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 785 {
 786         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
 787         file_t *fp;
 788 
 789         /*
 790          * If we hold the privilege to read from kernel memory, then
 791          * everything is readable.
 792          */
 793         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) {
 794                 DTRACE_RANGE_REMAIN(remain, addr, addr, sz);
 795                 return (1);
 796         }
 797 
 798         /*
 799          * You can obviously read that which you can store.
 800          */
 801         if (dtrace_canstore_remains(addr, sz, remain, mstate, vstate))
 802                 return (1);
 803 
 804         /*
 805          * We're allowed to read from our own string table.
 806          */
 807         if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
 808             mstate->dtms_difo->dtdo_strlen)) {
 809                 DTRACE_RANGE_REMAIN(remain, addr,
 810                     mstate->dtms_difo->dtdo_strtab,
 811                     mstate->dtms_difo->dtdo_strlen);
 812                 return (1);
 813         }
 814 
 815         if (vstate->dtvs_state != NULL &&
 816             dtrace_priv_proc(vstate->dtvs_state, mstate)) {
 817                 proc_t *p;
 818 
 819                 /*
 820                  * When we have privileges to the current process, there are
 821                  * several context-related kernel structures that are safe to
 822                  * read, even absent the privilege to read from kernel memory.
 823                  * These reads are safe because these structures contain only
 824                  * state that (1) we're permitted to read, (2) is harmless or
 825                  * (3) contains pointers to additional kernel state that we're
 826                  * not permitted to read (and as such, do not present an
 827                  * opportunity for privilege escalation).  Finally (and
 828                  * critically), because of the nature of their relation with
 829                  * the current thread context, the memory associated with these
 830                  * structures cannot change over the duration of probe context,
 831                  * and it is therefore impossible for this memory to be
 832                  * deallocated and reallocated as something else while it's
 833                  * being operated upon.
 834                  */
 835                 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t))) {
 836                         DTRACE_RANGE_REMAIN(remain, addr, curthread,
 837                             sizeof (kthread_t));
 838                         return (1);
 839                 }
 840 
 841                 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
 842                     sz, curthread->t_procp, sizeof (proc_t))) {
 843                         DTRACE_RANGE_REMAIN(remain, addr, curthread->t_procp,
 844                             sizeof (proc_t));
 845                         return (1);
 846                 }
 847 
 848                 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
 849                     curthread->t_cred, sizeof (cred_t))) {
 850                         DTRACE_RANGE_REMAIN(remain, addr, curthread->t_cred,
 851                             sizeof (cred_t));
 852                         return (1);
 853                 }
 854 
 855                 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
 856                     &(p->p_pidp->pid_id), sizeof (pid_t))) {
 857                         DTRACE_RANGE_REMAIN(remain, addr, &(p->p_pidp->pid_id),
 858                             sizeof (pid_t));
 859                         return (1);
 860                 }
 861 
 862                 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
 863                     curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
 864                         DTRACE_RANGE_REMAIN(remain, addr, curthread->t_cpu,
 865                             offsetof(cpu_t, cpu_pause_thread));
 866                         return (1);
 867                 }
 868         }
 869 
 870         if ((fp = mstate->dtms_getf) != NULL) {
 871                 uintptr_t psz = sizeof (void *);
 872                 vnode_t *vp;
 873                 vnodeops_t *op;
 874 
 875                 /*
 876                  * When getf() returns a file_t, the enabling is implicitly
 877                  * granted the (transient) right to read the returned file_t
 878                  * as well as the v_path and v_op->vnop_name of the underlying
 879                  * vnode.  These accesses are allowed after a successful
 880                  * getf() because the members that they refer to cannot change
 881                  * once set -- and the barrier logic in the kernel's closef()
 882                  * path assures that the file_t and its referenced vode_t
 883                  * cannot themselves be stale (that is, it impossible for
 884                  * either dtms_getf itself or its f_vnode member to reference
 885                  * freed memory).
 886                  */
 887                 if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t))) {
 888                         DTRACE_RANGE_REMAIN(remain, addr, fp, sizeof (file_t));
 889                         return (1);
 890                 }
 891 
 892                 if ((vp = fp->f_vnode) != NULL) {
 893                         size_t slen;
 894 
 895                         if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz)) {
 896                                 DTRACE_RANGE_REMAIN(remain, addr, &vp->v_path,
 897                                     psz);
 898                                 return (1);
 899                         }
 900 
 901                         slen = strlen(vp->v_path) + 1;
 902                         if (DTRACE_INRANGE(addr, sz, vp->v_path, slen)) {
 903                                 DTRACE_RANGE_REMAIN(remain, addr, vp->v_path,
 904                                     slen);
 905                                 return (1);
 906                         }
 907 
 908                         if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz)) {
 909                                 DTRACE_RANGE_REMAIN(remain, addr, &vp->v_op,
 910                                     psz);
 911                                 return (1);
 912                         }
 913 
 914                         if ((op = vp->v_op) != NULL &&
 915                             DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) {
 916                                 DTRACE_RANGE_REMAIN(remain, addr,
 917                                     &op->vnop_name, psz);
 918                                 return (1);
 919                         }
 920 
 921                         if (op != NULL && op->vnop_name != NULL &&
 922                             DTRACE_INRANGE(addr, sz, op->vnop_name,
 923                             (slen = strlen(op->vnop_name) + 1))) {
 924                                 DTRACE_RANGE_REMAIN(remain, addr,
 925                                     op->vnop_name, slen);
 926                                 return (1);
 927                         }
 928                 }
 929         }
 930 
 931         DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
 932         *illval = addr;
 933         return (0);
 934 }
 935 
 936 /*
 937  * Convenience routine to check to see if a given string is within a memory
 938  * region in which a load may be issued given the user's privilege level;
 939  * this exists so that we don't need to issue unnecessary dtrace_strlen()
 940  * calls in the event that the user has all privileges.
 941  */
 942 static int
 943 dtrace_strcanload(uint64_t addr, size_t sz, size_t *remain,
 944     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 945 {
 946         size_t rsize;
 947 
 948         /*
 949          * If we hold the privilege to read from kernel memory, then
 950          * everything is readable.
 951          */
 952         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) {
 953                 DTRACE_RANGE_REMAIN(remain, addr, addr, sz);
 954                 return (1);
 955         }
 956 
 957         /*
 958          * Even if the caller is uninterested in querying the remaining valid
 959          * range, it is required to ensure that the access is allowed.
 960          */
 961         if (remain == NULL) {
 962                 remain = &rsize;
 963         }
 964         if (dtrace_canload_remains(addr, 0, remain, mstate, vstate)) {
 965                 size_t strsz;
 966                 /*
 967                  * Perform the strlen after determining the length of the
 968                  * memory region which is accessible.  This prevents timing
 969                  * information from being used to find NULs in memory which is
 970                  * not accessible to the caller.
 971                  */
 972                 strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr,
 973                     MIN(sz, *remain));
 974                 if (strsz <= *remain) {
 975                         return (1);
 976                 }
 977         }
 978 
 979         return (0);
 980 }
 981 
 982 /*
 983  * Convenience routine to check to see if a given variable is within a memory
 984  * region in which a load may be issued given the user's privilege level.
 985  */
 986 static int
 987 dtrace_vcanload(void *src, dtrace_diftype_t *type, size_t *remain,
 988     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
 989 {
 990         size_t sz;
 991         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
 992 
 993         /*
 994          * Calculate the max size before performing any checks since even
 995          * DTRACE_ACCESS_KERNEL-credentialed callers expect that this function
 996          * return the max length via 'remain'.
 997          */
 998         if (type->dtdt_kind == DIF_TYPE_STRING) {
 999                 dtrace_state_t *state = vstate->dtvs_state;
1000 
1001                 if (state != NULL) {
1002                         sz = state->dts_options[DTRACEOPT_STRSIZE];
1003                 } else {
1004                         /*
1005                          * In helper context, we have a NULL state; fall back
1006                          * to using the system-wide default for the string size
1007                          * in this case.
1008                          */
1009                         sz = dtrace_strsize_default;
1010                 }
1011         } else {
1012                 sz = type->dtdt_size;
1013         }
1014 
1015         /*
1016          * If we hold the privilege to read from kernel memory, then
1017          * everything is readable.
1018          */
1019         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) {
1020                 DTRACE_RANGE_REMAIN(remain, (uintptr_t)src, src, sz);
1021                 return (1);
1022         }
1023 
1024         if (type->dtdt_kind == DIF_TYPE_STRING) {
1025                 return (dtrace_strcanload((uintptr_t)src, sz, remain, mstate,
1026                     vstate));
1027         }
1028         return (dtrace_canload_remains((uintptr_t)src, sz, remain, mstate,
1029             vstate));
1030 }
1031 
1032 /*
1033  * Convert a string to a signed integer using safe loads.
1034  *
1035  * NOTE: This function uses various macros from strtolctype.h to manipulate
1036  * digit values, etc -- these have all been checked to ensure they make
1037  * no additional function calls.
1038  */
1039 static int64_t
1040 dtrace_strtoll(char *input, int base, size_t limit)
1041 {
1042         uintptr_t pos = (uintptr_t)input;
1043         int64_t val = 0;
1044         int x;
1045         boolean_t neg = B_FALSE;
1046         char c, cc, ccc;
1047         uintptr_t end = pos + limit;
1048 
1049         /*
1050          * Consume any whitespace preceding digits.
1051          */
1052         while ((c = dtrace_load8(pos)) == ' ' || c == '\t')
1053                 pos++;
1054 
1055         /*
1056          * Handle an explicit sign if one is present.
1057          */
1058         if (c == '-' || c == '+') {
1059                 if (c == '-')
1060                         neg = B_TRUE;
1061                 c = dtrace_load8(++pos);
1062         }
1063 
1064         /*
1065          * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it
1066          * if present.
1067          */
1068         if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' ||
1069             cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) {
1070                 pos += 2;
1071                 c = ccc;
1072         }
1073 
1074         /*
1075          * Read in contiguous digits until the first non-digit character.
1076          */
1077         for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base;
1078             c = dtrace_load8(++pos))
1079                 val = val * base + x;
1080 
1081         return (neg ? -val : val);
1082 }
1083 
1084 /*
1085  * Compare two strings using safe loads.
1086  */
1087 static int
1088 dtrace_strncmp(char *s1, char *s2, size_t limit)
1089 {
1090         uint8_t c1, c2;
1091         volatile uint16_t *flags;
1092 
1093         if (s1 == s2 || limit == 0)
1094                 return (0);
1095 
1096         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
1097 
1098         do {
1099                 if (s1 == NULL) {
1100                         c1 = '\0';
1101                 } else {
1102                         c1 = dtrace_load8((uintptr_t)s1++);
1103                 }
1104 
1105                 if (s2 == NULL) {
1106                         c2 = '\0';
1107                 } else {
1108                         c2 = dtrace_load8((uintptr_t)s2++);
1109                 }
1110 
1111                 if (c1 != c2)
1112                         return (c1 - c2);
1113         } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
1114 
1115         return (0);
1116 }
1117 
1118 /*
1119  * Compute strlen(s) for a string using safe memory accesses.  The additional
1120  * len parameter is used to specify a maximum length to ensure completion.
1121  */
1122 static size_t
1123 dtrace_strlen(const char *s, size_t lim)
1124 {
1125         uint_t len;
1126 
1127         for (len = 0; len != lim; len++) {
1128                 if (dtrace_load8((uintptr_t)s++) == '\0')
1129                         break;
1130         }
1131 
1132         return (len);
1133 }
1134 
1135 /*
1136  * Check if an address falls within a toxic region.
1137  */
1138 static int
1139 dtrace_istoxic(uintptr_t kaddr, size_t size)
1140 {
1141         uintptr_t taddr, tsize;
1142         int i;
1143 
1144         for (i = 0; i < dtrace_toxranges; i++) {
1145                 taddr = dtrace_toxrange[i].dtt_base;
1146                 tsize = dtrace_toxrange[i].dtt_limit - taddr;
1147 
1148                 if (kaddr - taddr < tsize) {
1149                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
1150                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
1151                         return (1);
1152                 }
1153 
1154                 if (taddr - kaddr < size) {
1155                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
1156                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
1157                         return (1);
1158                 }
1159         }
1160 
1161         return (0);
1162 }
1163 
1164 /*
1165  * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
1166  * memory specified by the DIF program.  The dst is assumed to be safe memory
1167  * that we can store to directly because it is managed by DTrace.  As with
1168  * standard bcopy, overlapping copies are handled properly.
1169  */
1170 static void
1171 dtrace_bcopy(const void *src, void *dst, size_t len)
1172 {
1173         if (len != 0) {
1174                 uint8_t *s1 = dst;
1175                 const uint8_t *s2 = src;
1176 
1177                 if (s1 <= s2) {
1178                         do {
1179                                 *s1++ = dtrace_load8((uintptr_t)s2++);
1180                         } while (--len != 0);
1181                 } else {
1182                         s2 += len;
1183                         s1 += len;
1184 
1185                         do {
1186                                 *--s1 = dtrace_load8((uintptr_t)--s2);
1187                         } while (--len != 0);
1188                 }
1189         }
1190 }
1191 
1192 /*
1193  * Copy src to dst using safe memory accesses, up to either the specified
1194  * length, or the point that a nul byte is encountered.  The src is assumed to
1195  * be unsafe memory specified by the DIF program.  The dst is assumed to be
1196  * safe memory that we can store to directly because it is managed by DTrace.
1197  * Unlike dtrace_bcopy(), overlapping regions are not handled.
1198  */
1199 static void
1200 dtrace_strcpy(const void *src, void *dst, size_t len)
1201 {
1202         if (len != 0) {
1203                 uint8_t *s1 = dst, c;
1204                 const uint8_t *s2 = src;
1205 
1206                 do {
1207                         *s1++ = c = dtrace_load8((uintptr_t)s2++);
1208                 } while (--len != 0 && c != '\0');
1209         }
1210 }
1211 
1212 /*
1213  * Copy src to dst, deriving the size and type from the specified (BYREF)
1214  * variable type.  The src is assumed to be unsafe memory specified by the DIF
1215  * program.  The dst is assumed to be DTrace variable memory that is of the
1216  * specified type; we assume that we can store to directly.
1217  */
1218 static void
1219 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type, size_t limit)
1220 {
1221         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
1222 
1223         if (type->dtdt_kind == DIF_TYPE_STRING) {
1224                 dtrace_strcpy(src, dst, MIN(type->dtdt_size, limit));
1225         } else {
1226                 dtrace_bcopy(src, dst, MIN(type->dtdt_size, limit));
1227         }
1228 }
1229 
1230 /*
1231  * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
1232  * unsafe memory specified by the DIF program.  The s2 data is assumed to be
1233  * safe memory that we can access directly because it is managed by DTrace.
1234  */
1235 static int
1236 dtrace_bcmp(const void *s1, const void *s2, size_t len)
1237 {
1238         volatile uint16_t *flags;
1239 
1240         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
1241 
1242         if (s1 == s2)
1243                 return (0);
1244 
1245         if (s1 == NULL || s2 == NULL)
1246                 return (1);
1247 
1248         if (s1 != s2 && len != 0) {
1249                 const uint8_t *ps1 = s1;
1250                 const uint8_t *ps2 = s2;
1251 
1252                 do {
1253                         if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
1254                                 return (1);
1255                 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
1256         }
1257         return (0);
1258 }
1259 
1260 /*
1261  * Zero the specified region using a simple byte-by-byte loop.  Note that this
1262  * is for safe DTrace-managed memory only.
1263  */
1264 static void
1265 dtrace_bzero(void *dst, size_t len)
1266 {
1267         uchar_t *cp;
1268 
1269         for (cp = dst; len != 0; len--)
1270                 *cp++ = 0;
1271 }
1272 
1273 static void
1274 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
1275 {
1276         uint64_t result[2];
1277 
1278         result[0] = addend1[0] + addend2[0];
1279         result[1] = addend1[1] + addend2[1] +
1280             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
1281 
1282         sum[0] = result[0];
1283         sum[1] = result[1];
1284 }
1285 
1286 /*
1287  * Shift the 128-bit value in a by b. If b is positive, shift left.
1288  * If b is negative, shift right.
1289  */
1290 static void
1291 dtrace_shift_128(uint64_t *a, int b)
1292 {
1293         uint64_t mask;
1294 
1295         if (b == 0)
1296                 return;
1297 
1298         if (b < 0) {
1299                 b = -b;
1300                 if (b >= 64) {
1301                         a[0] = a[1] >> (b - 64);
1302                         a[1] = 0;
1303                 } else {
1304                         a[0] >>= b;
1305                         mask = 1LL << (64 - b);
1306                         mask -= 1;
1307                         a[0] |= ((a[1] & mask) << (64 - b));
1308                         a[1] >>= b;
1309                 }
1310         } else {
1311                 if (b >= 64) {
1312                         a[1] = a[0] << (b - 64);
1313                         a[0] = 0;
1314                 } else {
1315                         a[1] <<= b;
1316                         mask = a[0] >> (64 - b);
1317                         a[1] |= mask;
1318                         a[0] <<= b;
1319                 }
1320         }
1321 }
1322 
1323 /*
1324  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1325  * use native multiplication on those, and then re-combine into the
1326  * resulting 128-bit value.
1327  *
1328  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1329  *     hi1 * hi2 << 64 +
1330  *     hi1 * lo2 << 32 +
1331  *     hi2 * lo1 << 32 +
1332  *     lo1 * lo2
1333  */
1334 static void
1335 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
1336 {
1337         uint64_t hi1, hi2, lo1, lo2;
1338         uint64_t tmp[2];
1339 
1340         hi1 = factor1 >> 32;
1341         hi2 = factor2 >> 32;
1342 
1343         lo1 = factor1 & DT_MASK_LO;
1344         lo2 = factor2 & DT_MASK_LO;
1345 
1346         product[0] = lo1 * lo2;
1347         product[1] = hi1 * hi2;
1348 
1349         tmp[0] = hi1 * lo2;
1350         tmp[1] = 0;
1351         dtrace_shift_128(tmp, 32);
1352         dtrace_add_128(product, tmp, product);
1353 
1354         tmp[0] = hi2 * lo1;
1355         tmp[1] = 0;
1356         dtrace_shift_128(tmp, 32);
1357         dtrace_add_128(product, tmp, product);
1358 }
1359 
1360 /*
1361  * This privilege check should be used by actions and subroutines to
1362  * verify that the user credentials of the process that enabled the
1363  * invoking ECB match the target credentials
1364  */
1365 static int
1366 dtrace_priv_proc_common_user(dtrace_state_t *state)
1367 {
1368         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1369 
1370         /*
1371          * We should always have a non-NULL state cred here, since if cred
1372          * is null (anonymous tracing), we fast-path bypass this routine.
1373          */
1374         ASSERT(s_cr != NULL);
1375 
1376         if ((cr = CRED()) != NULL &&
1377             s_cr->cr_uid == cr->cr_uid &&
1378             s_cr->cr_uid == cr->cr_ruid &&
1379             s_cr->cr_uid == cr->cr_suid &&
1380             s_cr->cr_gid == cr->cr_gid &&
1381             s_cr->cr_gid == cr->cr_rgid &&
1382             s_cr->cr_gid == cr->cr_sgid)
1383                 return (1);
1384 
1385         return (0);
1386 }
1387 
1388 /*
1389  * This privilege check should be used by actions and subroutines to
1390  * verify that the zone of the process that enabled the invoking ECB
1391  * matches the target credentials
1392  */
1393 static int
1394 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1395 {
1396         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1397 
1398         /*
1399          * We should always have a non-NULL state cred here, since if cred
1400          * is null (anonymous tracing), we fast-path bypass this routine.
1401          */
1402         ASSERT(s_cr != NULL);
1403 
1404         if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone)
1405                 return (1);
1406 
1407         return (0);
1408 }
1409 
1410 /*
1411  * This privilege check should be used by actions and subroutines to
1412  * verify that the process has not setuid or changed credentials.
1413  */
1414 static int
1415 dtrace_priv_proc_common_nocd()
1416 {
1417         proc_t *proc;
1418 
1419         if ((proc = ttoproc(curthread)) != NULL &&
1420             !(proc->p_flag & SNOCD))
1421                 return (1);
1422 
1423         return (0);
1424 }
1425 
1426 static int
1427 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate)
1428 {
1429         int action = state->dts_cred.dcr_action;
1430 
1431         if (!(mstate->dtms_access & DTRACE_ACCESS_PROC))
1432                 goto bad;
1433 
1434         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
1435             dtrace_priv_proc_common_zone(state) == 0)
1436                 goto bad;
1437 
1438         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
1439             dtrace_priv_proc_common_user(state) == 0)
1440                 goto bad;
1441 
1442         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
1443             dtrace_priv_proc_common_nocd() == 0)
1444                 goto bad;
1445 
1446         return (1);
1447 
1448 bad:
1449         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1450 
1451         return (0);
1452 }
1453 
1454 static int
1455 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate)
1456 {
1457         if (mstate->dtms_access & DTRACE_ACCESS_PROC) {
1458                 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1459                         return (1);
1460 
1461                 if (dtrace_priv_proc_common_zone(state) &&
1462                     dtrace_priv_proc_common_user(state) &&
1463                     dtrace_priv_proc_common_nocd())
1464                         return (1);
1465         }
1466 
1467         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1468 
1469         return (0);
1470 }
1471 
1472 static int
1473 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate)
1474 {
1475         if ((mstate->dtms_access & DTRACE_ACCESS_PROC) &&
1476             (state->dts_cred.dcr_action & DTRACE_CRA_PROC))
1477                 return (1);
1478 
1479         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1480 
1481         return (0);
1482 }
1483 
1484 static int
1485 dtrace_priv_kernel(dtrace_state_t *state)
1486 {
1487         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1488                 return (1);
1489 
1490         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1491 
1492         return (0);
1493 }
1494 
1495 static int
1496 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1497 {
1498         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1499                 return (1);
1500 
1501         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1502 
1503         return (0);
1504 }
1505 
1506 /*
1507  * Determine if the dte_cond of the specified ECB allows for processing of
1508  * the current probe to continue.  Note that this routine may allow continued
1509  * processing, but with access(es) stripped from the mstate's dtms_access
1510  * field.
1511  */
1512 static int
1513 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
1514     dtrace_ecb_t *ecb)
1515 {
1516         dtrace_probe_t *probe = ecb->dte_probe;
1517         dtrace_provider_t *prov = probe->dtpr_provider;
1518         dtrace_pops_t *pops = &prov->dtpv_pops;
1519         int mode = DTRACE_MODE_NOPRIV_DROP;
1520 
1521         ASSERT(ecb->dte_cond);
1522 
1523         if (pops->dtps_mode != NULL) {
1524                 mode = pops->dtps_mode(prov->dtpv_arg,
1525                     probe->dtpr_id, probe->dtpr_arg);
1526 
1527                 ASSERT(mode & (DTRACE_MODE_USER | DTRACE_MODE_KERNEL));
1528                 ASSERT(mode & (DTRACE_MODE_NOPRIV_RESTRICT |
1529                     DTRACE_MODE_NOPRIV_DROP));
1530         }
1531 
1532         /*
1533          * If the dte_cond bits indicate that this consumer is only allowed to
1534          * see user-mode firings of this probe, check that the probe was fired
1535          * while in a user context.  If that's not the case, use the policy
1536          * specified by the provider to determine if we drop the probe or
1537          * merely restrict operation.
1538          */
1539         if (ecb->dte_cond & DTRACE_COND_USERMODE) {
1540                 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
1541 
1542                 if (!(mode & DTRACE_MODE_USER)) {
1543                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1544                                 return (0);
1545 
1546                         mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1547                 }
1548         }
1549 
1550         /*
1551          * This is more subtle than it looks. We have to be absolutely certain
1552          * that CRED() isn't going to change out from under us so it's only
1553          * legit to examine that structure if we're in constrained situations.
1554          * Currently, the only times we'll this check is if a non-super-user
1555          * has enabled the profile or syscall providers -- providers that
1556          * allow visibility of all processes. For the profile case, the check
1557          * above will ensure that we're examining a user context.
1558          */
1559         if (ecb->dte_cond & DTRACE_COND_OWNER) {
1560                 cred_t *cr;
1561                 cred_t *s_cr = state->dts_cred.dcr_cred;
1562                 proc_t *proc;
1563 
1564                 ASSERT(s_cr != NULL);
1565 
1566                 if ((cr = CRED()) == NULL ||
1567                     s_cr->cr_uid != cr->cr_uid ||
1568                     s_cr->cr_uid != cr->cr_ruid ||
1569                     s_cr->cr_uid != cr->cr_suid ||
1570                     s_cr->cr_gid != cr->cr_gid ||
1571                     s_cr->cr_gid != cr->cr_rgid ||
1572                     s_cr->cr_gid != cr->cr_sgid ||
1573                     (proc = ttoproc(curthread)) == NULL ||
1574                     (proc->p_flag & SNOCD)) {
1575                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1576                                 return (0);
1577 
1578                         mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
1579                 }
1580         }
1581 
1582         /*
1583          * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1584          * in our zone, check to see if our mode policy is to restrict rather
1585          * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1586          * and DTRACE_ACCESS_ARGS
1587          */
1588         if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
1589                 cred_t *cr;
1590                 cred_t *s_cr = state->dts_cred.dcr_cred;
1591 
1592                 ASSERT(s_cr != NULL);
1593 
1594                 if ((cr = CRED()) == NULL ||
1595                     s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
1596                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1597                                 return (0);
1598 
1599                         mstate->dtms_access &=
1600                             ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
1601                 }
1602         }
1603 
1604         /*
1605          * By merits of being in this code path at all, we have limited
1606          * privileges.  If the provider has indicated that limited privileges
1607          * are to denote restricted operation, strip off the ability to access
1608          * arguments.
1609          */
1610         if (mode & DTRACE_MODE_LIMITEDPRIV_RESTRICT)
1611                 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1612 
1613         return (1);
1614 }
1615 
1616 /*
1617  * Note:  not called from probe context.  This function is called
1618  * asynchronously (and at a regular interval) from outside of probe context to
1619  * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
1620  * cleaning is explained in detail in <sys/dtrace_impl.h>.
1621  */
1622 void
1623 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
1624 {
1625         dtrace_dynvar_t *dirty;
1626         dtrace_dstate_percpu_t *dcpu;
1627         dtrace_dynvar_t **rinsep;
1628         int i, j, work = 0;
1629 
1630         for (i = 0; i < NCPU; i++) {
1631                 dcpu = &dstate->dtds_percpu[i];
1632                 rinsep = &dcpu->dtdsc_rinsing;
1633 
1634                 /*
1635                  * If the dirty list is NULL, there is no dirty work to do.
1636                  */
1637                 if (dcpu->dtdsc_dirty == NULL)
1638                         continue;
1639 
1640                 if (dcpu->dtdsc_rinsing != NULL) {
1641                         /*
1642                          * If the rinsing list is non-NULL, then it is because
1643                          * this CPU was selected to accept another CPU's
1644                          * dirty list -- and since that time, dirty buffers
1645                          * have accumulated.  This is a highly unlikely
1646                          * condition, but we choose to ignore the dirty
1647                          * buffers -- they'll be picked up a future cleanse.
1648                          */
1649                         continue;
1650                 }
1651 
1652                 if (dcpu->dtdsc_clean != NULL) {
1653                         /*
1654                          * If the clean list is non-NULL, then we're in a
1655                          * situation where a CPU has done deallocations (we
1656                          * have a non-NULL dirty list) but no allocations (we
1657                          * also have a non-NULL clean list).  We can't simply
1658                          * move the dirty list into the clean list on this
1659                          * CPU, yet we also don't want to allow this condition
1660                          * to persist, lest a short clean list prevent a
1661                          * massive dirty list from being cleaned (which in
1662                          * turn could lead to otherwise avoidable dynamic
1663                          * drops).  To deal with this, we look for some CPU
1664                          * with a NULL clean list, NULL dirty list, and NULL
1665                          * rinsing list -- and then we borrow this CPU to
1666                          * rinse our dirty list.
1667                          */
1668                         for (j = 0; j < NCPU; j++) {
1669                                 dtrace_dstate_percpu_t *rinser;
1670 
1671                                 rinser = &dstate->dtds_percpu[j];
1672 
1673                                 if (rinser->dtdsc_rinsing != NULL)
1674                                         continue;
1675 
1676                                 if (rinser->dtdsc_dirty != NULL)
1677                                         continue;
1678 
1679                                 if (rinser->dtdsc_clean != NULL)
1680                                         continue;
1681 
1682                                 rinsep = &rinser->dtdsc_rinsing;
1683                                 break;
1684                         }
1685 
1686                         if (j == NCPU) {
1687                                 /*
1688                                  * We were unable to find another CPU that
1689                                  * could accept this dirty list -- we are
1690                                  * therefore unable to clean it now.
1691                                  */
1692                                 dtrace_dynvar_failclean++;
1693                                 continue;
1694                         }
1695                 }
1696 
1697                 work = 1;
1698 
1699                 /*
1700                  * Atomically move the dirty list aside.
1701                  */
1702                 do {
1703                         dirty = dcpu->dtdsc_dirty;
1704 
1705                         /*
1706                          * Before we zap the dirty list, set the rinsing list.
1707                          * (This allows for a potential assertion in
1708                          * dtrace_dynvar():  if a free dynamic variable appears
1709                          * on a hash chain, either the dirty list or the
1710                          * rinsing list for some CPU must be non-NULL.)
1711                          */
1712                         *rinsep = dirty;
1713                         dtrace_membar_producer();
1714                 } while (dtrace_casptr(&dcpu->dtdsc_dirty,
1715                     dirty, NULL) != dirty);
1716         }
1717 
1718         if (!work) {
1719                 /*
1720                  * We have no work to do; we can simply return.
1721                  */
1722                 return;
1723         }
1724 
1725         dtrace_sync();
1726 
1727         for (i = 0; i < NCPU; i++) {
1728                 dcpu = &dstate->dtds_percpu[i];
1729 
1730                 if (dcpu->dtdsc_rinsing == NULL)
1731                         continue;
1732 
1733                 /*
1734                  * We are now guaranteed that no hash chain contains a pointer
1735                  * into this dirty list; we can make it clean.
1736                  */
1737                 ASSERT(dcpu->dtdsc_clean == NULL);
1738                 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
1739                 dcpu->dtdsc_rinsing = NULL;
1740         }
1741 
1742         /*
1743          * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1744          * sure that all CPUs have seen all of the dtdsc_clean pointers.
1745          * This prevents a race whereby a CPU incorrectly decides that
1746          * the state should be something other than DTRACE_DSTATE_CLEAN
1747          * after dtrace_dynvar_clean() has completed.
1748          */
1749         dtrace_sync();
1750 
1751         dstate->dtds_state = DTRACE_DSTATE_CLEAN;
1752 }
1753 
1754 /*
1755  * Depending on the value of the op parameter, this function looks-up,
1756  * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
1757  * allocation is requested, this function will return a pointer to a
1758  * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1759  * variable can be allocated.  If NULL is returned, the appropriate counter
1760  * will be incremented.
1761  */
1762 dtrace_dynvar_t *
1763 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
1764     dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
1765     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
1766 {
1767         uint64_t hashval = DTRACE_DYNHASH_VALID;
1768         dtrace_dynhash_t *hash = dstate->dtds_hash;
1769         dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
1770         processorid_t me = CPU->cpu_id, cpu = me;
1771         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
1772         size_t bucket, ksize;
1773         size_t chunksize = dstate->dtds_chunksize;
1774         uintptr_t kdata, lock, nstate;
1775         uint_t i;
1776 
1777         ASSERT(nkeys != 0);
1778 
1779         /*
1780          * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
1781          * algorithm.  For the by-value portions, we perform the algorithm in
1782          * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
1783          * bit, and seems to have only a minute effect on distribution.  For
1784          * the by-reference data, we perform "One-at-a-time" iterating (safely)
1785          * over each referenced byte.  It's painful to do this, but it's much
1786          * better than pathological hash distribution.  The efficacy of the
1787          * hashing algorithm (and a comparison with other algorithms) may be
1788          * found by running the ::dtrace_dynstat MDB dcmd.
1789          */
1790         for (i = 0; i < nkeys; i++) {
1791                 if (key[i].dttk_size == 0) {
1792                         uint64_t val = key[i].dttk_value;
1793 
1794                         hashval += (val >> 48) & 0xffff;
1795                         hashval += (hashval << 10);
1796                         hashval ^= (hashval >> 6);
1797 
1798                         hashval += (val >> 32) & 0xffff;
1799                         hashval += (hashval << 10);
1800                         hashval ^= (hashval >> 6);
1801 
1802                         hashval += (val >> 16) & 0xffff;
1803                         hashval += (hashval << 10);
1804                         hashval ^= (hashval >> 6);
1805 
1806                         hashval += val & 0xffff;
1807                         hashval += (hashval << 10);
1808                         hashval ^= (hashval >> 6);
1809                 } else {
1810                         /*
1811                          * This is incredibly painful, but it beats the hell
1812                          * out of the alternative.
1813                          */
1814                         uint64_t j, size = key[i].dttk_size;
1815                         uintptr_t base = (uintptr_t)key[i].dttk_value;
1816 
1817                         if (!dtrace_canload(base, size, mstate, vstate))
1818                                 break;
1819 
1820                         for (j = 0; j < size; j++) {
1821                                 hashval += dtrace_load8(base + j);
1822                                 hashval += (hashval << 10);
1823                                 hashval ^= (hashval >> 6);
1824                         }
1825                 }
1826         }
1827 
1828         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
1829                 return (NULL);
1830 
1831         hashval += (hashval << 3);
1832         hashval ^= (hashval >> 11);
1833         hashval += (hashval << 15);
1834 
1835         /*
1836          * There is a remote chance (ideally, 1 in 2^31) that our hashval
1837          * comes out to be one of our two sentinel hash values.  If this
1838          * actually happens, we set the hashval to be a value known to be a
1839          * non-sentinel value.
1840          */
1841         if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
1842                 hashval = DTRACE_DYNHASH_VALID;
1843 
1844         /*
1845          * Yes, it's painful to do a divide here.  If the cycle count becomes
1846          * important here, tricks can be pulled to reduce it.  (However, it's
1847          * critical that hash collisions be kept to an absolute minimum;
1848          * they're much more painful than a divide.)  It's better to have a
1849          * solution that generates few collisions and still keeps things
1850          * relatively simple.
1851          */
1852         bucket = hashval % dstate->dtds_hashsize;
1853 
1854         if (op == DTRACE_DYNVAR_DEALLOC) {
1855                 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1856 
1857                 for (;;) {
1858                         while ((lock = *lockp) & 1)
1859                                 continue;
1860 
1861                         if (dtrace_casptr((void *)lockp,
1862                             (void *)lock, (void *)(lock + 1)) == (void *)lock)
1863                                 break;
1864                 }
1865 
1866                 dtrace_membar_producer();
1867         }
1868 
1869 top:
1870         prev = NULL;
1871         lock = hash[bucket].dtdh_lock;
1872 
1873         dtrace_membar_consumer();
1874 
1875         start = hash[bucket].dtdh_chain;
1876         ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
1877             start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
1878             op != DTRACE_DYNVAR_DEALLOC));
1879 
1880         for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1881                 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1882                 dtrace_key_t *dkey = &dtuple->dtt_key[0];
1883 
1884                 if (dvar->dtdv_hashval != hashval) {
1885                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
1886                                 /*
1887                                  * We've reached the sink, and therefore the
1888                                  * end of the hash chain; we can kick out of
1889                                  * the loop knowing that we have seen a valid
1890                                  * snapshot of state.
1891                                  */
1892                                 ASSERT(dvar->dtdv_next == NULL);
1893                                 ASSERT(dvar == &dtrace_dynhash_sink);
1894                                 break;
1895                         }
1896 
1897                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
1898                                 /*
1899                                  * We've gone off the rails:  somewhere along
1900                                  * the line, one of the members of this hash
1901                                  * chain was deleted.  Note that we could also
1902                                  * detect this by simply letting this loop run
1903                                  * to completion, as we would eventually hit
1904                                  * the end of the dirty list.  However, we
1905                                  * want to avoid running the length of the
1906                                  * dirty list unnecessarily (it might be quite
1907                                  * long), so we catch this as early as
1908                                  * possible by detecting the hash marker.  In
1909                                  * this case, we simply set dvar to NULL and
1910                                  * break; the conditional after the loop will
1911                                  * send us back to top.
1912                                  */
1913                                 dvar = NULL;
1914                                 break;
1915                         }
1916 
1917                         goto next;
1918                 }
1919 
1920                 if (dtuple->dtt_nkeys != nkeys)
1921                         goto next;
1922 
1923                 for (i = 0; i < nkeys; i++, dkey++) {
1924                         if (dkey->dttk_size != key[i].dttk_size)
1925                                 goto next; /* size or type mismatch */
1926 
1927                         if (dkey->dttk_size != 0) {
1928                                 if (dtrace_bcmp(
1929                                     (void *)(uintptr_t)key[i].dttk_value,
1930                                     (void *)(uintptr_t)dkey->dttk_value,
1931                                     dkey->dttk_size))
1932                                         goto next;
1933                         } else {
1934                                 if (dkey->dttk_value != key[i].dttk_value)
1935                                         goto next;
1936                         }
1937                 }
1938 
1939                 if (op != DTRACE_DYNVAR_DEALLOC)
1940                         return (dvar);
1941 
1942                 ASSERT(dvar->dtdv_next == NULL ||
1943                     dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
1944 
1945                 if (prev != NULL) {
1946                         ASSERT(hash[bucket].dtdh_chain != dvar);
1947                         ASSERT(start != dvar);
1948                         ASSERT(prev->dtdv_next == dvar);
1949                         prev->dtdv_next = dvar->dtdv_next;
1950                 } else {
1951                         if (dtrace_casptr(&hash[bucket].dtdh_chain,
1952                             start, dvar->dtdv_next) != start) {
1953                                 /*
1954                                  * We have failed to atomically swing the
1955                                  * hash table head pointer, presumably because
1956                                  * of a conflicting allocation on another CPU.
1957                                  * We need to reread the hash chain and try
1958                                  * again.
1959                                  */
1960                                 goto top;
1961                         }
1962                 }
1963 
1964                 dtrace_membar_producer();
1965 
1966                 /*
1967                  * Now set the hash value to indicate that it's free.
1968                  */
1969                 ASSERT(hash[bucket].dtdh_chain != dvar);
1970                 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1971 
1972                 dtrace_membar_producer();
1973 
1974                 /*
1975                  * Set the next pointer to point at the dirty list, and
1976                  * atomically swing the dirty pointer to the newly freed dvar.
1977                  */
1978                 do {
1979                         next = dcpu->dtdsc_dirty;
1980                         dvar->dtdv_next = next;
1981                 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1982 
1983                 /*
1984                  * Finally, unlock this hash bucket.
1985                  */
1986                 ASSERT(hash[bucket].dtdh_lock == lock);
1987                 ASSERT(lock & 1);
1988                 hash[bucket].dtdh_lock++;
1989 
1990                 return (NULL);
1991 next:
1992                 prev = dvar;
1993                 continue;
1994         }
1995 
1996         if (dvar == NULL) {
1997                 /*
1998                  * If dvar is NULL, it is because we went off the rails:
1999                  * one of the elements that we traversed in the hash chain
2000                  * was deleted while we were traversing it.  In this case,
2001                  * we assert that we aren't doing a dealloc (deallocs lock
2002                  * the hash bucket to prevent themselves from racing with
2003                  * one another), and retry the hash chain traversal.
2004                  */
2005                 ASSERT(op != DTRACE_DYNVAR_DEALLOC);
2006                 goto top;
2007         }
2008 
2009         if (op != DTRACE_DYNVAR_ALLOC) {
2010                 /*
2011                  * If we are not to allocate a new variable, we want to
2012                  * return NULL now.  Before we return, check that the value
2013                  * of the lock word hasn't changed.  If it has, we may have
2014                  * seen an inconsistent snapshot.
2015                  */
2016                 if (op == DTRACE_DYNVAR_NOALLOC) {
2017                         if (hash[bucket].dtdh_lock != lock)
2018                                 goto top;
2019                 } else {
2020                         ASSERT(op == DTRACE_DYNVAR_DEALLOC);
2021                         ASSERT(hash[bucket].dtdh_lock == lock);
2022                         ASSERT(lock & 1);
2023                         hash[bucket].dtdh_lock++;
2024                 }
2025 
2026                 return (NULL);
2027         }
2028 
2029         /*
2030          * We need to allocate a new dynamic variable.  The size we need is the
2031          * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
2032          * size of any auxiliary key data (rounded up to 8-byte alignment) plus
2033          * the size of any referred-to data (dsize).  We then round the final
2034          * size up to the chunksize for allocation.
2035          */
2036         for (ksize = 0, i = 0; i < nkeys; i++)
2037                 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
2038 
2039         /*
2040          * This should be pretty much impossible, but could happen if, say,
2041          * strange DIF specified the tuple.  Ideally, this should be an
2042          * assertion and not an error condition -- but that requires that the
2043          * chunksize calculation in dtrace_difo_chunksize() be absolutely
2044          * bullet-proof.  (That is, it must not be able to be fooled by
2045          * malicious DIF.)  Given the lack of backwards branches in DIF,
2046          * solving this would presumably not amount to solving the Halting
2047          * Problem -- but it still seems awfully hard.
2048          */
2049         if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
2050             ksize + dsize > chunksize) {
2051                 dcpu->dtdsc_drops++;
2052                 return (NULL);
2053         }
2054 
2055         nstate = DTRACE_DSTATE_EMPTY;
2056 
2057         do {
2058 retry:
2059                 free = dcpu->dtdsc_free;
2060 
2061                 if (free == NULL) {
2062                         dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
2063                         void *rval;
2064 
2065                         if (clean == NULL) {
2066                                 /*
2067                                  * We're out of dynamic variable space on
2068                                  * this CPU.  Unless we have tried all CPUs,
2069                                  * we'll try to allocate from a different
2070                                  * CPU.
2071                                  */
2072                                 switch (dstate->dtds_state) {
2073                                 case DTRACE_DSTATE_CLEAN: {
2074                                         void *sp = &dstate->dtds_state;
2075 
2076                                         if (++cpu >= NCPU)
2077                                                 cpu = 0;
2078 
2079                                         if (dcpu->dtdsc_dirty != NULL &&
2080                                             nstate == DTRACE_DSTATE_EMPTY)
2081                                                 nstate = DTRACE_DSTATE_DIRTY;
2082 
2083                                         if (dcpu->dtdsc_rinsing != NULL)
2084                                                 nstate = DTRACE_DSTATE_RINSING;
2085 
2086                                         dcpu = &dstate->dtds_percpu[cpu];
2087 
2088                                         if (cpu != me)
2089                                                 goto retry;
2090 
2091                                         (void) dtrace_cas32(sp,
2092                                             DTRACE_DSTATE_CLEAN, nstate);
2093 
2094                                         /*
2095                                          * To increment the correct bean
2096                                          * counter, take another lap.
2097                                          */
2098                                         goto retry;
2099                                 }
2100 
2101                                 case DTRACE_DSTATE_DIRTY:
2102                                         dcpu->dtdsc_dirty_drops++;
2103                                         break;
2104 
2105                                 case DTRACE_DSTATE_RINSING:
2106                                         dcpu->dtdsc_rinsing_drops++;
2107                                         break;
2108 
2109                                 case DTRACE_DSTATE_EMPTY:
2110                                         dcpu->dtdsc_drops++;
2111                                         break;
2112                                 }
2113 
2114                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
2115                                 return (NULL);
2116                         }
2117 
2118                         /*
2119                          * The clean list appears to be non-empty.  We want to
2120                          * move the clean list to the free list; we start by
2121                          * moving the clean pointer aside.
2122                          */
2123                         if (dtrace_casptr(&dcpu->dtdsc_clean,
2124                             clean, NULL) != clean) {
2125                                 /*
2126                                  * We are in one of two situations:
2127                                  *
2128                                  *  (a) The clean list was switched to the
2129                                  *      free list by another CPU.
2130                                  *
2131                                  *  (b) The clean list was added to by the
2132                                  *      cleansing cyclic.
2133                                  *
2134                                  * In either of these situations, we can
2135                                  * just reattempt the free list allocation.
2136                                  */
2137                                 goto retry;
2138                         }
2139 
2140                         ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
2141 
2142                         /*
2143                          * Now we'll move the clean list to our free list.
2144                          * It's impossible for this to fail:  the only way
2145                          * the free list can be updated is through this
2146                          * code path, and only one CPU can own the clean list.
2147                          * Thus, it would only be possible for this to fail if
2148                          * this code were racing with dtrace_dynvar_clean().
2149                          * (That is, if dtrace_dynvar_clean() updated the clean
2150                          * list, and we ended up racing to update the free
2151                          * list.)  This race is prevented by the dtrace_sync()
2152                          * in dtrace_dynvar_clean() -- which flushes the
2153                          * owners of the clean lists out before resetting
2154                          * the clean lists.
2155                          */
2156                         dcpu = &dstate->dtds_percpu[me];
2157                         rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
2158                         ASSERT(rval == NULL);
2159                         goto retry;
2160                 }
2161 
2162                 dvar = free;
2163                 new_free = dvar->dtdv_next;
2164         } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
2165 
2166         /*
2167          * We have now allocated a new chunk.  We copy the tuple keys into the
2168          * tuple array and copy any referenced key data into the data space
2169          * following the tuple array.  As we do this, we relocate dttk_value
2170          * in the final tuple to point to the key data address in the chunk.
2171          */
2172         kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
2173         dvar->dtdv_data = (void *)(kdata + ksize);
2174         dvar->dtdv_tuple.dtt_nkeys = nkeys;
2175 
2176         for (i = 0; i < nkeys; i++) {
2177                 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
2178                 size_t kesize = key[i].dttk_size;
2179 
2180                 if (kesize != 0) {
2181                         dtrace_bcopy(
2182                             (const void *)(uintptr_t)key[i].dttk_value,
2183                             (void *)kdata, kesize);
2184                         dkey->dttk_value = kdata;
2185                         kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
2186                 } else {
2187                         dkey->dttk_value = key[i].dttk_value;
2188                 }
2189 
2190                 dkey->dttk_size = kesize;
2191         }
2192 
2193         ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
2194         dvar->dtdv_hashval = hashval;
2195         dvar->dtdv_next = start;
2196 
2197         if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
2198                 return (dvar);
2199 
2200         /*
2201          * The cas has failed.  Either another CPU is adding an element to
2202          * this hash chain, or another CPU is deleting an element from this
2203          * hash chain.  The simplest way to deal with both of these cases
2204          * (though not necessarily the most efficient) is to free our
2205          * allocated block and re-attempt it all.  Note that the free is
2206          * to the dirty list and _not_ to the free list.  This is to prevent
2207          * races with allocators, above.
2208          */
2209         dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
2210 
2211         dtrace_membar_producer();
2212 
2213         do {
2214                 free = dcpu->dtdsc_dirty;
2215                 dvar->dtdv_next = free;
2216         } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
2217 
2218         goto top;
2219 }
2220 
2221 /*ARGSUSED*/
2222 static void
2223 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
2224 {
2225         if ((int64_t)nval < (int64_t)*oval)
2226                 *oval = nval;
2227 }
2228 
2229 /*ARGSUSED*/
2230 static void
2231 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
2232 {
2233         if ((int64_t)nval > (int64_t)*oval)
2234                 *oval = nval;
2235 }
2236 
2237 static void
2238 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
2239 {
2240         int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
2241         int64_t val = (int64_t)nval;
2242 
2243         if (val < 0) {
2244                 for (i = 0; i < zero; i++) {
2245                         if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
2246                                 quanta[i] += incr;
2247                                 return;
2248                         }
2249                 }
2250         } else {
2251                 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
2252                         if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
2253                                 quanta[i - 1] += incr;
2254                                 return;
2255                         }
2256                 }
2257 
2258                 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
2259                 return;
2260         }
2261 
2262         ASSERT(0);
2263 }
2264 
2265 static void
2266 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
2267 {
2268         uint64_t arg = *lquanta++;
2269         int32_t base = DTRACE_LQUANTIZE_BASE(arg);
2270         uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
2271         uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
2272         int32_t val = (int32_t)nval, level;
2273 
2274         ASSERT(step != 0);
2275         ASSERT(levels != 0);
2276 
2277         if (val < base) {
2278                 /*
2279                  * This is an underflow.
2280                  */
2281                 lquanta[0] += incr;
2282                 return;
2283         }
2284 
2285         level = (val - base) / step;
2286 
2287         if (level < levels) {
2288                 lquanta[level + 1] += incr;
2289                 return;
2290         }
2291 
2292         /*
2293          * This is an overflow.
2294          */
2295         lquanta[levels + 1] += incr;
2296 }
2297 
2298 static int
2299 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
2300     uint16_t high, uint16_t nsteps, int64_t value)
2301 {
2302         int64_t this = 1, last, next;
2303         int base = 1, order;
2304 
2305         ASSERT(factor <= nsteps);
2306         ASSERT(nsteps % factor == 0);
2307 
2308         for (order = 0; order < low; order++)
2309                 this *= factor;
2310 
2311         /*
2312          * If our value is less than our factor taken to the power of the
2313          * low order of magnitude, it goes into the zeroth bucket.
2314          */
2315         if (value < (last = this))
2316                 return (0);
2317 
2318         for (this *= factor; order <= high; order++) {
2319                 int nbuckets = this > nsteps ? nsteps : this;
2320 
2321                 if ((next = this * factor) < this) {
2322                         /*
2323                          * We should not generally get log/linear quantizations
2324                          * with a high magnitude that allows 64-bits to
2325                          * overflow, but we nonetheless protect against this
2326                          * by explicitly checking for overflow, and clamping
2327                          * our value accordingly.
2328                          */
2329                         value = this - 1;
2330                 }
2331 
2332                 if (value < this) {
2333                         /*
2334                          * If our value lies within this order of magnitude,
2335                          * determine its position by taking the offset within
2336                          * the order of magnitude, dividing by the bucket
2337                          * width, and adding to our (accumulated) base.
2338                          */
2339                         return (base + (value - last) / (this / nbuckets));
2340                 }
2341 
2342                 base += nbuckets - (nbuckets / factor);
2343                 last = this;
2344                 this = next;
2345         }
2346 
2347         /*
2348          * Our value is greater than or equal to our factor taken to the
2349          * power of one plus the high magnitude -- return the top bucket.
2350          */
2351         return (base);
2352 }
2353 
2354 static void
2355 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
2356 {
2357         uint64_t arg = *llquanta++;
2358         uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2359         uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2360         uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2361         uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2362 
2363         llquanta[dtrace_aggregate_llquantize_bucket(factor,
2364             low, high, nsteps, nval)] += incr;
2365 }
2366 
2367 /*ARGSUSED*/
2368 static void
2369 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
2370 {
2371         data[0]++;
2372         data[1] += nval;
2373 }
2374 
2375 /*ARGSUSED*/
2376 static void
2377 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
2378 {
2379         int64_t snval = (int64_t)nval;
2380         uint64_t tmp[2];
2381 
2382         data[0]++;
2383         data[1] += nval;
2384 
2385         /*
2386          * What we want to say here is:
2387          *
2388          * data[2] += nval * nval;
2389          *
2390          * But given that nval is 64-bit, we could easily overflow, so
2391          * we do this as 128-bit arithmetic.
2392          */
2393         if (snval < 0)
2394                 snval = -snval;
2395 
2396         dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
2397         dtrace_add_128(data + 2, tmp, data + 2);
2398 }
2399 
2400 /*ARGSUSED*/
2401 static void
2402 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
2403 {
2404         *oval = *oval + 1;
2405 }
2406 
2407 /*ARGSUSED*/
2408 static void
2409 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
2410 {
2411         *oval += nval;
2412 }
2413 
2414 /*
2415  * Aggregate given the tuple in the principal data buffer, and the aggregating
2416  * action denoted by the specified dtrace_aggregation_t.  The aggregation
2417  * buffer is specified as the buf parameter.  This routine does not return
2418  * failure; if there is no space in the aggregation buffer, the data will be
2419  * dropped, and a corresponding counter incremented.
2420  */
2421 static void
2422 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
2423     intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
2424 {
2425         dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
2426         uint32_t i, ndx, size, fsize;
2427         uint32_t align = sizeof (uint64_t) - 1;
2428         dtrace_aggbuffer_t *agb;
2429         dtrace_aggkey_t *key;
2430         uint32_t hashval = 0, limit, isstr;
2431         caddr_t tomax, data, kdata;
2432         dtrace_actkind_t action;
2433         dtrace_action_t *act;
2434         uintptr_t offs;
2435 
2436         if (buf == NULL)
2437                 return;
2438 
2439         if (!agg->dtag_hasarg) {
2440                 /*
2441                  * Currently, only quantize() and lquantize() take additional
2442                  * arguments, and they have the same semantics:  an increment
2443                  * value that defaults to 1 when not present.  If additional
2444                  * aggregating actions take arguments, the setting of the
2445                  * default argument value will presumably have to become more
2446                  * sophisticated...
2447                  */
2448                 arg = 1;
2449         }
2450 
2451         action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
2452         size = rec->dtrd_offset - agg->dtag_base;
2453         fsize = size + rec->dtrd_size;
2454 
2455         ASSERT(dbuf->dtb_tomax != NULL);
2456         data = dbuf->dtb_tomax + offset + agg->dtag_base;
2457 
2458         if ((tomax = buf->dtb_tomax) == NULL) {
2459                 dtrace_buffer_drop(buf);
2460                 return;
2461         }
2462 
2463         /*
2464          * The metastructure is always at the bottom of the buffer.
2465          */
2466         agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
2467             sizeof (dtrace_aggbuffer_t));
2468 
2469         if (buf->dtb_offset == 0) {
2470                 /*
2471                  * We just kludge up approximately 1/8th of the size to be
2472                  * buckets.  If this guess ends up being routinely
2473                  * off-the-mark, we may need to dynamically readjust this
2474                  * based on past performance.
2475                  */
2476                 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
2477 
2478                 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
2479                     (uintptr_t)tomax || hashsize == 0) {
2480                         /*
2481                          * We've been given a ludicrously small buffer;
2482                          * increment our drop count and leave.
2483                          */
2484                         dtrace_buffer_drop(buf);
2485                         return;
2486                 }
2487 
2488                 /*
2489                  * And now, a pathetic attempt to try to get a an odd (or
2490                  * perchance, a prime) hash size for better hash distribution.
2491                  */
2492                 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
2493                         hashsize -= DTRACE_AGGHASHSIZE_SLEW;
2494 
2495                 agb->dtagb_hashsize = hashsize;
2496                 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
2497                     agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
2498                 agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
2499 
2500                 for (i = 0; i < agb->dtagb_hashsize; i++)
2501                         agb->dtagb_hash[i] = NULL;
2502         }
2503 
2504         ASSERT(agg->dtag_first != NULL);
2505         ASSERT(agg->dtag_first->dta_intuple);
2506 
2507         /*
2508          * Calculate the hash value based on the key.  Note that we _don't_
2509          * include the aggid in the hashing (but we will store it as part of
2510          * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
2511          * algorithm: a simple, quick algorithm that has no known funnels, and
2512          * gets good distribution in practice.  The efficacy of the hashing
2513          * algorithm (and a comparison with other algorithms) may be found by
2514          * running the ::dtrace_aggstat MDB dcmd.
2515          */
2516         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2517                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2518                 limit = i + act->dta_rec.dtrd_size;
2519                 ASSERT(limit <= size);
2520                 isstr = DTRACEACT_ISSTRING(act);
2521 
2522                 for (; i < limit; i++) {
2523                         hashval += data[i];
2524                         hashval += (hashval << 10);
2525                         hashval ^= (hashval >> 6);
2526 
2527                         if (isstr && data[i] == '\0')
2528                                 break;
2529                 }
2530         }
2531 
2532         hashval += (hashval << 3);
2533         hashval ^= (hashval >> 11);
2534         hashval += (hashval << 15);
2535 
2536         /*
2537          * Yes, the divide here is expensive -- but it's generally the least
2538          * of the performance issues given the amount of data that we iterate
2539          * over to compute hash values, compare data, etc.
2540          */
2541         ndx = hashval % agb->dtagb_hashsize;
2542 
2543         for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
2544                 ASSERT((caddr_t)key >= tomax);
2545                 ASSERT((caddr_t)key < tomax + buf->dtb_size);
2546 
2547                 if (hashval != key->dtak_hashval || key->dtak_size != size)
2548                         continue;
2549 
2550                 kdata = key->dtak_data;
2551                 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
2552 
2553                 for (act = agg->dtag_first; act->dta_intuple;
2554                     act = act->dta_next) {
2555                         i = act->dta_rec.dtrd_offset - agg->dtag_base;
2556                         limit = i + act->dta_rec.dtrd_size;
2557                         ASSERT(limit <= size);
2558                         isstr = DTRACEACT_ISSTRING(act);
2559 
2560                         for (; i < limit; i++) {
2561                                 if (kdata[i] != data[i])
2562                                         goto next;
2563 
2564                                 if (isstr && data[i] == '\0')
2565                                         break;
2566                         }
2567                 }
2568 
2569                 if (action != key->dtak_action) {
2570                         /*
2571                          * We are aggregating on the same value in the same
2572                          * aggregation with two different aggregating actions.
2573                          * (This should have been picked up in the compiler,
2574                          * so we may be dealing with errant or devious DIF.)
2575                          * This is an error condition; we indicate as much,
2576                          * and return.
2577                          */
2578                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2579                         return;
2580                 }
2581 
2582                 /*
2583                  * This is a hit:  we need to apply the aggregator to
2584                  * the value at this key.
2585                  */
2586                 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
2587                 return;
2588 next:
2589                 continue;
2590         }
2591 
2592         /*
2593          * We didn't find it.  We need to allocate some zero-filled space,
2594          * link it into the hash table appropriately, and apply the aggregator
2595          * to the (zero-filled) value.
2596          */
2597         offs = buf->dtb_offset;
2598         while (offs & (align - 1))
2599                 offs += sizeof (uint32_t);
2600 
2601         /*
2602          * If we don't have enough room to both allocate a new key _and_
2603          * its associated data, increment the drop count and return.
2604          */
2605         if ((uintptr_t)tomax + offs + fsize >
2606             agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
2607                 dtrace_buffer_drop(buf);
2608                 return;
2609         }
2610 
2611         /*CONSTCOND*/
2612         ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
2613         key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
2614         agb->dtagb_free -= sizeof (dtrace_aggkey_t);
2615 
2616         key->dtak_data = kdata = tomax + offs;
2617         buf->dtb_offset = offs + fsize;
2618 
2619         /*
2620          * Now copy the data across.
2621          */
2622         *((dtrace_aggid_t *)kdata) = agg->dtag_id;
2623 
2624         for (i = sizeof (dtrace_aggid_t); i < size; i++)
2625                 kdata[i] = data[i];
2626 
2627         /*
2628          * Because strings are not zeroed out by default, we need to iterate
2629          * looking for actions that store strings, and we need to explicitly
2630          * pad these strings out with zeroes.
2631          */
2632         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2633                 int nul;
2634 
2635                 if (!DTRACEACT_ISSTRING(act))
2636                         continue;
2637 
2638                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2639                 limit = i + act->dta_rec.dtrd_size;
2640                 ASSERT(limit <= size);
2641 
2642                 for (nul = 0; i < limit; i++) {
2643                         if (nul) {
2644                                 kdata[i] = '\0';
2645                                 continue;
2646                         }
2647 
2648                         if (data[i] != '\0')
2649                                 continue;
2650 
2651                         nul = 1;
2652                 }
2653         }
2654 
2655         for (i = size; i < fsize; i++)
2656                 kdata[i] = 0;
2657 
2658         key->dtak_hashval = hashval;
2659         key->dtak_size = size;
2660         key->dtak_action = action;
2661         key->dtak_next = agb->dtagb_hash[ndx];
2662         agb->dtagb_hash[ndx] = key;
2663 
2664         /*
2665          * Finally, apply the aggregator.
2666          */
2667         *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
2668         agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
2669 }
2670 
2671 /*
2672  * Given consumer state, this routine finds a speculation in the INACTIVE
2673  * state and transitions it into the ACTIVE state.  If there is no speculation
2674  * in the INACTIVE state, 0 is returned.  In this case, no error counter is
2675  * incremented -- it is up to the caller to take appropriate action.
2676  */
2677 static int
2678 dtrace_speculation(dtrace_state_t *state)
2679 {
2680         int i = 0;
2681         dtrace_speculation_state_t current;
2682         uint32_t *stat = &state->dts_speculations_unavail, count;
2683 
2684         while (i < state->dts_nspeculations) {
2685                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2686 
2687                 current = spec->dtsp_state;
2688 
2689                 if (current != DTRACESPEC_INACTIVE) {
2690                         if (current == DTRACESPEC_COMMITTINGMANY ||
2691                             current == DTRACESPEC_COMMITTING ||
2692                             current == DTRACESPEC_DISCARDING)
2693                                 stat = &state->dts_speculations_busy;
2694                         i++;
2695                         continue;
2696                 }
2697 
2698                 if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2699                     current, DTRACESPEC_ACTIVE) == current)
2700                         return (i + 1);
2701         }
2702 
2703         /*
2704          * We couldn't find a speculation.  If we found as much as a single
2705          * busy speculation buffer, we'll attribute this failure as "busy"
2706          * instead of "unavail".
2707          */
2708         do {
2709                 count = *stat;
2710         } while (dtrace_cas32(stat, count, count + 1) != count);
2711 
2712         return (0);
2713 }
2714 
2715 /*
2716  * This routine commits an active speculation.  If the specified speculation
2717  * is not in a valid state to perform a commit(), this routine will silently do
2718  * nothing.  The state of the specified speculation is transitioned according
2719  * to the state transition diagram outlined in <sys/dtrace_impl.h>
2720  */
2721 static void
2722 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2723     dtrace_specid_t which)
2724 {
2725         dtrace_speculation_t *spec;
2726         dtrace_buffer_t *src, *dest;
2727         uintptr_t daddr, saddr, dlimit, slimit;
2728         dtrace_speculation_state_t current, new;
2729         intptr_t offs;
2730         uint64_t timestamp;
2731 
2732         if (which == 0)
2733                 return;
2734 
2735         if (which > state->dts_nspeculations) {
2736                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2737                 return;
2738         }
2739 
2740         spec = &state->dts_speculations[which - 1];
2741         src = &spec->dtsp_buffer[cpu];
2742         dest = &state->dts_buffer[cpu];
2743 
2744         do {
2745                 current = spec->dtsp_state;
2746 
2747                 if (current == DTRACESPEC_COMMITTINGMANY)
2748                         break;
2749 
2750                 switch (current) {
2751                 case DTRACESPEC_INACTIVE:
2752                 case DTRACESPEC_DISCARDING:
2753                         return;
2754 
2755                 case DTRACESPEC_COMMITTING:
2756                         /*
2757                          * This is only possible if we are (a) commit()'ing
2758                          * without having done a prior speculate() on this CPU
2759                          * and (b) racing with another commit() on a different
2760                          * CPU.  There's nothing to do -- we just assert that
2761                          * our offset is 0.
2762                          */
2763                         ASSERT(src->dtb_offset == 0);
2764                         return;
2765 
2766                 case DTRACESPEC_ACTIVE:
2767                         new = DTRACESPEC_COMMITTING;
2768                         break;
2769 
2770                 case DTRACESPEC_ACTIVEONE:
2771                         /*
2772                          * This speculation is active on one CPU.  If our
2773                          * buffer offset is non-zero, we know that the one CPU
2774                          * must be us.  Otherwise, we are committing on a
2775                          * different CPU from the speculate(), and we must
2776                          * rely on being asynchronously cleaned.
2777                          */
2778                         if (src->dtb_offset != 0) {
2779                                 new = DTRACESPEC_COMMITTING;
2780                                 break;
2781                         }
2782                         /*FALLTHROUGH*/
2783 
2784                 case DTRACESPEC_ACTIVEMANY:
2785                         new = DTRACESPEC_COMMITTINGMANY;
2786                         break;
2787 
2788                 default:
2789                         ASSERT(0);
2790                 }
2791         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2792             current, new) != current);
2793 
2794         /*
2795          * We have set the state to indicate that we are committing this
2796          * speculation.  Now reserve the necessary space in the destination
2797          * buffer.
2798          */
2799         if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
2800             sizeof (uint64_t), state, NULL)) < 0) {
2801                 dtrace_buffer_drop(dest);
2802                 goto out;
2803         }
2804 
2805         /*
2806          * We have sufficient space to copy the speculative buffer into the
2807          * primary buffer.  First, modify the speculative buffer, filling
2808          * in the timestamp of all entries with the current time.  The data
2809          * must have the commit() time rather than the time it was traced,
2810          * so that all entries in the primary buffer are in timestamp order.
2811          */
2812         timestamp = dtrace_gethrtime();
2813         saddr = (uintptr_t)src->dtb_tomax;
2814         slimit = saddr + src->dtb_offset;
2815         while (saddr < slimit) {
2816                 size_t size;
2817                 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
2818 
2819                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2820                         saddr += sizeof (dtrace_epid_t);
2821                         continue;
2822                 }
2823                 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
2824                 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
2825 
2826                 ASSERT3U(saddr + size, <=, slimit);
2827                 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
2828                 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
2829 
2830                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
2831 
2832                 saddr += size;
2833         }
2834 
2835         /*
2836          * Copy the buffer across.  (Note that this is a
2837          * highly subobtimal bcopy(); in the unlikely event that this becomes
2838          * a serious performance issue, a high-performance DTrace-specific
2839          * bcopy() should obviously be invented.)
2840          */
2841         daddr = (uintptr_t)dest->dtb_tomax + offs;
2842         dlimit = daddr + src->dtb_offset;
2843         saddr = (uintptr_t)src->dtb_tomax;
2844 
2845         /*
2846          * First, the aligned portion.
2847          */
2848         while (dlimit - daddr >= sizeof (uint64_t)) {
2849                 *((uint64_t *)daddr) = *((uint64_t *)saddr);
2850 
2851                 daddr += sizeof (uint64_t);
2852                 saddr += sizeof (uint64_t);
2853         }
2854 
2855         /*
2856          * Now any left-over bit...
2857          */
2858         while (dlimit - daddr)
2859                 *((uint8_t *)daddr++) = *((uint8_t *)saddr++);
2860 
2861         /*
2862          * Finally, commit the reserved space in the destination buffer.
2863          */
2864         dest->dtb_offset = offs + src->dtb_offset;
2865 
2866 out:
2867         /*
2868          * If we're lucky enough to be the only active CPU on this speculation
2869          * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2870          */
2871         if (current == DTRACESPEC_ACTIVE ||
2872             (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
2873                 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
2874                     DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
2875 
2876                 ASSERT(rval == DTRACESPEC_COMMITTING);
2877         }
2878 
2879         src->dtb_offset = 0;
2880         src->dtb_xamot_drops += src->dtb_drops;
2881         src->dtb_drops = 0;
2882 }
2883 
2884 /*
2885  * This routine discards an active speculation.  If the specified speculation
2886  * is not in a valid state to perform a discard(), this routine will silently
2887  * do nothing.  The state of the specified speculation is transitioned
2888  * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2889  */
2890 static void
2891 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2892     dtrace_specid_t which)
2893 {
2894         dtrace_speculation_t *spec;
2895         dtrace_speculation_state_t current, new;
2896         dtrace_buffer_t *buf;
2897 
2898         if (which == 0)
2899                 return;
2900 
2901         if (which > state->dts_nspeculations) {
2902                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2903                 return;
2904         }
2905 
2906         spec = &state->dts_speculations[which - 1];
2907         buf = &spec->dtsp_buffer[cpu];
2908 
2909         do {
2910                 current = spec->dtsp_state;
2911 
2912                 switch (current) {
2913                 case DTRACESPEC_INACTIVE:
2914                 case DTRACESPEC_COMMITTINGMANY:
2915                 case DTRACESPEC_COMMITTING:
2916                 case DTRACESPEC_DISCARDING:
2917                         return;
2918 
2919                 case DTRACESPEC_ACTIVE:
2920                 case DTRACESPEC_ACTIVEMANY:
2921                         new = DTRACESPEC_DISCARDING;
2922                         break;
2923 
2924                 case DTRACESPEC_ACTIVEONE:
2925                         if (buf->dtb_offset != 0) {
2926                                 new = DTRACESPEC_INACTIVE;
2927                         } else {
2928                                 new = DTRACESPEC_DISCARDING;
2929                         }
2930                         break;
2931 
2932                 default:
2933                         ASSERT(0);
2934                 }
2935         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2936             current, new) != current);
2937 
2938         buf->dtb_offset = 0;
2939         buf->dtb_drops = 0;
2940 }
2941 
2942 /*
2943  * Note:  not called from probe context.  This function is called
2944  * asynchronously from cross call context to clean any speculations that are
2945  * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
2946  * transitioned back to the INACTIVE state until all CPUs have cleaned the
2947  * speculation.
2948  */
2949 static void
2950 dtrace_speculation_clean_here(dtrace_state_t *state)
2951 {
2952         dtrace_icookie_t cookie;
2953         processorid_t cpu = CPU->cpu_id;
2954         dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2955         dtrace_specid_t i;
2956 
2957         cookie = dtrace_interrupt_disable();
2958 
2959         if (dest->dtb_tomax == NULL) {
2960                 dtrace_interrupt_enable(cookie);
2961                 return;
2962         }
2963 
2964         for (i = 0; i < state->dts_nspeculations; i++) {
2965                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2966                 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
2967 
2968                 if (src->dtb_tomax == NULL)
2969                         continue;
2970 
2971                 if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
2972                         src->dtb_offset = 0;
2973                         continue;
2974                 }
2975 
2976                 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2977                         continue;
2978 
2979                 if (src->dtb_offset == 0)
2980                         continue;
2981 
2982                 dtrace_speculation_commit(state, cpu, i + 1);
2983         }
2984 
2985         dtrace_interrupt_enable(cookie);
2986 }
2987 
2988 /*
2989  * Note:  not called from probe context.  This function is called
2990  * asynchronously (and at a regular interval) to clean any speculations that
2991  * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
2992  * is work to be done, it cross calls all CPUs to perform that work;
2993  * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2994  * INACTIVE state until they have been cleaned by all CPUs.
2995  */
2996 static void
2997 dtrace_speculation_clean(dtrace_state_t *state)
2998 {
2999         int work = 0, rv;
3000         dtrace_specid_t i;
3001 
3002         for (i = 0; i < state->dts_nspeculations; i++) {
3003                 dtrace_speculation_t *spec = &state->dts_speculations[i];
3004 
3005                 ASSERT(!spec->dtsp_cleaning);
3006 
3007                 if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
3008                     spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
3009                         continue;
3010 
3011                 work++;
3012                 spec->dtsp_cleaning = 1;
3013         }
3014 
3015         if (!work)
3016                 return;
3017 
3018         dtrace_xcall(DTRACE_CPUALL,
3019             (dtrace_xcall_t)dtrace_speculation_clean_here, state);
3020 
3021         /*
3022          * We now know that all CPUs have committed or discarded their
3023          * speculation buffers, as appropriate.  We can now set the state
3024          * to inactive.
3025          */
3026         for (i = 0; i < state->dts_nspeculations; i++) {
3027                 dtrace_speculation_t *spec = &state->dts_speculations[i];
3028                 dtrace_speculation_state_t current, new;
3029 
3030                 if (!spec->dtsp_cleaning)
3031                         continue;
3032 
3033                 current = spec->dtsp_state;
3034                 ASSERT(current == DTRACESPEC_DISCARDING ||
3035                     current == DTRACESPEC_COMMITTINGMANY);
3036 
3037                 new = DTRACESPEC_INACTIVE;
3038 
3039                 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
3040                 ASSERT(rv == current);
3041                 spec->dtsp_cleaning = 0;
3042         }
3043 }
3044 
3045 /*
3046  * Called as part of a speculate() to get the speculative buffer associated
3047  * with a given speculation.  Returns NULL if the specified speculation is not
3048  * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
3049  * the active CPU is not the specified CPU -- the speculation will be
3050  * atomically transitioned into the ACTIVEMANY state.
3051  */
3052 static dtrace_buffer_t *
3053 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
3054     dtrace_specid_t which)
3055 {
3056         dtrace_speculation_t *spec;
3057         dtrace_speculation_state_t current, new;
3058         dtrace_buffer_t *buf;
3059 
3060         if (which == 0)
3061                 return (NULL);
3062 
3063         if (which > state->dts_nspeculations) {
3064                 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
3065                 return (NULL);
3066         }
3067 
3068         spec = &state->dts_speculations[which - 1];
3069         buf = &spec->dtsp_buffer[cpuid];
3070 
3071         do {
3072                 current = spec->dtsp_state;
3073 
3074                 switch (current) {
3075                 case DTRACESPEC_INACTIVE:
3076                 case DTRACESPEC_COMMITTINGMANY:
3077                 case DTRACESPEC_DISCARDING:
3078                         return (NULL);
3079 
3080                 case DTRACESPEC_COMMITTING:
3081                         ASSERT(buf->dtb_offset == 0);
3082                         return (NULL);
3083 
3084                 case DTRACESPEC_ACTIVEONE:
3085                         /*
3086                          * This speculation is currently active on one CPU.
3087                          * Check the offset in the buffer; if it's non-zero,
3088                          * that CPU must be us (and we leave the state alone).
3089                          * If it's zero, assume that we're starting on a new
3090                          * CPU -- and change the state to indicate that the
3091                          * speculation is active on more than one CPU.
3092                          */
3093                         if (buf->dtb_offset != 0)
3094                                 return (buf);
3095 
3096                         new = DTRACESPEC_ACTIVEMANY;
3097                         break;
3098 
3099                 case DTRACESPEC_ACTIVEMANY:
3100                         return (buf);
3101 
3102                 case DTRACESPEC_ACTIVE:
3103                         new = DTRACESPEC_ACTIVEONE;
3104                         break;
3105 
3106                 default:
3107                         ASSERT(0);
3108                 }
3109         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
3110             current, new) != current);
3111 
3112         ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
3113         return (buf);
3114 }
3115 
3116 /*
3117  * Return a string.  In the event that the user lacks the privilege to access
3118  * arbitrary kernel memory, we copy the string out to scratch memory so that we
3119  * don't fail access checking.
3120  *
3121  * dtrace_dif_variable() uses this routine as a helper for various
3122  * builtin values such as 'execname' and 'probefunc.'
3123  */
3124 uintptr_t
3125 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
3126     dtrace_mstate_t *mstate)
3127 {
3128         uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3129         uintptr_t ret;
3130         size_t strsz;
3131 
3132         /*
3133          * The easy case: this probe is allowed to read all of memory, so
3134          * we can just return this as a vanilla pointer.
3135          */
3136         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
3137                 return (addr);
3138 
3139         /*
3140          * This is the tougher case: we copy the string in question from
3141          * kernel memory into scratch memory and return it that way: this
3142          * ensures that we won't trip up when access checking tests the
3143          * BYREF return value.
3144          */
3145         strsz = dtrace_strlen((char *)addr, size) + 1;
3146 
3147         if (mstate->dtms_scratch_ptr + strsz >
3148             mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3149                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3150                 return (0);
3151         }
3152 
3153         dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
3154             strsz);
3155         ret = mstate->dtms_scratch_ptr;
3156         mstate->dtms_scratch_ptr += strsz;
3157         return (ret);
3158 }
3159 
3160 /*
3161  * This function implements the DIF emulator's variable lookups.  The emulator
3162  * passes a reserved variable identifier and optional built-in array index.
3163  */
3164 static uint64_t
3165 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
3166     uint64_t ndx)
3167 {
3168         /*
3169          * If we're accessing one of the uncached arguments, we'll turn this
3170          * into a reference in the args array.
3171          */
3172         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
3173                 ndx = v - DIF_VAR_ARG0;
3174                 v = DIF_VAR_ARGS;
3175         }
3176 
3177         switch (v) {
3178         case DIF_VAR_ARGS:
3179                 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) {
3180                         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |=
3181                             CPU_DTRACE_KPRIV;
3182                         return (0);
3183                 }
3184 
3185                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
3186                 if (ndx >= sizeof (mstate->dtms_arg) /
3187                     sizeof (mstate->dtms_arg[0])) {
3188                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3189                         dtrace_provider_t *pv;
3190                         uint64_t val;
3191 
3192                         pv = mstate->dtms_probe->dtpr_provider;
3193                         if (pv->dtpv_pops.dtps_getargval != NULL)
3194                                 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
3195                                     mstate->dtms_probe->dtpr_id,
3196                                     mstate->dtms_probe->dtpr_arg, ndx, aframes);
3197                         else
3198                                 val = dtrace_getarg(ndx, aframes);
3199 
3200                         /*
3201                          * This is regrettably required to keep the compiler
3202                          * from tail-optimizing the call to dtrace_getarg().
3203                          * The condition always evaluates to true, but the
3204                          * compiler has no way of figuring that out a priori.
3205                          * (None of this would be necessary if the compiler
3206                          * could be relied upon to _always_ tail-optimize
3207                          * the call to dtrace_getarg() -- but it can't.)
3208                          */
3209                         if (mstate->dtms_probe != NULL)
3210                                 return (val);
3211 
3212                         ASSERT(0);
3213                 }
3214 
3215                 return (mstate->dtms_arg[ndx]);
3216 
3217         case DIF_VAR_UREGS: {
3218                 klwp_t *lwp;
3219 
3220                 if (!dtrace_priv_proc(state, mstate))
3221                         return (0);
3222 
3223                 if ((lwp = curthread->t_lwp) == NULL) {
3224                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
3225                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = 0;
3226                         return (0);
3227                 }
3228 
3229                 return (dtrace_getreg(lwp->lwp_regs, ndx));
3230         }
3231 
3232         case DIF_VAR_VMREGS: {
3233                 uint64_t rval;
3234 
3235                 if (!dtrace_priv_kernel(state))
3236                         return (0);
3237 
3238                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3239 
3240                 rval = dtrace_getvmreg(ndx,
3241                     &cpu_core[CPU->cpu_id].cpuc_dtrace_flags);
3242 
3243                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3244 
3245                 return (rval);
3246         }
3247 
3248         case DIF_VAR_CURTHREAD:
3249                 if (!dtrace_priv_proc(state, mstate))
3250                         return (0);
3251                 return ((uint64_t)(uintptr_t)curthread);
3252 
3253         case DIF_VAR_TIMESTAMP:
3254                 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
3255                         mstate->dtms_timestamp = dtrace_gethrtime();
3256                         mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
3257                 }
3258                 return (mstate->dtms_timestamp);
3259 
3260         case DIF_VAR_VTIMESTAMP:
3261                 ASSERT(dtrace_vtime_references != 0);
3262                 return (curthread->t_dtrace_vtime);
3263 
3264         case DIF_VAR_WALLTIMESTAMP:
3265                 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
3266                         mstate->dtms_walltimestamp = dtrace_gethrestime();
3267                         mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
3268                 }
3269                 return (mstate->dtms_walltimestamp);
3270 
3271         case DIF_VAR_IPL:
3272                 if (!dtrace_priv_kernel(state))
3273                         return (0);
3274                 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
3275                         mstate->dtms_ipl = dtrace_getipl();
3276                         mstate->dtms_present |= DTRACE_MSTATE_IPL;
3277                 }
3278                 return (mstate->dtms_ipl);
3279 
3280         case DIF_VAR_EPID:
3281                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
3282                 return (mstate->dtms_epid);
3283 
3284         case DIF_VAR_ID:
3285                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3286                 return (mstate->dtms_probe->dtpr_id);
3287 
3288         case DIF_VAR_STACKDEPTH:
3289                 if (!dtrace_priv_kernel(state))
3290                         return (0);
3291                 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
3292                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3293 
3294                         mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
3295                         mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
3296                 }
3297                 return (mstate->dtms_stackdepth);
3298 
3299         case DIF_VAR_USTACKDEPTH:
3300                 if (!dtrace_priv_proc(state, mstate))
3301                         return (0);
3302                 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
3303                         /*
3304                          * See comment in DIF_VAR_PID.
3305                          */
3306                         if (DTRACE_ANCHORED(mstate->dtms_probe) &&
3307                             CPU_ON_INTR(CPU)) {
3308                                 mstate->dtms_ustackdepth = 0;
3309                         } else {
3310                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3311                                 mstate->dtms_ustackdepth =
3312                                     dtrace_getustackdepth();
3313                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3314                         }
3315                         mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
3316                 }
3317                 return (mstate->dtms_ustackdepth);
3318 
3319         case DIF_VAR_CALLER:
3320                 if (!dtrace_priv_kernel(state))
3321                         return (0);
3322                 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
3323                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3324 
3325                         if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
3326                                 /*
3327                                  * If this is an unanchored probe, we are
3328                                  * required to go through the slow path:
3329                                  * dtrace_caller() only guarantees correct
3330                                  * results for anchored probes.
3331                                  */
3332                                 pc_t caller[2];
3333 
3334                                 dtrace_getpcstack(caller, 2, aframes,
3335                                     (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
3336                                 mstate->dtms_caller = caller[1];
3337                         } else if ((mstate->dtms_caller =
3338                             dtrace_caller(aframes)) == -1) {
3339                                 /*
3340                                  * We have failed to do this the quick way;
3341                                  * we must resort to the slower approach of
3342                                  * calling dtrace_getpcstack().
3343                                  */
3344                                 pc_t caller;
3345 
3346                                 dtrace_getpcstack(&caller, 1, aframes, NULL);
3347                                 mstate->dtms_caller = caller;
3348                         }
3349 
3350                         mstate->dtms_present |= DTRACE_MSTATE_CALLER;
3351                 }
3352                 return (mstate->dtms_caller);
3353 
3354         case DIF_VAR_UCALLER:
3355                 if (!dtrace_priv_proc(state, mstate))
3356                         return (0);
3357 
3358                 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
3359                         uint64_t ustack[3];
3360 
3361                         /*
3362                          * dtrace_getupcstack() fills in the first uint64_t
3363                          * with the current PID.  The second uint64_t will
3364                          * be the program counter at user-level.  The third
3365                          * uint64_t will contain the caller, which is what
3366                          * we're after.
3367                          */
3368                         ustack[2] = 0;
3369                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3370                         dtrace_getupcstack(ustack, 3);
3371                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3372                         mstate->dtms_ucaller = ustack[2];
3373                         mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
3374                 }
3375 
3376                 return (mstate->dtms_ucaller);
3377 
3378         case DIF_VAR_PROBEPROV:
3379                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3380                 return (dtrace_dif_varstr(
3381                     (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
3382                     state, mstate));
3383 
3384         case DIF_VAR_PROBEMOD:
3385                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3386                 return (dtrace_dif_varstr(
3387                     (uintptr_t)mstate->dtms_probe->dtpr_mod,
3388                     state, mstate));
3389 
3390         case DIF_VAR_PROBEFUNC:
3391                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3392                 return (dtrace_dif_varstr(
3393                     (uintptr_t)mstate->dtms_probe->dtpr_func,
3394                     state, mstate));
3395 
3396         case DIF_VAR_PROBENAME:
3397                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3398                 return (dtrace_dif_varstr(
3399                     (uintptr_t)mstate->dtms_probe->dtpr_name,
3400                     state, mstate));
3401 
3402         case DIF_VAR_PID:
3403                 if (!dtrace_priv_proc(state, mstate))
3404                         return (0);
3405 
3406                 /*
3407                  * Note that we are assuming that an unanchored probe is
3408                  * always due to a high-level interrupt.  (And we're assuming
3409                  * that there is only a single high level interrupt.)
3410                  */
3411                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3412                         return (pid0.pid_id);
3413 
3414                 /*
3415                  * It is always safe to dereference one's own t_procp pointer:
3416                  * it always points to a valid, allocated proc structure.
3417                  * Further, it is always safe to dereference the p_pidp member
3418                  * of one's own proc structure.  (These are truisms becuase
3419                  * threads and processes don't clean up their own state --
3420                  * they leave that task to whomever reaps them.)
3421                  */
3422                 return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
3423 
3424         case DIF_VAR_PPID:
3425                 if (!dtrace_priv_proc(state, mstate))
3426                         return (0);
3427 
3428                 /*
3429                  * See comment in DIF_VAR_PID.
3430                  */
3431                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3432                         return (pid0.pid_id);
3433 
3434                 /*
3435                  * It is always safe to dereference one's own t_procp pointer:
3436                  * it always points to a valid, allocated proc structure.
3437                  * (This is true because threads don't clean up their own
3438                  * state -- they leave that task to whomever reaps them.)
3439                  */
3440                 return ((uint64_t)curthread->t_procp->p_ppid);
3441 
3442         case DIF_VAR_TID:
3443                 /*
3444                  * See comment in DIF_VAR_PID.
3445                  */
3446                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3447                         return (0);
3448 
3449                 return ((uint64_t)curthread->t_tid);
3450 
3451         case DIF_VAR_EXECNAME:
3452                 if (!dtrace_priv_proc(state, mstate))
3453                         return (0);
3454 
3455                 /*
3456                  * See comment in DIF_VAR_PID.
3457                  */
3458                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3459                         return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
3460 
3461                 /*
3462                  * It is always safe to dereference one's own t_procp pointer:
3463                  * it always points to a valid, allocated proc structure.
3464                  * (This is true because threads don't clean up their own
3465                  * state -- they leave that task to whomever reaps them.)
3466                  */
3467                 return (dtrace_dif_varstr(
3468                     (uintptr_t)curthread->t_procp->p_user.u_comm,
3469                     state, mstate));
3470 
3471         case DIF_VAR_ZONENAME:
3472                 if (!dtrace_priv_proc(state, mstate))
3473                         return (0);
3474 
3475                 /*
3476                  * See comment in DIF_VAR_PID.
3477                  */
3478                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3479                         return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
3480 
3481                 /*
3482                  * It is always safe to dereference one's own t_procp pointer:
3483                  * it always points to a valid, allocated proc structure.
3484                  * (This is true because threads don't clean up their own
3485                  * state -- they leave that task to whomever reaps them.)
3486                  */
3487                 return (dtrace_dif_varstr(
3488                     (uintptr_t)curthread->t_procp->p_zone->zone_name,
3489                     state, mstate));
3490 
3491         case DIF_VAR_UID:
3492                 if (!dtrace_priv_proc(state, mstate))
3493                         return (0);
3494 
3495                 /*
3496                  * See comment in DIF_VAR_PID.
3497                  */
3498                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3499                         return ((uint64_t)p0.p_cred->cr_uid);
3500 
3501                 /*
3502                  * It is always safe to dereference one's own t_procp pointer:
3503                  * it always points to a valid, allocated proc structure.
3504                  * (This is true because threads don't clean up their own
3505                  * state -- they leave that task to whomever reaps them.)
3506                  *
3507                  * Additionally, it is safe to dereference one's own process
3508                  * credential, since this is never NULL after process birth.
3509                  */
3510                 return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
3511 
3512         case DIF_VAR_GID:
3513                 if (!dtrace_priv_proc(state, mstate))
3514                         return (0);
3515 
3516                 /*
3517                  * See comment in DIF_VAR_PID.
3518                  */
3519                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3520                         return ((uint64_t)p0.p_cred->cr_gid);
3521 
3522                 /*
3523                  * It is always safe to dereference one's own t_procp pointer:
3524                  * it always points to a valid, allocated proc structure.
3525                  * (This is true because threads don't clean up their own
3526                  * state -- they leave that task to whomever reaps them.)
3527                  *
3528                  * Additionally, it is safe to dereference one's own process
3529                  * credential, since this is never NULL after process birth.
3530                  */
3531                 return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
3532 
3533         case DIF_VAR_ERRNO: {
3534                 klwp_t *lwp;
3535                 if (!dtrace_priv_proc(state, mstate))
3536                         return (0);
3537 
3538                 /*
3539                  * See comment in DIF_VAR_PID.
3540                  */
3541                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3542                         return (0);
3543 
3544                 /*
3545                  * It is always safe to dereference one's own t_lwp pointer in
3546                  * the event that this pointer is non-NULL.  (This is true
3547                  * because threads and lwps don't clean up their own state --
3548                  * they leave that task to whomever reaps them.)
3549                  */
3550                 if ((lwp = curthread->t_lwp) == NULL)
3551                         return (0);
3552 
3553                 return ((uint64_t)lwp->lwp_errno);
3554         }
3555 
3556         case DIF_VAR_THREADNAME:
3557                 /*
3558                  * See comment in DIF_VAR_PID.
3559                  */
3560                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3561                         return (0);
3562 
3563                 if (curthread->t_name == NULL)
3564                         return (0);
3565 
3566                 /*
3567                  * Once set, ->t_name itself is never changed: any updates are
3568                  * made to the same buffer that we are pointing out.  So we are
3569                  * safe to dereference it here.
3570                  */
3571                 return (dtrace_dif_varstr((uintptr_t)curthread->t_name,
3572                     state, mstate));
3573 
3574         default:
3575                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3576                 return (0);
3577         }
3578 }
3579 
3580 static void
3581 dtrace_dif_variable_write(dtrace_mstate_t *mstate, dtrace_state_t *state,
3582     uint64_t v, uint64_t ndx, uint64_t data)
3583 {
3584         switch (v) {
3585         case DIF_VAR_UREGS: {
3586                 klwp_t *lwp;
3587 
3588                 if (dtrace_destructive_disallow ||
3589                     !dtrace_priv_proc_control(state, mstate)) {
3590                         return;
3591                 }
3592 
3593                 if ((lwp = curthread->t_lwp) == NULL) {
3594                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
3595                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = 0;
3596                         return;
3597                 }
3598 
3599                 dtrace_setreg(lwp->lwp_regs, ndx, data);
3600                 return;
3601         }
3602 
3603         default:
3604                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3605                 return;
3606         }
3607 }
3608 
3609 typedef enum dtrace_json_state {
3610         DTRACE_JSON_REST = 1,
3611         DTRACE_JSON_OBJECT,
3612         DTRACE_JSON_STRING,
3613         DTRACE_JSON_STRING_ESCAPE,
3614         DTRACE_JSON_STRING_ESCAPE_UNICODE,
3615         DTRACE_JSON_COLON,
3616         DTRACE_JSON_COMMA,
3617         DTRACE_JSON_VALUE,
3618         DTRACE_JSON_IDENTIFIER,
3619         DTRACE_JSON_NUMBER,
3620         DTRACE_JSON_NUMBER_FRAC,
3621         DTRACE_JSON_NUMBER_EXP,
3622         DTRACE_JSON_COLLECT_OBJECT
3623 } dtrace_json_state_t;
3624 
3625 /*
3626  * This function possesses just enough knowledge about JSON to extract a single
3627  * value from a JSON string and store it in the scratch buffer.  It is able
3628  * to extract nested object values, and members of arrays by index.
3629  *
3630  * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to
3631  * be looked up as we descend into the object tree.  e.g.
3632  *
3633  *    foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL
3634  *       with nelems = 5.
3635  *
3636  * The run time of this function must be bounded above by strsize to limit the
3637  * amount of work done in probe context.  As such, it is implemented as a
3638  * simple state machine, reading one character at a time using safe loads
3639  * until we find the requested element, hit a parsing error or run off the
3640  * end of the object or string.
3641  *
3642  * As there is no way for a subroutine to return an error without interrupting
3643  * clause execution, we simply return NULL in the event of a missing key or any
3644  * other error condition.  Each NULL return in this function is commented with
3645  * the error condition it represents -- parsing or otherwise.
3646  *
3647  * The set of states for the state machine closely matches the JSON
3648  * specification (http://json.org/).  Briefly:
3649  *
3650  *   DTRACE_JSON_REST:
3651  *     Skip whitespace until we find either a top-level Object, moving
3652  *     to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE.
3653  *
3654  *   DTRACE_JSON_OBJECT:
3655  *     Locate the next key String in an Object.  Sets a flag to denote
3656  *     the next String as a key string and moves to DTRACE_JSON_STRING.
3657  *
3658  *   DTRACE_JSON_COLON:
3659  *     Skip whitespace until we find the colon that separates key Strings
3660  *     from their values.  Once found, move to DTRACE_JSON_VALUE.
3661  *
3662  *   DTRACE_JSON_VALUE:
3663  *     Detects the type of the next value (String, Number, Identifier, Object
3664  *     or Array) and routes to the states that process that type.  Here we also
3665  *     deal with the element selector list if we are requested to traverse down
3666  *     into the object tree.
3667  *
3668  *   DTRACE_JSON_COMMA:
3669  *     Skip whitespace until we find the comma that separates key-value pairs
3670  *     in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays
3671  *     (similarly DTRACE_JSON_VALUE).  All following literal value processing
3672  *     states return to this state at the end of their value, unless otherwise
3673  *     noted.
3674  *
3675  *   DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP:
3676  *     Processes a Number literal from the JSON, including any exponent
3677  *     component that may be present.  Numbers are returned as strings, which
3678  *     may be passed to strtoll() if an integer is required.
3679  *
3680  *   DTRACE_JSON_IDENTIFIER:
3681  *     Processes a "true", "false" or "null" literal in the JSON.
3682  *
3683  *   DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE,
3684  *   DTRACE_JSON_STRING_ESCAPE_UNICODE:
3685  *     Processes a String literal from the JSON, whether the String denotes
3686  *     a key, a value or part of a larger Object.  Handles all escape sequences
3687  *     present in the specification, including four-digit unicode characters,
3688  *     but merely includes the escape sequence without converting it to the
3689  *     actual escaped character.  If the String is flagged as a key, we
3690  *     move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA.
3691  *
3692  *   DTRACE_JSON_COLLECT_OBJECT:
3693  *     This state collects an entire Object (or Array), correctly handling
3694  *     embedded strings.  If the full element selector list matches this nested
3695  *     object, we return the Object in full as a string.  If not, we use this
3696  *     state to skip to the next value at this level and continue processing.
3697  *
3698  * NOTE: This function uses various macros from strtolctype.h to manipulate
3699  * digit values, etc -- these have all been checked to ensure they make
3700  * no additional function calls.
3701  */
3702 static char *
3703 dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems,
3704     char *dest)
3705 {
3706         dtrace_json_state_t state = DTRACE_JSON_REST;
3707         int64_t array_elem = INT64_MIN;
3708         int64_t array_pos = 0;
3709         uint8_t escape_unicount = 0;
3710         boolean_t string_is_key = B_FALSE;
3711         boolean_t collect_object = B_FALSE;
3712         boolean_t found_key = B_FALSE;
3713         boolean_t in_array = B_FALSE;
3714         uint32_t braces = 0, brackets = 0;
3715         char *elem = elemlist;
3716         char *dd = dest;
3717         uintptr_t cur;
3718 
3719         for (cur = json; cur < json + size; cur++) {
3720                 char cc = dtrace_load8(cur);
3721                 if (cc == '\0')
3722                         return (NULL);
3723 
3724                 switch (state) {
3725                 case DTRACE_JSON_REST:
3726                         if (isspace(cc))
3727                                 break;
3728 
3729                         if (cc == '{') {
3730                                 state = DTRACE_JSON_OBJECT;
3731                                 break;
3732                         }
3733 
3734                         if (cc == '[') {
3735                                 in_array = B_TRUE;
3736                                 array_pos = 0;
3737                                 array_elem = dtrace_strtoll(elem, 10, size);
3738                                 found_key = array_elem == 0 ? B_TRUE : B_FALSE;
3739                                 state = DTRACE_JSON_VALUE;
3740                                 break;
3741                         }
3742 
3743                         /*
3744                          * ERROR: expected to find a top-level object or array.
3745                          */
3746                         return (NULL);
3747                 case DTRACE_JSON_OBJECT:
3748                         if (isspace(cc))
3749                                 break;
3750 
3751                         if (cc == '"') {
3752                                 state = DTRACE_JSON_STRING;
3753                                 string_is_key = B_TRUE;
3754                                 break;
3755                         }
3756 
3757                         /*
3758                          * ERROR: either the object did not start with a key
3759                          * string, or we've run off the end of the object
3760                          * without finding the requested key.
3761                          */
3762                         return (NULL);
3763                 case DTRACE_JSON_STRING:
3764                         if (cc == '\\') {
3765                                 *dd++ = '\\';
3766                                 state = DTRACE_JSON_STRING_ESCAPE;
3767                                 break;
3768                         }
3769 
3770                         if (cc == '"') {
3771                                 if (collect_object) {
3772                                         /*
3773                                          * We don't reset the dest here, as
3774                                          * the string is part of a larger
3775                                          * object being collected.
3776                                          */
3777                                         *dd++ = cc;
3778                                         collect_object = B_FALSE;
3779                                         state = DTRACE_JSON_COLLECT_OBJECT;
3780                                         break;
3781                                 }
3782                                 *dd = '\0';
3783                                 dd = dest; /* reset string buffer */
3784                                 if (string_is_key) {
3785                                         if (dtrace_strncmp(dest, elem,
3786                                             size) == 0)
3787                                                 found_key = B_TRUE;
3788                                 } else if (found_key) {
3789                                         if (nelems > 1) {
3790                                                 /*
3791                                                  * We expected an object, not
3792                                                  * this string.
3793                                                  */
3794                                                 return (NULL);
3795                                         }
3796                                         return (dest);
3797                                 }
3798                                 state = string_is_key ? DTRACE_JSON_COLON :
3799                                     DTRACE_JSON_COMMA;
3800                                 string_is_key = B_FALSE;
3801                                 break;
3802                         }
3803 
3804                         *dd++ = cc;
3805                         break;
3806                 case DTRACE_JSON_STRING_ESCAPE:
3807                         *dd++ = cc;
3808                         if (cc == 'u') {
3809                                 escape_unicount = 0;
3810                                 state = DTRACE_JSON_STRING_ESCAPE_UNICODE;
3811                         } else {
3812                                 state = DTRACE_JSON_STRING;
3813                         }
3814                         break;
3815                 case DTRACE_JSON_STRING_ESCAPE_UNICODE:
3816                         if (!isxdigit(cc)) {
3817                                 /*
3818                                  * ERROR: invalid unicode escape, expected
3819                                  * four valid hexidecimal digits.
3820                                  */
3821                                 return (NULL);
3822                         }
3823 
3824                         *dd++ = cc;
3825                         if (++escape_unicount == 4)
3826                                 state = DTRACE_JSON_STRING;
3827                         break;
3828                 case DTRACE_JSON_COLON:
3829                         if (isspace(cc))
3830                                 break;
3831 
3832                         if (cc == ':') {
3833                                 state = DTRACE_JSON_VALUE;
3834                                 break;
3835                         }
3836 
3837                         /*
3838                          * ERROR: expected a colon.
3839                          */
3840                         return (NULL);
3841                 case DTRACE_JSON_COMMA:
3842                         if (isspace(cc))
3843                                 break;
3844 
3845                         if (cc == ',') {
3846                                 if (in_array) {
3847                                         state = DTRACE_JSON_VALUE;
3848                                         if (++array_pos == array_elem)
3849                                                 found_key = B_TRUE;
3850                                 } else {
3851                                         state = DTRACE_JSON_OBJECT;
3852                                 }
3853                                 break;
3854                         }
3855 
3856                         /*
3857                          * ERROR: either we hit an unexpected character, or
3858                          * we reached the end of the object or array without
3859                          * finding the requested key.
3860                          */
3861                         return (NULL);
3862                 case DTRACE_JSON_IDENTIFIER:
3863                         if (islower(cc)) {
3864                                 *dd++ = cc;
3865                                 break;
3866                         }
3867 
3868                         *dd = '\0';
3869                         dd = dest; /* reset string buffer */
3870 
3871                         if (dtrace_strncmp(dest, "true", 5) == 0 ||
3872                             dtrace_strncmp(dest, "false", 6) == 0 ||
3873                             dtrace_strncmp(dest, "null", 5) == 0) {
3874                                 if (found_key) {
3875                                         if (nelems > 1) {
3876                                                 /*
3877                                                  * ERROR: We expected an object,
3878                                                  * not this identifier.
3879                                                  */
3880                                                 return (NULL);
3881                                         }
3882                                         return (dest);
3883                                 } else {
3884                                         cur--;
3885                                         state = DTRACE_JSON_COMMA;
3886                                         break;
3887                                 }
3888                         }
3889 
3890                         /*
3891                          * ERROR: we did not recognise the identifier as one
3892                          * of those in the JSON specification.
3893                          */
3894                         return (NULL);
3895                 case DTRACE_JSON_NUMBER:
3896                         if (cc == '.') {
3897                                 *dd++ = cc;
3898                                 state = DTRACE_JSON_NUMBER_FRAC;
3899                                 break;
3900                         }
3901 
3902                         if (cc == 'x' || cc == 'X') {
3903                                 /*
3904                                  * ERROR: specification explicitly excludes
3905                                  * hexidecimal or octal numbers.
3906                                  */
3907                                 return (NULL);
3908                         }
3909 
3910                         /* FALLTHRU */
3911                 case DTRACE_JSON_NUMBER_FRAC:
3912                         if (cc == 'e' || cc == 'E') {
3913                                 *dd++ = cc;
3914                                 state = DTRACE_JSON_NUMBER_EXP;
3915                                 break;
3916                         }
3917 
3918                         if (cc == '+' || cc == '-') {
3919                                 /*
3920                                  * ERROR: expect sign as part of exponent only.
3921                                  */
3922                                 return (NULL);
3923                         }
3924                         /* FALLTHRU */
3925                 case DTRACE_JSON_NUMBER_EXP:
3926                         if (isdigit(cc) || cc == '+' || cc == '-') {
3927                                 *dd++ = cc;
3928                                 break;
3929                         }
3930 
3931                         *dd = '\0';
3932                         dd = dest; /* reset string buffer */
3933                         if (found_key) {
3934                                 if (nelems > 1) {
3935                                         /*
3936                                          * ERROR: We expected an object, not
3937                                          * this number.
3938                                          */
3939                                         return (NULL);
3940                                 }
3941                                 return (dest);
3942                         }
3943 
3944                         cur--;
3945                         state = DTRACE_JSON_COMMA;
3946                         break;
3947                 case DTRACE_JSON_VALUE:
3948                         if (isspace(cc))
3949                                 break;
3950 
3951                         if (cc == '{' || cc == '[') {
3952                                 if (nelems > 1 && found_key) {
3953                                         in_array = cc == '[' ? B_TRUE : B_FALSE;
3954                                         /*
3955                                          * If our element selector directs us
3956                                          * to descend into this nested object,
3957                                          * then move to the next selector
3958                                          * element in the list and restart the
3959                                          * state machine.
3960                                          */
3961                                         while (*elem != '\0')
3962                                                 elem++;
3963                                         elem++; /* skip the inter-element NUL */
3964                                         nelems--;
3965                                         dd = dest;
3966                                         if (in_array) {
3967                                                 state = DTRACE_JSON_VALUE;
3968                                                 array_pos = 0;
3969                                                 array_elem = dtrace_strtoll(
3970                                                     elem, 10, size);
3971                                                 found_key = array_elem == 0 ?
3972                                                     B_TRUE : B_FALSE;
3973                                         } else {
3974                                                 found_key = B_FALSE;
3975                                                 state = DTRACE_JSON_OBJECT;
3976                                         }
3977                                         break;
3978                                 }
3979 
3980                                 /*
3981                                  * Otherwise, we wish to either skip this
3982                                  * nested object or return it in full.
3983                                  */
3984                                 if (cc == '[')
3985                                         brackets = 1;
3986                                 else
3987                                         braces = 1;
3988                                 *dd++ = cc;
3989                                 state = DTRACE_JSON_COLLECT_OBJECT;
3990                                 break;
3991                         }
3992 
3993                         if (cc == '"') {
3994                                 state = DTRACE_JSON_STRING;
3995                                 break;
3996                         }
3997 
3998                         if (islower(cc)) {
3999                                 /*
4000                                  * Here we deal with true, false and null.
4001                                  */
4002                                 *dd++ = cc;
4003                                 state = DTRACE_JSON_IDENTIFIER;
4004                                 break;
4005                         }
4006 
4007                         if (cc == '-' || isdigit(cc)) {
4008                                 *dd++ = cc;
4009                                 state = DTRACE_JSON_NUMBER;
4010                                 break;
4011                         }
4012 
4013                         /*
4014                          * ERROR: unexpected character at start of value.
4015                          */
4016                         return (NULL);
4017                 case DTRACE_JSON_COLLECT_OBJECT:
4018                         if (cc == '\0')
4019                                 /*
4020                                  * ERROR: unexpected end of input.
4021                                  */
4022                                 return (NULL);
4023 
4024                         *dd++ = cc;
4025                         if (cc == '"') {
4026                                 collect_object = B_TRUE;
4027                                 state = DTRACE_JSON_STRING;
4028                                 break;
4029                         }
4030 
4031                         if (cc == ']') {
4032                                 if (brackets-- == 0) {
4033                                         /*
4034                                          * ERROR: unbalanced brackets.
4035                                          */
4036                                         return (NULL);
4037                                 }
4038                         } else if (cc == '}') {
4039                                 if (braces-- == 0) {
4040                                         /*
4041                                          * ERROR: unbalanced braces.
4042                                          */
4043                                         return (NULL);
4044                                 }
4045                         } else if (cc == '{') {
4046                                 braces++;
4047                         } else if (cc == '[') {
4048                                 brackets++;
4049                         }
4050 
4051                         if (brackets == 0 && braces == 0) {
4052                                 if (found_key) {
4053                                         *dd = '\0';
4054                                         return (dest);
4055                                 }
4056                                 dd = dest; /* reset string buffer */
4057                                 state = DTRACE_JSON_COMMA;
4058                         }
4059                         break;
4060                 }
4061         }
4062         return (NULL);
4063 }
4064 
4065 /*
4066  * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
4067  * Notice that we don't bother validating the proper number of arguments or
4068  * their types in the tuple stack.  This isn't needed because all argument
4069  * interpretation is safe because of our load safety -- the worst that can
4070  * happen is that a bogus program can obtain bogus results.
4071  */
4072 static void
4073 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
4074     dtrace_key_t *tupregs, int nargs,
4075     dtrace_mstate_t *mstate, dtrace_state_t *state)
4076 {
4077         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
4078         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
4079         dtrace_vstate_t *vstate = &state->dts_vstate;
4080 
4081         union {
4082                 mutex_impl_t mi;
4083                 uint64_t mx;
4084         } m;
4085 
4086         union {
4087                 krwlock_t ri;
4088                 uintptr_t rw;
4089         } r;
4090 
4091         switch (subr) {
4092         case DIF_SUBR_RAND:
4093                 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
4094                 break;
4095 
4096         case DIF_SUBR_MUTEX_OWNED:
4097                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
4098                     mstate, vstate)) {
4099                         regs[rd] = 0;
4100                         break;
4101                 }
4102 
4103                 m.mx = dtrace_load64(tupregs[0].dttk_value);
4104                 if (MUTEX_TYPE_ADAPTIVE(&m.mi))
4105                         regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
4106                 else
4107                         regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
4108                 break;
4109 
4110         case DIF_SUBR_MUTEX_OWNER:
4111                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
4112                     mstate, vstate)) {
4113                         regs[rd] = 0;
4114                         break;
4115                 }
4116 
4117                 m.mx = dtrace_load64(tupregs[0].dttk_value);
4118                 if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
4119                     MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
4120                         regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
4121                 else
4122                         regs[rd] = 0;
4123                 break;
4124 
4125         case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
4126                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
4127                     mstate, vstate)) {
4128                         regs[rd] = 0;
4129                         break;
4130                 }
4131 
4132                 m.mx = dtrace_load64(tupregs[0].dttk_value);
4133                 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
4134                 break;
4135 
4136         case DIF_SUBR_MUTEX_TYPE_SPIN:
4137                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
4138                     mstate, vstate)) {
4139                         regs[rd] = 0;
4140                         break;
4141                 }
4142 
4143                 m.mx = dtrace_load64(tupregs[0].dttk_value);
4144                 regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
4145                 break;
4146 
4147         case DIF_SUBR_RW_READ_HELD: {
4148                 uintptr_t tmp;
4149 
4150                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
4151                     mstate, vstate)) {
4152                         regs[rd] = 0;
4153                         break;
4154                 }
4155 
4156                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
4157                 regs[rd] = _RW_READ_HELD(&r.ri, tmp);
4158                 break;
4159         }
4160 
4161         case DIF_SUBR_RW_WRITE_HELD:
4162                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
4163                     mstate, vstate)) {
4164                         regs[rd] = 0;
4165                         break;
4166                 }
4167 
4168                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
4169                 regs[rd] = _RW_WRITE_HELD(&r.ri);
4170                 break;
4171 
4172         case DIF_SUBR_RW_ISWRITER:
4173                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
4174                     mstate, vstate)) {
4175                         regs[rd] = 0;
4176                         break;
4177                 }
4178 
4179                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
4180                 regs[rd] = _RW_ISWRITER(&r.ri);
4181                 break;
4182 
4183         case DIF_SUBR_BCOPY: {
4184                 /*
4185                  * We need to be sure that the destination is in the scratch
4186                  * region -- no other region is allowed.
4187                  */
4188                 uintptr_t src = tupregs[0].dttk_value;
4189                 uintptr_t dest = tupregs[1].dttk_value;
4190                 size_t size = tupregs[2].dttk_value;
4191 
4192                 if (!dtrace_inscratch(dest, size, mstate)) {
4193                         *flags |= CPU_DTRACE_BADADDR;
4194                         *illval = regs[rd];
4195                         break;
4196                 }
4197 
4198                 if (!dtrace_canload(src, size, mstate, vstate)) {
4199                         regs[rd] = 0;
4200                         break;
4201                 }
4202 
4203                 dtrace_bcopy((void *)src, (void *)dest, size);
4204                 break;
4205         }
4206 
4207         case DIF_SUBR_ALLOCA:
4208         case DIF_SUBR_COPYIN: {
4209                 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
4210                 uint64_t size =
4211                     tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
4212                 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
4213 
4214                 /*
4215                  * This action doesn't require any credential checks since
4216                  * probes will not activate in user contexts to which the
4217                  * enabling user does not have permissions.
4218                  */
4219 
4220                 /*
4221                  * Rounding up the user allocation size could have overflowed
4222                  * a large, bogus allocation (like -1ULL) to 0.
4223                  */
4224                 if (scratch_size < size ||
4225                     !DTRACE_INSCRATCH(mstate, scratch_size)) {
4226                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4227                         regs[rd] = 0;
4228                         break;
4229                 }
4230 
4231                 if (subr == DIF_SUBR_COPYIN) {
4232                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4233                         dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4234                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4235                 }
4236 
4237                 mstate->dtms_scratch_ptr += scratch_size;
4238                 regs[rd] = dest;
4239                 break;
4240         }
4241 
4242         case DIF_SUBR_COPYINTO: {
4243                 uint64_t size = tupregs[1].dttk_value;
4244                 uintptr_t dest = tupregs[2].dttk_value;
4245 
4246                 /*
4247                  * This action doesn't require any credential checks since
4248                  * probes will not activate in user contexts to which the
4249                  * enabling user does not have permissions.
4250                  */
4251                 if (!dtrace_inscratch(dest, size, mstate)) {
4252                         *flags |= CPU_DTRACE_BADADDR;
4253                         *illval = regs[rd];
4254                         break;
4255                 }
4256 
4257                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4258                 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
4259                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4260                 break;
4261         }
4262 
4263         case DIF_SUBR_COPYINSTR: {
4264                 uintptr_t dest = mstate->dtms_scratch_ptr;
4265                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4266 
4267                 if (nargs > 1 && tupregs[1].dttk_value < size)
4268                         size = tupregs[1].dttk_value + 1;
4269 
4270                 /*
4271                  * This action doesn't require any credential checks since
4272                  * probes will not activate in user contexts to which the
4273                  * enabling user does not have permissions.
4274                  */
4275                 if (!DTRACE_INSCRATCH(mstate, size)) {
4276                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4277                         regs[rd] = 0;
4278                         break;
4279                 }
4280 
4281                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4282                 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
4283                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4284 
4285                 ((char *)dest)[size - 1] = '\0';
4286                 mstate->dtms_scratch_ptr += size;
4287                 regs[rd] = dest;
4288                 break;
4289         }
4290 
4291         case DIF_SUBR_MSGSIZE:
4292         case DIF_SUBR_MSGDSIZE: {
4293                 uintptr_t baddr = tupregs[0].dttk_value, daddr;
4294                 uintptr_t wptr, rptr;
4295                 size_t count = 0;
4296                 int cont = 0;
4297 
4298                 while (baddr != 0 && !(*flags & CPU_DTRACE_FAULT)) {
4299 
4300                         if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
4301                             vstate)) {
4302                                 regs[rd] = 0;
4303                                 break;
4304                         }
4305 
4306                         wptr = dtrace_loadptr(baddr +
4307                             offsetof(mblk_t, b_wptr));
4308 
4309                         rptr = dtrace_loadptr(baddr +
4310                             offsetof(mblk_t, b_rptr));
4311 
4312                         if (wptr < rptr) {
4313                                 *flags |= CPU_DTRACE_BADADDR;
4314                                 *illval = tupregs[0].dttk_value;
4315                                 break;
4316                         }
4317 
4318                         daddr = dtrace_loadptr(baddr +
4319                             offsetof(mblk_t, b_datap));
4320 
4321                         baddr = dtrace_loadptr(baddr +
4322                             offsetof(mblk_t, b_cont));
4323 
4324                         /*
4325                          * We want to prevent against denial-of-service here,
4326                          * so we're only going to search the list for
4327                          * dtrace_msgdsize_max mblks.
4328                          */
4329                         if (cont++ > dtrace_msgdsize_max) {
4330                                 *flags |= CPU_DTRACE_ILLOP;
4331                                 break;
4332                         }
4333 
4334                         if (subr == DIF_SUBR_MSGDSIZE) {
4335                                 if (dtrace_load8(daddr +
4336                                     offsetof(dblk_t, db_type)) != M_DATA)
4337                                         continue;
4338                         }
4339 
4340                         count += wptr - rptr;
4341                 }
4342 
4343                 if (!(*flags & CPU_DTRACE_FAULT))
4344                         regs[rd] = count;
4345 
4346                 break;
4347         }
4348 
4349         case DIF_SUBR_PROGENYOF: {
4350                 pid_t pid = tupregs[0].dttk_value;
4351                 proc_t *p;
4352                 int rval = 0;
4353 
4354                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4355 
4356                 for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
4357                         if (p->p_pidp->pid_id == pid) {
4358                                 rval = 1;
4359                                 break;
4360                         }
4361                 }
4362 
4363                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4364 
4365                 regs[rd] = rval;
4366                 break;
4367         }
4368 
4369         case DIF_SUBR_SPECULATION:
4370                 regs[rd] = dtrace_speculation(state);
4371                 break;
4372 
4373         case DIF_SUBR_COPYOUT: {
4374                 uintptr_t kaddr = tupregs[0].dttk_value;
4375                 uintptr_t uaddr = tupregs[1].dttk_value;
4376                 uint64_t size = tupregs[2].dttk_value;
4377 
4378                 if (!dtrace_destructive_disallow &&
4379                     dtrace_priv_proc_control(state, mstate) &&
4380                     !dtrace_istoxic(kaddr, size) &&
4381                     dtrace_canload(kaddr, size, mstate, vstate)) {
4382                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4383                         dtrace_copyout(kaddr, uaddr, size, flags);
4384                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4385                 }
4386                 break;
4387         }
4388 
4389         case DIF_SUBR_COPYOUTSTR: {
4390                 uintptr_t kaddr = tupregs[0].dttk_value;
4391                 uintptr_t uaddr = tupregs[1].dttk_value;
4392                 uint64_t size = tupregs[2].dttk_value;
4393                 size_t lim;
4394 
4395                 if (!dtrace_destructive_disallow &&
4396                     dtrace_priv_proc_control(state, mstate) &&
4397                     !dtrace_istoxic(kaddr, size) &&
4398                     dtrace_strcanload(kaddr, size, &lim, mstate, vstate)) {
4399                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4400                         dtrace_copyoutstr(kaddr, uaddr, lim, flags);
4401                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4402                 }
4403                 break;
4404         }
4405 
4406         case DIF_SUBR_STRLEN: {
4407                 size_t size = state->dts_options[DTRACEOPT_STRSIZE];
4408                 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
4409                 size_t lim;
4410 
4411                 if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) {
4412                         regs[rd] = 0;
4413                         break;
4414                 }
4415                 regs[rd] = dtrace_strlen((char *)addr, lim);
4416 
4417                 break;
4418         }
4419 
4420         case DIF_SUBR_STRCHR:
4421         case DIF_SUBR_STRRCHR: {
4422                 /*
4423                  * We're going to iterate over the string looking for the
4424                  * specified character.  We will iterate until we have reached
4425                  * the string length or we have found the character.  If this
4426                  * is DIF_SUBR_STRRCHR, we will look for the last occurrence
4427                  * of the specified character instead of the first.
4428                  */
4429                 uintptr_t addr = tupregs[0].dttk_value;
4430                 uintptr_t addr_limit;
4431                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4432                 size_t lim;
4433                 char c, target = (char)tupregs[1].dttk_value;
4434 
4435                 if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) {
4436                         regs[rd] = 0;
4437                         break;
4438                 }
4439                 addr_limit = addr + lim;
4440 
4441                 for (regs[rd] = 0; addr < addr_limit; addr++) {
4442                         if ((c = dtrace_load8(addr)) == target) {
4443                                 regs[rd] = addr;
4444 
4445                                 if (subr == DIF_SUBR_STRCHR)
4446                                         break;
4447                         }
4448                         if (c == '\0')
4449                                 break;
4450                 }
4451 
4452                 break;
4453         }
4454 
4455         case DIF_SUBR_STRSTR:
4456         case DIF_SUBR_INDEX:
4457         case DIF_SUBR_RINDEX: {
4458                 /*
4459                  * We're going to iterate over the string looking for the
4460                  * specified string.  We will iterate until we have reached
4461                  * the string length or we have found the string.  (Yes, this
4462                  * is done in the most naive way possible -- but considering
4463                  * that the string we're searching for is likely to be
4464                  * relatively short, the complexity of Rabin-Karp or similar
4465                  * hardly seems merited.)
4466                  */
4467                 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
4468                 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
4469                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4470                 size_t len = dtrace_strlen(addr, size);
4471                 size_t sublen = dtrace_strlen(substr, size);
4472                 char *limit = addr + len, *orig = addr;
4473                 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
4474                 int inc = 1;
4475 
4476                 regs[rd] = notfound;
4477 
4478                 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
4479                         regs[rd] = 0;
4480                         break;
4481                 }
4482 
4483                 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
4484                     vstate)) {
4485                         regs[rd] = 0;
4486                         break;
4487                 }
4488 
4489                 /*
4490                  * strstr() and index()/rindex() have similar semantics if
4491                  * both strings are the empty string: strstr() returns a
4492                  * pointer to the (empty) string, and index() and rindex()
4493                  * both return index 0 (regardless of any position argument).
4494                  */
4495                 if (sublen == 0 && len == 0) {
4496                         if (subr == DIF_SUBR_STRSTR)
4497                                 regs[rd] = (uintptr_t)addr;
4498                         else
4499                                 regs[rd] = 0;
4500                         break;
4501                 }
4502 
4503                 if (subr != DIF_SUBR_STRSTR) {
4504                         if (subr == DIF_SUBR_RINDEX) {
4505                                 limit = orig - 1;
4506                                 addr += len;
4507                                 inc = -1;
4508                         }
4509 
4510                         /*
4511                          * Both index() and rindex() take an optional position
4512                          * argument that denotes the starting position.
4513                          */
4514                         if (nargs == 3) {
4515                                 int64_t pos = (int64_t)tupregs[2].dttk_value;
4516 
4517                                 /*
4518                                  * If the position argument to index() is
4519                                  * negative, Perl implicitly clamps it at
4520                                  * zero.  This semantic is a little surprising
4521                                  * given the special meaning of negative
4522                                  * positions to similar Perl functions like
4523                                  * substr(), but it appears to reflect a
4524                                  * notion that index() can start from a
4525                                  * negative index and increment its way up to
4526                                  * the string.  Given this notion, Perl's
4527                                  * rindex() is at least self-consistent in
4528                                  * that it implicitly clamps positions greater
4529                                  * than the string length to be the string
4530                                  * length.  Where Perl completely loses
4531                                  * coherence, however, is when the specified
4532                                  * substring is the empty string ("").  In
4533                                  * this case, even if the position is
4534                                  * negative, rindex() returns 0 -- and even if
4535                                  * the position is greater than the length,
4536                                  * index() returns the string length.  These
4537                                  * semantics violate the notion that index()
4538                                  * should never return a value less than the
4539                                  * specified position and that rindex() should
4540                                  * never return a value greater than the
4541                                  * specified position.  (One assumes that
4542                                  * these semantics are artifacts of Perl's
4543                                  * implementation and not the results of
4544                                  * deliberate design -- it beggars belief that
4545                                  * even Larry Wall could desire such oddness.)
4546                                  * While in the abstract one would wish for
4547                                  * consistent position semantics across
4548                                  * substr(), index() and rindex() -- or at the
4549                                  * very least self-consistent position
4550                                  * semantics for index() and rindex() -- we
4551                                  * instead opt to keep with the extant Perl
4552                                  * semantics, in all their broken glory.  (Do
4553                                  * we have more desire to maintain Perl's
4554                                  * semantics than Perl does?  Probably.)
4555                                  */
4556                                 if (subr == DIF_SUBR_RINDEX) {
4557                                         if (pos < 0) {
4558                                                 if (sublen == 0)
4559                                                         regs[rd] = 0;
4560                                                 break;
4561                                         }
4562 
4563                                         if (pos > len)
4564                                                 pos = len;
4565                                 } else {
4566                                         if (pos < 0)
4567                                                 pos = 0;
4568 
4569                                         if (pos >= len) {
4570                                                 if (sublen == 0)
4571                                                         regs[rd] = len;
4572                                                 break;
4573                                         }
4574                                 }
4575 
4576                                 addr = orig + pos;
4577                         }
4578                 }
4579 
4580                 for (regs[rd] = notfound; addr != limit; addr += inc) {
4581                         if (dtrace_strncmp(addr, substr, sublen) == 0) {
4582                                 if (subr != DIF_SUBR_STRSTR) {
4583                                         /*
4584                                          * As D index() and rindex() are
4585                                          * modeled on Perl (and not on awk),
4586                                          * we return a zero-based (and not a
4587                                          * one-based) index.  (For you Perl
4588                                          * weenies: no, we're not going to add
4589                                          * $[ -- and shouldn't you be at a con
4590                                          * or something?)
4591                                          */
4592                                         regs[rd] = (uintptr_t)(addr - orig);
4593                                         break;
4594                                 }
4595 
4596                                 ASSERT(subr == DIF_SUBR_STRSTR);
4597                                 regs[rd] = (uintptr_t)addr;
4598                                 break;
4599                         }
4600                 }
4601 
4602                 break;
4603         }
4604 
4605         case DIF_SUBR_STRTOK: {
4606                 uintptr_t addr = tupregs[0].dttk_value;
4607                 uintptr_t tokaddr = tupregs[1].dttk_value;
4608                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4609                 uintptr_t limit, toklimit;
4610                 size_t clim;
4611                 uint8_t c, tokmap[32];   /* 256 / 8 */
4612                 char *dest = (char *)mstate->dtms_scratch_ptr;
4613                 int i;
4614 
4615                 /*
4616                  * Check both the token buffer and (later) the input buffer,
4617                  * since both could be non-scratch addresses.
4618                  */
4619                 if (!dtrace_strcanload(tokaddr, size, &clim, mstate, vstate)) {
4620                         regs[rd] = 0;
4621                         break;
4622                 }
4623                 toklimit = tokaddr + clim;
4624 
4625                 if (!DTRACE_INSCRATCH(mstate, size)) {
4626                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4627                         regs[rd] = 0;
4628                         break;
4629                 }
4630 
4631                 if (addr == 0) {
4632                         /*
4633                          * If the address specified is NULL, we use our saved
4634                          * strtok pointer from the mstate.  Note that this
4635                          * means that the saved strtok pointer is _only_
4636                          * valid within multiple enablings of the same probe --
4637                          * it behaves like an implicit clause-local variable.
4638                          */
4639                         addr = mstate->dtms_strtok;
4640                         limit = mstate->dtms_strtok_limit;
4641                 } else {
4642                         /*
4643                          * If the user-specified address is non-NULL we must
4644                          * access check it.  This is the only time we have
4645                          * a chance to do so, since this address may reside
4646                          * in the string table of this clause-- future calls
4647                          * (when we fetch addr from mstate->dtms_strtok)
4648                          * would fail this access check.
4649                          */
4650                         if (!dtrace_strcanload(addr, size, &clim, mstate,
4651                             vstate)) {
4652                                 regs[rd] = 0;
4653                                 break;
4654                         }
4655                         limit = addr + clim;
4656                 }
4657 
4658                 /*
4659                  * First, zero the token map, and then process the token
4660                  * string -- setting a bit in the map for every character
4661                  * found in the token string.
4662                  */
4663                 for (i = 0; i < sizeof (tokmap); i++)
4664                         tokmap[i] = 0;
4665 
4666                 for (; tokaddr < toklimit; tokaddr++) {
4667                         if ((c = dtrace_load8(tokaddr)) == '\0')
4668                                 break;
4669 
4670                         ASSERT((c >> 3) < sizeof (tokmap));
4671                         tokmap[c >> 3] |= (1 << (c & 0x7));
4672                 }
4673 
4674                 for (; addr < limit; addr++) {
4675                         /*
4676                          * We're looking for a character that is _not_
4677                          * contained in the token string.
4678                          */
4679                         if ((c = dtrace_load8(addr)) == '\0')
4680                                 break;
4681 
4682                         if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
4683                                 break;
4684                 }
4685 
4686                 if (c == '\0') {
4687                         /*
4688                          * We reached the end of the string without finding
4689                          * any character that was not in the token string.
4690                          * We return NULL in this case, and we set the saved
4691                          * address to NULL as well.
4692                          */
4693                         regs[rd] = 0;
4694                         mstate->dtms_strtok = 0;
4695                         mstate->dtms_strtok_limit = 0;
4696                         break;
4697                 }
4698 
4699                 /*
4700                  * From here on, we're copying into the destination string.
4701                  */
4702                 for (i = 0; addr < limit && i < size - 1; addr++) {
4703                         if ((c = dtrace_load8(addr)) == '\0')
4704                                 break;
4705 
4706                         if (tokmap[c >> 3] & (1 << (c & 0x7)))
4707                                 break;
4708 
4709                         ASSERT(i < size);
4710                         dest[i++] = c;
4711                 }
4712 
4713                 ASSERT(i < size);
4714                 dest[i] = '\0';
4715                 regs[rd] = (uintptr_t)dest;
4716                 mstate->dtms_scratch_ptr += size;
4717                 mstate->dtms_strtok = addr;
4718                 mstate->dtms_strtok_limit = limit;
4719                 break;
4720         }
4721 
4722         case DIF_SUBR_SUBSTR: {
4723                 uintptr_t s = tupregs[0].dttk_value;
4724                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4725                 char *d = (char *)mstate->dtms_scratch_ptr;
4726                 int64_t index = (int64_t)tupregs[1].dttk_value;
4727                 int64_t remaining = (int64_t)tupregs[2].dttk_value;
4728                 size_t len = dtrace_strlen((char *)s, size);
4729                 int64_t i;
4730 
4731                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4732                         regs[rd] = 0;
4733                         break;
4734                 }
4735 
4736                 if (!DTRACE_INSCRATCH(mstate, size)) {
4737                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4738                         regs[rd] = 0;
4739                         break;
4740                 }
4741 
4742                 if (nargs <= 2)
4743                         remaining = (int64_t)size;
4744 
4745                 if (index < 0) {
4746                         index += len;
4747 
4748                         if (index < 0 && index + remaining > 0) {
4749                                 remaining += index;
4750                                 index = 0;
4751                         }
4752                 }
4753 
4754                 if (index >= len || index < 0) {
4755                         remaining = 0;
4756                 } else if (remaining < 0) {
4757                         remaining += len - index;
4758                 } else if (index + remaining > size) {
4759                         remaining = size - index;
4760                 }
4761 
4762                 for (i = 0; i < remaining; i++) {
4763                         if ((d[i] = dtrace_load8(s + index + i)) == '\0')
4764                                 break;
4765                 }
4766 
4767                 d[i] = '\0';
4768 
4769                 mstate->dtms_scratch_ptr += size;
4770                 regs[rd] = (uintptr_t)d;
4771                 break;
4772         }
4773 
4774         case DIF_SUBR_JSON: {
4775                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4776                 uintptr_t json = tupregs[0].dttk_value;
4777                 size_t jsonlen = dtrace_strlen((char *)json, size);
4778                 uintptr_t elem = tupregs[1].dttk_value;
4779                 size_t elemlen = dtrace_strlen((char *)elem, size);
4780 
4781                 char *dest = (char *)mstate->dtms_scratch_ptr;
4782                 char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1;
4783                 char *ee = elemlist;
4784                 int nelems = 1;
4785                 uintptr_t cur;
4786 
4787                 if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) ||
4788                     !dtrace_canload(elem, elemlen + 1, mstate, vstate)) {
4789                         regs[rd] = 0;
4790                         break;
4791                 }
4792 
4793                 if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) {
4794                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4795                         regs[rd] = 0;
4796                         break;
4797                 }
4798 
4799                 /*
4800                  * Read the element selector and split it up into a packed list
4801                  * of strings.
4802                  */
4803                 for (cur = elem; cur < elem + elemlen; cur++) {
4804                         char cc = dtrace_load8(cur);
4805 
4806                         if (cur == elem && cc == '[') {
4807                                 /*
4808                                  * If the first element selector key is
4809                                  * actually an array index then ignore the
4810                                  * bracket.
4811                                  */
4812                                 continue;
4813                         }
4814 
4815                         if (cc == ']')
4816                                 continue;
4817 
4818                         if (cc == '.' || cc == '[') {
4819                                 nelems++;
4820                                 cc = '\0';
4821                         }
4822 
4823                         *ee++ = cc;
4824                 }
4825                 *ee++ = '\0';
4826 
4827                 if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist,
4828                     nelems, dest)) != 0)
4829                         mstate->dtms_scratch_ptr += jsonlen + 1;
4830                 break;
4831         }
4832 
4833         case DIF_SUBR_TOUPPER:
4834         case DIF_SUBR_TOLOWER: {
4835                 uintptr_t s = tupregs[0].dttk_value;
4836                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4837                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
4838                 size_t len = dtrace_strlen((char *)s, size);
4839                 char lower, upper, convert;
4840                 int64_t i;
4841 
4842                 if (subr == DIF_SUBR_TOUPPER) {
4843                         lower = 'a';
4844                         upper = 'z';
4845                         convert = 'A';
4846                 } else {
4847                         lower = 'A';
4848                         upper = 'Z';
4849                         convert = 'a';
4850                 }
4851 
4852                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
4853                         regs[rd] = 0;
4854                         break;
4855                 }
4856 
4857                 if (!DTRACE_INSCRATCH(mstate, size)) {
4858                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4859                         regs[rd] = 0;
4860                         break;
4861                 }
4862 
4863                 for (i = 0; i < size - 1; i++) {
4864                         if ((c = dtrace_load8(s + i)) == '\0')
4865                                 break;
4866 
4867                         if (c >= lower && c <= upper)
4868                                 c = convert + (c - lower);
4869 
4870                         dest[i] = c;
4871                 }
4872 
4873                 ASSERT(i < size);
4874                 dest[i] = '\0';
4875                 regs[rd] = (uintptr_t)dest;
4876                 mstate->dtms_scratch_ptr += size;
4877                 break;
4878         }
4879 
4880 case DIF_SUBR_GETMAJOR:
4881 #ifdef _LP64
4882                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
4883 #else
4884                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
4885 #endif
4886                 break;
4887 
4888         case DIF_SUBR_GETMINOR:
4889 #ifdef _LP64
4890                 regs[rd] = tupregs[0].dttk_value & MAXMIN64;
4891 #else
4892                 regs[rd] = tupregs[0].dttk_value & MAXMIN;
4893 #endif
4894                 break;
4895 
4896         case DIF_SUBR_DDI_PATHNAME: {
4897                 /*
4898                  * This one is a galactic mess.  We are going to roughly
4899                  * emulate ddi_pathname(), but it's made more complicated
4900                  * by the fact that we (a) want to include the minor name and
4901                  * (b) must proceed iteratively instead of recursively.
4902                  */
4903                 uintptr_t dest = mstate->dtms_scratch_ptr;
4904                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4905                 char *start = (char *)dest, *end = start + size - 1;
4906                 uintptr_t daddr = tupregs[0].dttk_value;
4907                 int64_t minor = (int64_t)tupregs[1].dttk_value;
4908                 char *s;
4909                 int i, len, depth = 0;
4910 
4911                 /*
4912                  * Due to all the pointer jumping we do and context we must
4913                  * rely upon, we just mandate that the user must have kernel
4914                  * read privileges to use this routine.
4915                  */
4916                 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
4917                         *flags |= CPU_DTRACE_KPRIV;
4918                         *illval = daddr;
4919                         regs[rd] = 0;
4920                 }
4921 
4922                 if (!DTRACE_INSCRATCH(mstate, size)) {
4923                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4924                         regs[rd] = 0;
4925                         break;
4926                 }
4927 
4928                 *end = '\0';
4929 
4930                 /*
4931                  * We want to have a name for the minor.  In order to do this,
4932                  * we need to walk the minor list from the devinfo.  We want
4933                  * to be sure that we don't infinitely walk a circular list,
4934                  * so we check for circularity by sending a scout pointer
4935                  * ahead two elements for every element that we iterate over;
4936                  * if the list is circular, these will ultimately point to the
4937                  * same element.  You may recognize this little trick as the
4938                  * answer to a stupid interview question -- one that always
4939                  * seems to be asked by those who had to have it laboriously
4940                  * explained to them, and who can't even concisely describe
4941                  * the conditions under which one would be forced to resort to
4942                  * this technique.  Needless to say, those conditions are
4943                  * found here -- and probably only here.  Is this the only use
4944                  * of this infamous trick in shipping, production code?  If it
4945                  * isn't, it probably should be...
4946                  */
4947                 if (minor != -1) {
4948                         uintptr_t maddr = dtrace_loadptr(daddr +
4949                             offsetof(struct dev_info, devi_minor));
4950 
4951                         uintptr_t next = offsetof(struct ddi_minor_data, next);
4952                         uintptr_t name = offsetof(struct ddi_minor_data,
4953                             d_minor) + offsetof(struct ddi_minor, name);
4954                         uintptr_t dev = offsetof(struct ddi_minor_data,
4955                             d_minor) + offsetof(struct ddi_minor, dev);
4956                         uintptr_t scout;
4957 
4958                         if (maddr != 0)
4959                                 scout = dtrace_loadptr(maddr + next);
4960 
4961                         while (maddr != 0 && !(*flags & CPU_DTRACE_FAULT)) {
4962                                 uint64_t m;
4963 #ifdef _LP64
4964                                 m = dtrace_load64(maddr + dev) & MAXMIN64;
4965 #else
4966                                 m = dtrace_load32(maddr + dev) & MAXMIN;
4967 #endif
4968                                 if (m != minor) {
4969                                         maddr = dtrace_loadptr(maddr + next);
4970 
4971                                         if (scout == 0)
4972                                                 continue;
4973 
4974                                         scout = dtrace_loadptr(scout + next);
4975 
4976                                         if (scout == 0)
4977                                                 continue;
4978 
4979                                         scout = dtrace_loadptr(scout + next);
4980 
4981                                         if (scout == 0)
4982                                                 continue;
4983 
4984                                         if (scout == maddr) {
4985                                                 *flags |= CPU_DTRACE_ILLOP;
4986                                                 break;
4987                                         }
4988 
4989                                         continue;
4990                                 }
4991 
4992                                 /*
4993                                  * We have the minor data.  Now we need to
4994                                  * copy the minor's name into the end of the
4995                                  * pathname.
4996                                  */
4997                                 s = (char *)dtrace_loadptr(maddr + name);
4998                                 len = dtrace_strlen(s, size);
4999 
5000                                 if (*flags & CPU_DTRACE_FAULT)
5001                                         break;
5002 
5003                                 if (len != 0) {
5004                                         if ((end -= (len + 1)) < start)
5005                                                 break;
5006 
5007                                         *end = ':';
5008                                 }
5009 
5010                                 for (i = 1; i <= len; i++)
5011                                         end[i] = dtrace_load8((uintptr_t)s++);
5012                                 break;
5013                         }
5014                 }
5015 
5016                 while (daddr != 0 && !(*flags & CPU_DTRACE_FAULT)) {
5017                         ddi_node_state_t devi_state;
5018 
5019                         devi_state = dtrace_load32(daddr +
5020                             offsetof(struct dev_info, devi_node_state));
5021 
5022                         if (*flags & CPU_DTRACE_FAULT)
5023                                 break;
5024 
5025                         if (devi_state >= DS_INITIALIZED) {
5026                                 s = (char *)dtrace_loadptr(daddr +
5027                                     offsetof(struct dev_info, devi_addr));
5028                                 len = dtrace_strlen(s, size);
5029 
5030                                 if (*flags & CPU_DTRACE_FAULT)
5031                                         break;
5032 
5033                                 if (len != 0) {
5034                                         if ((end -= (len + 1)) < start)
5035                                                 break;
5036 
5037                                         *end = '@';
5038                                 }
5039 
5040                                 for (i = 1; i <= len; i++)
5041                                         end[i] = dtrace_load8((uintptr_t)s++);
5042                         }
5043 
5044                         /*
5045                          * Now for the node name...
5046                          */
5047                         s = (char *)dtrace_loadptr(daddr +
5048                             offsetof(struct dev_info, devi_node_name));
5049 
5050                         daddr = dtrace_loadptr(daddr +
5051                             offsetof(struct dev_info, devi_parent));
5052 
5053                         /*
5054                          * If our parent is NULL (that is, if we're the root
5055                          * node), we're going to use the special path
5056                          * "devices".
5057                          */
5058                         if (daddr == 0)
5059                                 s = "devices";
5060 
5061                         len = dtrace_strlen(s, size);
5062                         if (*flags & CPU_DTRACE_FAULT)
5063                                 break;
5064 
5065                         if ((end -= (len + 1)) < start)
5066                                 break;
5067 
5068                         for (i = 1; i <= len; i++)
5069                                 end[i] = dtrace_load8((uintptr_t)s++);
5070                         *end = '/';
5071 
5072                         if (depth++ > dtrace_devdepth_max) {
5073                                 *flags |= CPU_DTRACE_ILLOP;
5074                                 break;
5075                         }
5076                 }
5077 
5078                 if (end < start)
5079                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5080 
5081                 if (daddr == 0) {
5082                         regs[rd] = (uintptr_t)end;
5083                         mstate->dtms_scratch_ptr += size;
5084                 }
5085 
5086                 break;
5087         }
5088 
5089         case DIF_SUBR_STRJOIN: {
5090                 char *d = (char *)mstate->dtms_scratch_ptr;
5091                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5092                 uintptr_t s1 = tupregs[0].dttk_value;
5093                 uintptr_t s2 = tupregs[1].dttk_value;
5094                 int i = 0, j = 0;
5095                 size_t lim1, lim2;
5096                 char c;
5097 
5098                 if (!dtrace_strcanload(s1, size, &lim1, mstate, vstate) ||
5099                     !dtrace_strcanload(s2, size, &lim2, mstate, vstate)) {
5100                         regs[rd] = 0;
5101                         break;
5102                 }
5103 
5104                 if (!DTRACE_INSCRATCH(mstate, size)) {
5105                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5106                         regs[rd] = 0;
5107                         break;
5108                 }
5109 
5110                 for (;;) {
5111                         if (i >= size) {
5112                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5113                                 regs[rd] = 0;
5114                                 break;
5115                         }
5116                         c = (i >= lim1) ? '\0' : dtrace_load8(s1++);
5117                         if ((d[i++] = c) == '\0') {
5118                                 i--;
5119                                 break;
5120                         }
5121                 }
5122 
5123                 for (;;) {
5124                         if (i >= size) {
5125                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5126                                 regs[rd] = 0;
5127                                 break;
5128                         }
5129 
5130                         c = (j++ >= lim2) ? '\0' : dtrace_load8(s2++);
5131                         if ((d[i++] = c) == '\0')
5132                                 break;
5133                 }
5134 
5135                 if (i < size) {
5136                         mstate->dtms_scratch_ptr += i;
5137                         regs[rd] = (uintptr_t)d;
5138                 }
5139 
5140                 break;
5141         }
5142 
5143         case DIF_SUBR_STRTOLL: {
5144                 uintptr_t s = tupregs[0].dttk_value;
5145                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5146                 size_t lim;
5147                 int base = 10;
5148 
5149                 if (nargs > 1) {
5150                         if ((base = tupregs[1].dttk_value) <= 1 ||
5151                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
5152                                 *flags |= CPU_DTRACE_ILLOP;
5153                                 break;
5154                         }
5155                 }
5156 
5157                 if (!dtrace_strcanload(s, size, &lim, mstate, vstate)) {
5158                         regs[rd] = INT64_MIN;
5159                         break;
5160                 }
5161 
5162                 regs[rd] = dtrace_strtoll((char *)s, base, lim);
5163                 break;
5164         }
5165 
5166         case DIF_SUBR_LLTOSTR: {
5167                 int64_t i = (int64_t)tupregs[0].dttk_value;
5168                 uint64_t val, digit;
5169                 uint64_t size = 65;     /* enough room for 2^64 in binary */
5170                 char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
5171                 int base = 10;
5172 
5173                 if (nargs > 1) {
5174                         if ((base = tupregs[1].dttk_value) <= 1 ||
5175                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
5176                                 *flags |= CPU_DTRACE_ILLOP;
5177                                 break;
5178                         }
5179                 }
5180 
5181                 val = (base == 10 && i < 0) ? i * -1 : i;
5182 
5183                 if (!DTRACE_INSCRATCH(mstate, size)) {
5184                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5185                         regs[rd] = 0;
5186                         break;
5187                 }
5188 
5189                 for (*end-- = '\0'; val; val /= base) {
5190                         if ((digit = val % base) <= '9' - '0') {
5191                                 *end-- = '0' + digit;
5192                         } else {
5193                                 *end-- = 'a' + (digit - ('9' - '0') - 1);
5194                         }
5195                 }
5196 
5197                 if (i == 0 && base == 16)
5198                         *end-- = '0';
5199 
5200                 if (base == 16)
5201                         *end-- = 'x';
5202 
5203                 if (i == 0 || base == 8 || base == 16)
5204                         *end-- = '0';
5205 
5206                 if (i < 0 && base == 10)
5207                         *end-- = '-';
5208 
5209                 regs[rd] = (uintptr_t)end + 1;
5210                 mstate->dtms_scratch_ptr += size;
5211                 break;
5212         }
5213 
5214         case DIF_SUBR_HTONS:
5215         case DIF_SUBR_NTOHS:
5216 #ifdef _BIG_ENDIAN
5217                 regs[rd] = (uint16_t)tupregs[0].dttk_value;
5218 #else
5219                 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
5220 #endif
5221                 break;
5222 
5223 
5224         case DIF_SUBR_HTONL:
5225         case DIF_SUBR_NTOHL:
5226 #ifdef _BIG_ENDIAN
5227                 regs[rd] = (uint32_t)tupregs[0].dttk_value;
5228 #else
5229                 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
5230 #endif
5231                 break;
5232 
5233 
5234         case DIF_SUBR_HTONLL:
5235         case DIF_SUBR_NTOHLL:
5236 #ifdef _BIG_ENDIAN
5237                 regs[rd] = (uint64_t)tupregs[0].dttk_value;
5238 #else
5239                 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
5240 #endif
5241                 break;
5242 
5243 
5244         case DIF_SUBR_DIRNAME:
5245         case DIF_SUBR_BASENAME: {
5246                 char *dest = (char *)mstate->dtms_scratch_ptr;
5247                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5248                 uintptr_t src = tupregs[0].dttk_value;
5249                 int i, j, len = dtrace_strlen((char *)src, size);
5250                 int lastbase = -1, firstbase = -1, lastdir = -1;
5251                 int start, end;
5252 
5253                 if (!dtrace_canload(src, len + 1, mstate, vstate)) {
5254                         regs[rd] = 0;
5255                         break;
5256                 }
5257 
5258                 if (!DTRACE_INSCRATCH(mstate, size)) {
5259                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5260                         regs[rd] = 0;
5261                         break;
5262                 }
5263 
5264                 /*
5265                  * The basename and dirname for a zero-length string is
5266                  * defined to be "."
5267                  */
5268                 if (len == 0) {
5269                         len = 1;
5270                         src = (uintptr_t)".";
5271                 }
5272 
5273                 /*
5274                  * Start from the back of the string, moving back toward the
5275                  * front until we see a character that isn't a slash.  That
5276                  * character is the last character in the basename.
5277                  */
5278                 for (i = len - 1; i >= 0; i--) {
5279                         if (dtrace_load8(src + i) != '/')
5280                                 break;
5281                 }
5282 
5283                 if (i >= 0)
5284                         lastbase = i;
5285 
5286                 /*
5287                  * Starting from the last character in the basename, move
5288                  * towards the front until we find a slash.  The character
5289                  * that we processed immediately before that is the first
5290                  * character in the basename.
5291                  */
5292                 for (; i >= 0; i--) {
5293                         if (dtrace_load8(src + i) == '/')
5294                                 break;
5295                 }
5296 
5297                 if (i >= 0)
5298                         firstbase = i + 1;
5299 
5300                 /*
5301                  * Now keep going until we find a non-slash character.  That
5302                  * character is the last character in the dirname.
5303                  */
5304                 for (; i >= 0; i--) {
5305                         if (dtrace_load8(src + i) != '/')
5306                                 break;
5307                 }
5308 
5309                 if (i >= 0)
5310                         lastdir = i;
5311 
5312                 ASSERT(!(lastbase == -1 && firstbase != -1));
5313                 ASSERT(!(firstbase == -1 && lastdir != -1));
5314 
5315                 if (lastbase == -1) {
5316                         /*
5317                          * We didn't find a non-slash character.  We know that
5318                          * the length is non-zero, so the whole string must be
5319                          * slashes.  In either the dirname or the basename
5320                          * case, we return '/'.
5321                          */
5322                         ASSERT(firstbase == -1);
5323                         firstbase = lastbase = lastdir = 0;
5324                 }
5325 
5326                 if (firstbase == -1) {
5327                         /*
5328                          * The entire string consists only of a basename
5329                          * component.  If we're looking for dirname, we need
5330                          * to change our string to be just "."; if we're
5331                          * looking for a basename, we'll just set the first
5332                          * character of the basename to be 0.
5333                          */
5334                         if (subr == DIF_SUBR_DIRNAME) {
5335                                 ASSERT(lastdir == -1);
5336                                 src = (uintptr_t)".";
5337                                 lastdir = 0;
5338                         } else {
5339                                 firstbase = 0;
5340                         }
5341                 }
5342 
5343                 if (subr == DIF_SUBR_DIRNAME) {
5344                         if (lastdir == -1) {
5345                                 /*
5346                                  * We know that we have a slash in the name --
5347                                  * or lastdir would be set to 0, above.  And
5348                                  * because lastdir is -1, we know that this
5349                                  * slash must be the first character.  (That
5350                                  * is, the full string must be of the form
5351                                  * "/basename".)  In this case, the last
5352                                  * character of the directory name is 0.
5353                                  */
5354                                 lastdir = 0;
5355                         }
5356 
5357                         start = 0;
5358                         end = lastdir;
5359                 } else {
5360                         ASSERT(subr == DIF_SUBR_BASENAME);
5361                         ASSERT(firstbase != -1 && lastbase != -1);
5362                         start = firstbase;
5363                         end = lastbase;
5364                 }
5365 
5366                 for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
5367                         dest[j] = dtrace_load8(src + i);
5368 
5369                 dest[j] = '\0';
5370                 regs[rd] = (uintptr_t)dest;
5371                 mstate->dtms_scratch_ptr += size;
5372                 break;
5373         }
5374 
5375         case DIF_SUBR_GETF: {
5376                 uintptr_t fd = tupregs[0].dttk_value;
5377                 uf_info_t *finfo = &curthread->t_procp->p_user.u_finfo;
5378                 file_t *fp;
5379 
5380                 if (!dtrace_priv_proc(state, mstate)) {
5381                         regs[rd] = 0;
5382                         break;
5383                 }
5384 
5385                 /*
5386                  * This is safe because fi_nfiles only increases, and the
5387                  * fi_list array is not freed when the array size doubles.
5388                  * (See the comment in flist_grow() for details on the
5389                  * management of the u_finfo structure.)
5390                  */
5391                 fp = fd < finfo->fi_nfiles ? finfo->fi_list[fd].uf_file : NULL;
5392 
5393                 mstate->dtms_getf = fp;
5394                 regs[rd] = (uintptr_t)fp;
5395                 break;
5396         }
5397 
5398         case DIF_SUBR_CLEANPATH: {
5399                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
5400                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
5401                 uintptr_t src = tupregs[0].dttk_value;
5402                 size_t lim;
5403                 int i = 0, j = 0;
5404                 zone_t *z;
5405 
5406                 if (!dtrace_strcanload(src, size, &lim, mstate, vstate)) {
5407                         regs[rd] = 0;
5408                         break;
5409                 }
5410 
5411                 if (!DTRACE_INSCRATCH(mstate, size)) {
5412                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5413                         regs[rd] = 0;
5414                         break;
5415                 }
5416 
5417                 /*
5418                  * Move forward, loading each character.
5419                  */
5420                 do {
5421                         c = (i >= lim) ? '\0' : dtrace_load8(src + i++);
5422 next:
5423                         if (j + 5 >= size)   /* 5 = strlen("/..c\0") */
5424                                 break;
5425 
5426                         if (c != '/') {
5427                                 dest[j++] = c;
5428                                 continue;
5429                         }
5430 
5431                         c = (i >= lim) ? '\0' : dtrace_load8(src + i++);
5432 
5433                         if (c == '/') {
5434                                 /*
5435                                  * We have two slashes -- we can just advance
5436                                  * to the next character.
5437                                  */
5438                                 goto next;
5439                         }
5440 
5441                         if (c != '.') {
5442                                 /*
5443                                  * This is not "." and it's not ".." -- we can
5444                                  * just store the "/" and this character and
5445                                  * drive on.
5446                                  */
5447                                 dest[j++] = '/';
5448                                 dest[j++] = c;
5449                                 continue;
5450                         }
5451 
5452                         c = (i >= lim) ? '\0' : dtrace_load8(src + i++);
5453 
5454                         if (c == '/') {
5455                                 /*
5456                                  * This is a "/./" component.  We're not going
5457                                  * to store anything in the destination buffer;
5458                                  * we're just going to go to the next component.
5459                                  */
5460                                 goto next;
5461                         }
5462 
5463                         if (c != '.') {
5464                                 /*
5465                                  * This is not ".." -- we can just store the
5466                                  * "/." and this character and continue
5467                                  * processing.
5468                                  */
5469                                 dest[j++] = '/';
5470                                 dest[j++] = '.';
5471                                 dest[j++] = c;
5472                                 continue;
5473                         }
5474 
5475                         c = (i >= lim) ? '\0' : dtrace_load8(src + i++);
5476 
5477                         if (c != '/' && c != '\0') {
5478                                 /*
5479                                  * This is not ".." -- it's "..[mumble]".
5480                                  * We'll store the "/.." and this character
5481                                  * and continue processing.
5482                                  */
5483                                 dest[j++] = '/';
5484                                 dest[j++] = '.';
5485                                 dest[j++] = '.';
5486                                 dest[j++] = c;
5487                                 continue;
5488                         }
5489 
5490                         /*
5491                          * This is "/../" or "/..\0".  We need to back up
5492                          * our destination pointer until we find a "/".
5493                          */
5494                         i--;
5495                         while (j != 0 && dest[--j] != '/')
5496                                 continue;
5497 
5498                         if (c == '\0')
5499                                 dest[++j] = '/';
5500                 } while (c != '\0');
5501 
5502                 dest[j] = '\0';
5503 
5504                 if (mstate->dtms_getf != NULL &&
5505                     !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) &&
5506                     (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) {
5507                         /*
5508                          * If we've done a getf() as a part of this ECB and we
5509                          * don't have kernel access (and we're not in the global
5510                          * zone), check if the path we cleaned up begins with
5511                          * the zone's root path, and trim it off if so.  Note
5512                          * that this is an output cleanliness issue, not a
5513                          * security issue: knowing one's zone root path does
5514                          * not enable privilege escalation.
5515                          */
5516                         if (strstr(dest, z->zone_rootpath) == dest)
5517                                 dest += strlen(z->zone_rootpath) - 1;
5518                 }
5519 
5520                 regs[rd] = (uintptr_t)dest;
5521                 mstate->dtms_scratch_ptr += size;
5522                 break;
5523         }
5524 
5525         case DIF_SUBR_INET_NTOA:
5526         case DIF_SUBR_INET_NTOA6:
5527         case DIF_SUBR_INET_NTOP: {
5528                 size_t size;
5529                 int af, argi, i;
5530                 char *base, *end;
5531 
5532                 if (subr == DIF_SUBR_INET_NTOP) {
5533                         af = (int)tupregs[0].dttk_value;
5534                         argi = 1;
5535                 } else {
5536                         af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
5537                         argi = 0;
5538                 }
5539 
5540                 if (af == AF_INET) {
5541                         ipaddr_t ip4;
5542                         uint8_t *ptr8, val;
5543 
5544                         if (!dtrace_canload(tupregs[argi].dttk_value,
5545                             sizeof (ipaddr_t), mstate, vstate)) {
5546                                 regs[rd] = 0;
5547                                 break;
5548                         }
5549 
5550                         /*
5551                          * Safely load the IPv4 address.
5552                          */
5553                         ip4 = dtrace_load32(tupregs[argi].dttk_value);
5554 
5555                         /*
5556                          * Check an IPv4 string will fit in scratch.
5557                          */
5558                         size = INET_ADDRSTRLEN;
5559                         if (!DTRACE_INSCRATCH(mstate, size)) {
5560                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5561                                 regs[rd] = 0;
5562                                 break;
5563                         }
5564                         base = (char *)mstate->dtms_scratch_ptr;
5565                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5566 
5567                         /*
5568                          * Stringify as a dotted decimal quad.
5569                          */
5570                         *end-- = '\0';
5571                         ptr8 = (uint8_t *)&ip4;
5572                         for (i = 3; i >= 0; i--) {
5573                                 val = ptr8[i];
5574 
5575                                 if (val == 0) {
5576                                         *end-- = '0';
5577                                 } else {
5578                                         for (; val; val /= 10) {
5579                                                 *end-- = '0' + (val % 10);
5580                                         }
5581                                 }
5582 
5583                                 if (i > 0)
5584                                         *end-- = '.';
5585                         }
5586                         ASSERT(end + 1 >= base);
5587 
5588                 } else if (af == AF_INET6) {
5589                         struct in6_addr ip6;
5590                         int firstzero, tryzero, numzero, v6end;
5591                         uint16_t val;
5592                         const char digits[] = "0123456789abcdef";
5593 
5594                         /*
5595                          * Stringify using RFC 1884 convention 2 - 16 bit
5596                          * hexadecimal values with a zero-run compression.
5597                          * Lower case hexadecimal digits are used.
5598                          *      eg, fe80::214:4fff:fe0b:76c8.
5599                          * The IPv4 embedded form is returned for inet_ntop,
5600                          * just the IPv4 string is returned for inet_ntoa6.
5601                          */
5602 
5603                         if (!dtrace_canload(tupregs[argi].dttk_value,
5604                             sizeof (struct in6_addr), mstate, vstate)) {
5605                                 regs[rd] = 0;
5606                                 break;
5607                         }
5608 
5609                         /*
5610                          * Safely load the IPv6 address.
5611                          */
5612                         dtrace_bcopy(
5613                             (void *)(uintptr_t)tupregs[argi].dttk_value,
5614                             (void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
5615 
5616                         /*
5617                          * Check an IPv6 string will fit in scratch.
5618                          */
5619                         size = INET6_ADDRSTRLEN;
5620                         if (!DTRACE_INSCRATCH(mstate, size)) {
5621                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5622                                 regs[rd] = 0;
5623                                 break;
5624                         }
5625                         base = (char *)mstate->dtms_scratch_ptr;
5626                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
5627                         *end-- = '\0';
5628 
5629                         /*
5630                          * Find the longest run of 16 bit zero values
5631                          * for the single allowed zero compression - "::".
5632                          */
5633                         firstzero = -1;
5634                         tryzero = -1;
5635                         numzero = 1;
5636                         for (i = 0; i < sizeof (struct in6_addr); i++) {
5637                                 if (ip6._S6_un._S6_u8[i] == 0 &&
5638                                     tryzero == -1 && i % 2 == 0) {
5639                                         tryzero = i;
5640                                         continue;
5641                                 }
5642 
5643                                 if (tryzero != -1 &&
5644                                     (ip6._S6_un._S6_u8[i] != 0 ||
5645                                     i == sizeof (struct in6_addr) - 1)) {
5646 
5647                                         if (i - tryzero <= numzero) {
5648                                                 tryzero = -1;
5649                                                 continue;
5650                                         }
5651 
5652                                         firstzero = tryzero;
5653                                         numzero = i - i % 2 - tryzero;
5654                                         tryzero = -1;
5655 
5656                                         if (ip6._S6_un._S6_u8[i] == 0 &&
5657                                             i == sizeof (struct in6_addr) - 1)
5658                                                 numzero += 2;
5659                                 }
5660                         }
5661                         ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
5662 
5663                         /*
5664                          * Check for an IPv4 embedded address.
5665                          */
5666                         v6end = sizeof (struct in6_addr) - 2;
5667                         if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
5668                             IN6_IS_ADDR_V4COMPAT(&ip6)) {
5669                                 for (i = sizeof (struct in6_addr) - 1;
5670                                     i >= DTRACE_V4MAPPED_OFFSET; i--) {
5671                                         ASSERT(end >= base);
5672 
5673                                         val = ip6._S6_un._S6_u8[i];
5674 
5675                                         if (val == 0) {
5676                                                 *end-- = '0';
5677                                         } else {
5678                                                 for (; val; val /= 10) {
5679                                                         *end-- = '0' + val % 10;
5680                                                 }
5681                                         }
5682 
5683                                         if (i > DTRACE_V4MAPPED_OFFSET)
5684                                                 *end-- = '.';
5685                                 }
5686 
5687                                 if (subr == DIF_SUBR_INET_NTOA6)
5688                                         goto inetout;
5689 
5690                                 /*
5691                                  * Set v6end to skip the IPv4 address that
5692                                  * we have already stringified.
5693                                  */
5694                                 v6end = 10;
5695                         }
5696 
5697                         /*
5698                          * Build the IPv6 string by working through the
5699                          * address in reverse.
5700                          */
5701                         for (i = v6end; i >= 0; i -= 2) {
5702                                 ASSERT(end >= base);
5703 
5704                                 if (i == firstzero + numzero - 2) {
5705                                         *end-- = ':';
5706                                         *end-- = ':';
5707                                         i -= numzero - 2;
5708                                         continue;
5709                                 }
5710 
5711                                 if (i < 14 && i != firstzero - 2)
5712                                         *end-- = ':';
5713 
5714                                 val = (ip6._S6_un._S6_u8[i] << 8) +
5715                                     ip6._S6_un._S6_u8[i + 1];
5716 
5717                                 if (val == 0) {
5718                                         *end-- = '0';
5719                                 } else {
5720                                         for (; val; val /= 16) {
5721                                                 *end-- = digits[val % 16];
5722                                         }
5723                                 }
5724                         }
5725                         ASSERT(end + 1 >= base);
5726 
5727                 } else {
5728                         /*
5729                          * The user didn't use AH_INET or AH_INET6.
5730                          */
5731                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
5732                         regs[rd] = 0;
5733                         break;
5734                 }
5735 
5736 inetout:        regs[rd] = (uintptr_t)end + 1;
5737                 mstate->dtms_scratch_ptr += size;
5738                 break;
5739         }
5740 
5741         }
5742 }
5743 
5744 /*
5745  * Emulate the execution of DTrace IR instructions specified by the given
5746  * DIF object.  This function is deliberately void of assertions as all of
5747  * the necessary checks are handled by a call to dtrace_difo_validate().
5748  */
5749 static uint64_t
5750 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
5751     dtrace_vstate_t *vstate, dtrace_state_t *state)
5752 {
5753         const dif_instr_t *text = difo->dtdo_buf;
5754         const uint_t textlen = difo->dtdo_len;
5755         const char *strtab = difo->dtdo_strtab;
5756         const uint64_t *inttab = difo->dtdo_inttab;
5757 
5758         uint64_t rval = 0;
5759         dtrace_statvar_t *svar;
5760         dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
5761         dtrace_difv_t *v;
5762         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5763         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
5764 
5765         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
5766         uint64_t regs[DIF_DIR_NREGS];
5767         uint64_t *tmp;
5768 
5769         uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
5770         int64_t cc_r;
5771         uint_t pc = 0, id, opc;
5772         uint8_t ttop = 0;
5773         dif_instr_t instr;
5774         uint_t r1, r2, rd;
5775 
5776         /*
5777          * We stash the current DIF object into the machine state: we need it
5778          * for subsequent access checking.
5779          */
5780         mstate->dtms_difo = difo;
5781 
5782         regs[DIF_REG_R0] = 0;           /* %r0 is fixed at zero */
5783 
5784         while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
5785                 opc = pc;
5786 
5787                 instr = text[pc++];
5788                 r1 = DIF_INSTR_R1(instr);
5789                 r2 = DIF_INSTR_R2(instr);
5790                 rd = DIF_INSTR_RD(instr);
5791 
5792                 switch (DIF_INSTR_OP(instr)) {
5793                 case DIF_OP_OR:
5794                         regs[rd] = regs[r1] | regs[r2];
5795                         break;
5796                 case DIF_OP_XOR:
5797                         regs[rd] = regs[r1] ^ regs[r2];
5798                         break;
5799                 case DIF_OP_AND:
5800                         regs[rd] = regs[r1] & regs[r2];
5801                         break;
5802                 case DIF_OP_SLL:
5803                         regs[rd] = regs[r1] << regs[r2];
5804                         break;
5805                 case DIF_OP_SRL:
5806                         regs[rd] = regs[r1] >> regs[r2];
5807                         break;
5808                 case DIF_OP_SUB:
5809                         regs[rd] = regs[r1] - regs[r2];
5810                         break;
5811                 case DIF_OP_ADD:
5812                         regs[rd] = regs[r1] + regs[r2];
5813                         break;
5814                 case DIF_OP_MUL:
5815                         regs[rd] = regs[r1] * regs[r2];
5816                         break;
5817                 case DIF_OP_SDIV:
5818                         if (regs[r2] == 0) {
5819                                 regs[rd] = 0;
5820                                 *flags |= CPU_DTRACE_DIVZERO;
5821                         } else {
5822                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5823                                 regs[rd] = (int64_t)regs[r1] /
5824                                     (int64_t)regs[r2];
5825                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5826                         }
5827                         break;
5828 
5829                 case DIF_OP_UDIV:
5830                         if (regs[r2] == 0) {
5831                                 regs[rd] = 0;
5832                                 *flags |= CPU_DTRACE_DIVZERO;
5833                         } else {
5834                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5835                                 regs[rd] = regs[r1] / regs[r2];
5836                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5837                         }
5838                         break;
5839 
5840                 case DIF_OP_SREM:
5841                         if (regs[r2] == 0) {
5842                                 regs[rd] = 0;
5843                                 *flags |= CPU_DTRACE_DIVZERO;
5844                         } else {
5845                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5846                                 regs[rd] = (int64_t)regs[r1] %
5847                                     (int64_t)regs[r2];
5848                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5849                         }
5850                         break;
5851 
5852                 case DIF_OP_UREM:
5853                         if (regs[r2] == 0) {
5854                                 regs[rd] = 0;
5855                                 *flags |= CPU_DTRACE_DIVZERO;
5856                         } else {
5857                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5858                                 regs[rd] = regs[r1] % regs[r2];
5859                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5860                         }
5861                         break;
5862 
5863                 case DIF_OP_NOT:
5864                         regs[rd] = ~regs[r1];
5865                         break;
5866                 case DIF_OP_MOV:
5867                         regs[rd] = regs[r1];
5868                         break;
5869                 case DIF_OP_CMP:
5870                         cc_r = regs[r1] - regs[r2];
5871                         cc_n = cc_r < 0;
5872                         cc_z = cc_r == 0;
5873                         cc_v = 0;
5874                         cc_c = regs[r1] < regs[r2];
5875                         break;
5876                 case DIF_OP_TST:
5877                         cc_n = cc_v = cc_c = 0;
5878                         cc_z = regs[r1] == 0;
5879                         break;
5880                 case DIF_OP_BA:
5881                         pc = DIF_INSTR_LABEL(instr);
5882                         break;
5883                 case DIF_OP_BE:
5884                         if (cc_z)
5885                                 pc = DIF_INSTR_LABEL(instr);
5886                         break;
5887                 case DIF_OP_BNE:
5888                         if (cc_z == 0)
5889                                 pc = DIF_INSTR_LABEL(instr);
5890                         break;
5891                 case DIF_OP_BG:
5892                         if ((cc_z | (cc_n ^ cc_v)) == 0)
5893                                 pc = DIF_INSTR_LABEL(instr);
5894                         break;
5895                 case DIF_OP_BGU:
5896                         if ((cc_c | cc_z) == 0)
5897                                 pc = DIF_INSTR_LABEL(instr);
5898                         break;
5899                 case DIF_OP_BGE:
5900                         if ((cc_n ^ cc_v) == 0)
5901                                 pc = DIF_INSTR_LABEL(instr);
5902                         break;
5903                 case DIF_OP_BGEU:
5904                         if (cc_c == 0)
5905                                 pc = DIF_INSTR_LABEL(instr);
5906                         break;
5907                 case DIF_OP_BL:
5908                         if (cc_n ^ cc_v)
5909                                 pc = DIF_INSTR_LABEL(instr);
5910                         break;
5911                 case DIF_OP_BLU:
5912                         if (cc_c)
5913                                 pc = DIF_INSTR_LABEL(instr);
5914                         break;
5915                 case DIF_OP_BLE:
5916                         if (cc_z | (cc_n ^ cc_v))
5917                                 pc = DIF_INSTR_LABEL(instr);
5918                         break;
5919                 case DIF_OP_BLEU:
5920                         if (cc_c | cc_z)
5921                                 pc = DIF_INSTR_LABEL(instr);
5922                         break;
5923                 case DIF_OP_RLDSB:
5924                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5925                                 break;
5926                         /*FALLTHROUGH*/
5927                 case DIF_OP_LDSB:
5928                         regs[rd] = (int8_t)dtrace_load8(regs[r1]);
5929                         break;
5930                 case DIF_OP_RLDSH:
5931                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5932                                 break;
5933                         /*FALLTHROUGH*/
5934                 case DIF_OP_LDSH:
5935                         regs[rd] = (int16_t)dtrace_load16(regs[r1]);
5936                         break;
5937                 case DIF_OP_RLDSW:
5938                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5939                                 break;
5940                         /*FALLTHROUGH*/
5941                 case DIF_OP_LDSW:
5942                         regs[rd] = (int32_t)dtrace_load32(regs[r1]);
5943                         break;
5944                 case DIF_OP_RLDUB:
5945                         if (!dtrace_canload(regs[r1], 1, mstate, vstate))
5946                                 break;
5947                         /*FALLTHROUGH*/
5948                 case DIF_OP_LDUB:
5949                         regs[rd] = dtrace_load8(regs[r1]);
5950                         break;
5951                 case DIF_OP_RLDUH:
5952                         if (!dtrace_canload(regs[r1], 2, mstate, vstate))
5953                                 break;
5954                         /*FALLTHROUGH*/
5955                 case DIF_OP_LDUH:
5956                         regs[rd] = dtrace_load16(regs[r1]);
5957                         break;
5958                 case DIF_OP_RLDUW:
5959                         if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5960                                 break;
5961                         /*FALLTHROUGH*/
5962                 case DIF_OP_LDUW:
5963                         regs[rd] = dtrace_load32(regs[r1]);
5964                         break;
5965                 case DIF_OP_RLDX:
5966                         if (!dtrace_canload(regs[r1], 8, mstate, vstate))
5967                                 break;
5968                         /*FALLTHROUGH*/
5969                 case DIF_OP_LDX:
5970                         regs[rd] = dtrace_load64(regs[r1]);
5971                         break;
5972                 case DIF_OP_ULDSB:
5973                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5974                         regs[rd] = (int8_t)
5975                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5976                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5977                         break;
5978                 case DIF_OP_ULDSH:
5979                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5980                         regs[rd] = (int16_t)
5981                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5982                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5983                         break;
5984                 case DIF_OP_ULDSW:
5985                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5986                         regs[rd] = (int32_t)
5987                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5988                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5989                         break;
5990                 case DIF_OP_ULDUB:
5991                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5992                         regs[rd] =
5993                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5994                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5995                         break;
5996                 case DIF_OP_ULDUH:
5997                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5998                         regs[rd] =
5999                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
6000                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6001                         break;
6002                 case DIF_OP_ULDUW:
6003                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6004                         regs[rd] =
6005                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
6006                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6007                         break;
6008                 case DIF_OP_ULDX:
6009                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6010                         regs[rd] =
6011                             dtrace_fuword64((void *)(uintptr_t)regs[r1]);
6012                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6013                         break;
6014                 case DIF_OP_RET:
6015                         rval = regs[rd];
6016                         pc = textlen;
6017                         break;
6018                 case DIF_OP_NOP:
6019                         break;
6020                 case DIF_OP_SETX:
6021                         regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
6022                         break;
6023                 case DIF_OP_SETS:
6024                         regs[rd] = (uint64_t)(uintptr_t)
6025                             (strtab + DIF_INSTR_STRING(instr));
6026                         break;
6027                 case DIF_OP_SCMP: {
6028                         size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
6029                         uintptr_t s1 = regs[r1];
6030                         uintptr_t s2 = regs[r2];
6031                         size_t lim1, lim2;
6032 
6033                         if (s1 != 0 &&
6034                             !dtrace_strcanload(s1, sz, &lim1, mstate, vstate))
6035                                 break;
6036                         if (s2 != 0 &&
6037                             !dtrace_strcanload(s2, sz, &lim2, mstate, vstate))
6038                                 break;
6039 
6040                         cc_r = dtrace_strncmp((char *)s1, (char *)s2,
6041                             MIN(lim1, lim2));
6042 
6043                         cc_n = cc_r < 0;
6044                         cc_z = cc_r == 0;
6045                         cc_v = cc_c = 0;
6046                         break;
6047                 }
6048                 case DIF_OP_LDGA:
6049                         regs[rd] = dtrace_dif_variable(mstate, state,
6050                             r1, regs[r2]);
6051                         break;
6052                 case DIF_OP_LDGS:
6053                         id = DIF_INSTR_VAR(instr);
6054 
6055                         if (id >= DIF_VAR_OTHER_UBASE) {
6056                                 uintptr_t a;
6057 
6058                                 id -= DIF_VAR_OTHER_UBASE;
6059                                 svar = vstate->dtvs_globals[id];
6060                                 ASSERT(svar != NULL);
6061                                 v = &svar->dtsv_var;
6062 
6063                                 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
6064                                         regs[rd] = svar->dtsv_data;
6065                                         break;
6066                                 }
6067 
6068                                 a = (uintptr_t)svar->dtsv_data;
6069 
6070                                 if (*(uint8_t *)a == UINT8_MAX) {
6071                                         /*
6072                                          * If the 0th byte is set to UINT8_MAX
6073                                          * then this is to be treated as a
6074                                          * reference to a NULL variable.
6075                                          */
6076                                         regs[rd] = 0;
6077                                 } else {
6078                                         regs[rd] = a + sizeof (uint64_t);
6079                                 }
6080 
6081                                 break;
6082                         }
6083 
6084                         regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
6085                         break;
6086 
6087                 case DIF_OP_STGA:
6088                         dtrace_dif_variable_write(mstate, state, r1, regs[r2],
6089                             regs[rd]);
6090                         break;
6091 
6092                 case DIF_OP_STGS:
6093                         id = DIF_INSTR_VAR(instr);
6094 
6095                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6096                         id -= DIF_VAR_OTHER_UBASE;
6097 
6098                         VERIFY(id < vstate->dtvs_nglobals);
6099                         svar = vstate->dtvs_globals[id];
6100                         ASSERT(svar != NULL);
6101                         v = &svar->dtsv_var;
6102 
6103                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6104                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
6105                                 size_t lim;
6106 
6107                                 ASSERT(a != (uintptr_t)NULL);
6108                                 ASSERT(svar->dtsv_size != 0);
6109 
6110                                 if (regs[rd] == 0) {
6111                                         *(uint8_t *)a = UINT8_MAX;
6112                                         break;
6113                                 } else {
6114                                         *(uint8_t *)a = 0;
6115                                         a += sizeof (uint64_t);
6116                                 }
6117                                 if (!dtrace_vcanload(
6118                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
6119                                     &lim, mstate, vstate))
6120                                         break;
6121 
6122                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6123                                     (void *)a, &v->dtdv_type, lim);
6124                                 break;
6125                         }
6126 
6127                         svar->dtsv_data = regs[rd];
6128                         break;
6129 
6130                 case DIF_OP_LDTA:
6131                         /*
6132                          * There are no DTrace built-in thread-local arrays at
6133                          * present.  This opcode is saved for future work.
6134                          */
6135                         *flags |= CPU_DTRACE_ILLOP;
6136                         regs[rd] = 0;
6137                         break;
6138 
6139                 case DIF_OP_LDLS:
6140                         id = DIF_INSTR_VAR(instr);
6141 
6142                         if (id < DIF_VAR_OTHER_UBASE) {
6143                                 /*
6144                                  * For now, this has no meaning.
6145                                  */
6146                                 regs[rd] = 0;
6147                                 break;
6148                         }
6149 
6150                         id -= DIF_VAR_OTHER_UBASE;
6151 
6152                         ASSERT(id < vstate->dtvs_nlocals);
6153                         ASSERT(vstate->dtvs_locals != NULL);
6154 
6155                         svar = vstate->dtvs_locals[id];
6156                         ASSERT(svar != NULL);
6157                         v = &svar->dtsv_var;
6158 
6159                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6160                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
6161                                 size_t sz = v->dtdv_type.dtdt_size;
6162 
6163                                 sz += sizeof (uint64_t);
6164                                 ASSERT(svar->dtsv_size == NCPU * sz);
6165                                 a += CPU->cpu_id * sz;
6166 
6167                                 if (*(uint8_t *)a == UINT8_MAX) {
6168                                         /*
6169                                          * If the 0th byte is set to UINT8_MAX
6170                                          * then this is to be treated as a
6171                                          * reference to a NULL variable.
6172                                          */
6173                                         regs[rd] = 0;
6174                                 } else {
6175                                         regs[rd] = a + sizeof (uint64_t);
6176                                 }
6177 
6178                                 break;
6179                         }
6180 
6181                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
6182                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
6183                         regs[rd] = tmp[CPU->cpu_id];
6184                         break;
6185 
6186                 case DIF_OP_STLS:
6187                         id = DIF_INSTR_VAR(instr);
6188 
6189                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6190                         id -= DIF_VAR_OTHER_UBASE;
6191                         VERIFY(id < vstate->dtvs_nlocals);
6192 
6193                         ASSERT(vstate->dtvs_locals != NULL);
6194                         svar = vstate->dtvs_locals[id];
6195                         ASSERT(svar != NULL);
6196                         v = &svar->dtsv_var;
6197 
6198                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6199                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
6200                                 size_t sz = v->dtdv_type.dtdt_size;
6201                                 size_t lim;
6202 
6203                                 sz += sizeof (uint64_t);
6204                                 ASSERT(svar->dtsv_size == NCPU * sz);
6205                                 a += CPU->cpu_id * sz;
6206 
6207                                 if (regs[rd] == 0) {
6208                                         *(uint8_t *)a = UINT8_MAX;
6209                                         break;
6210                                 } else {
6211                                         *(uint8_t *)a = 0;
6212                                         a += sizeof (uint64_t);
6213                                 }
6214 
6215                                 if (!dtrace_vcanload(
6216                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
6217                                     &lim, mstate, vstate))
6218                                         break;
6219 
6220                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6221                                     (void *)a, &v->dtdv_type, lim);
6222                                 break;
6223                         }
6224 
6225                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
6226                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
6227                         tmp[CPU->cpu_id] = regs[rd];
6228                         break;
6229 
6230                 case DIF_OP_LDTS: {
6231                         dtrace_dynvar_t *dvar;
6232                         dtrace_key_t *key;
6233 
6234                         id = DIF_INSTR_VAR(instr);
6235                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6236                         id -= DIF_VAR_OTHER_UBASE;
6237                         v = &vstate->dtvs_tlocals[id];
6238 
6239                         key = &tupregs[DIF_DTR_NREGS];
6240                         key[0].dttk_value = (uint64_t)id;
6241                         key[0].dttk_size = 0;
6242                         DTRACE_TLS_THRKEY(key[1].dttk_value);
6243                         key[1].dttk_size = 0;
6244 
6245                         dvar = dtrace_dynvar(dstate, 2, key,
6246                             sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
6247                             mstate, vstate);
6248 
6249                         if (dvar == NULL) {
6250                                 regs[rd] = 0;
6251                                 break;
6252                         }
6253 
6254                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6255                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
6256                         } else {
6257                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
6258                         }
6259 
6260                         break;
6261                 }
6262 
6263                 case DIF_OP_STTS: {
6264                         dtrace_dynvar_t *dvar;
6265                         dtrace_key_t *key;
6266 
6267                         id = DIF_INSTR_VAR(instr);
6268                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6269                         id -= DIF_VAR_OTHER_UBASE;
6270                         VERIFY(id < vstate->dtvs_ntlocals);
6271 
6272                         key = &tupregs[DIF_DTR_NREGS];
6273                         key[0].dttk_value = (uint64_t)id;
6274                         key[0].dttk_size = 0;
6275                         DTRACE_TLS_THRKEY(key[1].dttk_value);
6276                         key[1].dttk_size = 0;
6277                         v = &vstate->dtvs_tlocals[id];
6278 
6279                         dvar = dtrace_dynvar(dstate, 2, key,
6280                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6281                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6282                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6283                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6284 
6285                         /*
6286                          * Given that we're storing to thread-local data,
6287                          * we need to flush our predicate cache.
6288                          */
6289                         curthread->t_predcache = DTRACE_CACHEIDNONE;
6290 
6291                         if (dvar == NULL)
6292                                 break;
6293 
6294                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6295                                 size_t lim;
6296 
6297                                 if (!dtrace_vcanload(
6298                                     (void *)(uintptr_t)regs[rd],
6299                                     &v->dtdv_type, &lim, mstate, vstate))
6300                                         break;
6301 
6302                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6303                                     dvar->dtdv_data, &v->dtdv_type, lim);
6304                         } else {
6305                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6306                         }
6307 
6308                         break;
6309                 }
6310 
6311                 case DIF_OP_SRA:
6312                         regs[rd] = (int64_t)regs[r1] >> regs[r2];
6313                         break;
6314 
6315                 case DIF_OP_CALL:
6316                         dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
6317                             regs, tupregs, ttop, mstate, state);
6318                         break;
6319 
6320                 case DIF_OP_PUSHTR:
6321                         if (ttop == DIF_DTR_NREGS) {
6322                                 *flags |= CPU_DTRACE_TUPOFLOW;
6323                                 break;
6324                         }
6325 
6326                         if (r1 == DIF_TYPE_STRING) {
6327                                 /*
6328                                  * If this is a string type and the size is 0,
6329                                  * we'll use the system-wide default string
6330                                  * size.  Note that we are _not_ looking at
6331                                  * the value of the DTRACEOPT_STRSIZE option;
6332                                  * had this been set, we would expect to have
6333                                  * a non-zero size value in the "pushtr".
6334                                  */
6335                                 tupregs[ttop].dttk_size =
6336                                     dtrace_strlen((char *)(uintptr_t)regs[rd],
6337                                     regs[r2] ? regs[r2] :
6338                                     dtrace_strsize_default) + 1;
6339                         } else {
6340                                 if (regs[r2] > LONG_MAX) {
6341                                         *flags |= CPU_DTRACE_ILLOP;
6342                                         break;
6343                                 }
6344 
6345                                 tupregs[ttop].dttk_size = regs[r2];
6346                         }
6347 
6348                         tupregs[ttop++].dttk_value = regs[rd];
6349                         break;
6350 
6351                 case DIF_OP_PUSHTV:
6352                         if (ttop == DIF_DTR_NREGS) {
6353                                 *flags |= CPU_DTRACE_TUPOFLOW;
6354                                 break;
6355                         }
6356 
6357                         tupregs[ttop].dttk_value = regs[rd];
6358                         tupregs[ttop++].dttk_size = 0;
6359                         break;
6360 
6361                 case DIF_OP_POPTS:
6362                         if (ttop != 0)
6363                                 ttop--;
6364                         break;
6365 
6366                 case DIF_OP_FLUSHTS:
6367                         ttop = 0;
6368                         break;
6369 
6370                 case DIF_OP_LDGAA:
6371                 case DIF_OP_LDTAA: {
6372                         dtrace_dynvar_t *dvar;
6373                         dtrace_key_t *key = tupregs;
6374                         uint_t nkeys = ttop;
6375 
6376                         id = DIF_INSTR_VAR(instr);
6377                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6378                         id -= DIF_VAR_OTHER_UBASE;
6379 
6380                         key[nkeys].dttk_value = (uint64_t)id;
6381                         key[nkeys++].dttk_size = 0;
6382 
6383                         if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
6384                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6385                                 key[nkeys++].dttk_size = 0;
6386                                 VERIFY(id < vstate->dtvs_ntlocals);
6387                                 v = &vstate->dtvs_tlocals[id];
6388                         } else {
6389                                 VERIFY(id < vstate->dtvs_nglobals);
6390                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6391                         }
6392 
6393                         dvar = dtrace_dynvar(dstate, nkeys, key,
6394                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6395                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6396                             DTRACE_DYNVAR_NOALLOC, mstate, vstate);
6397 
6398                         if (dvar == NULL) {
6399                                 regs[rd] = 0;
6400                                 break;
6401                         }
6402 
6403                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6404                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
6405                         } else {
6406                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
6407                         }
6408 
6409                         break;
6410                 }
6411 
6412                 case DIF_OP_STGAA:
6413                 case DIF_OP_STTAA: {
6414                         dtrace_dynvar_t *dvar;
6415                         dtrace_key_t *key = tupregs;
6416                         uint_t nkeys = ttop;
6417 
6418                         id = DIF_INSTR_VAR(instr);
6419                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
6420                         id -= DIF_VAR_OTHER_UBASE;
6421 
6422                         key[nkeys].dttk_value = (uint64_t)id;
6423                         key[nkeys++].dttk_size = 0;
6424 
6425                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
6426                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
6427                                 key[nkeys++].dttk_size = 0;
6428                                 VERIFY(id < vstate->dtvs_ntlocals);
6429                                 v = &vstate->dtvs_tlocals[id];
6430                         } else {
6431                                 VERIFY(id < vstate->dtvs_nglobals);
6432                                 v = &vstate->dtvs_globals[id]->dtsv_var;
6433                         }
6434 
6435                         dvar = dtrace_dynvar(dstate, nkeys, key,
6436                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
6437                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
6438                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
6439                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
6440 
6441                         if (dvar == NULL)
6442                                 break;
6443 
6444                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
6445                                 size_t lim;
6446 
6447                                 if (!dtrace_vcanload(
6448                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
6449                                     &lim, mstate, vstate))
6450                                         break;
6451 
6452                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
6453                                     dvar->dtdv_data, &v->dtdv_type, lim);
6454                         } else {
6455                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
6456                         }
6457 
6458                         break;
6459                 }
6460 
6461                 case DIF_OP_ALLOCS: {
6462                         uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6463                         size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
6464 
6465                         /*
6466                          * Rounding up the user allocation size could have
6467                          * overflowed large, bogus allocations (like -1ULL) to
6468                          * 0.
6469                          */
6470                         if (size < regs[r1] ||
6471                             !DTRACE_INSCRATCH(mstate, size)) {
6472                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6473                                 regs[rd] = 0;
6474                                 break;
6475                         }
6476 
6477                         dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
6478                         mstate->dtms_scratch_ptr += size;
6479                         regs[rd] = ptr;
6480                         break;
6481                 }
6482 
6483                 case DIF_OP_COPYS:
6484                         if (!dtrace_canstore(regs[rd], regs[r2],
6485                             mstate, vstate)) {
6486                                 *flags |= CPU_DTRACE_BADADDR;
6487                                 *illval = regs[rd];
6488                                 break;
6489                         }
6490 
6491                         if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
6492                                 break;
6493 
6494                         dtrace_bcopy((void *)(uintptr_t)regs[r1],
6495                             (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
6496                         break;
6497 
6498                 case DIF_OP_STB:
6499                         if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
6500                                 *flags |= CPU_DTRACE_BADADDR;
6501                                 *illval = regs[rd];
6502                                 break;
6503                         }
6504                         *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
6505                         break;
6506 
6507                 case DIF_OP_STH:
6508                         if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
6509                                 *flags |= CPU_DTRACE_BADADDR;
6510                                 *illval = regs[rd];
6511                                 break;
6512                         }
6513                         if (regs[rd] & 1) {
6514                                 *flags |= CPU_DTRACE_BADALIGN;
6515                                 *illval = regs[rd];
6516                                 break;
6517                         }
6518                         *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
6519                         break;
6520 
6521                 case DIF_OP_STW:
6522                         if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
6523                                 *flags |= CPU_DTRACE_BADADDR;
6524                                 *illval = regs[rd];
6525                                 break;
6526                         }
6527                         if (regs[rd] & 3) {
6528                                 *flags |= CPU_DTRACE_BADALIGN;
6529                                 *illval = regs[rd];
6530                                 break;
6531                         }
6532                         *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
6533                         break;
6534 
6535                 case DIF_OP_STX:
6536                         if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
6537                                 *flags |= CPU_DTRACE_BADADDR;
6538                                 *illval = regs[rd];
6539                                 break;
6540                         }
6541                         if (regs[rd] & 7) {
6542                                 *flags |= CPU_DTRACE_BADALIGN;
6543                                 *illval = regs[rd];
6544                                 break;
6545                         }
6546                         *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
6547                         break;
6548                 }
6549         }
6550 
6551         if (!(*flags & CPU_DTRACE_FAULT))
6552                 return (rval);
6553 
6554         mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
6555         mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
6556 
6557         return (0);
6558 }
6559 
6560 static void
6561 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
6562 {
6563         dtrace_probe_t *probe = ecb->dte_probe;
6564         dtrace_provider_t *prov = probe->dtpr_provider;
6565         char c[DTRACE_FULLNAMELEN + 80], *str;
6566         char *msg = "dtrace: breakpoint action at probe ";
6567         char *ecbmsg = " (ecb ";
6568         uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
6569         uintptr_t val = (uintptr_t)ecb;
6570         int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
6571 
6572         if (dtrace_destructive_disallow)
6573                 return;
6574 
6575         /*
6576          * It's impossible to be taking action on the NULL probe.
6577          */
6578         ASSERT(probe != NULL);
6579 
6580         /*
6581          * This is a poor man's (destitute man's?) sprintf():  we want to
6582          * print the provider name, module name, function name and name of
6583          * the probe, along with the hex address of the ECB with the breakpoint
6584          * action -- all of which we must place in the character buffer by
6585          * hand.
6586          */
6587         while (*msg != '\0')
6588                 c[i++] = *msg++;
6589 
6590         for (str = prov->dtpv_name; *str != '\0'; str++)
6591                 c[i++] = *str;
6592         c[i++] = ':';
6593 
6594         for (str = probe->dtpr_mod; *str != '\0'; str++)
6595                 c[i++] = *str;
6596         c[i++] = ':';
6597 
6598         for (str = probe->dtpr_func; *str != '\0'; str++)
6599                 c[i++] = *str;
6600         c[i++] = ':';
6601 
6602         for (str = probe->dtpr_name; *str != '\0'; str++)
6603                 c[i++] = *str;
6604 
6605         while (*ecbmsg != '\0')
6606                 c[i++] = *ecbmsg++;
6607 
6608         while (shift >= 0) {
6609                 mask = (uintptr_t)0xf << shift;
6610 
6611                 if (val >= ((uintptr_t)1 << shift))
6612                         c[i++] = "0123456789abcdef"[(val & mask) >> shift];
6613                 shift -= 4;
6614         }
6615 
6616         c[i++] = ')';
6617         c[i] = '\0';
6618 
6619         debug_enter(c);
6620 }
6621 
6622 static void
6623 dtrace_action_panic(dtrace_ecb_t *ecb)
6624 {
6625         dtrace_probe_t *probe = ecb->dte_probe;
6626 
6627         /*
6628          * It's impossible to be taking action on the NULL probe.
6629          */
6630         ASSERT(probe != NULL);
6631 
6632         if (dtrace_destructive_disallow)
6633                 return;
6634 
6635         if (dtrace_panicked != NULL)
6636                 return;
6637 
6638         if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
6639                 return;
6640 
6641         /*
6642          * We won the right to panic.  (We want to be sure that only one
6643          * thread calls panic() from dtrace_probe(), and that panic() is
6644          * called exactly once.)
6645          */
6646         dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
6647             probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
6648             probe->dtpr_func, probe->dtpr_name, (void *)ecb);
6649 }
6650 
6651 static void
6652 dtrace_action_raise(uint64_t sig)
6653 {
6654         if (dtrace_destructive_disallow)
6655                 return;
6656 
6657         if (sig >= NSIG) {
6658                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
6659                 return;
6660         }
6661 
6662         /*
6663          * raise() has a queue depth of 1 -- we ignore all subsequent
6664          * invocations of the raise() action.
6665          */
6666         if (curthread->t_dtrace_sig == 0)
6667                 curthread->t_dtrace_sig = (uint8_t)sig;
6668 
6669         curthread->t_sig_check = 1;
6670         aston(curthread);
6671 }
6672 
6673 static void
6674 dtrace_action_stop(void)
6675 {
6676         if (dtrace_destructive_disallow)
6677                 return;
6678 
6679         if (!curthread->t_dtrace_stop) {
6680                 curthread->t_dtrace_stop = 1;
6681                 curthread->t_sig_check = 1;
6682                 aston(curthread);
6683         }
6684 }
6685 
6686 static void
6687 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
6688 {
6689         hrtime_t now;
6690         volatile uint16_t *flags;
6691         cpu_t *cpu = CPU;
6692 
6693         if (dtrace_destructive_disallow)
6694                 return;
6695 
6696         flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
6697 
6698         now = dtrace_gethrtime();
6699 
6700         if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
6701                 /*
6702                  * We need to advance the mark to the current time.
6703                  */
6704                 cpu->cpu_dtrace_chillmark = now;
6705                 cpu->cpu_dtrace_chilled = 0;
6706         }
6707 
6708         /*
6709          * Now check to see if the requested chill time would take us over
6710          * the maximum amount of time allowed in the chill interval.  (Or
6711          * worse, if the calculation itself induces overflow.)
6712          */
6713         if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
6714             cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
6715                 *flags |= CPU_DTRACE_ILLOP;
6716                 return;
6717         }
6718 
6719         while (dtrace_gethrtime() - now < val)
6720                 continue;
6721 
6722         /*
6723          * Normally, we assure that the value of the variable "timestamp" does
6724          * not change within an ECB.  The presence of chill() represents an
6725          * exception to this rule, however.
6726          */
6727         mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
6728         cpu->cpu_dtrace_chilled += val;
6729 }
6730 
6731 static void
6732 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
6733     uint64_t *buf, uint64_t arg)
6734 {
6735         int nframes = DTRACE_USTACK_NFRAMES(arg);
6736         int strsize = DTRACE_USTACK_STRSIZE(arg);
6737         uint64_t *pcs = &buf[1], *fps;
6738         char *str = (char *)&pcs[nframes];
6739         int size, offs = 0, i, j;
6740         size_t rem;
6741         uintptr_t old = mstate->dtms_scratch_ptr, saved;
6742         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6743         char *sym;
6744 
6745         /*
6746          * Should be taking a faster path if string space has not been
6747          * allocated.
6748          */
6749         ASSERT(strsize != 0);
6750 
6751         /*
6752          * We will first allocate some temporary space for the frame pointers.
6753          */
6754         fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
6755         size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
6756             (nframes * sizeof (uint64_t));
6757 
6758         if (!DTRACE_INSCRATCH(mstate, size)) {
6759                 /*
6760                  * Not enough room for our frame pointers -- need to indicate
6761                  * that we ran out of scratch space.
6762                  */
6763                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
6764                 return;
6765         }
6766 
6767         mstate->dtms_scratch_ptr += size;
6768         saved = mstate->dtms_scratch_ptr;
6769 
6770         /*
6771          * Now get a stack with both program counters and frame pointers.
6772          */
6773         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6774         dtrace_getufpstack(buf, fps, nframes + 1);
6775         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6776 
6777         /*
6778          * If that faulted, we're cooked.
6779          */
6780         if (*flags & CPU_DTRACE_FAULT)
6781                 goto out;
6782 
6783         /*
6784          * Now we want to walk up the stack, calling the USTACK helper.  For
6785          * each iteration, we restore the scratch pointer.
6786          */
6787         for (i = 0; i < nframes; i++) {
6788                 mstate->dtms_scratch_ptr = saved;
6789 
6790                 if (offs >= strsize)
6791                         break;
6792 
6793                 sym = (char *)(uintptr_t)dtrace_helper(
6794                     DTRACE_HELPER_ACTION_USTACK,
6795                     mstate, state, pcs[i], fps[i]);
6796 
6797                 /*
6798                  * If we faulted while running the helper, we're going to
6799                  * clear the fault and null out the corresponding string.
6800                  */
6801                 if (*flags & CPU_DTRACE_FAULT) {
6802                         *flags &= ~CPU_DTRACE_FAULT;
6803                         str[offs++] = '\0';
6804                         continue;
6805                 }
6806 
6807                 if (sym == NULL) {
6808                         str[offs++] = '\0';
6809                         continue;
6810                 }
6811 
6812                 if (!dtrace_strcanload((uintptr_t)sym, strsize, &rem, mstate,
6813                     &(state->dts_vstate))) {
6814                         str[offs++] = '\0';
6815                         continue;
6816                 }
6817 
6818                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6819 
6820                 /*
6821                  * Now copy in the string that the helper returned to us.
6822                  */
6823                 for (j = 0; offs + j < strsize && j < rem; j++) {
6824                         if ((str[offs + j] = sym[j]) == '\0')
6825                                 break;
6826                 }
6827 
6828                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6829 
6830                 offs += j + 1;
6831         }
6832 
6833         if (offs >= strsize) {
6834                 /*
6835                  * If we didn't have room for all of the strings, we don't
6836                  * abort processing -- this needn't be a fatal error -- but we
6837                  * still want to increment a counter (dts_stkstroverflows) to
6838                  * allow this condition to be warned about.  (If this is from
6839                  * a jstack() action, it is easily tuned via jstackstrsize.)
6840                  */
6841                 dtrace_error(&state->dts_stkstroverflows);
6842         }
6843 
6844         while (offs < strsize)
6845                 str[offs++] = '\0';
6846 
6847 out:
6848         mstate->dtms_scratch_ptr = old;
6849 }
6850 
6851 static void
6852 dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size,
6853     size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind)
6854 {
6855         volatile uint16_t *flags;
6856         uint64_t val = *valp;
6857         size_t valoffs = *valoffsp;
6858 
6859         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6860         ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF);
6861 
6862         /*
6863          * If this is a string, we're going to only load until we find the zero
6864          * byte -- after which we'll store zero bytes.
6865          */
6866         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
6867                 char c = '\0' + 1;
6868                 size_t s;
6869 
6870                 for (s = 0; s < size; s++) {
6871                         if (c != '\0' && dtkind == DIF_TF_BYREF) {
6872                                 c = dtrace_load8(val++);
6873                         } else if (c != '\0' && dtkind == DIF_TF_BYUREF) {
6874                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6875                                 c = dtrace_fuword8((void *)(uintptr_t)val++);
6876                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6877                                 if (*flags & CPU_DTRACE_FAULT)
6878                                         break;
6879                         }
6880 
6881                         DTRACE_STORE(uint8_t, tomax, valoffs++, c);
6882 
6883                         if (c == '\0' && intuple)
6884                                 break;
6885                 }
6886         } else {
6887                 uint8_t c;
6888                 while (valoffs < end) {
6889                         if (dtkind == DIF_TF_BYREF) {
6890                                 c = dtrace_load8(val++);
6891                         } else if (dtkind == DIF_TF_BYUREF) {
6892                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6893                                 c = dtrace_fuword8((void *)(uintptr_t)val++);
6894                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6895                                 if (*flags & CPU_DTRACE_FAULT)
6896                                         break;
6897                         }
6898 
6899                         DTRACE_STORE(uint8_t, tomax,
6900                             valoffs++, c);
6901                 }
6902         }
6903 
6904         *valp = val;
6905         *valoffsp = valoffs;
6906 }
6907 
6908 /*
6909  * If you're looking for the epicenter of DTrace, you just found it.  This
6910  * is the function called by the provider to fire a probe -- from which all
6911  * subsequent probe-context DTrace activity emanates.
6912  */
6913 void
6914 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
6915     uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
6916 {
6917         processorid_t cpuid;
6918         dtrace_icookie_t cookie;
6919         dtrace_probe_t *probe;
6920         dtrace_mstate_t mstate;
6921         dtrace_ecb_t *ecb;
6922         dtrace_action_t *act;
6923         intptr_t offs;
6924         size_t size;
6925         int vtime, onintr;
6926         volatile uint16_t *flags;
6927         hrtime_t now, end;
6928 
6929         /*
6930          * Kick out immediately if this CPU is still being born (in which case
6931          * curthread will be set to -1) or the current thread can't allow
6932          * probes in its current context.
6933          */
6934         if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
6935                 return;
6936 
6937         cookie = dtrace_interrupt_disable();
6938 
6939         /*
6940          * Also refuse to process any probe firings that might happen on a
6941          * disabled CPU.
6942          */
6943         if (CPU->cpu_flags & CPU_DISABLED) {
6944                 dtrace_interrupt_enable(cookie);
6945                 return;
6946         }
6947 
6948         probe = dtrace_probes[id - 1];
6949         cpuid = CPU->cpu_id;
6950         onintr = CPU_ON_INTR(CPU);
6951 
6952         CPU->cpu_dtrace_probes++;
6953 
6954         if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
6955             probe->dtpr_predcache == curthread->t_predcache) {
6956                 /*
6957                  * We have hit in the predicate cache; we know that
6958                  * this predicate would evaluate to be false.
6959                  */
6960                 dtrace_interrupt_enable(cookie);
6961                 return;
6962         }
6963 
6964         if (panic_quiesce) {
6965                 /*
6966                  * We don't trace anything if we're panicking.
6967                  */
6968                 dtrace_interrupt_enable(cookie);
6969                 return;
6970         }
6971 
6972         now = mstate.dtms_timestamp = dtrace_gethrtime();
6973         mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
6974         vtime = dtrace_vtime_references != 0;
6975 
6976         if (vtime && curthread->t_dtrace_start)
6977                 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
6978 
6979         mstate.dtms_difo = NULL;
6980         mstate.dtms_probe = probe;
6981         mstate.dtms_strtok = 0;
6982         mstate.dtms_arg[0] = arg0;
6983         mstate.dtms_arg[1] = arg1;
6984         mstate.dtms_arg[2] = arg2;
6985         mstate.dtms_arg[3] = arg3;
6986         mstate.dtms_arg[4] = arg4;
6987 
6988         flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
6989 
6990         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
6991                 dtrace_predicate_t *pred = ecb->dte_predicate;
6992                 dtrace_state_t *state = ecb->dte_state;
6993                 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
6994                 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
6995                 dtrace_vstate_t *vstate = &state->dts_vstate;
6996                 dtrace_provider_t *prov = probe->dtpr_provider;
6997                 uint64_t tracememsize = 0;
6998                 int committed = 0;
6999                 caddr_t tomax;
7000 
7001                 /*
7002                  * A little subtlety with the following (seemingly innocuous)
7003                  * declaration of the automatic 'val':  by looking at the
7004                  * code, you might think that it could be declared in the
7005                  * action processing loop, below.  (That is, it's only used in
7006                  * the action processing loop.)  However, it must be declared
7007                  * out of that scope because in the case of DIF expression
7008                  * arguments to aggregating actions, one iteration of the
7009                  * action loop will use the last iteration's value.
7010                  */
7011 #ifdef lint
7012                 uint64_t val = 0;
7013 #else
7014                 uint64_t val;
7015 #endif
7016 
7017                 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
7018                 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC;
7019                 mstate.dtms_getf = NULL;
7020 
7021                 *flags &= ~CPU_DTRACE_ERROR;
7022 
7023                 if (prov == dtrace_provider) {
7024                         /*
7025                          * If dtrace itself is the provider of this probe,
7026                          * we're only going to continue processing the ECB if
7027                          * arg0 (the dtrace_state_t) is equal to the ECB's
7028                          * creating state.  (This prevents disjoint consumers
7029                          * from seeing one another's metaprobes.)
7030                          */
7031                         if (arg0 != (uint64_t)(uintptr_t)state)
7032                                 continue;
7033                 }
7034 
7035                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
7036                         /*
7037                          * We're not currently active.  If our provider isn't
7038                          * the dtrace pseudo provider, we're not interested.
7039                          */
7040                         if (prov != dtrace_provider)
7041                                 continue;
7042 
7043                         /*
7044                          * Now we must further check if we are in the BEGIN
7045                          * probe.  If we are, we will only continue processing
7046                          * if we're still in WARMUP -- if one BEGIN enabling
7047                          * has invoked the exit() action, we don't want to
7048                          * evaluate subsequent BEGIN enablings.
7049                          */
7050                         if (probe->dtpr_id == dtrace_probeid_begin &&
7051                             state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
7052                                 ASSERT(state->dts_activity ==
7053                                     DTRACE_ACTIVITY_DRAINING);
7054                                 continue;
7055                         }
7056                 }
7057 
7058                 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb))
7059                         continue;
7060 
7061                 if (now - state->dts_alive > dtrace_deadman_timeout) {
7062                         /*
7063                          * We seem to be dead.  Unless we (a) have kernel
7064                          * destructive permissions (b) have explicitly enabled
7065                          * destructive actions and (c) destructive actions have
7066                          * not been disabled, we're going to transition into
7067                          * the KILLED state, from which no further processing
7068                          * on this state will be performed.
7069                          */
7070                         if (!dtrace_priv_kernel_destructive(state) ||
7071                             !state->dts_cred.dcr_destructive ||
7072                             dtrace_destructive_disallow) {
7073                                 void *activity = &state->dts_activity;
7074                                 dtrace_activity_t current;
7075 
7076                                 do {
7077                                         current = state->dts_activity;
7078                                 } while (dtrace_cas32(activity, current,
7079                                     DTRACE_ACTIVITY_KILLED) != current);
7080 
7081                                 continue;
7082                         }
7083                 }
7084 
7085                 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
7086                     ecb->dte_alignment, state, &mstate)) < 0)
7087                         continue;
7088 
7089                 tomax = buf->dtb_tomax;
7090                 ASSERT(tomax != NULL);
7091 
7092                 if (ecb->dte_size != 0) {
7093                         dtrace_rechdr_t dtrh;
7094                         if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
7095                                 mstate.dtms_timestamp = dtrace_gethrtime();
7096                                 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
7097                         }
7098                         ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
7099                         dtrh.dtrh_epid = ecb->dte_epid;
7100                         DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
7101                             mstate.dtms_timestamp);
7102                         *((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
7103                 }
7104 
7105                 mstate.dtms_epid = ecb->dte_epid;
7106                 mstate.dtms_present |= DTRACE_MSTATE_EPID;
7107 
7108                 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
7109                         mstate.dtms_access |= DTRACE_ACCESS_KERNEL;
7110 
7111                 if (pred != NULL) {
7112                         dtrace_difo_t *dp = pred->dtp_difo;
7113                         int rval;
7114 
7115                         rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
7116 
7117                         if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
7118                                 dtrace_cacheid_t cid = probe->dtpr_predcache;
7119 
7120                                 if (cid != DTRACE_CACHEIDNONE && !onintr) {
7121                                         /*
7122                                          * Update the predicate cache...
7123                                          */
7124                                         ASSERT(cid == pred->dtp_cacheid);
7125                                         curthread->t_predcache = cid;
7126                                 }
7127 
7128                                 continue;
7129                         }
7130                 }
7131 
7132                 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
7133                     act != NULL; act = act->dta_next) {
7134                         size_t valoffs;
7135                         dtrace_difo_t *dp;
7136                         dtrace_recdesc_t *rec = &act->dta_rec;
7137 
7138                         size = rec->dtrd_size;
7139                         valoffs = offs + rec->dtrd_offset;
7140 
7141                         if (DTRACEACT_ISAGG(act->dta_kind)) {
7142                                 uint64_t v = 0xbad;
7143                                 dtrace_aggregation_t *agg;
7144 
7145                                 agg = (dtrace_aggregation_t *)act;
7146 
7147                                 if ((dp = act->dta_difo) != NULL)
7148                                         v = dtrace_dif_emulate(dp,
7149                                             &mstate, vstate, state);
7150 
7151                                 if (*flags & CPU_DTRACE_ERROR)
7152                                         continue;
7153 
7154                                 /*
7155                                  * Note that we always pass the expression
7156                                  * value from the previous iteration of the
7157                                  * action loop.  This value will only be used
7158                                  * if there is an expression argument to the
7159                                  * aggregating action, denoted by the
7160                                  * dtag_hasarg field.
7161                                  */
7162                                 dtrace_aggregate(agg, buf,
7163                                     offs, aggbuf, v, val);
7164                                 continue;
7165                         }
7166 
7167                         switch (act->dta_kind) {
7168                         case DTRACEACT_STOP:
7169                                 if (dtrace_priv_proc_destructive(state,
7170                                     &mstate))
7171                                         dtrace_action_stop();
7172                                 continue;
7173 
7174                         case DTRACEACT_BREAKPOINT:
7175                                 if (dtrace_priv_kernel_destructive(state))
7176                                         dtrace_action_breakpoint(ecb);
7177                                 continue;
7178 
7179                         case DTRACEACT_PANIC:
7180                                 if (dtrace_priv_kernel_destructive(state))
7181                                         dtrace_action_panic(ecb);
7182                                 continue;
7183 
7184                         case DTRACEACT_STACK:
7185                                 if (!dtrace_priv_kernel(state))
7186                                         continue;
7187 
7188                                 dtrace_getpcstack((pc_t *)(tomax + valoffs),
7189                                     size / sizeof (pc_t), probe->dtpr_aframes,
7190                                     DTRACE_ANCHORED(probe) ? NULL :
7191                                     (uint32_t *)arg0);
7192 
7193                                 continue;
7194 
7195                         case DTRACEACT_JSTACK:
7196                         case DTRACEACT_USTACK:
7197                                 if (!dtrace_priv_proc(state, &mstate))
7198                                         continue;
7199 
7200                                 /*
7201                                  * See comment in DIF_VAR_PID.
7202                                  */
7203                                 if (DTRACE_ANCHORED(mstate.dtms_probe) &&
7204                                     CPU_ON_INTR(CPU)) {
7205                                         int depth = DTRACE_USTACK_NFRAMES(
7206                                             rec->dtrd_arg) + 1;
7207 
7208                                         dtrace_bzero((void *)(tomax + valoffs),
7209                                             DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
7210                                             + depth * sizeof (uint64_t));
7211 
7212                                         continue;
7213                                 }
7214 
7215                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
7216                                     curproc->p_dtrace_helpers != NULL) {
7217                                         /*
7218                                          * This is the slow path -- we have
7219                                          * allocated string space, and we're
7220                                          * getting the stack of a process that
7221                                          * has helpers.  Call into a separate
7222                                          * routine to perform this processing.
7223                                          */
7224                                         dtrace_action_ustack(&mstate, state,
7225                                             (uint64_t *)(tomax + valoffs),
7226                                             rec->dtrd_arg);
7227                                         continue;
7228                                 }
7229 
7230                                 /*
7231                                  * Clear the string space, since there's no
7232                                  * helper to do it for us.
7233                                  */
7234                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) {
7235                                         int depth = DTRACE_USTACK_NFRAMES(
7236                                             rec->dtrd_arg);
7237                                         size_t strsize = DTRACE_USTACK_STRSIZE(
7238                                             rec->dtrd_arg);
7239                                         uint64_t *buf = (uint64_t *)(tomax +
7240                                             valoffs);
7241                                         void *strspace = &buf[depth + 1];
7242 
7243                                         dtrace_bzero(strspace,
7244                                             MIN(depth, strsize));
7245                                 }
7246 
7247                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
7248                                 dtrace_getupcstack((uint64_t *)
7249                                     (tomax + valoffs),
7250                                     DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
7251                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
7252                                 continue;
7253 
7254                         default:
7255                                 break;
7256                         }
7257 
7258                         dp = act->dta_difo;
7259                         ASSERT(dp != NULL);
7260 
7261                         val = dtrace_dif_emulate(dp, &mstate, vstate, state);
7262 
7263                         if (*flags & CPU_DTRACE_ERROR)
7264                                 continue;
7265 
7266                         switch (act->dta_kind) {
7267                         case DTRACEACT_SPECULATE: {
7268                                 dtrace_rechdr_t *dtrh;
7269 
7270                                 ASSERT(buf == &state->dts_buffer[cpuid]);
7271                                 buf = dtrace_speculation_buffer(state,
7272                                     cpuid, val);
7273 
7274                                 if (buf == NULL) {
7275                                         *flags |= CPU_DTRACE_DROP;
7276                                         continue;
7277                                 }
7278 
7279                                 offs = dtrace_buffer_reserve(buf,
7280                                     ecb->dte_needed, ecb->dte_alignment,
7281                                     state, NULL);
7282 
7283                                 if (offs < 0) {
7284                                         *flags |= CPU_DTRACE_DROP;
7285                                         continue;
7286                                 }
7287 
7288                                 tomax = buf->dtb_tomax;
7289                                 ASSERT(tomax != NULL);
7290 
7291                                 if (ecb->dte_size == 0)
7292                                         continue;
7293 
7294                                 ASSERT3U(ecb->dte_size, >=,
7295                                     sizeof (dtrace_rechdr_t));
7296                                 dtrh = ((void *)(tomax + offs));
7297                                 dtrh->dtrh_epid = ecb->dte_epid;
7298                                 /*
7299                                  * When the speculation is committed, all of
7300                                  * the records in the speculative buffer will
7301                                  * have their timestamps set to the commit
7302                                  * time.  Until then, it is set to a sentinel
7303                                  * value, for debugability.
7304                                  */
7305                                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
7306                                 continue;
7307                         }
7308 
7309                         case DTRACEACT_CHILL:
7310                                 if (dtrace_priv_kernel_destructive(state))
7311                                         dtrace_action_chill(&mstate, val);
7312                                 continue;
7313 
7314                         case DTRACEACT_RAISE:
7315                                 if (dtrace_priv_proc_destructive(state,
7316                                     &mstate))
7317                                         dtrace_action_raise(val);
7318                                 continue;
7319 
7320                         case DTRACEACT_COMMIT:
7321                                 ASSERT(!committed);
7322 
7323                                 /*
7324                                  * We need to commit our buffer state.
7325                                  */
7326                                 if (ecb->dte_size)
7327                                         buf->dtb_offset = offs + ecb->dte_size;
7328                                 buf = &state->dts_buffer[cpuid];
7329                                 dtrace_speculation_commit(state, cpuid, val);
7330                                 committed = 1;
7331                                 continue;
7332 
7333                         case DTRACEACT_DISCARD:
7334                                 dtrace_speculation_discard(state, cpuid, val);
7335                                 continue;
7336 
7337                         case DTRACEACT_DIFEXPR:
7338                         case DTRACEACT_LIBACT:
7339                         case DTRACEACT_PRINTF:
7340                         case DTRACEACT_PRINTA:
7341                         case DTRACEACT_SYSTEM:
7342                         case DTRACEACT_FREOPEN:
7343                         case DTRACEACT_TRACEMEM:
7344                                 break;
7345 
7346                         case DTRACEACT_TRACEMEM_DYNSIZE:
7347                                 tracememsize = val;
7348                                 break;
7349 
7350                         case DTRACEACT_SYM:
7351                         case DTRACEACT_MOD:
7352                                 if (!dtrace_priv_kernel(state))
7353                                         continue;
7354                                 break;
7355 
7356                         case DTRACEACT_USYM:
7357                         case DTRACEACT_UMOD:
7358                         case DTRACEACT_UADDR: {
7359                                 struct pid *pid = curthread->t_procp->p_pidp;
7360 
7361                                 if (!dtrace_priv_proc(state, &mstate))
7362                                         continue;
7363 
7364                                 DTRACE_STORE(uint64_t, tomax,
7365                                     valoffs, (uint64_t)pid->pid_id);
7366                                 DTRACE_STORE(uint64_t, tomax,
7367                                     valoffs + sizeof (uint64_t), val);
7368 
7369                                 continue;
7370                         }
7371 
7372                         case DTRACEACT_EXIT: {
7373                                 /*
7374                                  * For the exit action, we are going to attempt
7375                                  * to atomically set our activity to be
7376                                  * draining.  If this fails (either because
7377                                  * another CPU has beat us to the exit action,
7378                                  * or because our current activity is something
7379                                  * other than ACTIVE or WARMUP), we will
7380                                  * continue.  This assures that the exit action
7381                                  * can be successfully recorded at most once
7382                                  * when we're in the ACTIVE state.  If we're
7383                                  * encountering the exit() action while in
7384                                  * COOLDOWN, however, we want to honor the new
7385                                  * status code.  (We know that we're the only
7386                                  * thread in COOLDOWN, so there is no race.)
7387                                  */
7388                                 void *activity = &state->dts_activity;
7389                                 dtrace_activity_t current = state->dts_activity;
7390 
7391                                 if (current == DTRACE_ACTIVITY_COOLDOWN)
7392                                         break;
7393 
7394                                 if (current != DTRACE_ACTIVITY_WARMUP)
7395                                         current = DTRACE_ACTIVITY_ACTIVE;
7396 
7397                                 if (dtrace_cas32(activity, current,
7398                                     DTRACE_ACTIVITY_DRAINING) != current) {
7399                                         *flags |= CPU_DTRACE_DROP;
7400                                         continue;
7401                                 }
7402 
7403                                 break;
7404                         }
7405 
7406                         default:
7407                                 ASSERT(0);
7408                         }
7409 
7410                         if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ||
7411                             dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) {
7412                                 uintptr_t end = valoffs + size;
7413 
7414                                 if (tracememsize != 0 &&
7415                                     valoffs + tracememsize < end) {
7416                                         end = valoffs + tracememsize;
7417                                         tracememsize = 0;
7418                                 }
7419 
7420                                 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF &&
7421                                     !dtrace_vcanload((void *)(uintptr_t)val,
7422                                     &dp->dtdo_rtype, NULL, &mstate, vstate))
7423                                         continue;
7424 
7425                                 dtrace_store_by_ref(dp, tomax, size, &valoffs,
7426                                     &val, end, act->dta_intuple,
7427                                     dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ?
7428                                     DIF_TF_BYREF: DIF_TF_BYUREF);
7429                                 continue;
7430                         }
7431 
7432                         switch (size) {
7433                         case 0:
7434                                 break;
7435 
7436                         case sizeof (uint8_t):
7437                                 DTRACE_STORE(uint8_t, tomax, valoffs, val);
7438                                 break;
7439                         case sizeof (uint16_t):
7440                                 DTRACE_STORE(uint16_t, tomax, valoffs, val);
7441                                 break;
7442                         case sizeof (uint32_t):
7443                                 DTRACE_STORE(uint32_t, tomax, valoffs, val);
7444                                 break;
7445                         case sizeof (uint64_t):
7446                                 DTRACE_STORE(uint64_t, tomax, valoffs, val);
7447                                 break;
7448                         default:
7449                                 /*
7450                                  * Any other size should have been returned by
7451                                  * reference, not by value.
7452                                  */
7453                                 ASSERT(0);
7454                                 break;
7455                         }
7456                 }
7457 
7458                 if (*flags & CPU_DTRACE_DROP)
7459                         continue;
7460 
7461                 if (*flags & CPU_DTRACE_FAULT) {
7462                         int ndx;
7463                         dtrace_action_t *err;
7464 
7465                         buf->dtb_errors++;
7466 
7467                         if (probe->dtpr_id == dtrace_probeid_error) {
7468                                 /*
7469                                  * There's nothing we can do -- we had an
7470                                  * error on the error probe.  We bump an
7471                                  * error counter to at least indicate that
7472                                  * this condition happened.
7473                                  */
7474                                 dtrace_error(&state->dts_dblerrors);
7475                                 continue;
7476                         }
7477 
7478                         if (vtime) {
7479                                 /*
7480                                  * Before recursing on dtrace_probe(), we
7481                                  * need to explicitly clear out our start
7482                                  * time to prevent it from being accumulated
7483                                  * into t_dtrace_vtime.
7484                                  */
7485                                 curthread->t_dtrace_start = 0;
7486                         }
7487 
7488                         /*
7489                          * Iterate over the actions to figure out which action
7490                          * we were processing when we experienced the error.
7491                          * Note that act points _past_ the faulting action; if
7492                          * act is ecb->dte_action, the fault was in the
7493                          * predicate, if it's ecb->dte_action->dta_next it's
7494                          * in action #1, and so on.
7495                          */
7496                         for (err = ecb->dte_action, ndx = 0;
7497                             err != act; err = err->dta_next, ndx++)
7498                                 continue;
7499 
7500                         dtrace_probe_error(state, ecb->dte_epid, ndx,
7501                             (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
7502                             mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
7503                             cpu_core[cpuid].cpuc_dtrace_illval);
7504 
7505                         continue;
7506                 }
7507 
7508                 if (!committed)
7509                         buf->dtb_offset = offs + ecb->dte_size;
7510         }
7511 
7512         end = dtrace_gethrtime();
7513         if (vtime)
7514                 curthread->t_dtrace_start = end;
7515 
7516         CPU->cpu_dtrace_nsec += end - now;
7517 
7518         dtrace_interrupt_enable(cookie);
7519 }
7520 
7521 /*
7522  * DTrace Probe Hashing Functions
7523  *
7524  * The functions in this section (and indeed, the functions in remaining
7525  * sections) are not _called_ from probe context.  (Any exceptions to this are
7526  * marked with a "Note:".)  Rather, they are called from elsewhere in the
7527  * DTrace framework to look-up probes in, add probes to and remove probes from
7528  * the DTrace probe hashes.  (Each probe is hashed by each element of the
7529  * probe tuple -- allowing for fast lookups, regardless of what was
7530  * specified.)
7531  */
7532 static uint_t
7533 dtrace_hash_str(char *p)
7534 {
7535         unsigned int g;
7536         uint_t hval = 0;
7537 
7538         while (*p) {
7539                 hval = (hval << 4) + *p++;
7540                 if ((g = (hval & 0xf0000000)) != 0)
7541                         hval ^= g >> 24;
7542                 hval &= ~g;
7543         }
7544         return (hval);
7545 }
7546 
7547 static dtrace_hash_t *
7548 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
7549 {
7550         dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
7551 
7552         hash->dth_stroffs = stroffs;
7553         hash->dth_nextoffs = nextoffs;
7554         hash->dth_prevoffs = prevoffs;
7555 
7556         hash->dth_size = 1;
7557         hash->dth_mask = hash->dth_size - 1;
7558 
7559         hash->dth_tab = kmem_zalloc(hash->dth_size *
7560             sizeof (dtrace_hashbucket_t *), KM_SLEEP);
7561 
7562         return (hash);
7563 }
7564 
7565 static void
7566 dtrace_hash_destroy(dtrace_hash_t *hash)
7567 {
7568 #ifdef DEBUG
7569         int i;
7570 
7571         for (i = 0; i < hash->dth_size; i++)
7572                 ASSERT(hash->dth_tab[i] == NULL);
7573 #endif
7574 
7575         kmem_free(hash->dth_tab,
7576             hash->dth_size * sizeof (dtrace_hashbucket_t *));
7577         kmem_free(hash, sizeof (dtrace_hash_t));
7578 }
7579 
7580 static void
7581 dtrace_hash_resize(dtrace_hash_t *hash)
7582 {
7583         int size = hash->dth_size, i, ndx;
7584         int new_size = hash->dth_size << 1;
7585         int new_mask = new_size - 1;
7586         dtrace_hashbucket_t **new_tab, *bucket, *next;
7587 
7588         ASSERT((new_size & new_mask) == 0);
7589 
7590         new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
7591 
7592         for (i = 0; i < size; i++) {
7593                 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
7594                         dtrace_probe_t *probe = bucket->dthb_chain;
7595 
7596                         ASSERT(probe != NULL);
7597                         ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
7598 
7599                         next = bucket->dthb_next;
7600                         bucket->dthb_next = new_tab[ndx];
7601                         new_tab[ndx] = bucket;
7602                 }
7603         }
7604 
7605         kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
7606         hash->dth_tab = new_tab;
7607         hash->dth_size = new_size;
7608         hash->dth_mask = new_mask;
7609 }
7610 
7611 static void
7612 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
7613 {
7614         int hashval = DTRACE_HASHSTR(hash, new);
7615         int ndx = hashval & hash->dth_mask;
7616         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7617         dtrace_probe_t **nextp, **prevp;
7618 
7619         for (; bucket != NULL; bucket = bucket->dthb_next) {
7620                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
7621                         goto add;
7622         }
7623 
7624         if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
7625                 dtrace_hash_resize(hash);
7626                 dtrace_hash_add(hash, new);
7627                 return;
7628         }
7629 
7630         bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
7631         bucket->dthb_next = hash->dth_tab[ndx];
7632         hash->dth_tab[ndx] = bucket;
7633         hash->dth_nbuckets++;
7634 
7635 add:
7636         nextp = DTRACE_HASHNEXT(hash, new);
7637         ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
7638         *nextp = bucket->dthb_chain;
7639 
7640         if (bucket->dthb_chain != NULL) {
7641                 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
7642                 ASSERT(*prevp == NULL);
7643                 *prevp = new;
7644         }
7645 
7646         bucket->dthb_chain = new;
7647         bucket->dthb_len++;
7648 }
7649 
7650 static dtrace_probe_t *
7651 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
7652 {
7653         int hashval = DTRACE_HASHSTR(hash, template);
7654         int ndx = hashval & hash->dth_mask;
7655         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7656 
7657         for (; bucket != NULL; bucket = bucket->dthb_next) {
7658                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7659                         return (bucket->dthb_chain);
7660         }
7661 
7662         return (NULL);
7663 }
7664 
7665 static int
7666 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
7667 {
7668         int hashval = DTRACE_HASHSTR(hash, template);
7669         int ndx = hashval & hash->dth_mask;
7670         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7671 
7672         for (; bucket != NULL; bucket = bucket->dthb_next) {
7673                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
7674                         return (bucket->dthb_len);
7675         }
7676 
7677         return (0);
7678 }
7679 
7680 static void
7681 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
7682 {
7683         int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
7684         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
7685 
7686         dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
7687         dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
7688 
7689         /*
7690          * Find the bucket that we're removing this probe from.
7691          */
7692         for (; bucket != NULL; bucket = bucket->dthb_next) {
7693                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
7694                         break;
7695         }
7696 
7697         ASSERT(bucket != NULL);
7698 
7699         if (*prevp == NULL) {
7700                 if (*nextp == NULL) {
7701                         /*
7702                          * The removed probe was the only probe on this
7703                          * bucket; we need to remove the bucket.
7704                          */
7705                         dtrace_hashbucket_t *b = hash->dth_tab[ndx];
7706 
7707                         ASSERT(bucket->dthb_chain == probe);
7708                         ASSERT(b != NULL);
7709 
7710                         if (b == bucket) {
7711                                 hash->dth_tab[ndx] = bucket->dthb_next;
7712                         } else {
7713                                 while (b->dthb_next != bucket)
7714                                         b = b->dthb_next;
7715                                 b->dthb_next = bucket->dthb_next;
7716                         }
7717 
7718                         ASSERT(hash->dth_nbuckets > 0);
7719                         hash->dth_nbuckets--;
7720                         kmem_free(bucket, sizeof (dtrace_hashbucket_t));
7721                         return;
7722                 }
7723 
7724                 bucket->dthb_chain = *nextp;
7725         } else {
7726                 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
7727         }
7728 
7729         if (*nextp != NULL)
7730                 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
7731 }
7732 
7733 /*
7734  * DTrace Utility Functions
7735  *
7736  * These are random utility functions that are _not_ called from probe context.
7737  */
7738 static int
7739 dtrace_badattr(const dtrace_attribute_t *a)
7740 {
7741         return (a->dtat_name > DTRACE_STABILITY_MAX ||
7742             a->dtat_data > DTRACE_STABILITY_MAX ||
7743             a->dtat_class > DTRACE_CLASS_MAX);
7744 }
7745 
7746 /*
7747  * Return a duplicate copy of a string.  If the specified string is NULL,
7748  * this function returns a zero-length string.
7749  */
7750 static char *
7751 dtrace_strdup(const char *str)
7752 {
7753         char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
7754 
7755         if (str != NULL)
7756                 (void) strcpy(new, str);
7757 
7758         return (new);
7759 }
7760 
7761 #define DTRACE_ISALPHA(c)       \
7762         (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
7763 
7764 static int
7765 dtrace_badname(const char *s)
7766 {
7767         char c;
7768 
7769         if (s == NULL || (c = *s++) == '\0')
7770                 return (0);
7771 
7772         if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
7773                 return (1);
7774 
7775         while ((c = *s++) != '\0') {
7776                 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
7777                     c != '-' && c != '_' && c != '.' && c != '`')
7778                         return (1);
7779         }
7780 
7781         return (0);
7782 }
7783 
7784 static void
7785 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
7786 {
7787         uint32_t priv;
7788 
7789         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
7790                 /*
7791                  * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
7792                  */
7793                 priv = DTRACE_PRIV_ALL;
7794         } else {
7795                 *uidp = crgetuid(cr);
7796                 *zoneidp = crgetzonedid(cr);
7797 
7798                 priv = 0;
7799                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
7800                         priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
7801                 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
7802                         priv |= DTRACE_PRIV_USER;
7803                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
7804                         priv |= DTRACE_PRIV_PROC;
7805                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
7806                         priv |= DTRACE_PRIV_OWNER;
7807                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
7808                         priv |= DTRACE_PRIV_ZONEOWNER;
7809         }
7810 
7811         *privp = priv;
7812 }
7813 
7814 #ifdef DTRACE_ERRDEBUG
7815 static void
7816 dtrace_errdebug(const char *str)
7817 {
7818         int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
7819         int occupied = 0;
7820 
7821         mutex_enter(&dtrace_errlock);
7822         dtrace_errlast = str;
7823         dtrace_errthread = curthread;
7824 
7825         while (occupied++ < DTRACE_ERRHASHSZ) {
7826                 if (dtrace_errhash[hval].dter_msg == str) {
7827                         dtrace_errhash[hval].dter_count++;
7828                         goto out;
7829                 }
7830 
7831                 if (dtrace_errhash[hval].dter_msg != NULL) {
7832                         hval = (hval + 1) % DTRACE_ERRHASHSZ;
7833                         continue;
7834                 }
7835 
7836                 dtrace_errhash[hval].dter_msg = str;
7837                 dtrace_errhash[hval].dter_count = 1;
7838                 goto out;
7839         }
7840 
7841         panic("dtrace: undersized error hash");
7842 out:
7843         mutex_exit(&dtrace_errlock);
7844 }
7845 #endif
7846 
7847 /*
7848  * DTrace Matching Functions
7849  *
7850  * These functions are used to match groups of probes, given some elements of
7851  * a probe tuple, or some globbed expressions for elements of a probe tuple.
7852  */
7853 static int
7854 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
7855     zoneid_t zoneid)
7856 {
7857         if (priv != DTRACE_PRIV_ALL) {
7858                 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
7859                 uint32_t match = priv & ppriv;
7860 
7861                 /*
7862                  * No PRIV_DTRACE_* privileges...
7863                  */
7864                 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
7865                     DTRACE_PRIV_KERNEL)) == 0)
7866                         return (0);
7867 
7868                 /*
7869                  * No matching bits, but there were bits to match...
7870                  */
7871                 if (match == 0 && ppriv != 0)
7872                         return (0);
7873 
7874                 /*
7875                  * Need to have permissions to the process, but don't...
7876                  */
7877                 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
7878                     uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
7879                         return (0);
7880                 }
7881 
7882                 /*
7883                  * Need to be in the same zone unless we possess the
7884                  * privilege to examine all zones.
7885                  */
7886                 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
7887                     zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
7888                         return (0);
7889                 }
7890         }
7891 
7892         return (1);
7893 }
7894 
7895 /*
7896  * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
7897  * consists of input pattern strings and an ops-vector to evaluate them.
7898  * This function returns >0 for match, 0 for no match, and <0 for error.
7899  */
7900 static int
7901 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
7902     uint32_t priv, uid_t uid, zoneid_t zoneid)
7903 {
7904         dtrace_provider_t *pvp = prp->dtpr_provider;
7905         int rv;
7906 
7907         if (pvp->dtpv_defunct)
7908                 return (0);
7909 
7910         if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
7911                 return (rv);
7912 
7913         if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
7914                 return (rv);
7915 
7916         if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
7917                 return (rv);
7918 
7919         if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
7920                 return (rv);
7921 
7922         if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
7923                 return (0);
7924 
7925         return (rv);
7926 }
7927 
7928 /*
7929  * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
7930  * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
7931  * libc's version, the kernel version only applies to 8-bit ASCII strings.
7932  * In addition, all of the recursion cases except for '*' matching have been
7933  * unwound.  For '*', we still implement recursive evaluation, but a depth
7934  * counter is maintained and matching is aborted if we recurse too deep.
7935  * The function returns 0 if no match, >0 if match, and <0 if recursion error.
7936  */
7937 static int
7938 dtrace_match_glob(const char *s, const char *p, int depth)
7939 {
7940         const char *olds;
7941         char s1, c;
7942         int gs;
7943 
7944         if (depth > DTRACE_PROBEKEY_MAXDEPTH)
7945                 return (-1);
7946 
7947         if (s == NULL)
7948                 s = ""; /* treat NULL as empty string */
7949 
7950 top:
7951         olds = s;
7952         s1 = *s++;
7953 
7954         if (p == NULL)
7955                 return (0);
7956 
7957         if ((c = *p++) == '\0')
7958                 return (s1 == '\0');
7959 
7960         switch (c) {
7961         case '[': {
7962                 int ok = 0, notflag = 0;
7963                 char lc = '\0';
7964 
7965                 if (s1 == '\0')
7966                         return (0);
7967 
7968                 if (*p == '!') {
7969                         notflag = 1;
7970                         p++;
7971                 }
7972 
7973                 if ((c = *p++) == '\0')
7974                         return (0);
7975 
7976                 do {
7977                         if (c == '-' && lc != '\0' && *p != ']') {
7978                                 if ((c = *p++) == '\0')
7979                                         return (0);
7980                                 if (c == '\\' && (c = *p++) == '\0')
7981                                         return (0);
7982 
7983                                 if (notflag) {
7984                                         if (s1 < lc || s1 > c)
7985                                                 ok++;
7986                                         else
7987                                                 return (0);
7988                                 } else if (lc <= s1 && s1 <= c)
7989                                         ok++;
7990 
7991                         } else if (c == '\\' && (c = *p++) == '\0')
7992                                 return (0);
7993 
7994                         lc = c; /* save left-hand 'c' for next iteration */
7995 
7996                         if (notflag) {
7997                                 if (s1 != c)
7998                                         ok++;
7999                                 else
8000                                         return (0);
8001                         } else if (s1 == c)
8002                                 ok++;
8003 
8004                         if ((c = *p++) == '\0')
8005                                 return (0);
8006 
8007                 } while (c != ']');
8008 
8009                 if (ok)
8010                         goto top;
8011 
8012                 return (0);
8013         }
8014 
8015         case '\\':
8016                 if ((c = *p++) == '\0')
8017                         return (0);
8018                 /*FALLTHRU*/
8019 
8020         default:
8021                 if (c != s1)
8022                         return (0);
8023                 /*FALLTHRU*/
8024 
8025         case '?':
8026                 if (s1 != '\0')
8027                         goto top;
8028                 return (0);
8029 
8030         case '*':
8031                 while (*p == '*')
8032                         p++; /* consecutive *'s are identical to a single one */
8033 
8034                 if (*p == '\0')
8035                         return (1);
8036 
8037                 for (s = olds; *s != '\0'; s++) {
8038                         if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
8039                                 return (gs);
8040                 }
8041 
8042                 return (0);
8043         }
8044 }
8045 
8046 /*ARGSUSED*/
8047 static int
8048 dtrace_match_string(const char *s, const char *p, int depth)
8049 {
8050         return (s != NULL && strcmp(s, p) == 0);
8051 }
8052 
8053 /*ARGSUSED*/
8054 static int
8055 dtrace_match_nul(const char *s, const char *p, int depth)
8056 {
8057         return (1); /* always match the empty pattern */
8058 }
8059 
8060 /*ARGSUSED*/
8061 static int
8062 dtrace_match_nonzero(const char *s, const char *p, int depth)
8063 {
8064         return (s != NULL && s[0] != '\0');
8065 }
8066 
8067 static int
8068 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
8069     zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
8070 {
8071         dtrace_probe_t template, *probe;
8072         dtrace_hash_t *hash = NULL;
8073         int len, rc, best = INT_MAX, nmatched = 0;
8074         dtrace_id_t i;
8075 
8076         ASSERT(MUTEX_HELD(&dtrace_lock));
8077 
8078         /*
8079          * If the probe ID is specified in the key, just lookup by ID and
8080          * invoke the match callback once if a matching probe is found.
8081          */
8082         if (pkp->dtpk_id != DTRACE_IDNONE) {
8083                 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
8084                     dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
8085                         if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
8086                                 return (DTRACE_MATCH_FAIL);
8087                         nmatched++;
8088                 }
8089                 return (nmatched);
8090         }
8091 
8092         template.dtpr_mod = (char *)pkp->dtpk_mod;
8093         template.dtpr_func = (char *)pkp->dtpk_func;
8094         template.dtpr_name = (char *)pkp->dtpk_name;
8095 
8096         /*
8097          * We want to find the most distinct of the module name, function
8098          * name, and name.  So for each one that is not a glob pattern or
8099          * empty string, we perform a lookup in the corresponding hash and
8100          * use the hash table with the fewest collisions to do our search.
8101          */
8102         if (pkp->dtpk_mmatch == &dtrace_match_string &&
8103             (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
8104                 best = len;
8105                 hash = dtrace_bymod;
8106         }
8107 
8108         if (pkp->dtpk_fmatch == &dtrace_match_string &&
8109             (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
8110                 best = len;
8111                 hash = dtrace_byfunc;
8112         }
8113 
8114         if (pkp->dtpk_nmatch == &dtrace_match_string &&
8115             (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
8116                 best = len;
8117                 hash = dtrace_byname;
8118         }
8119 
8120         /*
8121          * If we did not select a hash table, iterate over every probe and
8122          * invoke our callback for each one that matches our input probe key.
8123          */
8124         if (hash == NULL) {
8125                 for (i = 0; i < dtrace_nprobes; i++) {
8126                         if ((probe = dtrace_probes[i]) == NULL ||
8127                             dtrace_match_probe(probe, pkp, priv, uid,
8128                             zoneid) <= 0)
8129                                 continue;
8130 
8131                         nmatched++;
8132 
8133                         if ((rc = (*matched)(probe, arg)) !=
8134                             DTRACE_MATCH_NEXT) {
8135                                 if (rc == DTRACE_MATCH_FAIL)
8136                                         return (DTRACE_MATCH_FAIL);
8137                                 break;
8138                         }
8139                 }
8140 
8141                 return (nmatched);
8142         }
8143 
8144         /*
8145          * If we selected a hash table, iterate over each probe of the same key
8146          * name and invoke the callback for every probe that matches the other
8147          * attributes of our input probe key.
8148          */
8149         for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
8150             probe = *(DTRACE_HASHNEXT(hash, probe))) {
8151 
8152                 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
8153                         continue;
8154 
8155                 nmatched++;
8156 
8157                 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
8158                         if (rc == DTRACE_MATCH_FAIL)
8159                                 return (DTRACE_MATCH_FAIL);
8160                         break;
8161                 }
8162         }
8163 
8164         return (nmatched);
8165 }
8166 
8167 /*
8168  * Return the function pointer dtrace_probecmp() should use to compare the
8169  * specified pattern with a string.  For NULL or empty patterns, we select
8170  * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
8171  * For non-empty non-glob strings, we use dtrace_match_string().
8172  */
8173 static dtrace_probekey_f *
8174 dtrace_probekey_func(const char *p)
8175 {
8176         char c;
8177 
8178         if (p == NULL || *p == '\0')
8179                 return (&dtrace_match_nul);
8180 
8181         while ((c = *p++) != '\0') {
8182                 if (c == '[' || c == '?' || c == '*' || c == '\\')
8183                         return (&dtrace_match_glob);
8184         }
8185 
8186         return (&dtrace_match_string);
8187 }
8188 
8189 /*
8190  * Build a probe comparison key for use with dtrace_match_probe() from the
8191  * given probe description.  By convention, a null key only matches anchored
8192  * probes: if each field is the empty string, reset dtpk_fmatch to
8193  * dtrace_match_nonzero().
8194  */
8195 static void
8196 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
8197 {
8198         pkp->dtpk_prov = pdp->dtpd_provider;
8199         pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
8200 
8201         pkp->dtpk_mod = pdp->dtpd_mod;
8202         pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
8203 
8204         pkp->dtpk_func = pdp->dtpd_func;
8205         pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
8206 
8207         pkp->dtpk_name = pdp->dtpd_name;
8208         pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
8209 
8210         pkp->dtpk_id = pdp->dtpd_id;
8211 
8212         if (pkp->dtpk_id == DTRACE_IDNONE &&
8213             pkp->dtpk_pmatch == &dtrace_match_nul &&
8214             pkp->dtpk_mmatch == &dtrace_match_nul &&
8215             pkp->dtpk_fmatch == &dtrace_match_nul &&
8216             pkp->dtpk_nmatch == &dtrace_match_nul)
8217                 pkp->dtpk_fmatch = &dtrace_match_nonzero;
8218 }
8219 
8220 /*
8221  * DTrace Provider-to-Framework API Functions
8222  *
8223  * These functions implement much of the Provider-to-Framework API, as
8224  * described in <sys/dtrace.h>.  The parts of the API not in this section are
8225  * the functions in the API for probe management (found below), and
8226  * dtrace_probe() itself (found above).
8227  */
8228 
8229 /*
8230  * Register the calling provider with the DTrace framework.  This should
8231  * generally be called by DTrace providers in their attach(9E) entry point.
8232  */
8233 int
8234 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
8235     cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
8236 {
8237         dtrace_provider_t *provider;
8238 
8239         if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
8240                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
8241                     "arguments", name ? name : "<NULL>");
8242                 return (EINVAL);
8243         }
8244 
8245         if (name[0] == '\0' || dtrace_badname(name)) {
8246                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
8247                     "provider name", name);
8248                 return (EINVAL);
8249         }
8250 
8251         if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
8252             pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
8253             pops->dtps_destroy == NULL ||
8254             ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
8255                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
8256                     "provider ops", name);
8257                 return (EINVAL);
8258         }
8259 
8260         if (dtrace_badattr(&pap->dtpa_provider) ||
8261             dtrace_badattr(&pap->dtpa_mod) ||
8262             dtrace_badattr(&pap->dtpa_func) ||
8263             dtrace_badattr(&pap->dtpa_name) ||
8264             dtrace_badattr(&pap->dtpa_args)) {
8265                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
8266                     "provider attributes", name);
8267                 return (EINVAL);
8268         }
8269 
8270         if (priv & ~DTRACE_PRIV_ALL) {
8271                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
8272                     "privilege attributes", name);
8273                 return (EINVAL);
8274         }
8275 
8276         if ((priv & DTRACE_PRIV_KERNEL) &&
8277             (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
8278             pops->dtps_mode == NULL) {
8279                 cmn_err(CE_WARN, "failed to register provider '%s': need "
8280                     "dtps_mode() op for given privilege attributes", name);
8281                 return (EINVAL);
8282         }
8283 
8284         provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
8285         provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
8286         (void) strcpy(provider->dtpv_name, name);
8287 
8288         provider->dtpv_attr = *pap;
8289         provider->dtpv_priv.dtpp_flags = priv;
8290         if (cr != NULL) {
8291                 provider->dtpv_priv.dtpp_uid = crgetuid(cr);
8292                 provider->dtpv_priv.dtpp_zoneid = crgetzonedid(cr);
8293         }
8294         provider->dtpv_pops = *pops;
8295 
8296         if (pops->dtps_provide == NULL) {
8297                 ASSERT(pops->dtps_provide_module != NULL);
8298                 provider->dtpv_pops.dtps_provide = dtrace_nullop_provide;
8299         }
8300 
8301         if (pops->dtps_provide_module == NULL) {
8302                 ASSERT(pops->dtps_provide != NULL);
8303                 provider->dtpv_pops.dtps_provide_module = dtrace_nullop_module;
8304         }
8305 
8306         if (pops->dtps_suspend == NULL) {
8307                 ASSERT(pops->dtps_resume == NULL);
8308                 provider->dtpv_pops.dtps_suspend = dtrace_nullop;
8309                 provider->dtpv_pops.dtps_resume = dtrace_nullop;
8310         }
8311 
8312         provider->dtpv_arg = arg;
8313         *idp = (dtrace_provider_id_t)provider;
8314 
8315         if (pops == &dtrace_provider_ops) {
8316                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8317                 ASSERT(MUTEX_HELD(&dtrace_lock));
8318                 ASSERT(dtrace_anon.dta_enabling == NULL);
8319 
8320                 /*
8321                  * We make sure that the DTrace provider is at the head of
8322                  * the provider chain.
8323                  */
8324                 provider->dtpv_next = dtrace_provider;
8325                 dtrace_provider = provider;
8326                 return (0);
8327         }
8328 
8329         mutex_enter(&dtrace_provider_lock);
8330         mutex_enter(&dtrace_lock);
8331 
8332         /*
8333          * If there is at least one provider registered, we'll add this
8334          * provider after the first provider.
8335          */
8336         if (dtrace_provider != NULL) {
8337                 provider->dtpv_next = dtrace_provider->dtpv_next;
8338                 dtrace_provider->dtpv_next = provider;
8339         } else {
8340                 dtrace_provider = provider;
8341         }
8342 
8343         if (dtrace_retained != NULL) {
8344                 dtrace_enabling_provide(provider);
8345 
8346                 /*
8347                  * Now we need to call dtrace_enabling_matchall() -- which
8348                  * will acquire cpu_lock and dtrace_lock.  We therefore need
8349                  * to drop all of our locks before calling into it...
8350                  */
8351                 mutex_exit(&dtrace_lock);
8352                 mutex_exit(&dtrace_provider_lock);
8353                 dtrace_enabling_matchall();
8354 
8355                 return (0);
8356         }
8357 
8358         mutex_exit(&dtrace_lock);
8359         mutex_exit(&dtrace_provider_lock);
8360 
8361         return (0);
8362 }
8363 
8364 /*
8365  * Unregister the specified provider from the DTrace framework.  This should
8366  * generally be called by DTrace providers in their detach(9E) entry point.
8367  */
8368 int
8369 dtrace_unregister(dtrace_provider_id_t id)
8370 {
8371         dtrace_provider_t *old = (dtrace_provider_t *)id;
8372         dtrace_provider_t *prev = NULL;
8373         int i, self = 0, noreap = 0;
8374         dtrace_probe_t *probe, *first = NULL;
8375 
8376         if (old->dtpv_pops.dtps_enable == dtrace_enable_nullop) {
8377                 /*
8378                  * If DTrace itself is the provider, we're called with locks
8379                  * already held.
8380                  */
8381                 ASSERT(old == dtrace_provider);
8382                 ASSERT(dtrace_devi != NULL);
8383                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8384                 ASSERT(MUTEX_HELD(&dtrace_lock));
8385                 self = 1;
8386 
8387                 if (dtrace_provider->dtpv_next != NULL) {
8388                         /*
8389                          * There's another provider here; return failure.
8390                          */
8391                         return (EBUSY);
8392                 }
8393         } else {
8394                 mutex_enter(&dtrace_provider_lock);
8395                 mutex_enter(&mod_lock);
8396                 mutex_enter(&dtrace_lock);
8397         }
8398 
8399         /*
8400          * If anyone has /dev/dtrace open, or if there are anonymous enabled
8401          * probes, we refuse to let providers slither away, unless this
8402          * provider has already been explicitly invalidated.
8403          */
8404         if (!old->dtpv_defunct &&
8405             (dtrace_opens || (dtrace_anon.dta_state != NULL &&
8406             dtrace_anon.dta_state->dts_necbs > 0))) {
8407                 if (!self) {
8408                         mutex_exit(&dtrace_lock);
8409                         mutex_exit(&mod_lock);
8410                         mutex_exit(&dtrace_provider_lock);
8411                 }
8412                 return (EBUSY);
8413         }
8414 
8415         /*
8416          * Attempt to destroy the probes associated with this provider.
8417          */
8418         for (i = 0; i < dtrace_nprobes; i++) {
8419                 if ((probe = dtrace_probes[i]) == NULL)
8420                         continue;
8421 
8422                 if (probe->dtpr_provider != old)
8423                         continue;
8424 
8425                 if (probe->dtpr_ecb == NULL)
8426                         continue;
8427 
8428                 /*
8429                  * If we are trying to unregister a defunct provider, and the
8430                  * provider was made defunct within the interval dictated by
8431                  * dtrace_unregister_defunct_reap, we'll (asynchronously)
8432                  * attempt to reap our enablings.  To denote that the provider
8433                  * should reattempt to unregister itself at some point in the
8434                  * future, we will return a differentiable error code (EAGAIN
8435                  * instead of EBUSY) in this case.
8436                  */
8437                 if (dtrace_gethrtime() - old->dtpv_defunct >
8438                     dtrace_unregister_defunct_reap)
8439                         noreap = 1;
8440 
8441                 if (!self) {
8442                         mutex_exit(&dtrace_lock);
8443                         mutex_exit(&mod_lock);
8444                         mutex_exit(&dtrace_provider_lock);
8445                 }
8446 
8447                 if (noreap)
8448                         return (EBUSY);
8449 
8450                 (void) taskq_dispatch(dtrace_taskq,
8451                     (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
8452 
8453                 return (EAGAIN);
8454         }
8455 
8456         /*
8457          * All of the probes for this provider are disabled; we can safely
8458          * remove all of them from their hash chains and from the probe array.
8459          */
8460         for (i = 0; i < dtrace_nprobes; i++) {
8461                 if ((probe = dtrace_probes[i]) == NULL)
8462                         continue;
8463 
8464                 if (probe->dtpr_provider != old)
8465                         continue;
8466 
8467                 dtrace_probes[i] = NULL;
8468 
8469                 dtrace_hash_remove(dtrace_bymod, probe);
8470                 dtrace_hash_remove(dtrace_byfunc, probe);
8471                 dtrace_hash_remove(dtrace_byname, probe);
8472 
8473                 if (first == NULL) {
8474                         first = probe;
8475                         probe->dtpr_nextmod = NULL;
8476                 } else {
8477                         probe->dtpr_nextmod = first;
8478                         first = probe;
8479                 }
8480         }
8481 
8482         /*
8483          * The provider's probes have been removed from the hash chains and
8484          * from the probe array.  Now issue a dtrace_sync() to be sure that
8485          * everyone has cleared out from any probe array processing.
8486          */
8487         dtrace_sync();
8488 
8489         for (probe = first; probe != NULL; probe = first) {
8490                 first = probe->dtpr_nextmod;
8491 
8492                 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
8493                     probe->dtpr_arg);
8494                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8495                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8496                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8497                 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
8498                 kmem_free(probe, sizeof (dtrace_probe_t));
8499         }
8500 
8501         if ((prev = dtrace_provider) == old) {
8502                 ASSERT(self || dtrace_devi == NULL);
8503                 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
8504                 dtrace_provider = old->dtpv_next;
8505         } else {
8506                 while (prev != NULL && prev->dtpv_next != old)
8507                         prev = prev->dtpv_next;
8508 
8509                 if (prev == NULL) {
8510                         panic("attempt to unregister non-existent "
8511                             "dtrace provider %p\n", (void *)id);
8512                 }
8513 
8514                 prev->dtpv_next = old->dtpv_next;
8515         }
8516 
8517         if (!self) {
8518                 mutex_exit(&dtrace_lock);
8519                 mutex_exit(&mod_lock);
8520                 mutex_exit(&dtrace_provider_lock);
8521         }
8522 
8523         kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
8524         kmem_free(old, sizeof (dtrace_provider_t));
8525 
8526         return (0);
8527 }
8528 
8529 /*
8530  * Invalidate the specified provider.  All subsequent probe lookups for the
8531  * specified provider will fail, but its probes will not be removed.
8532  */
8533 void
8534 dtrace_invalidate(dtrace_provider_id_t id)
8535 {
8536         dtrace_provider_t *pvp = (dtrace_provider_t *)id;
8537 
8538         ASSERT(pvp->dtpv_pops.dtps_enable != dtrace_enable_nullop);
8539 
8540         mutex_enter(&dtrace_provider_lock);
8541         mutex_enter(&dtrace_lock);
8542 
8543         pvp->dtpv_defunct = dtrace_gethrtime();
8544 
8545         mutex_exit(&dtrace_lock);
8546         mutex_exit(&dtrace_provider_lock);
8547 }
8548 
8549 /*
8550  * Indicate whether or not DTrace has attached.
8551  */
8552 int
8553 dtrace_attached(void)
8554 {
8555         /*
8556          * dtrace_provider will be non-NULL iff the DTrace driver has
8557          * attached.  (It's non-NULL because DTrace is always itself a
8558          * provider.)
8559          */
8560         return (dtrace_provider != NULL);
8561 }
8562 
8563 /*
8564  * Remove all the unenabled probes for the given provider.  This function is
8565  * not unlike dtrace_unregister(), except that it doesn't remove the provider
8566  * -- just as many of its associated probes as it can.
8567  */
8568 int
8569 dtrace_condense(dtrace_provider_id_t id)
8570 {
8571         dtrace_provider_t *prov = (dtrace_provider_t *)id;
8572         int i;
8573         dtrace_probe_t *probe;
8574 
8575         /*
8576          * Make sure this isn't the dtrace provider itself.
8577          */
8578         ASSERT(prov->dtpv_pops.dtps_enable != dtrace_enable_nullop);
8579 
8580         mutex_enter(&dtrace_provider_lock);
8581         mutex_enter(&dtrace_lock);
8582 
8583         /*
8584          * Attempt to destroy the probes associated with this provider.
8585          */
8586         for (i = 0; i < dtrace_nprobes; i++) {
8587                 if ((probe = dtrace_probes[i]) == NULL)
8588                         continue;
8589 
8590                 if (probe->dtpr_provider != prov)
8591                         continue;
8592 
8593                 if (probe->dtpr_ecb != NULL)
8594                         continue;
8595 
8596                 dtrace_probes[i] = NULL;
8597 
8598                 dtrace_hash_remove(dtrace_bymod, probe);
8599                 dtrace_hash_remove(dtrace_byfunc, probe);
8600                 dtrace_hash_remove(dtrace_byname, probe);
8601 
8602                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
8603                     probe->dtpr_arg);
8604                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
8605                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
8606                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
8607                 kmem_free(probe, sizeof (dtrace_probe_t));
8608                 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
8609         }
8610 
8611         mutex_exit(&dtrace_lock);
8612         mutex_exit(&dtrace_provider_lock);
8613 
8614         return (0);
8615 }
8616 
8617 /*
8618  * DTrace Probe Management Functions
8619  *
8620  * The functions in this section perform the DTrace probe management,
8621  * including functions to create probes, look-up probes, and call into the
8622  * providers to request that probes be provided.  Some of these functions are
8623  * in the Provider-to-Framework API; these functions can be identified by the
8624  * fact that they are not declared "static".
8625  */
8626 
8627 /*
8628  * Create a probe with the specified module name, function name, and name.
8629  */
8630 dtrace_id_t
8631 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
8632     const char *func, const char *name, int aframes, void *arg)
8633 {
8634         dtrace_probe_t *probe, **probes;
8635         dtrace_provider_t *provider = (dtrace_provider_t *)prov;
8636         dtrace_id_t id;
8637 
8638         if (provider == dtrace_provider) {
8639                 ASSERT(MUTEX_HELD(&dtrace_lock));
8640         } else {
8641                 mutex_enter(&dtrace_lock);
8642         }
8643 
8644         id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
8645             VM_BESTFIT | VM_SLEEP);
8646         probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
8647 
8648         probe->dtpr_id = id;
8649         probe->dtpr_gen = dtrace_probegen++;
8650         probe->dtpr_mod = dtrace_strdup(mod);
8651         probe->dtpr_func = dtrace_strdup(func);
8652         probe->dtpr_name = dtrace_strdup(name);
8653         probe->dtpr_arg = arg;
8654         probe->dtpr_aframes = aframes;
8655         probe->dtpr_provider = provider;
8656 
8657         dtrace_hash_add(dtrace_bymod, probe);
8658         dtrace_hash_add(dtrace_byfunc, probe);
8659         dtrace_hash_add(dtrace_byname, probe);
8660 
8661         if (id - 1 >= dtrace_nprobes) {
8662                 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
8663                 size_t nsize = osize << 1;
8664 
8665                 if (nsize == 0) {
8666                         ASSERT(osize == 0);
8667                         ASSERT(dtrace_probes == NULL);
8668                         nsize = sizeof (dtrace_probe_t *);
8669                 }
8670 
8671                 probes = kmem_zalloc(nsize, KM_SLEEP);
8672 
8673                 if (dtrace_probes == NULL) {
8674                         ASSERT(osize == 0);
8675                         dtrace_probes = probes;
8676                         dtrace_nprobes = 1;
8677                 } else {
8678                         dtrace_probe_t **oprobes = dtrace_probes;
8679 
8680                         bcopy(oprobes, probes, osize);
8681                         dtrace_membar_producer();
8682                         dtrace_probes = probes;
8683 
8684                         dtrace_sync();
8685 
8686                         /*
8687                          * All CPUs are now seeing the new probes array; we can
8688                          * safely free the old array.
8689                          */
8690                         kmem_free(oprobes, osize);
8691                         dtrace_nprobes <<= 1;
8692                 }
8693 
8694                 ASSERT(id - 1 < dtrace_nprobes);
8695         }
8696 
8697         ASSERT(dtrace_probes[id - 1] == NULL);
8698         dtrace_probes[id - 1] = probe;
8699 
8700         if (provider != dtrace_provider)
8701                 mutex_exit(&dtrace_lock);
8702 
8703         return (id);
8704 }
8705 
8706 static dtrace_probe_t *
8707 dtrace_probe_lookup_id(dtrace_id_t id)
8708 {
8709         ASSERT(MUTEX_HELD(&dtrace_lock));
8710 
8711         if (id == 0 || id > dtrace_nprobes)
8712                 return (NULL);
8713 
8714         return (dtrace_probes[id - 1]);
8715 }
8716 
8717 static int
8718 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
8719 {
8720         *((dtrace_id_t *)arg) = probe->dtpr_id;
8721 
8722         return (DTRACE_MATCH_DONE);
8723 }
8724 
8725 /*
8726  * Look up a probe based on provider and one or more of module name, function
8727  * name and probe name.
8728  */
8729 dtrace_id_t
8730 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
8731     const char *func, const char *name)
8732 {
8733         dtrace_probekey_t pkey;
8734         dtrace_id_t id;
8735         int match;
8736 
8737         pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
8738         pkey.dtpk_pmatch = &dtrace_match_string;
8739         pkey.dtpk_mod = mod;
8740         pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
8741         pkey.dtpk_func = func;
8742         pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
8743         pkey.dtpk_name = name;
8744         pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
8745         pkey.dtpk_id = DTRACE_IDNONE;
8746 
8747         mutex_enter(&dtrace_lock);
8748         match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
8749             dtrace_probe_lookup_match, &id);
8750         mutex_exit(&dtrace_lock);
8751 
8752         ASSERT(match == 1 || match == 0);
8753         return (match ? id : 0);
8754 }
8755 
8756 /*
8757  * Returns the probe argument associated with the specified probe.
8758  */
8759 void *
8760 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
8761 {
8762         dtrace_probe_t *probe;
8763         void *rval = NULL;
8764 
8765         mutex_enter(&dtrace_lock);
8766 
8767         if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
8768             probe->dtpr_provider == (dtrace_provider_t *)id)
8769                 rval = probe->dtpr_arg;
8770 
8771         mutex_exit(&dtrace_lock);
8772 
8773         return (rval);
8774 }
8775 
8776 /*
8777  * Copy a probe into a probe description.
8778  */
8779 static void
8780 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
8781 {
8782         bzero(pdp, sizeof (dtrace_probedesc_t));
8783         pdp->dtpd_id = prp->dtpr_id;
8784 
8785         (void) strncpy(pdp->dtpd_provider,
8786             prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
8787 
8788         (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
8789         (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
8790         (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
8791 }
8792 
8793 /*
8794  * Called to indicate that a probe -- or probes -- should be provided by a
8795  * specfied provider.  If the specified description is NULL, the provider will
8796  * be told to provide all of its probes.  (This is done whenever a new
8797  * consumer comes along, or whenever a retained enabling is to be matched.) If
8798  * the specified description is non-NULL, the provider is given the
8799  * opportunity to dynamically provide the specified probe, allowing providers
8800  * to support the creation of probes on-the-fly.  (So-called _autocreated_
8801  * probes.)  If the provider is NULL, the operations will be applied to all
8802  * providers; if the provider is non-NULL the operations will only be applied
8803  * to the specified provider.  The dtrace_provider_lock must be held, and the
8804  * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
8805  * will need to grab the dtrace_lock when it reenters the framework through
8806  * dtrace_probe_lookup(), dtrace_probe_create(), etc.
8807  */
8808 static void
8809 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
8810 {
8811         struct modctl *ctl;
8812         int all = 0;
8813 
8814         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
8815 
8816         if (prv == NULL) {
8817                 all = 1;
8818                 prv = dtrace_provider;
8819         }
8820 
8821         do {
8822                 /*
8823                  * First, call the blanket provide operation.
8824                  */
8825                 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
8826 
8827                 /*
8828                  * Now call the per-module provide operation.  We will grab
8829                  * mod_lock to prevent the list from being modified.  Note
8830                  * that this also prevents the mod_busy bits from changing.
8831                  * (mod_busy can only be changed with mod_lock held.)
8832                  */
8833                 mutex_enter(&mod_lock);
8834 
8835                 ctl = &modules;
8836                 do {
8837                         if (ctl->mod_busy || ctl->mod_mp == NULL)
8838                                 continue;
8839 
8840                         prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
8841 
8842                 } while ((ctl = ctl->mod_next) != &modules);
8843 
8844                 mutex_exit(&mod_lock);
8845         } while (all && (prv = prv->dtpv_next) != NULL);
8846 }
8847 
8848 /*
8849  * Iterate over each probe, and call the Framework-to-Provider API function
8850  * denoted by offs.
8851  */
8852 static void
8853 dtrace_probe_foreach(uintptr_t offs)
8854 {
8855         dtrace_provider_t *prov;
8856         void (*func)(void *, dtrace_id_t, void *);
8857         dtrace_probe_t *probe;
8858         dtrace_icookie_t cookie;
8859         int i;
8860 
8861         /*
8862          * We disable interrupts to walk through the probe array.  This is
8863          * safe -- the dtrace_sync() in dtrace_unregister() assures that we
8864          * won't see stale data.
8865          */
8866         cookie = dtrace_interrupt_disable();
8867 
8868         for (i = 0; i < dtrace_nprobes; i++) {
8869                 if ((probe = dtrace_probes[i]) == NULL)
8870                         continue;
8871 
8872                 if (probe->dtpr_ecb == NULL) {
8873                         /*
8874                          * This probe isn't enabled -- don't call the function.
8875                          */
8876                         continue;
8877                 }
8878 
8879                 prov = probe->dtpr_provider;
8880                 func = *((void(**)(void *, dtrace_id_t, void *))
8881                     ((uintptr_t)&prov->dtpv_pops + offs));
8882 
8883                 func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
8884         }
8885 
8886         dtrace_interrupt_enable(cookie);
8887 }
8888 
8889 static int
8890 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
8891 {
8892         dtrace_probekey_t pkey;
8893         uint32_t priv;
8894         uid_t uid;
8895         zoneid_t zoneid;
8896         dtrace_state_t *state = enab->dten_vstate->dtvs_state;
8897 
8898         ASSERT(MUTEX_HELD(&dtrace_lock));
8899         dtrace_ecb_create_cache = NULL;
8900 
8901         if (desc == NULL) {
8902                 /*
8903                  * If we're passed a NULL description, we're being asked to
8904                  * create an ECB with a NULL probe.
8905                  */
8906                 (void) dtrace_ecb_create_enable(NULL, enab);
8907                 return (0);
8908         }
8909 
8910         dtrace_probekey(desc, &pkey);
8911         dtrace_cred2priv(state->dts_cred.dcr_cred, &priv, &uid, &zoneid);
8912 
8913         if ((priv & DTRACE_PRIV_ZONEOWNER) &&
8914             state->dts_options[DTRACEOPT_ZONE] != DTRACEOPT_UNSET) {
8915                 /*
8916                  * If we have the privilege of instrumenting all zones but we
8917                  * have been told to instrument but one, we will spoof this up
8918                  * depriving ourselves of DTRACE_PRIV_ZONEOWNER for purposes
8919                  * of dtrace_match().  (Note that DTRACEOPT_ZONE is not for
8920                  * security but rather for performance: it allows the global
8921                  * zone to instrument USDT probes in a local zone without
8922                  * requiring all zones to be instrumented.)
8923                  */
8924                 priv &= ~DTRACE_PRIV_ZONEOWNER;
8925                 zoneid = state->dts_options[DTRACEOPT_ZONE];
8926         }
8927 
8928         return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
8929             enab));
8930 }
8931 
8932 /*
8933  * DTrace Helper Provider Functions
8934  */
8935 static void
8936 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
8937 {
8938         attr->dtat_name = DOF_ATTR_NAME(dofattr);
8939         attr->dtat_data = DOF_ATTR_DATA(dofattr);
8940         attr->dtat_class = DOF_ATTR_CLASS(dofattr);
8941 }
8942 
8943 static void
8944 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
8945     const dof_provider_t *dofprov, char *strtab)
8946 {
8947         hprov->dthpv_provname = strtab + dofprov->dofpv_name;
8948         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
8949             dofprov->dofpv_provattr);
8950         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
8951             dofprov->dofpv_modattr);
8952         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
8953             dofprov->dofpv_funcattr);
8954         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
8955             dofprov->dofpv_nameattr);
8956         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
8957             dofprov->dofpv_argsattr);
8958 }
8959 
8960 static void
8961 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8962 {
8963         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8964         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8965         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
8966         dof_provider_t *provider;
8967         dof_probe_t *probe;
8968         uint32_t *off, *enoff;
8969         uint8_t *arg;
8970         char *strtab;
8971         uint_t i, nprobes;
8972         dtrace_helper_provdesc_t dhpv;
8973         dtrace_helper_probedesc_t dhpb;
8974         dtrace_meta_t *meta = dtrace_meta_pid;
8975         dtrace_mops_t *mops = &meta->dtm_mops;
8976         void *parg;
8977 
8978         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8979         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8980             provider->dofpv_strtab * dof->dofh_secsize);
8981         prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8982             provider->dofpv_probes * dof->dofh_secsize);
8983         arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8984             provider->dofpv_prargs * dof->dofh_secsize);
8985         off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8986             provider->dofpv_proffs * dof->dofh_secsize);
8987 
8988         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8989         off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
8990         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
8991         enoff = NULL;
8992 
8993         /*
8994          * See dtrace_helper_provider_validate().
8995          */
8996         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
8997             provider->dofpv_prenoffs != DOF_SECT_NONE) {
8998                 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8999                     provider->dofpv_prenoffs * dof->dofh_secsize);
9000                 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
9001         }
9002 
9003         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
9004 
9005         /*
9006          * Create the provider.
9007          */
9008         dtrace_dofprov2hprov(&dhpv, provider, strtab);
9009 
9010         if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
9011                 return;
9012 
9013         meta->dtm_count++;
9014 
9015         /*
9016          * Create the probes.
9017          */
9018         for (i = 0; i < nprobes; i++) {
9019                 probe = (dof_probe_t *)(uintptr_t)(daddr +
9020                     prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
9021 
9022                 dhpb.dthpb_mod = dhp->dofhp_mod;
9023                 dhpb.dthpb_func = strtab + probe->dofpr_func;
9024                 dhpb.dthpb_name = strtab + probe->dofpr_name;
9025                 dhpb.dthpb_base = probe->dofpr_addr;
9026                 dhpb.dthpb_offs = off + probe->dofpr_offidx;
9027                 dhpb.dthpb_noffs = probe->dofpr_noffs;
9028                 if (enoff != NULL) {
9029                         dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
9030                         dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
9031                 } else {
9032                         dhpb.dthpb_enoffs = NULL;
9033                         dhpb.dthpb_nenoffs = 0;
9034                 }
9035                 dhpb.dthpb_args = arg + probe->dofpr_argidx;
9036                 dhpb.dthpb_nargc = probe->dofpr_nargc;
9037                 dhpb.dthpb_xargc = probe->dofpr_xargc;
9038                 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
9039                 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
9040 
9041                 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
9042         }
9043 }
9044 
9045 static void
9046 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
9047 {
9048         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
9049         dof_hdr_t *dof = (dof_hdr_t *)daddr;
9050         int i;
9051 
9052         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
9053 
9054         for (i = 0; i < dof->dofh_secnum; i++) {
9055                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
9056                     dof->dofh_secoff + i * dof->dofh_secsize);
9057 
9058                 if (sec->dofs_type != DOF_SECT_PROVIDER)
9059                         continue;
9060 
9061                 dtrace_helper_provide_one(dhp, sec, pid);
9062         }
9063 
9064         /*
9065          * We may have just created probes, so we must now rematch against
9066          * any retained enablings.  Note that this call will acquire both
9067          * cpu_lock and dtrace_lock; the fact that we are holding
9068          * dtrace_meta_lock now is what defines the ordering with respect to
9069          * these three locks.
9070          */
9071         dtrace_enabling_matchall();
9072 }
9073 
9074 static void
9075 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
9076 {
9077         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
9078         dof_hdr_t *dof = (dof_hdr_t *)daddr;
9079         dof_sec_t *str_sec;
9080         dof_provider_t *provider;
9081         char *strtab;
9082         dtrace_helper_provdesc_t dhpv;
9083         dtrace_meta_t *meta = dtrace_meta_pid;
9084         dtrace_mops_t *mops = &meta->dtm_mops;
9085 
9086         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
9087         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
9088             provider->dofpv_strtab * dof->dofh_secsize);
9089 
9090         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
9091 
9092         /*
9093          * Create the provider.
9094          */
9095         dtrace_dofprov2hprov(&dhpv, provider, strtab);
9096 
9097         mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
9098 
9099         meta->dtm_count--;
9100 }
9101 
9102 static void
9103 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
9104 {
9105         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
9106         dof_hdr_t *dof = (dof_hdr_t *)daddr;
9107         int i;
9108 
9109         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
9110 
9111         for (i = 0; i < dof->dofh_secnum; i++) {
9112                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
9113                     dof->dofh_secoff + i * dof->dofh_secsize);
9114 
9115                 if (sec->dofs_type != DOF_SECT_PROVIDER)
9116                         continue;
9117 
9118                 dtrace_helper_provider_remove_one(dhp, sec, pid);
9119         }
9120 }
9121 
9122 /*
9123  * DTrace Meta Provider-to-Framework API Functions
9124  *
9125  * These functions implement the Meta Provider-to-Framework API, as described
9126  * in <sys/dtrace.h>.
9127  */
9128 int
9129 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
9130     dtrace_meta_provider_id_t *idp)
9131 {
9132         dtrace_meta_t *meta;
9133         dtrace_helpers_t *help, *next;
9134         int i;
9135 
9136         *idp = DTRACE_METAPROVNONE;
9137 
9138         /*
9139          * We strictly don't need the name, but we hold onto it for
9140          * debuggability. All hail error queues!
9141          */
9142         if (name == NULL) {
9143                 cmn_err(CE_WARN, "failed to register meta-provider: "
9144                     "invalid name");
9145                 return (EINVAL);
9146         }
9147 
9148         if (mops == NULL ||
9149             mops->dtms_create_probe == NULL ||
9150             mops->dtms_provide_pid == NULL ||
9151             mops->dtms_remove_pid == NULL) {
9152                 cmn_err(CE_WARN, "failed to register meta-register %s: "
9153                     "invalid ops", name);
9154                 return (EINVAL);
9155         }
9156 
9157         meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
9158         meta->dtm_mops = *mops;
9159         meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
9160         (void) strcpy(meta->dtm_name, name);
9161         meta->dtm_arg = arg;
9162 
9163         mutex_enter(&dtrace_meta_lock);
9164         mutex_enter(&dtrace_lock);
9165 
9166         if (dtrace_meta_pid != NULL) {
9167                 mutex_exit(&dtrace_lock);
9168                 mutex_exit(&dtrace_meta_lock);
9169                 cmn_err(CE_WARN, "failed to register meta-register %s: "
9170                     "user-land meta-provider exists", name);
9171                 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
9172                 kmem_free(meta, sizeof (dtrace_meta_t));
9173                 return (EINVAL);
9174         }
9175 
9176         dtrace_meta_pid = meta;
9177         *idp = (dtrace_meta_provider_id_t)meta;
9178 
9179         /*
9180          * If there are providers and probes ready to go, pass them
9181          * off to the new meta provider now.
9182          */
9183 
9184         help = dtrace_deferred_pid;
9185         dtrace_deferred_pid = NULL;
9186 
9187         mutex_exit(&dtrace_lock);
9188 
9189         while (help != NULL) {
9190                 for (i = 0; i < help->dthps_nprovs; i++) {
9191                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
9192                             help->dthps_pid);
9193                 }
9194 
9195                 next = help->dthps_next;
9196                 help->dthps_next = NULL;
9197                 help->dthps_prev = NULL;
9198                 help->dthps_deferred = 0;
9199                 help = next;
9200         }
9201 
9202         mutex_exit(&dtrace_meta_lock);
9203 
9204         return (0);
9205 }
9206 
9207 int
9208 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
9209 {
9210         dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
9211 
9212         mutex_enter(&dtrace_meta_lock);
9213         mutex_enter(&dtrace_lock);
9214 
9215         if (old == dtrace_meta_pid) {
9216                 pp = &dtrace_meta_pid;
9217         } else {
9218                 panic("attempt to unregister non-existent "
9219                     "dtrace meta-provider %p\n", (void *)old);
9220         }
9221 
9222         if (old->dtm_count != 0) {
9223                 mutex_exit(&dtrace_lock);
9224                 mutex_exit(&dtrace_meta_lock);
9225                 return (EBUSY);
9226         }
9227 
9228         *pp = NULL;
9229 
9230         mutex_exit(&dtrace_lock);
9231         mutex_exit(&dtrace_meta_lock);
9232 
9233         kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
9234         kmem_free(old, sizeof (dtrace_meta_t));
9235 
9236         return (0);
9237 }
9238 
9239 
9240 /*
9241  * DTrace DIF Object Functions
9242  */
9243 static int
9244 dtrace_difo_err(uint_t pc, const char *format, ...)
9245 {
9246         if (dtrace_err_verbose) {
9247                 va_list alist;
9248 
9249                 (void) uprintf("dtrace DIF object error: [%u]: ", pc);
9250                 va_start(alist, format);
9251                 (void) vuprintf(format, alist);
9252                 va_end(alist);
9253         }
9254 
9255 #ifdef DTRACE_ERRDEBUG
9256         dtrace_errdebug(format);
9257 #endif
9258         return (1);
9259 }
9260 
9261 /*
9262  * Validate a DTrace DIF object by checking the IR instructions.  The following
9263  * rules are currently enforced by dtrace_difo_validate():
9264  *
9265  * 1. Each instruction must have a valid opcode
9266  * 2. Each register, string, variable, or subroutine reference must be valid
9267  * 3. No instruction can modify register %r0 (must be zero)
9268  * 4. All instruction reserved bits must be set to zero
9269  * 5. The last instruction must be a "ret" instruction
9270  * 6. All branch targets must reference a valid instruction _after_ the branch
9271  */
9272 static int
9273 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
9274     cred_t *cr)
9275 {
9276         int err = 0, i;
9277         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
9278         int kcheckload;
9279         uint_t pc;
9280         int maxglobal = -1, maxlocal = -1, maxtlocal = -1;
9281 
9282         kcheckload = cr == NULL ||
9283             (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
9284 
9285         dp->dtdo_destructive = 0;
9286 
9287         for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
9288                 dif_instr_t instr = dp->dtdo_buf[pc];
9289 
9290                 uint_t r1 = DIF_INSTR_R1(instr);
9291                 uint_t r2 = DIF_INSTR_R2(instr);
9292                 uint_t rd = DIF_INSTR_RD(instr);
9293                 uint_t rs = DIF_INSTR_RS(instr);
9294                 uint_t label = DIF_INSTR_LABEL(instr);
9295                 uint_t v = DIF_INSTR_VAR(instr);
9296                 uint_t subr = DIF_INSTR_SUBR(instr);
9297                 uint_t type = DIF_INSTR_TYPE(instr);
9298                 uint_t op = DIF_INSTR_OP(instr);
9299 
9300                 switch (op) {
9301                 case DIF_OP_OR:
9302                 case DIF_OP_XOR:
9303                 case DIF_OP_AND:
9304                 case DIF_OP_SLL:
9305                 case DIF_OP_SRL:
9306                 case DIF_OP_SRA:
9307                 case DIF_OP_SUB:
9308                 case DIF_OP_ADD:
9309                 case DIF_OP_MUL:
9310                 case DIF_OP_SDIV:
9311                 case DIF_OP_UDIV:
9312                 case DIF_OP_SREM:
9313                 case DIF_OP_UREM:
9314                 case DIF_OP_COPYS:
9315                         if (r1 >= nregs)
9316                                 err += efunc(pc, "invalid register %u\n", r1);
9317                         if (r2 >= nregs)
9318                                 err += efunc(pc, "invalid register %u\n", r2);
9319                         if (rd >= nregs)
9320                                 err += efunc(pc, "invalid register %u\n", rd);
9321                         if (rd == 0)
9322                                 err += efunc(pc, "cannot write to %r0\n");
9323                         break;
9324                 case DIF_OP_NOT:
9325                 case DIF_OP_MOV:
9326                 case DIF_OP_ALLOCS:
9327                         if (r1 >= nregs)
9328                                 err += efunc(pc, "invalid register %u\n", r1);
9329                         if (r2 != 0)
9330                                 err += efunc(pc, "non-zero reserved bits\n");
9331                         if (rd >= nregs)
9332                                 err += efunc(pc, "invalid register %u\n", rd);
9333                         if (rd == 0)
9334                                 err += efunc(pc, "cannot write to %r0\n");
9335                         break;
9336                 case DIF_OP_LDSB:
9337                 case DIF_OP_LDSH:
9338                 case DIF_OP_LDSW:
9339                 case DIF_OP_LDUB:
9340                 case DIF_OP_LDUH:
9341                 case DIF_OP_LDUW:
9342                 case DIF_OP_LDX:
9343                         if (r1 >= nregs)
9344                                 err += efunc(pc, "invalid register %u\n", r1);
9345                         if (r2 != 0)
9346                                 err += efunc(pc, "non-zero reserved bits\n");
9347                         if (rd >= nregs)
9348                                 err += efunc(pc, "invalid register %u\n", rd);
9349                         if (rd == 0)
9350                                 err += efunc(pc, "cannot write to %r0\n");
9351                         if (kcheckload)
9352                                 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
9353                                     DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
9354                         break;
9355                 case DIF_OP_RLDSB:
9356                 case DIF_OP_RLDSH:
9357                 case DIF_OP_RLDSW:
9358                 case DIF_OP_RLDUB:
9359                 case DIF_OP_RLDUH:
9360                 case DIF_OP_RLDUW:
9361                 case DIF_OP_RLDX:
9362                         if (r1 >= nregs)
9363                                 err += efunc(pc, "invalid register %u\n", r1);
9364                         if (r2 != 0)
9365                                 err += efunc(pc, "non-zero reserved bits\n");
9366                         if (rd >= nregs)
9367                                 err += efunc(pc, "invalid register %u\n", rd);
9368                         if (rd == 0)
9369                                 err += efunc(pc, "cannot write to %r0\n");
9370                         break;
9371                 case DIF_OP_ULDSB:
9372                 case DIF_OP_ULDSH:
9373                 case DIF_OP_ULDSW:
9374                 case DIF_OP_ULDUB:
9375                 case DIF_OP_ULDUH:
9376                 case DIF_OP_ULDUW:
9377                 case DIF_OP_ULDX:
9378                         if (r1 >= nregs)
9379                                 err += efunc(pc, "invalid register %u\n", r1);
9380                         if (r2 != 0)
9381                                 err += efunc(pc, "non-zero reserved bits\n");
9382                         if (rd >= nregs)
9383                                 err += efunc(pc, "invalid register %u\n", rd);
9384                         if (rd == 0)
9385                                 err += efunc(pc, "cannot write to %r0\n");
9386                         break;
9387                 case DIF_OP_STB:
9388                 case DIF_OP_STH:
9389                 case DIF_OP_STW:
9390                 case DIF_OP_STX:
9391                         if (r1 >= nregs)
9392                                 err += efunc(pc, "invalid register %u\n", r1);
9393                         if (r2 != 0)
9394                                 err += efunc(pc, "non-zero reserved bits\n");
9395                         if (rd >= nregs)
9396                                 err += efunc(pc, "invalid register %u\n", rd);
9397                         if (rd == 0)
9398                                 err += efunc(pc, "cannot write to 0 address\n");
9399                         break;
9400                 case DIF_OP_CMP:
9401                 case DIF_OP_SCMP:
9402                         if (r1 >= nregs)
9403                                 err += efunc(pc, "invalid register %u\n", r1);
9404                         if (r2 >= nregs)
9405                                 err += efunc(pc, "invalid register %u\n", r2);
9406                         if (rd != 0)
9407                                 err += efunc(pc, "non-zero reserved bits\n");
9408                         break;
9409                 case DIF_OP_TST:
9410                         if (r1 >= nregs)
9411                                 err += efunc(pc, "invalid register %u\n", r1);
9412                         if (r2 != 0 || rd != 0)
9413                                 err += efunc(pc, "non-zero reserved bits\n");
9414                         break;
9415                 case DIF_OP_BA:
9416                 case DIF_OP_BE:
9417                 case DIF_OP_BNE:
9418                 case DIF_OP_BG:
9419                 case DIF_OP_BGU:
9420                 case DIF_OP_BGE:
9421                 case DIF_OP_BGEU:
9422                 case DIF_OP_BL:
9423                 case DIF_OP_BLU:
9424                 case DIF_OP_BLE:
9425                 case DIF_OP_BLEU:
9426                         if (label >= dp->dtdo_len) {
9427                                 err += efunc(pc, "invalid branch target %u\n",
9428                                     label);
9429                         }
9430                         if (label <= pc) {
9431                                 err += efunc(pc, "backward branch to %u\n",
9432                                     label);
9433                         }
9434                         break;
9435                 case DIF_OP_RET:
9436                         if (r1 != 0 || r2 != 0)
9437                                 err += efunc(pc, "non-zero reserved bits\n");
9438                         if (rd >= nregs)
9439                                 err += efunc(pc, "invalid register %u\n", rd);
9440                         break;
9441                 case DIF_OP_NOP:
9442                 case DIF_OP_POPTS:
9443                 case DIF_OP_FLUSHTS:
9444                         if (r1 != 0 || r2 != 0 || rd != 0)
9445                                 err += efunc(pc, "non-zero reserved bits\n");
9446                         break;
9447                 case DIF_OP_SETX:
9448                         if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
9449                                 err += efunc(pc, "invalid integer ref %u\n",
9450                                     DIF_INSTR_INTEGER(instr));
9451                         }
9452                         if (rd >= nregs)
9453                                 err += efunc(pc, "invalid register %u\n", rd);
9454                         if (rd == 0)
9455                                 err += efunc(pc, "cannot write to %r0\n");
9456                         break;
9457                 case DIF_OP_SETS:
9458                         if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
9459                                 err += efunc(pc, "invalid string ref %u\n",
9460                                     DIF_INSTR_STRING(instr));
9461                         }
9462                         if (rd >= nregs)
9463                                 err += efunc(pc, "invalid register %u\n", rd);
9464                         if (rd == 0)
9465                                 err += efunc(pc, "cannot write to %r0\n");
9466                         break;
9467                 case DIF_OP_LDGA:
9468                 case DIF_OP_LDTA:
9469                         if (r1 > DIF_VAR_ARRAY_MAX)
9470                                 err += efunc(pc, "invalid array %u\n", r1);
9471                         if (r2 >= nregs)
9472                                 err += efunc(pc, "invalid register %u\n", r2);
9473                         if (rd >= nregs)
9474                                 err += efunc(pc, "invalid register %u\n", rd);
9475                         if (rd == 0)
9476                                 err += efunc(pc, "cannot write to %r0\n");
9477                         break;
9478                 case DIF_OP_STGA:
9479                         if (r1 > DIF_VAR_ARRAY_MAX)
9480                                 err += efunc(pc, "invalid array %u\n", r1);
9481                         if (r2 >= nregs)
9482                                 err += efunc(pc, "invalid register %u\n", r2);
9483                         if (rd >= nregs)
9484                                 err += efunc(pc, "invalid register %u\n", rd);
9485                         dp->dtdo_destructive = 1;
9486                         break;
9487                 case DIF_OP_LDGS:
9488                 case DIF_OP_LDTS:
9489                 case DIF_OP_LDLS:
9490                 case DIF_OP_LDGAA:
9491                 case DIF_OP_LDTAA:
9492                         if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
9493                                 err += efunc(pc, "invalid variable %u\n", v);
9494                         if (rd >= nregs)
9495                                 err += efunc(pc, "invalid register %u\n", rd);
9496                         if (rd == 0)
9497                                 err += efunc(pc, "cannot write to %r0\n");
9498                         break;
9499                 case DIF_OP_STGS:
9500                 case DIF_OP_STTS:
9501                 case DIF_OP_STLS:
9502                 case DIF_OP_STGAA:
9503                 case DIF_OP_STTAA:
9504                         if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
9505                                 err += efunc(pc, "invalid variable %u\n", v);
9506                         if (rs >= nregs)
9507                                 err += efunc(pc, "invalid register %u\n", rd);
9508                         break;
9509                 case DIF_OP_CALL:
9510                         if (subr > DIF_SUBR_MAX)
9511                                 err += efunc(pc, "invalid subr %u\n", subr);
9512                         if (rd >= nregs)
9513                                 err += efunc(pc, "invalid register %u\n", rd);
9514                         if (rd == 0)
9515                                 err += efunc(pc, "cannot write to %r0\n");
9516 
9517                         if (subr == DIF_SUBR_COPYOUT ||
9518                             subr == DIF_SUBR_COPYOUTSTR) {
9519                                 dp->dtdo_destructive = 1;
9520                         }
9521 
9522                         if (subr == DIF_SUBR_GETF) {
9523                                 /*
9524                                  * If we have a getf() we need to record that
9525                                  * in our state.  Note that our state can be
9526                                  * NULL if this is a helper -- but in that
9527                                  * case, the call to getf() is itself illegal,
9528                                  * and will be caught (slightly later) when
9529                                  * the helper is validated.
9530                                  */
9531                                 if (vstate->dtvs_state != NULL)
9532                                         vstate->dtvs_state->dts_getf++;
9533                         }
9534 
9535                         break;
9536                 case DIF_OP_PUSHTR:
9537                         if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
9538                                 err += efunc(pc, "invalid ref type %u\n", type);
9539                         if (r2 >= nregs)
9540                                 err += efunc(pc, "invalid register %u\n", r2);
9541                         if (rs >= nregs)
9542                                 err += efunc(pc, "invalid register %u\n", rs);
9543                         break;
9544                 case DIF_OP_PUSHTV:
9545                         if (type != DIF_TYPE_CTF)
9546                                 err += efunc(pc, "invalid val type %u\n", type);
9547                         if (r2 >= nregs)
9548                                 err += efunc(pc, "invalid register %u\n", r2);
9549                         if (rs >= nregs)
9550                                 err += efunc(pc, "invalid register %u\n", rs);
9551                         break;
9552                 default:
9553                         err += efunc(pc, "invalid opcode %u\n",
9554                             DIF_INSTR_OP(instr));
9555                 }
9556         }
9557 
9558         if (dp->dtdo_len != 0 &&
9559             DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
9560                 err += efunc(dp->dtdo_len - 1,
9561                     "expected 'ret' as last DIF instruction\n");
9562         }
9563 
9564         if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) {
9565                 /*
9566                  * If we're not returning by reference, the size must be either
9567                  * 0 or the size of one of the base types.
9568                  */
9569                 switch (dp->dtdo_rtype.dtdt_size) {
9570                 case 0:
9571                 case sizeof (uint8_t):
9572                 case sizeof (uint16_t):
9573                 case sizeof (uint32_t):
9574                 case sizeof (uint64_t):
9575                         break;
9576 
9577                 default:
9578                         err += efunc(dp->dtdo_len - 1, "bad return size\n");
9579                 }
9580         }
9581 
9582         for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
9583                 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
9584                 dtrace_diftype_t *vt, *et;
9585                 uint_t id, ndx;
9586 
9587                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
9588                     v->dtdv_scope != DIFV_SCOPE_THREAD &&
9589                     v->dtdv_scope != DIFV_SCOPE_LOCAL) {
9590                         err += efunc(i, "unrecognized variable scope %d\n",
9591                             v->dtdv_scope);
9592                         break;
9593                 }
9594 
9595                 if (v->dtdv_kind != DIFV_KIND_ARRAY &&
9596                     v->dtdv_kind != DIFV_KIND_SCALAR) {
9597                         err += efunc(i, "unrecognized variable type %d\n",
9598                             v->dtdv_kind);
9599                         break;
9600                 }
9601 
9602                 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
9603                         err += efunc(i, "%d exceeds variable id limit\n", id);
9604                         break;
9605                 }
9606 
9607                 if (id < DIF_VAR_OTHER_UBASE)
9608                         continue;
9609 
9610                 /*
9611                  * For user-defined variables, we need to check that this
9612                  * definition is identical to any previous definition that we
9613                  * encountered.
9614                  */
9615                 ndx = id - DIF_VAR_OTHER_UBASE;
9616 
9617                 switch (v->dtdv_scope) {
9618                 case DIFV_SCOPE_GLOBAL:
9619                         if (maxglobal == -1 || ndx > maxglobal)
9620                                 maxglobal = ndx;
9621 
9622                         if (ndx < vstate->dtvs_nglobals) {
9623                                 dtrace_statvar_t *svar;
9624 
9625                                 if ((svar = vstate->dtvs_globals[ndx]) != NULL)
9626                                         existing = &svar->dtsv_var;
9627                         }
9628 
9629                         break;
9630 
9631                 case DIFV_SCOPE_THREAD:
9632                         if (maxtlocal == -1 || ndx > maxtlocal)
9633                                 maxtlocal = ndx;
9634 
9635                         if (ndx < vstate->dtvs_ntlocals)
9636                                 existing = &vstate->dtvs_tlocals[ndx];
9637                         break;
9638 
9639                 case DIFV_SCOPE_LOCAL:
9640                         if (maxlocal == -1 || ndx > maxlocal)
9641                                 maxlocal = ndx;
9642 
9643                         if (ndx < vstate->dtvs_nlocals) {
9644                                 dtrace_statvar_t *svar;
9645 
9646                                 if ((svar = vstate->dtvs_locals[ndx]) != NULL)
9647                                         existing = &svar->dtsv_var;
9648                         }
9649 
9650                         break;
9651                 }
9652 
9653                 vt = &v->dtdv_type;
9654 
9655                 if (vt->dtdt_flags & DIF_TF_BYREF) {
9656                         if (vt->dtdt_size == 0) {
9657                                 err += efunc(i, "zero-sized variable\n");
9658                                 break;
9659                         }
9660 
9661                         if ((v->dtdv_scope == DIFV_SCOPE_GLOBAL ||
9662                             v->dtdv_scope == DIFV_SCOPE_LOCAL) &&
9663                             vt->dtdt_size > dtrace_statvar_maxsize) {
9664                                 err += efunc(i, "oversized by-ref static\n");
9665                                 break;
9666                         }
9667                 }
9668 
9669                 if (existing == NULL || existing->dtdv_id == 0)
9670                         continue;
9671 
9672                 ASSERT(existing->dtdv_id == v->dtdv_id);
9673                 ASSERT(existing->dtdv_scope == v->dtdv_scope);
9674 
9675                 if (existing->dtdv_kind != v->dtdv_kind)
9676                         err += efunc(i, "%d changed variable kind\n", id);
9677 
9678                 et = &existing->dtdv_type;
9679 
9680                 if (vt->dtdt_flags != et->dtdt_flags) {
9681                         err += efunc(i, "%d changed variable type flags\n", id);
9682                         break;
9683                 }
9684 
9685                 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
9686                         err += efunc(i, "%d changed variable type size\n", id);
9687                         break;
9688                 }
9689         }
9690 
9691         for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
9692                 dif_instr_t instr = dp->dtdo_buf[pc];
9693 
9694                 uint_t v = DIF_INSTR_VAR(instr);
9695                 uint_t op = DIF_INSTR_OP(instr);
9696 
9697                 switch (op) {
9698                 case DIF_OP_LDGS:
9699                 case DIF_OP_LDGAA:
9700                 case DIF_OP_STGS:
9701                 case DIF_OP_STGAA:
9702                         if (v > DIF_VAR_OTHER_UBASE + maxglobal)
9703                                 err += efunc(pc, "invalid variable %u\n", v);
9704                         break;
9705                 case DIF_OP_LDTS:
9706                 case DIF_OP_LDTAA:
9707                 case DIF_OP_STTS:
9708                 case DIF_OP_STTAA:
9709                         if (v > DIF_VAR_OTHER_UBASE + maxtlocal)
9710                                 err += efunc(pc, "invalid variable %u\n", v);
9711                         break;
9712                 case DIF_OP_LDLS:
9713                 case DIF_OP_STLS:
9714                         if (v > DIF_VAR_OTHER_UBASE + maxlocal)
9715                                 err += efunc(pc, "invalid variable %u\n", v);
9716                         break;
9717                 default:
9718                         break;
9719                 }
9720         }
9721 
9722         return (err);
9723 }
9724 
9725 /*
9726  * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
9727  * are much more constrained than normal DIFOs.  Specifically, they may
9728  * not:
9729  *
9730  * 1. Make calls to subroutines other than copyin(), copyinstr() or
9731  *    miscellaneous string routines
9732  * 2. Access DTrace variables other than the args[] array, and the
9733  *    curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
9734  * 3. Have thread-local variables.
9735  * 4. Have dynamic variables.
9736  */
9737 static int
9738 dtrace_difo_validate_helper(dtrace_difo_t *dp)
9739 {
9740         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
9741         int err = 0;
9742         uint_t pc;
9743 
9744         for (pc = 0; pc < dp->dtdo_len; pc++) {
9745                 dif_instr_t instr = dp->dtdo_buf[pc];
9746 
9747                 uint_t v = DIF_INSTR_VAR(instr);
9748                 uint_t subr = DIF_INSTR_SUBR(instr);
9749                 uint_t op = DIF_INSTR_OP(instr);
9750 
9751                 switch (op) {
9752                 case DIF_OP_OR:
9753                 case DIF_OP_XOR:
9754                 case DIF_OP_AND:
9755                 case DIF_OP_SLL:
9756                 case DIF_OP_SRL:
9757                 case DIF_OP_SRA:
9758                 case DIF_OP_SUB:
9759                 case DIF_OP_ADD:
9760                 case DIF_OP_MUL:
9761                 case DIF_OP_SDIV:
9762                 case DIF_OP_UDIV:
9763                 case DIF_OP_SREM:
9764                 case DIF_OP_UREM:
9765                 case DIF_OP_COPYS:
9766                 case DIF_OP_NOT:
9767                 case DIF_OP_MOV:
9768                 case DIF_OP_RLDSB:
9769                 case DIF_OP_RLDSH:
9770                 case DIF_OP_RLDSW:
9771                 case DIF_OP_RLDUB:
9772                 case DIF_OP_RLDUH:
9773                 case DIF_OP_RLDUW:
9774                 case DIF_OP_RLDX:
9775                 case DIF_OP_ULDSB:
9776                 case DIF_OP_ULDSH:
9777                 case DIF_OP_ULDSW:
9778                 case DIF_OP_ULDUB:
9779                 case DIF_OP_ULDUH:
9780                 case DIF_OP_ULDUW:
9781                 case DIF_OP_ULDX:
9782                 case DIF_OP_STB:
9783                 case DIF_OP_STH:
9784                 case DIF_OP_STW:
9785                 case DIF_OP_STX:
9786                 case DIF_OP_ALLOCS:
9787                 case DIF_OP_CMP:
9788                 case DIF_OP_SCMP:
9789                 case DIF_OP_TST:
9790                 case DIF_OP_BA:
9791                 case DIF_OP_BE:
9792                 case DIF_OP_BNE:
9793                 case DIF_OP_BG:
9794                 case DIF_OP_BGU:
9795                 case DIF_OP_BGE:
9796                 case DIF_OP_BGEU:
9797                 case DIF_OP_BL:
9798                 case DIF_OP_BLU:
9799                 case DIF_OP_BLE:
9800                 case DIF_OP_BLEU:
9801                 case DIF_OP_RET:
9802                 case DIF_OP_NOP:
9803                 case DIF_OP_POPTS:
9804                 case DIF_OP_FLUSHTS:
9805                 case DIF_OP_SETX:
9806                 case DIF_OP_SETS:
9807                 case DIF_OP_LDGA:
9808                 case DIF_OP_LDLS:
9809                 case DIF_OP_STGS:
9810                 case DIF_OP_STLS:
9811                 case DIF_OP_PUSHTR:
9812                 case DIF_OP_PUSHTV:
9813                         break;
9814 
9815                 case DIF_OP_LDGS:
9816                         if (v >= DIF_VAR_OTHER_UBASE)
9817                                 break;
9818 
9819                         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
9820                                 break;
9821 
9822                         if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
9823                             v == DIF_VAR_PPID || v == DIF_VAR_TID ||
9824                             v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
9825                             v == DIF_VAR_UID || v == DIF_VAR_GID)
9826                                 break;
9827 
9828                         err += efunc(pc, "illegal variable %u\n", v);
9829                         break;
9830 
9831                 case DIF_OP_LDTA:
9832                         if (v < DIF_VAR_OTHER_UBASE) {
9833                                 err += efunc(pc, "illegal variable load\n");
9834                                 break;
9835                         }
9836                         /* FALLTHROUGH */
9837                 case DIF_OP_LDTS:
9838                 case DIF_OP_LDGAA:
9839                 case DIF_OP_LDTAA:
9840                         err += efunc(pc, "illegal dynamic variable load\n");
9841                         break;
9842 
9843                 case DIF_OP_STGA:
9844                         if (v < DIF_VAR_OTHER_UBASE) {
9845                                 err += efunc(pc, "illegal variable store\n");
9846                                 break;
9847                         }
9848                         /* FALLTHROUGH */
9849                 case DIF_OP_STTS:
9850                 case DIF_OP_STGAA:
9851                 case DIF_OP_STTAA:
9852                         err += efunc(pc, "illegal dynamic variable store\n");
9853                         break;
9854 
9855                 case DIF_OP_CALL:
9856                         if (subr == DIF_SUBR_ALLOCA ||
9857                             subr == DIF_SUBR_BCOPY ||
9858                             subr == DIF_SUBR_COPYIN ||
9859                             subr == DIF_SUBR_COPYINTO ||
9860                             subr == DIF_SUBR_COPYINSTR ||
9861                             subr == DIF_SUBR_INDEX ||
9862                             subr == DIF_SUBR_INET_NTOA ||
9863                             subr == DIF_SUBR_INET_NTOA6 ||
9864                             subr == DIF_SUBR_INET_NTOP ||
9865                             subr == DIF_SUBR_JSON ||
9866                             subr == DIF_SUBR_LLTOSTR ||
9867                             subr == DIF_SUBR_STRTOLL ||
9868                             subr == DIF_SUBR_RINDEX ||
9869                             subr == DIF_SUBR_STRCHR ||
9870                             subr == DIF_SUBR_STRJOIN ||
9871                             subr == DIF_SUBR_STRRCHR ||
9872                             subr == DIF_SUBR_STRSTR ||
9873                             subr == DIF_SUBR_HTONS ||
9874                             subr == DIF_SUBR_HTONL ||
9875                             subr == DIF_SUBR_HTONLL ||
9876                             subr == DIF_SUBR_NTOHS ||
9877                             subr == DIF_SUBR_NTOHL ||
9878                             subr == DIF_SUBR_NTOHLL)
9879                                 break;
9880 
9881                         err += efunc(pc, "invalid subr %u\n", subr);
9882                         break;
9883 
9884                 default:
9885                         err += efunc(pc, "invalid opcode %u\n",
9886                             DIF_INSTR_OP(instr));
9887                 }
9888         }
9889 
9890         return (err);
9891 }
9892 
9893 /*
9894  * Returns 1 if the expression in the DIF object can be cached on a per-thread
9895  * basis; 0 if not.
9896  */
9897 static int
9898 dtrace_difo_cacheable(dtrace_difo_t *dp)
9899 {
9900         int i;
9901 
9902         if (dp == NULL)
9903                 return (0);
9904 
9905         for (i = 0; i < dp->dtdo_varlen; i++) {
9906                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9907 
9908                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
9909                         continue;
9910 
9911                 switch (v->dtdv_id) {
9912                 case DIF_VAR_CURTHREAD:
9913                 case DIF_VAR_PID:
9914                 case DIF_VAR_TID:
9915                 case DIF_VAR_EXECNAME:
9916                 case DIF_VAR_ZONENAME:
9917                         break;
9918 
9919                 default:
9920                         return (0);
9921                 }
9922         }
9923 
9924         /*
9925          * This DIF object may be cacheable.  Now we need to look for any
9926          * array loading instructions, any memory loading instructions, or
9927          * any stores to thread-local variables.
9928          */
9929         for (i = 0; i < dp->dtdo_len; i++) {
9930                 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
9931 
9932                 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
9933                     (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
9934                     (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
9935                     op == DIF_OP_LDGA || op == DIF_OP_STTS)
9936                         return (0);
9937         }
9938 
9939         return (1);
9940 }
9941 
9942 static void
9943 dtrace_difo_hold(dtrace_difo_t *dp)
9944 {
9945         int i;
9946 
9947         ASSERT(MUTEX_HELD(&dtrace_lock));
9948 
9949         dp->dtdo_refcnt++;
9950         ASSERT(dp->dtdo_refcnt != 0);
9951 
9952         /*
9953          * We need to check this DIF object for references to the variable
9954          * DIF_VAR_VTIMESTAMP.
9955          */
9956         for (i = 0; i < dp->dtdo_varlen; i++) {
9957                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9958 
9959                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9960                         continue;
9961 
9962                 if (dtrace_vtime_references++ == 0)
9963                         dtrace_vtime_enable();
9964         }
9965 }
9966 
9967 /*
9968  * This routine calculates the dynamic variable chunksize for a given DIF
9969  * object.  The calculation is not fool-proof, and can probably be tricked by
9970  * malicious DIF -- but it works for all compiler-generated DIF.  Because this
9971  * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
9972  * if a dynamic variable size exceeds the chunksize.
9973  */
9974 static void
9975 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9976 {
9977         uint64_t sval;
9978         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
9979         const dif_instr_t *text = dp->dtdo_buf;
9980         uint_t pc, srd = 0;
9981         uint_t ttop = 0;
9982         size_t size, ksize;
9983         uint_t id, i;
9984 
9985         for (pc = 0; pc < dp->dtdo_len; pc++) {
9986                 dif_instr_t instr = text[pc];
9987                 uint_t op = DIF_INSTR_OP(instr);
9988                 uint_t rd = DIF_INSTR_RD(instr);
9989                 uint_t r1 = DIF_INSTR_R1(instr);
9990                 uint_t nkeys = 0;
9991                 uchar_t scope;
9992 
9993                 dtrace_key_t *key = tupregs;
9994 
9995                 switch (op) {
9996                 case DIF_OP_SETX:
9997                         sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
9998                         srd = rd;
9999                         continue;
10000 
10001                 case DIF_OP_STTS:
10002                         key = &tupregs[DIF_DTR_NREGS];
10003                         key[0].dttk_size = 0;
10004                         key[1].dttk_size = 0;
10005                         nkeys = 2;
10006                         scope = DIFV_SCOPE_THREAD;
10007                         break;
10008 
10009                 case DIF_OP_STGAA:
10010                 case DIF_OP_STTAA:
10011                         nkeys = ttop;
10012 
10013                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
10014                                 key[nkeys++].dttk_size = 0;
10015 
10016                         key[nkeys++].dttk_size = 0;
10017 
10018                         if (op == DIF_OP_STTAA) {
10019                                 scope = DIFV_SCOPE_THREAD;
10020                         } else {
10021                                 scope = DIFV_SCOPE_GLOBAL;
10022                         }
10023 
10024                         break;
10025 
10026                 case DIF_OP_PUSHTR:
10027                         if (ttop == DIF_DTR_NREGS)
10028                                 return;
10029 
10030                         if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
10031                                 /*
10032                                  * If the register for the size of the "pushtr"
10033                                  * is %r0 (or the value is 0) and the type is
10034                                  * a string, we'll use the system-wide default
10035                                  * string size.
10036                                  */
10037                                 tupregs[ttop++].dttk_size =
10038                                     dtrace_strsize_default;
10039                         } else {
10040                                 if (srd == 0)
10041                                         return;
10042 
10043                                 if (sval > LONG_MAX)
10044                                         return;
10045 
10046                                 tupregs[ttop++].dttk_size = sval;
10047                         }
10048 
10049                         break;
10050 
10051                 case DIF_OP_PUSHTV:
10052                         if (ttop == DIF_DTR_NREGS)
10053                                 return;
10054 
10055                         tupregs[ttop++].dttk_size = 0;
10056                         break;
10057 
10058                 case DIF_OP_FLUSHTS:
10059                         ttop = 0;
10060                         break;
10061 
10062                 case DIF_OP_POPTS:
10063                         if (ttop != 0)
10064                                 ttop--;
10065                         break;
10066                 }
10067 
10068                 sval = 0;
10069                 srd = 0;
10070 
10071                 if (nkeys == 0)
10072                         continue;
10073 
10074                 /*
10075                  * We have a dynamic variable allocation; calculate its size.
10076                  */
10077                 for (ksize = 0, i = 0; i < nkeys; i++)
10078                         ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
10079 
10080                 size = sizeof (dtrace_dynvar_t);
10081                 size += sizeof (dtrace_key_t) * (nkeys - 1);
10082                 size += ksize;
10083 
10084                 /*
10085                  * Now we need to determine the size of the stored data.
10086                  */
10087                 id = DIF_INSTR_VAR(instr);
10088 
10089                 for (i = 0; i < dp->dtdo_varlen; i++) {
10090                         dtrace_difv_t *v = &dp->dtdo_vartab[i];
10091 
10092                         if (v->dtdv_id == id && v->dtdv_scope == scope) {
10093                                 size += v->dtdv_type.dtdt_size;
10094                                 break;
10095                         }
10096                 }
10097 
10098                 if (i == dp->dtdo_varlen)
10099                         return;
10100 
10101                 /*
10102                  * We have the size.  If this is larger than the chunk size
10103                  * for our dynamic variable state, reset the chunk size.
10104                  */
10105                 size = P2ROUNDUP(size, sizeof (uint64_t));
10106 
10107                 /*
10108                  * Before setting the chunk size, check that we're not going
10109                  * to set it to a negative value...
10110                  */
10111                 if (size > LONG_MAX)
10112                         return;
10113 
10114                 /*
10115                  * ...and make certain that we didn't badly overflow.
10116                  */
10117                 if (size < ksize || size < sizeof (dtrace_dynvar_t))
10118                         return;
10119 
10120                 if (size > vstate->dtvs_dynvars.dtds_chunksize)
10121                         vstate->dtvs_dynvars.dtds_chunksize = size;
10122         }
10123 }
10124 
10125 static void
10126 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
10127 {
10128         int i, oldsvars, osz, nsz, otlocals, ntlocals;
10129         uint_t id;
10130 
10131         ASSERT(MUTEX_HELD(&dtrace_lock));
10132         ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
10133 
10134         for (i = 0; i < dp->dtdo_varlen; i++) {
10135                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
10136                 dtrace_statvar_t *svar, ***svarp;
10137                 size_t dsize = 0;
10138                 uint8_t scope = v->dtdv_scope;
10139                 int *np;
10140 
10141                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
10142                         continue;
10143 
10144                 id -= DIF_VAR_OTHER_UBASE;
10145 
10146                 switch (scope) {
10147                 case DIFV_SCOPE_THREAD:
10148                         while (id >= (otlocals = vstate->dtvs_ntlocals)) {
10149                                 dtrace_difv_t *tlocals;
10150 
10151                                 if ((ntlocals = (otlocals << 1)) == 0)
10152                                         ntlocals = 1;
10153 
10154                                 osz = otlocals * sizeof (dtrace_difv_t);
10155                                 nsz = ntlocals * sizeof (dtrace_difv_t);
10156 
10157                                 tlocals = kmem_zalloc(nsz, KM_SLEEP);
10158 
10159                                 if (osz != 0) {
10160                                         bcopy(vstate->dtvs_tlocals,
10161                                             tlocals, osz);
10162                                         kmem_free(vstate->dtvs_tlocals, osz);
10163                                 }
10164 
10165                                 vstate->dtvs_tlocals = tlocals;
10166                                 vstate->dtvs_ntlocals = ntlocals;
10167                         }
10168 
10169                         vstate->dtvs_tlocals[id] = *v;
10170                         continue;
10171 
10172                 case DIFV_SCOPE_LOCAL:
10173                         np = &vstate->dtvs_nlocals;
10174                         svarp = &vstate->dtvs_locals;
10175 
10176                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
10177                                 dsize = NCPU * (v->dtdv_type.dtdt_size +
10178                                     sizeof (uint64_t));
10179                         else
10180                                 dsize = NCPU * sizeof (uint64_t);
10181 
10182                         break;
10183 
10184                 case DIFV_SCOPE_GLOBAL:
10185                         np = &vstate->dtvs_nglobals;
10186                         svarp = &vstate->dtvs_globals;
10187 
10188                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
10189                                 dsize = v->dtdv_type.dtdt_size +
10190                                     sizeof (uint64_t);
10191 
10192                         break;
10193 
10194                 default:
10195                         ASSERT(0);
10196                 }
10197 
10198                 while (id >= (oldsvars = *np)) {
10199                         dtrace_statvar_t **statics;
10200                         int newsvars, oldsize, newsize;
10201 
10202                         if ((newsvars = (oldsvars << 1)) == 0)
10203                                 newsvars = 1;
10204 
10205                         oldsize = oldsvars * sizeof (dtrace_statvar_t *);
10206                         newsize = newsvars * sizeof (dtrace_statvar_t *);
10207 
10208                         statics = kmem_zalloc(newsize, KM_SLEEP);
10209 
10210                         if (oldsize != 0) {
10211                                 bcopy(*svarp, statics, oldsize);
10212                                 kmem_free(*svarp, oldsize);
10213                         }
10214 
10215                         *svarp = statics;
10216                         *np = newsvars;
10217                 }
10218 
10219                 if ((svar = (*svarp)[id]) == NULL) {
10220                         svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
10221                         svar->dtsv_var = *v;
10222 
10223                         if ((svar->dtsv_size = dsize) != 0) {
10224                                 svar->dtsv_data = (uint64_t)(uintptr_t)
10225                                     kmem_zalloc(dsize, KM_SLEEP);
10226                         }
10227 
10228                         (*svarp)[id] = svar;
10229                 }
10230 
10231                 svar->dtsv_refcnt++;
10232         }
10233 
10234         dtrace_difo_chunksize(dp, vstate);
10235         dtrace_difo_hold(dp);
10236 }
10237 
10238 static dtrace_difo_t *
10239 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
10240 {
10241         dtrace_difo_t *new;
10242         size_t sz;
10243 
10244         ASSERT(dp->dtdo_buf != NULL);
10245         ASSERT(dp->dtdo_refcnt != 0);
10246 
10247         new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
10248 
10249         ASSERT(dp->dtdo_buf != NULL);
10250         sz = dp->dtdo_len * sizeof (dif_instr_t);
10251         new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
10252         bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
10253         new->dtdo_len = dp->dtdo_len;
10254 
10255         if (dp->dtdo_strtab != NULL) {
10256                 ASSERT(dp->dtdo_strlen != 0);
10257                 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
10258                 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
10259                 new->dtdo_strlen = dp->dtdo_strlen;
10260         }
10261 
10262         if (dp->dtdo_inttab != NULL) {
10263                 ASSERT(dp->dtdo_intlen != 0);
10264                 sz = dp->dtdo_intlen * sizeof (uint64_t);
10265                 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
10266                 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
10267                 new->dtdo_intlen = dp->dtdo_intlen;
10268         }
10269 
10270         if (dp->dtdo_vartab != NULL) {
10271                 ASSERT(dp->dtdo_varlen != 0);
10272                 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
10273                 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
10274                 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
10275                 new->dtdo_varlen = dp->dtdo_varlen;
10276         }
10277 
10278         dtrace_difo_init(new, vstate);
10279         return (new);
10280 }
10281 
10282 static void
10283 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
10284 {
10285         int i;
10286 
10287         ASSERT(dp->dtdo_refcnt == 0);
10288 
10289         for (i = 0; i < dp->dtdo_varlen; i++) {
10290                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
10291                 dtrace_statvar_t *svar, **svarp;
10292                 uint_t id;
10293                 uint8_t scope = v->dtdv_scope;
10294                 int *np;
10295 
10296                 switch (scope) {
10297                 case DIFV_SCOPE_THREAD:
10298                         continue;
10299 
10300                 case DIFV_SCOPE_LOCAL:
10301                         np = &vstate->dtvs_nlocals;
10302                         svarp = vstate->dtvs_locals;
10303                         break;
10304 
10305                 case DIFV_SCOPE_GLOBAL:
10306                         np = &vstate->dtvs_nglobals;
10307                         svarp = vstate->dtvs_globals;
10308                         break;
10309 
10310                 default:
10311                         ASSERT(0);
10312                 }
10313 
10314                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
10315                         continue;
10316 
10317                 id -= DIF_VAR_OTHER_UBASE;
10318                 ASSERT(id < *np);
10319 
10320                 svar = svarp[id];
10321                 ASSERT(svar != NULL);
10322                 ASSERT(svar->dtsv_refcnt > 0);
10323 
10324                 if (--svar->dtsv_refcnt > 0)
10325                         continue;
10326 
10327                 if (svar->dtsv_size != 0) {
10328                         ASSERT(svar->dtsv_data != 0);
10329                         kmem_free((void *)(uintptr_t)svar->dtsv_data,
10330                             svar->dtsv_size);
10331                 }
10332 
10333                 kmem_free(svar, sizeof (dtrace_statvar_t));
10334                 svarp[id] = NULL;
10335         }
10336 
10337         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
10338         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
10339         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
10340         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
10341 
10342         kmem_free(dp, sizeof (dtrace_difo_t));
10343 }
10344 
10345 static void
10346 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
10347 {
10348         int i;
10349 
10350         ASSERT(MUTEX_HELD(&dtrace_lock));
10351         ASSERT(dp->dtdo_refcnt != 0);
10352 
10353         for (i = 0; i < dp->dtdo_varlen; i++) {
10354                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
10355 
10356                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
10357                         continue;
10358 
10359                 ASSERT(dtrace_vtime_references > 0);
10360                 if (--dtrace_vtime_references == 0)
10361                         dtrace_vtime_disable();
10362         }
10363 
10364         if (--dp->dtdo_refcnt == 0)
10365                 dtrace_difo_destroy(dp, vstate);
10366 }
10367 
10368 /*
10369  * DTrace Format Functions
10370  */
10371 static uint16_t
10372 dtrace_format_add(dtrace_state_t *state, char *str)
10373 {
10374         char *fmt, **new;
10375         uint16_t ndx, len = strlen(str) + 1;
10376 
10377         fmt = kmem_zalloc(len, KM_SLEEP);
10378         bcopy(str, fmt, len);
10379 
10380         for (ndx = 0; ndx < state->dts_nformats; ndx++) {
10381                 if (state->dts_formats[ndx] == NULL) {
10382                         state->dts_formats[ndx] = fmt;
10383                         return (ndx + 1);
10384                 }
10385         }
10386 
10387         if (state->dts_nformats == USHRT_MAX) {
10388                 /*
10389                  * This is only likely if a denial-of-service attack is being
10390                  * attempted.  As such, it's okay to fail silently here.
10391                  */
10392                 kmem_free(fmt, len);
10393                 return (0);
10394         }
10395 
10396         /*
10397          * For simplicity, we always resize the formats array to be exactly the
10398          * number of formats.
10399          */
10400         ndx = state->dts_nformats++;
10401         new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
10402 
10403         if (state->dts_formats != NULL) {
10404                 ASSERT(ndx != 0);
10405                 bcopy(state->dts_formats, new, ndx * sizeof (char *));
10406                 kmem_free(state->dts_formats, ndx * sizeof (char *));
10407         }
10408 
10409         state->dts_formats = new;
10410         state->dts_formats[ndx] = fmt;
10411 
10412         return (ndx + 1);
10413 }
10414 
10415 static void
10416 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
10417 {
10418         char *fmt;
10419 
10420         ASSERT(state->dts_formats != NULL);
10421         ASSERT(format <= state->dts_nformats);
10422         ASSERT(state->dts_formats[format - 1] != NULL);
10423 
10424         fmt = state->dts_formats[format - 1];
10425         kmem_free(fmt, strlen(fmt) + 1);
10426         state->dts_formats[format - 1] = NULL;
10427 }
10428 
10429 static void
10430 dtrace_format_destroy(dtrace_state_t *state)
10431 {
10432         int i;
10433 
10434         if (state->dts_nformats == 0) {
10435                 ASSERT(state->dts_formats == NULL);
10436                 return;
10437         }
10438 
10439         ASSERT(state->dts_formats != NULL);
10440 
10441         for (i = 0; i < state->dts_nformats; i++) {
10442                 char *fmt = state->dts_formats[i];
10443 
10444                 if (fmt == NULL)
10445                         continue;
10446 
10447                 kmem_free(fmt, strlen(fmt) + 1);
10448         }
10449 
10450         kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
10451         state->dts_nformats = 0;
10452         state->dts_formats = NULL;
10453 }
10454 
10455 /*
10456  * DTrace Predicate Functions
10457  */
10458 static dtrace_predicate_t *
10459 dtrace_predicate_create(dtrace_difo_t *dp)
10460 {
10461         dtrace_predicate_t *pred;
10462 
10463         ASSERT(MUTEX_HELD(&dtrace_lock));
10464         ASSERT(dp->dtdo_refcnt != 0);
10465 
10466         pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
10467         pred->dtp_difo = dp;
10468         pred->dtp_refcnt = 1;
10469 
10470         if (!dtrace_difo_cacheable(dp))
10471                 return (pred);
10472 
10473         if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
10474                 /*
10475                  * This is only theoretically possible -- we have had 2^32
10476                  * cacheable predicates on this machine.  We cannot allow any
10477                  * more predicates to become cacheable:  as unlikely as it is,
10478                  * there may be a thread caching a (now stale) predicate cache
10479                  * ID. (N.B.: the temptation is being successfully resisted to
10480                  * have this cmn_err() "Holy shit -- we executed this code!")
10481                  */
10482                 return (pred);
10483         }
10484 
10485         pred->dtp_cacheid = dtrace_predcache_id++;
10486 
10487         return (pred);
10488 }
10489 
10490 static void
10491 dtrace_predicate_hold(dtrace_predicate_t *pred)
10492 {
10493         ASSERT(MUTEX_HELD(&dtrace_lock));
10494         ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
10495         ASSERT(pred->dtp_refcnt > 0);
10496 
10497         pred->dtp_refcnt++;
10498 }
10499 
10500 static void
10501 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
10502 {
10503         dtrace_difo_t *dp = pred->dtp_difo;
10504 
10505         ASSERT(MUTEX_HELD(&dtrace_lock));
10506         ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
10507         ASSERT(pred->dtp_refcnt > 0);
10508 
10509         if (--pred->dtp_refcnt == 0) {
10510                 dtrace_difo_release(pred->dtp_difo, vstate);
10511                 kmem_free(pred, sizeof (dtrace_predicate_t));
10512         }
10513 }
10514 
10515 /*
10516  * DTrace Action Description Functions
10517  */
10518 static dtrace_actdesc_t *
10519 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
10520     uint64_t uarg, uint64_t arg)
10521 {
10522         dtrace_actdesc_t *act;
10523 
10524         ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != 0 &&
10525             arg >= KERNELBASE) || (arg == 0 && kind == DTRACEACT_PRINTA));
10526 
10527         act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
10528         act->dtad_kind = kind;
10529         act->dtad_ntuple = ntuple;
10530         act->dtad_uarg = uarg;
10531         act->dtad_arg = arg;
10532         act->dtad_refcnt = 1;
10533 
10534         return (act);
10535 }
10536 
10537 static void
10538 dtrace_actdesc_hold(dtrace_actdesc_t *act)
10539 {
10540         ASSERT(act->dtad_refcnt >= 1);
10541         act->dtad_refcnt++;
10542 }
10543 
10544 static void
10545 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
10546 {
10547         dtrace_actkind_t kind = act->dtad_kind;
10548         dtrace_difo_t *dp;
10549 
10550         ASSERT(act->dtad_refcnt >= 1);
10551 
10552         if (--act->dtad_refcnt != 0)
10553                 return;
10554 
10555         if ((dp = act->dtad_difo) != NULL)
10556                 dtrace_difo_release(dp, vstate);
10557 
10558         if (DTRACEACT_ISPRINTFLIKE(kind)) {
10559                 char *str = (char *)(uintptr_t)act->dtad_arg;
10560 
10561                 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
10562                     (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
10563 
10564                 if (str != NULL)
10565                         kmem_free(str, strlen(str) + 1);
10566         }
10567 
10568         kmem_free(act, sizeof (dtrace_actdesc_t));
10569 }
10570 
10571 /*
10572  * DTrace ECB Functions
10573  */
10574 static dtrace_ecb_t *
10575 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
10576 {
10577         dtrace_ecb_t *ecb;
10578         dtrace_epid_t epid;
10579 
10580         ASSERT(MUTEX_HELD(&dtrace_lock));
10581 
10582         ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
10583         ecb->dte_predicate = NULL;
10584         ecb->dte_probe = probe;
10585 
10586         /*
10587          * The default size is the size of the default action: recording
10588          * the header.
10589          */
10590         ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
10591         ecb->dte_alignment = sizeof (dtrace_epid_t);
10592 
10593         epid = state->dts_epid++;
10594 
10595         if (epid - 1 >= state->dts_necbs) {
10596                 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
10597                 int necbs = state->dts_necbs << 1;
10598 
10599                 ASSERT(epid == state->dts_necbs + 1);
10600 
10601                 if (necbs == 0) {
10602                         ASSERT(oecbs == NULL);
10603                         necbs = 1;
10604                 }
10605 
10606                 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
10607 
10608                 if (oecbs != NULL)
10609                         bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
10610 
10611                 dtrace_membar_producer();
10612                 state->dts_ecbs = ecbs;
10613 
10614                 if (oecbs != NULL) {
10615                         /*
10616                          * If this state is active, we must dtrace_sync()
10617                          * before we can free the old dts_ecbs array:  we're
10618                          * coming in hot, and there may be active ring
10619                          * buffer processing (which indexes into the dts_ecbs
10620                          * array) on another CPU.
10621                          */
10622                         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
10623                                 dtrace_sync();
10624 
10625                         kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
10626                 }
10627 
10628                 dtrace_membar_producer();
10629                 state->dts_necbs = necbs;
10630         }
10631 
10632         ecb->dte_state = state;
10633 
10634         ASSERT(state->dts_ecbs[epid - 1] == NULL);
10635         dtrace_membar_producer();
10636         state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
10637 
10638         return (ecb);
10639 }
10640 
10641 static int
10642 dtrace_ecb_enable(dtrace_ecb_t *ecb)
10643 {
10644         dtrace_probe_t *probe = ecb->dte_probe;
10645 
10646         ASSERT(MUTEX_HELD(&cpu_lock));
10647         ASSERT(MUTEX_HELD(&dtrace_lock));
10648         ASSERT(ecb->dte_next == NULL);
10649 
10650         if (probe == NULL) {
10651                 /*
10652                  * This is the NULL probe -- there's nothing to do.
10653                  */
10654                 return (0);
10655         }
10656 
10657         if (probe->dtpr_ecb == NULL) {
10658                 dtrace_provider_t *prov = probe->dtpr_provider;
10659 
10660                 /*
10661                  * We're the first ECB on this probe.
10662                  */
10663                 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
10664 
10665                 if (ecb->dte_predicate != NULL)
10666                         probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
10667 
10668                 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
10669                     probe->dtpr_id, probe->dtpr_arg));
10670         } else {
10671                 /*
10672                  * This probe is already active.  Swing the last pointer to
10673                  * point to the new ECB, and issue a dtrace_sync() to assure
10674                  * that all CPUs have seen the change.
10675                  */
10676                 ASSERT(probe->dtpr_ecb_last != NULL);
10677                 probe->dtpr_ecb_last->dte_next = ecb;
10678                 probe->dtpr_ecb_last = ecb;
10679                 probe->dtpr_predcache = 0;
10680 
10681                 dtrace_sync();
10682                 return (0);
10683         }
10684 }
10685 
10686 static int
10687 dtrace_ecb_resize(dtrace_ecb_t *ecb)
10688 {
10689         dtrace_action_t *act;
10690         uint32_t curneeded = UINT32_MAX;
10691         uint32_t aggbase = UINT32_MAX;
10692 
10693         /*
10694          * If we record anything, we always record the dtrace_rechdr_t.  (And
10695          * we always record it first.)
10696          */
10697         ecb->dte_size = sizeof (dtrace_rechdr_t);
10698         ecb->dte_alignment = sizeof (dtrace_epid_t);
10699 
10700         for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
10701                 dtrace_recdesc_t *rec = &act->dta_rec;
10702                 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
10703 
10704                 ecb->dte_alignment = MAX(ecb->dte_alignment,
10705                     rec->dtrd_alignment);
10706 
10707                 if (DTRACEACT_ISAGG(act->dta_kind)) {
10708                         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10709 
10710                         ASSERT(rec->dtrd_size != 0);
10711                         ASSERT(agg->dtag_first != NULL);
10712                         ASSERT(act->dta_prev->dta_intuple);
10713                         ASSERT(aggbase != UINT32_MAX);
10714                         ASSERT(curneeded != UINT32_MAX);
10715 
10716                         agg->dtag_base = aggbase;
10717 
10718                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10719                         rec->dtrd_offset = curneeded;
10720                         if (curneeded + rec->dtrd_size < curneeded)
10721                                 return (EINVAL);
10722                         curneeded += rec->dtrd_size;
10723                         ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
10724 
10725                         aggbase = UINT32_MAX;
10726                         curneeded = UINT32_MAX;
10727                 } else if (act->dta_intuple) {
10728                         if (curneeded == UINT32_MAX) {
10729                                 /*
10730                                  * This is the first record in a tuple.  Align
10731                                  * curneeded to be at offset 4 in an 8-byte
10732                                  * aligned block.
10733                                  */
10734                                 ASSERT(act->dta_prev == NULL ||
10735                                     !act->dta_prev->dta_intuple);
10736                                 ASSERT3U(aggbase, ==, UINT32_MAX);
10737                                 curneeded = P2PHASEUP(ecb->dte_size,
10738                                     sizeof (uint64_t), sizeof (dtrace_aggid_t));
10739 
10740                                 aggbase = curneeded - sizeof (dtrace_aggid_t);
10741                                 ASSERT(IS_P2ALIGNED(aggbase,
10742                                     sizeof (uint64_t)));
10743                         }
10744                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
10745                         rec->dtrd_offset = curneeded;
10746                         if (curneeded + rec->dtrd_size < curneeded)
10747                                 return (EINVAL);
10748                         curneeded += rec->dtrd_size;
10749                 } else {
10750                         /* tuples must be followed by an aggregation */
10751                         ASSERT(act->dta_prev == NULL ||
10752                             !act->dta_prev->dta_intuple);
10753 
10754                         ecb->dte_size = P2ROUNDUP(ecb->dte_size,
10755                             rec->dtrd_alignment);
10756                         rec->dtrd_offset = ecb->dte_size;
10757                         if (ecb->dte_size + rec->dtrd_size < ecb->dte_size)
10758                                 return (EINVAL);
10759                         ecb->dte_size += rec->dtrd_size;
10760                         ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
10761                 }
10762         }
10763 
10764         if ((act = ecb->dte_action) != NULL &&
10765             !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
10766             ecb->dte_size == sizeof (dtrace_rechdr_t)) {
10767                 /*
10768                  * If the size is still sizeof (dtrace_rechdr_t), then all
10769                  * actions store no data; set the size to 0.
10770                  */
10771                 ecb->dte_size = 0;
10772         }
10773 
10774         ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
10775         ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
10776         ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
10777             ecb->dte_needed);
10778         return (0);
10779 }
10780 
10781 static dtrace_action_t *
10782 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10783 {
10784         dtrace_aggregation_t *agg;
10785         size_t size = sizeof (uint64_t);
10786         int ntuple = desc->dtad_ntuple;
10787         dtrace_action_t *act;
10788         dtrace_recdesc_t *frec;
10789         dtrace_aggid_t aggid;
10790         dtrace_state_t *state = ecb->dte_state;
10791 
10792         agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
10793         agg->dtag_ecb = ecb;
10794 
10795         ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
10796 
10797         switch (desc->dtad_kind) {
10798         case DTRACEAGG_MIN:
10799                 agg->dtag_initial = INT64_MAX;
10800                 agg->dtag_aggregate = dtrace_aggregate_min;
10801                 break;
10802 
10803         case DTRACEAGG_MAX:
10804                 agg->dtag_initial = INT64_MIN;
10805                 agg->dtag_aggregate = dtrace_aggregate_max;
10806                 break;
10807 
10808         case DTRACEAGG_COUNT:
10809                 agg->dtag_aggregate = dtrace_aggregate_count;
10810                 break;
10811 
10812         case DTRACEAGG_QUANTIZE:
10813                 agg->dtag_aggregate = dtrace_aggregate_quantize;
10814                 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
10815                     sizeof (uint64_t);
10816                 break;
10817 
10818         case DTRACEAGG_LQUANTIZE: {
10819                 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
10820                 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
10821 
10822                 agg->dtag_initial = desc->dtad_arg;
10823                 agg->dtag_aggregate = dtrace_aggregate_lquantize;
10824 
10825                 if (step == 0 || levels == 0)
10826                         goto err;
10827 
10828                 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
10829                 break;
10830         }
10831 
10832         case DTRACEAGG_LLQUANTIZE: {
10833                 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
10834                 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
10835                 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
10836                 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
10837                 int64_t v;
10838 
10839                 agg->dtag_initial = desc->dtad_arg;
10840                 agg->dtag_aggregate = dtrace_aggregate_llquantize;
10841 
10842                 if (factor < 2 || low >= high || nsteps < factor)
10843                         goto err;
10844 
10845                 /*
10846                  * Now check that the number of steps evenly divides a power
10847                  * of the factor.  (This assures both integer bucket size and
10848                  * linearity within each magnitude.)
10849                  */
10850                 for (v = factor; v < nsteps; v *= factor)
10851                         continue;
10852 
10853                 if ((v % nsteps) || (nsteps % factor))
10854                         goto err;
10855 
10856                 size = (dtrace_aggregate_llquantize_bucket(factor,
10857                     low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
10858                 break;
10859         }
10860 
10861         case DTRACEAGG_AVG:
10862                 agg->dtag_aggregate = dtrace_aggregate_avg;
10863                 size = sizeof (uint64_t) * 2;
10864                 break;
10865 
10866         case DTRACEAGG_STDDEV:
10867                 agg->dtag_aggregate = dtrace_aggregate_stddev;
10868                 size = sizeof (uint64_t) * 4;
10869                 break;
10870 
10871         case DTRACEAGG_SUM:
10872                 agg->dtag_aggregate = dtrace_aggregate_sum;
10873                 break;
10874 
10875         default:
10876                 goto err;
10877         }
10878 
10879         agg->dtag_action.dta_rec.dtrd_size = size;
10880 
10881         if (ntuple == 0)
10882                 goto err;
10883 
10884         /*
10885          * We must make sure that we have enough actions for the n-tuple.
10886          */
10887         for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
10888                 if (DTRACEACT_ISAGG(act->dta_kind))
10889                         break;
10890 
10891                 if (--ntuple == 0) {
10892                         /*
10893                          * This is the action with which our n-tuple begins.
10894                          */
10895                         agg->dtag_first = act;
10896                         goto success;
10897                 }
10898         }
10899 
10900         /*
10901          * This n-tuple is short by ntuple elements.  Return failure.
10902          */
10903         ASSERT(ntuple != 0);
10904 err:
10905         kmem_free(agg, sizeof (dtrace_aggregation_t));
10906         return (NULL);
10907 
10908 success:
10909         /*
10910          * If the last action in the tuple has a size of zero, it's actually
10911          * an expression argument for the aggregating action.
10912          */
10913         ASSERT(ecb->dte_action_last != NULL);
10914         act = ecb->dte_action_last;
10915 
10916         if (act->dta_kind == DTRACEACT_DIFEXPR) {
10917                 ASSERT(act->dta_difo != NULL);
10918 
10919                 if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
10920                         agg->dtag_hasarg = 1;
10921         }
10922 
10923         /*
10924          * We need to allocate an id for this aggregation.
10925          */
10926         aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
10927             VM_BESTFIT | VM_SLEEP);
10928 
10929         if (aggid - 1 >= state->dts_naggregations) {
10930                 dtrace_aggregation_t **oaggs = state->dts_aggregations;
10931                 dtrace_aggregation_t **aggs;
10932                 int naggs = state->dts_naggregations << 1;
10933                 int onaggs = state->dts_naggregations;
10934 
10935                 ASSERT(aggid == state->dts_naggregations + 1);
10936 
10937                 if (naggs == 0) {
10938                         ASSERT(oaggs == NULL);
10939                         naggs = 1;
10940                 }
10941 
10942                 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
10943 
10944                 if (oaggs != NULL) {
10945                         bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
10946                         kmem_free(oaggs, onaggs * sizeof (*aggs));
10947                 }
10948 
10949                 state->dts_aggregations = aggs;
10950                 state->dts_naggregations = naggs;
10951         }
10952 
10953         ASSERT(state->dts_aggregations[aggid - 1] == NULL);
10954         state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
10955 
10956         frec = &agg->dtag_first->dta_rec;
10957         if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
10958                 frec->dtrd_alignment = sizeof (dtrace_aggid_t);
10959 
10960         for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
10961                 ASSERT(!act->dta_intuple);
10962                 act->dta_intuple = 1;
10963         }
10964 
10965         return (&agg->dtag_action);
10966 }
10967 
10968 static void
10969 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
10970 {
10971         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
10972         dtrace_state_t *state = ecb->dte_state;
10973         dtrace_aggid_t aggid = agg->dtag_id;
10974 
10975         ASSERT(DTRACEACT_ISAGG(act->dta_kind));
10976         vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
10977 
10978         ASSERT(state->dts_aggregations[aggid - 1] == agg);
10979         state->dts_aggregations[aggid - 1] = NULL;
10980 
10981         kmem_free(agg, sizeof (dtrace_aggregation_t));
10982 }
10983 
10984 static int
10985 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
10986 {
10987         dtrace_action_t *action, *last;
10988         dtrace_difo_t *dp = desc->dtad_difo;
10989         uint32_t size = 0, align = sizeof (uint8_t), mask;
10990         uint16_t format = 0;
10991         dtrace_recdesc_t *rec;
10992         dtrace_state_t *state = ecb->dte_state;
10993         dtrace_optval_t *opt = state->dts_options, nframes, strsize;
10994         uint64_t arg = desc->dtad_arg;
10995 
10996         ASSERT(MUTEX_HELD(&dtrace_lock));
10997         ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
10998 
10999         if (DTRACEACT_ISAGG(desc->dtad_kind)) {
11000                 /*
11001                  * If this is an aggregating action, there must be neither
11002                  * a speculate nor a commit on the action chain.
11003                  */
11004                 dtrace_action_t *act;
11005 
11006                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
11007                         if (act->dta_kind == DTRACEACT_COMMIT)
11008                                 return (EINVAL);
11009 
11010                         if (act->dta_kind == DTRACEACT_SPECULATE)
11011                                 return (EINVAL);
11012                 }
11013 
11014                 action = dtrace_ecb_aggregation_create(ecb, desc);
11015 
11016                 if (action == NULL)
11017                         return (EINVAL);
11018         } else {
11019                 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
11020                     (desc->dtad_kind == DTRACEACT_DIFEXPR &&
11021                     dp != NULL && dp->dtdo_destructive)) {
11022                         state->dts_destructive = 1;
11023                 }
11024 
11025                 switch (desc->dtad_kind) {
11026                 case DTRACEACT_PRINTF:
11027                 case DTRACEACT_PRINTA:
11028                 case DTRACEACT_SYSTEM:
11029                 case DTRACEACT_FREOPEN:
11030                 case DTRACEACT_DIFEXPR:
11031                         /*
11032                          * We know that our arg is a string -- turn it into a
11033                          * format.
11034                          */
11035                         if (arg == 0) {
11036                                 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
11037                                     desc->dtad_kind == DTRACEACT_DIFEXPR);
11038                                 format = 0;
11039                         } else {
11040                                 ASSERT(arg != 0);
11041                                 ASSERT(arg > KERNELBASE);
11042                                 format = dtrace_format_add(state,
11043                                     (char *)(uintptr_t)arg);
11044                         }
11045 
11046                         /*FALLTHROUGH*/
11047                 case DTRACEACT_LIBACT:
11048                 case DTRACEACT_TRACEMEM:
11049                 case DTRACEACT_TRACEMEM_DYNSIZE:
11050                         if (dp == NULL)
11051                                 return (EINVAL);
11052 
11053                         if ((size = dp->dtdo_rtype.dtdt_size) != 0)
11054                                 break;
11055 
11056                         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
11057                                 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
11058                                         return (EINVAL);
11059 
11060                                 size = opt[DTRACEOPT_STRSIZE];
11061                         }
11062 
11063                         break;
11064 
11065                 case DTRACEACT_STACK:
11066                         if ((nframes = arg) == 0) {
11067                                 nframes = opt[DTRACEOPT_STACKFRAMES];
11068                                 ASSERT(nframes > 0);
11069                                 arg = nframes;
11070                         }
11071 
11072                         size = nframes * sizeof (pc_t);
11073                         break;
11074 
11075                 case DTRACEACT_JSTACK:
11076                         if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
11077                                 strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
11078 
11079                         if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
11080                                 nframes = opt[DTRACEOPT_JSTACKFRAMES];
11081 
11082                         arg = DTRACE_USTACK_ARG(nframes, strsize);
11083 
11084                         /*FALLTHROUGH*/
11085                 case DTRACEACT_USTACK:
11086                         if (desc->dtad_kind != DTRACEACT_JSTACK &&
11087                             (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
11088                                 strsize = DTRACE_USTACK_STRSIZE(arg);
11089                                 nframes = opt[DTRACEOPT_USTACKFRAMES];
11090                                 ASSERT(nframes > 0);
11091                                 arg = DTRACE_USTACK_ARG(nframes, strsize);
11092                         }
11093 
11094                         /*
11095                          * Save a slot for the pid.
11096                          */
11097                         size = (nframes + 1) * sizeof (uint64_t);
11098                         size += DTRACE_USTACK_STRSIZE(arg);
11099                         size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
11100 
11101                         break;
11102 
11103                 case DTRACEACT_SYM:
11104                 case DTRACEACT_MOD:
11105                         if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
11106                             sizeof (uint64_t)) ||
11107                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
11108                                 return (EINVAL);
11109                         break;
11110 
11111                 case DTRACEACT_USYM:
11112                 case DTRACEACT_UMOD:
11113                 case DTRACEACT_UADDR:
11114                         if (dp == NULL ||
11115                             (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
11116                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
11117                                 return (EINVAL);
11118 
11119                         /*
11120                          * We have a slot for the pid, plus a slot for the
11121                          * argument.  To keep things simple (aligned with
11122                          * bitness-neutral sizing), we store each as a 64-bit
11123                          * quantity.
11124                          */
11125                         size = 2 * sizeof (uint64_t);
11126                         break;
11127 
11128                 case DTRACEACT_STOP:
11129                 case DTRACEACT_BREAKPOINT:
11130                 case DTRACEACT_PANIC:
11131                         break;
11132 
11133                 case DTRACEACT_CHILL:
11134                 case DTRACEACT_DISCARD:
11135                 case DTRACEACT_RAISE:
11136                         if (dp == NULL)
11137                                 return (EINVAL);
11138                         break;
11139 
11140                 case DTRACEACT_EXIT:
11141                         if (dp == NULL ||
11142                             (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
11143                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
11144                                 return (EINVAL);
11145                         break;
11146 
11147                 case DTRACEACT_SPECULATE:
11148                         if (ecb->dte_size > sizeof (dtrace_rechdr_t))
11149                                 return (EINVAL);
11150 
11151                         if (dp == NULL)
11152                                 return (EINVAL);
11153 
11154                         state->dts_speculates = 1;
11155                         break;
11156 
11157                 case DTRACEACT_COMMIT: {
11158                         dtrace_action_t *act = ecb->dte_action;
11159 
11160                         for (; act != NULL; act = act->dta_next) {
11161                                 if (act->dta_kind == DTRACEACT_COMMIT)
11162                                         return (EINVAL);
11163                         }
11164 
11165                         if (dp == NULL)
11166                                 return (EINVAL);
11167                         break;
11168                 }
11169 
11170                 default:
11171                         return (EINVAL);
11172                 }
11173 
11174                 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
11175                         /*
11176                          * If this is a data-storing action or a speculate,
11177                          * we must be sure that there isn't a commit on the
11178                          * action chain.
11179                          */
11180                         dtrace_action_t *act = ecb->dte_action;
11181 
11182                         for (; act != NULL; act = act->dta_next) {
11183                                 if (act->dta_kind == DTRACEACT_COMMIT)
11184                                         return (EINVAL);
11185                         }
11186                 }
11187 
11188                 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
11189                 action->dta_rec.dtrd_size = size;
11190         }
11191 
11192         action->dta_refcnt = 1;
11193         rec = &action->dta_rec;
11194         size = rec->dtrd_size;
11195 
11196         for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
11197                 if (!(size & mask)) {
11198                         align = mask + 1;
11199                         break;
11200                 }
11201         }
11202 
11203         action->dta_kind = desc->dtad_kind;
11204 
11205         if ((action->dta_difo = dp) != NULL)
11206                 dtrace_difo_hold(dp);
11207 
11208         rec->dtrd_action = action->dta_kind;
11209         rec->dtrd_arg = arg;
11210         rec->dtrd_uarg = desc->dtad_uarg;
11211         rec->dtrd_alignment = (uint16_t)align;
11212         rec->dtrd_format = format;
11213 
11214         if ((last = ecb->dte_action_last) != NULL) {
11215                 ASSERT(ecb->dte_action != NULL);
11216                 action->dta_prev = last;
11217                 last->dta_next = action;
11218         } else {
11219                 ASSERT(ecb->dte_action == NULL);
11220                 ecb->dte_action = action;
11221         }
11222 
11223         ecb->dte_action_last = action;
11224 
11225         return (0);
11226 }
11227 
11228 static void
11229 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
11230 {
11231         dtrace_action_t *act = ecb->dte_action, *next;
11232         dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
11233         dtrace_difo_t *dp;
11234         uint16_t format;
11235 
11236         if (act != NULL && act->dta_refcnt > 1) {
11237                 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
11238                 act->dta_refcnt--;
11239         } else {
11240                 for (; act != NULL; act = next) {
11241                         next = act->dta_next;
11242                         ASSERT(next != NULL || act == ecb->dte_action_last);
11243                         ASSERT(act->dta_refcnt == 1);
11244 
11245                         if ((format = act->dta_rec.dtrd_format) != 0)
11246                                 dtrace_format_remove(ecb->dte_state, format);
11247 
11248                         if ((dp = act->dta_difo) != NULL)
11249                                 dtrace_difo_release(dp, vstate);
11250 
11251                         if (DTRACEACT_ISAGG(act->dta_kind)) {
11252                                 dtrace_ecb_aggregation_destroy(ecb, act);
11253                         } else {
11254                                 kmem_free(act, sizeof (dtrace_action_t));
11255                         }
11256                 }
11257         }
11258 
11259         ecb->dte_action = NULL;
11260         ecb->dte_action_last = NULL;
11261         ecb->dte_size = 0;
11262 }
11263 
11264 static void
11265 dtrace_ecb_disable(dtrace_ecb_t *ecb)
11266 {
11267         /*
11268          * We disable the ECB by removing it from its probe.
11269          */
11270         dtrace_ecb_t *pecb, *prev = NULL;
11271         dtrace_probe_t *probe = ecb->dte_probe;
11272 
11273         ASSERT(MUTEX_HELD(&dtrace_lock));
11274 
11275         if (probe == NULL) {
11276                 /*
11277                  * This is the NULL probe; there is nothing to disable.
11278                  */
11279                 return;
11280         }
11281 
11282         for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
11283                 if (pecb == ecb)
11284                         break;
11285                 prev = pecb;
11286         }
11287 
11288         ASSERT(pecb != NULL);
11289 
11290         if (prev == NULL) {
11291                 probe->dtpr_ecb = ecb->dte_next;
11292         } else {
11293                 prev->dte_next = ecb->dte_next;
11294         }
11295 
11296         if (ecb == probe->dtpr_ecb_last) {
11297                 ASSERT(ecb->dte_next == NULL);
11298                 probe->dtpr_ecb_last = prev;
11299         }
11300 
11301         /*
11302          * The ECB has been disconnected from the probe; now sync to assure
11303          * that all CPUs have seen the change before returning.
11304          */
11305         dtrace_sync();
11306 
11307         if (probe->dtpr_ecb == NULL) {
11308                 /*
11309                  * That was the last ECB on the probe; clear the predicate
11310                  * cache ID for the probe, disable it and sync one more time
11311                  * to assure that we'll never hit it again.
11312                  */
11313                 dtrace_provider_t *prov = probe->dtpr_provider;
11314 
11315                 ASSERT(ecb->dte_next == NULL);
11316                 ASSERT(probe->dtpr_ecb_last == NULL);
11317                 probe->dtpr_predcache = DTRACE_CACHEIDNONE;
11318                 prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
11319                     probe->dtpr_id, probe->dtpr_arg);
11320                 dtrace_sync();
11321         } else {
11322                 /*
11323                  * There is at least one ECB remaining on the probe.  If there
11324                  * is _exactly_ one, set the probe's predicate cache ID to be
11325                  * the predicate cache ID of the remaining ECB.
11326                  */
11327                 ASSERT(probe->dtpr_ecb_last != NULL);
11328                 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
11329 
11330                 if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
11331                         dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
11332 
11333                         ASSERT(probe->dtpr_ecb->dte_next == NULL);
11334 
11335                         if (p != NULL)
11336                                 probe->dtpr_predcache = p->dtp_cacheid;
11337                 }
11338 
11339                 ecb->dte_next = NULL;
11340         }
11341 }
11342 
11343 static void
11344 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
11345 {
11346         dtrace_state_t *state = ecb->dte_state;
11347         dtrace_vstate_t *vstate = &state->dts_vstate;
11348         dtrace_predicate_t *pred;
11349         dtrace_epid_t epid = ecb->dte_epid;
11350 
11351         ASSERT(MUTEX_HELD(&dtrace_lock));
11352         ASSERT(ecb->dte_next == NULL);
11353         ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
11354 
11355         if ((pred = ecb->dte_predicate) != NULL)
11356                 dtrace_predicate_release(pred, vstate);
11357 
11358         dtrace_ecb_action_remove(ecb);
11359 
11360         ASSERT(state->dts_ecbs[epid - 1] == ecb);
11361         state->dts_ecbs[epid - 1] = NULL;
11362 
11363         kmem_free(ecb, sizeof (dtrace_ecb_t));
11364 }
11365 
11366 static dtrace_ecb_t *
11367 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
11368     dtrace_enabling_t *enab)
11369 {
11370         dtrace_ecb_t *ecb;
11371         dtrace_predicate_t *pred;
11372         dtrace_actdesc_t *act;
11373         dtrace_provider_t *prov;
11374         dtrace_ecbdesc_t *desc = enab->dten_current;
11375 
11376         ASSERT(MUTEX_HELD(&dtrace_lock));
11377         ASSERT(state != NULL);
11378 
11379         ecb = dtrace_ecb_add(state, probe);
11380         ecb->dte_uarg = desc->dted_uarg;
11381 
11382         if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
11383                 dtrace_predicate_hold(pred);
11384                 ecb->dte_predicate = pred;
11385         }
11386 
11387         if (probe != NULL) {
11388                 /*
11389                  * If the provider shows more leg than the consumer is old
11390                  * enough to see, we need to enable the appropriate implicit
11391                  * predicate bits to prevent the ecb from activating at
11392                  * revealing times.
11393                  *
11394                  * Providers specifying DTRACE_PRIV_USER at register time
11395                  * are stating that they need the /proc-style privilege
11396                  * model to be enforced, and this is what DTRACE_COND_OWNER
11397                  * and DTRACE_COND_ZONEOWNER will then do at probe time.
11398                  */
11399                 prov = probe->dtpr_provider;
11400                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
11401                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
11402                         ecb->dte_cond |= DTRACE_COND_OWNER;
11403 
11404                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
11405                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
11406                         ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
11407 
11408                 /*
11409                  * If the provider shows us kernel innards and the user
11410                  * is lacking sufficient privilege, enable the
11411                  * DTRACE_COND_USERMODE implicit predicate.
11412                  */
11413                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
11414                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
11415                         ecb->dte_cond |= DTRACE_COND_USERMODE;
11416         }
11417 
11418         if (dtrace_ecb_create_cache != NULL) {
11419                 /*
11420                  * If we have a cached ecb, we'll use its action list instead
11421                  * of creating our own (saving both time and space).
11422                  */
11423                 dtrace_ecb_t *cached = dtrace_ecb_create_cache;
11424                 dtrace_action_t *act = cached->dte_action;
11425 
11426                 if (act != NULL) {
11427                         ASSERT(act->dta_refcnt > 0);
11428                         act->dta_refcnt++;
11429                         ecb->dte_action = act;
11430                         ecb->dte_action_last = cached->dte_action_last;
11431                         ecb->dte_needed = cached->dte_needed;
11432                         ecb->dte_size = cached->dte_size;
11433                         ecb->dte_alignment = cached->dte_alignment;
11434                 }
11435 
11436                 return (ecb);
11437         }
11438 
11439         for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
11440                 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
11441                         dtrace_ecb_destroy(ecb);
11442                         return (NULL);
11443                 }
11444         }
11445 
11446         if ((enab->dten_error = dtrace_ecb_resize(ecb)) != 0) {
11447                 dtrace_ecb_destroy(ecb);
11448                 return (NULL);
11449         }
11450 
11451         return (dtrace_ecb_create_cache = ecb);
11452 }
11453 
11454 static int
11455 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
11456 {
11457         dtrace_ecb_t *ecb;
11458         dtrace_enabling_t *enab = arg;
11459         dtrace_state_t *state = enab->dten_vstate->dtvs_state;
11460 
11461         ASSERT(state != NULL);
11462 
11463         if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
11464                 /*
11465                  * This probe was created in a generation for which this
11466                  * enabling has previously created ECBs; we don't want to
11467                  * enable it again, so just kick out.
11468                  */
11469                 return (DTRACE_MATCH_NEXT);
11470         }
11471 
11472         if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
11473                 return (DTRACE_MATCH_DONE);
11474 
11475         if (dtrace_ecb_enable(ecb) < 0)
11476                 return (DTRACE_MATCH_FAIL);
11477 
11478         return (DTRACE_MATCH_NEXT);
11479 }
11480 
11481 static dtrace_ecb_t *
11482 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
11483 {
11484         dtrace_ecb_t *ecb;
11485 
11486         ASSERT(MUTEX_HELD(&dtrace_lock));
11487 
11488         if (id == 0 || id > state->dts_necbs)
11489                 return (NULL);
11490 
11491         ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
11492         ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
11493 
11494         return (state->dts_ecbs[id - 1]);
11495 }
11496 
11497 static dtrace_aggregation_t *
11498 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
11499 {
11500         dtrace_aggregation_t *agg;
11501 
11502         ASSERT(MUTEX_HELD(&dtrace_lock));
11503 
11504         if (id == 0 || id > state->dts_naggregations)
11505                 return (NULL);
11506 
11507         ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
11508         ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
11509             agg->dtag_id == id);
11510 
11511         return (state->dts_aggregations[id - 1]);
11512 }
11513 
11514 /*
11515  * DTrace Buffer Functions
11516  *
11517  * The following functions manipulate DTrace buffers.  Most of these functions
11518  * are called in the context of establishing or processing consumer state;
11519  * exceptions are explicitly noted.
11520  */
11521 
11522 /*
11523  * Note:  called from cross call context.  This function switches the two
11524  * buffers on a given CPU.  The atomicity of this operation is assured by
11525  * disabling interrupts while the actual switch takes place; the disabling of
11526  * interrupts serializes the execution with any execution of dtrace_probe() on
11527  * the same CPU.
11528  */
11529 static void
11530 dtrace_buffer_switch(dtrace_buffer_t *buf)
11531 {
11532         caddr_t tomax = buf->dtb_tomax;
11533         caddr_t xamot = buf->dtb_xamot;
11534         dtrace_icookie_t cookie;
11535         hrtime_t now;
11536 
11537         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
11538         ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
11539 
11540         cookie = dtrace_interrupt_disable();
11541         now = dtrace_gethrtime();
11542         buf->dtb_tomax = xamot;
11543         buf->dtb_xamot = tomax;
11544         buf->dtb_xamot_drops = buf->dtb_drops;
11545         buf->dtb_xamot_offset = buf->dtb_offset;
11546         buf->dtb_xamot_errors = buf->dtb_errors;
11547         buf->dtb_xamot_flags = buf->dtb_flags;
11548         buf->dtb_offset = 0;
11549         buf->dtb_drops = 0;
11550         buf->dtb_errors = 0;
11551         buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
11552         buf->dtb_interval = now - buf->dtb_switched;
11553         buf->dtb_switched = now;
11554         dtrace_interrupt_enable(cookie);
11555 }
11556 
11557 /*
11558  * Note:  called from cross call context.  This function activates a buffer
11559  * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
11560  * is guaranteed by the disabling of interrupts.
11561  */
11562 static void
11563 dtrace_buffer_activate(dtrace_state_t *state)
11564 {
11565         dtrace_buffer_t *buf;
11566         dtrace_icookie_t cookie = dtrace_interrupt_disable();
11567 
11568         buf = &state->dts_buffer[CPU->cpu_id];
11569 
11570         if (buf->dtb_tomax != NULL) {
11571                 /*
11572                  * We might like to assert that the buffer is marked inactive,
11573                  * but this isn't necessarily true:  the buffer for the CPU
11574                  * that processes the BEGIN probe has its buffer activated
11575                  * manually.  In this case, we take the (harmless) action
11576                  * re-clearing the bit INACTIVE bit.
11577                  */
11578                 buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
11579         }
11580 
11581         dtrace_interrupt_enable(cookie);
11582 }
11583 
11584 static int
11585 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
11586     processorid_t cpu, int *factor)
11587 {
11588         cpu_t *cp;
11589         dtrace_buffer_t *buf;
11590         int allocated = 0, desired = 0;
11591 
11592         ASSERT(MUTEX_HELD(&cpu_lock));
11593         ASSERT(MUTEX_HELD(&dtrace_lock));
11594 
11595         *factor = 1;
11596 
11597         if (size > dtrace_nonroot_maxsize &&
11598             !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
11599                 return (EFBIG);
11600 
11601         cp = cpu_list;
11602 
11603         do {
11604                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11605                         continue;
11606 
11607                 buf = &bufs[cp->cpu_id];
11608 
11609                 /*
11610                  * If there is already a buffer allocated for this CPU, it
11611                  * is only possible that this is a DR event.  In this case,
11612                  * the buffer size must match our specified size.
11613                  */
11614                 if (buf->dtb_tomax != NULL) {
11615                         ASSERT(buf->dtb_size == size);
11616                         continue;
11617                 }
11618 
11619                 ASSERT(buf->dtb_xamot == NULL);
11620 
11621                 if ((buf->dtb_tomax = kmem_zalloc(size,
11622                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11623                         goto err;
11624 
11625                 buf->dtb_size = size;
11626                 buf->dtb_flags = flags;
11627                 buf->dtb_offset = 0;
11628                 buf->dtb_drops = 0;
11629 
11630                 if (flags & DTRACEBUF_NOSWITCH)
11631                         continue;
11632 
11633                 if ((buf->dtb_xamot = kmem_zalloc(size,
11634                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
11635                         goto err;
11636         } while ((cp = cp->cpu_next) != cpu_list);
11637 
11638         return (0);
11639 
11640 err:
11641         cp = cpu_list;
11642 
11643         do {
11644                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
11645                         continue;
11646 
11647                 buf = &bufs[cp->cpu_id];
11648                 desired += 2;
11649 
11650                 if (buf->dtb_xamot != NULL) {
11651                         ASSERT(buf->dtb_tomax != NULL);
11652                         ASSERT(buf->dtb_size == size);
11653                         kmem_free(buf->dtb_xamot, size);
11654                         allocated++;
11655                 }
11656 
11657                 if (buf->dtb_tomax != NULL) {
11658                         ASSERT(buf->dtb_size == size);
11659                         kmem_free(buf->dtb_tomax, size);
11660                         allocated++;
11661                 }
11662 
11663                 buf->dtb_tomax = NULL;
11664                 buf->dtb_xamot = NULL;
11665                 buf->dtb_size = 0;
11666         } while ((cp = cp->cpu_next) != cpu_list);
11667 
11668         *factor = desired / (allocated > 0 ? allocated : 1);
11669 
11670         return (ENOMEM);
11671 }
11672 
11673 /*
11674  * Note:  called from probe context.  This function just increments the drop
11675  * count on a buffer.  It has been made a function to allow for the
11676  * possibility of understanding the source of mysterious drop counts.  (A
11677  * problem for which one may be particularly disappointed that DTrace cannot
11678  * be used to understand DTrace.)
11679  */
11680 static void
11681 dtrace_buffer_drop(dtrace_buffer_t *buf)
11682 {
11683         buf->dtb_drops++;
11684 }
11685 
11686 /*
11687  * Note:  called from probe context.  This function is called to reserve space
11688  * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
11689  * mstate.  Returns the new offset in the buffer, or a negative value if an
11690  * error has occurred.
11691  */
11692 static intptr_t
11693 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
11694     dtrace_state_t *state, dtrace_mstate_t *mstate)
11695 {
11696         intptr_t offs = buf->dtb_offset, soffs;
11697         intptr_t woffs;
11698         caddr_t tomax;
11699         size_t total;
11700 
11701         if (buf->dtb_flags & DTRACEBUF_INACTIVE)
11702                 return (-1);
11703 
11704         if ((tomax = buf->dtb_tomax) == NULL) {
11705                 dtrace_buffer_drop(buf);
11706                 return (-1);
11707         }
11708 
11709         if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
11710                 while (offs & (align - 1)) {
11711                         /*
11712                          * Assert that our alignment is off by a number which
11713                          * is itself sizeof (uint32_t) aligned.
11714                          */
11715                         ASSERT(!((align - (offs & (align - 1))) &
11716                             (sizeof (uint32_t) - 1)));
11717                         DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11718                         offs += sizeof (uint32_t);
11719                 }
11720 
11721                 if ((soffs = offs + needed) > buf->dtb_size) {
11722                         dtrace_buffer_drop(buf);
11723                         return (-1);
11724                 }
11725 
11726                 if (mstate == NULL)
11727                         return (offs);
11728 
11729                 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
11730                 mstate->dtms_scratch_size = buf->dtb_size - soffs;
11731                 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11732 
11733                 return (offs);
11734         }
11735 
11736         if (buf->dtb_flags & DTRACEBUF_FILL) {
11737                 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
11738                     (buf->dtb_flags & DTRACEBUF_FULL))
11739                         return (-1);
11740                 goto out;
11741         }
11742 
11743         total = needed + (offs & (align - 1));
11744 
11745         /*
11746          * For a ring buffer, life is quite a bit more complicated.  Before
11747          * we can store any padding, we need to adjust our wrapping offset.
11748          * (If we've never before wrapped or we're not about to, no adjustment
11749          * is required.)
11750          */
11751         if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
11752             offs + total > buf->dtb_size) {
11753                 woffs = buf->dtb_xamot_offset;
11754 
11755                 if (offs + total > buf->dtb_size) {
11756                         /*
11757                          * We can't fit in the end of the buffer.  First, a
11758                          * sanity check that we can fit in the buffer at all.
11759                          */
11760                         if (total > buf->dtb_size) {
11761                                 dtrace_buffer_drop(buf);
11762                                 return (-1);
11763                         }
11764 
11765                         /*
11766                          * We're going to be storing at the top of the buffer,
11767                          * so now we need to deal with the wrapped offset.  We
11768                          * only reset our wrapped offset to 0 if it is
11769                          * currently greater than the current offset.  If it
11770                          * is less than the current offset, it is because a
11771                          * previous allocation induced a wrap -- but the
11772                          * allocation didn't subsequently take the space due
11773                          * to an error or false predicate evaluation.  In this
11774                          * case, we'll just leave the wrapped offset alone: if
11775                          * the wrapped offset hasn't been advanced far enough
11776                          * for this allocation, it will be adjusted in the
11777                          * lower loop.
11778                          */
11779                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
11780                                 if (woffs >= offs)
11781                                         woffs = 0;
11782                         } else {
11783                                 woffs = 0;
11784                         }
11785 
11786                         /*
11787                          * Now we know that we're going to be storing to the
11788                          * top of the buffer and that there is room for us
11789                          * there.  We need to clear the buffer from the current
11790                          * offset to the end (there may be old gunk there).
11791                          */
11792                         while (offs < buf->dtb_size)
11793                                 tomax[offs++] = 0;
11794 
11795                         /*
11796                          * We need to set our offset to zero.  And because we
11797                          * are wrapping, we need to set the bit indicating as
11798                          * much.  We can also adjust our needed space back
11799                          * down to the space required by the ECB -- we know
11800                          * that the top of the buffer is aligned.
11801                          */
11802                         offs = 0;
11803                         total = needed;
11804                         buf->dtb_flags |= DTRACEBUF_WRAPPED;
11805                 } else {
11806                         /*
11807                          * There is room for us in the buffer, so we simply
11808                          * need to check the wrapped offset.
11809                          */
11810                         if (woffs < offs) {
11811                                 /*
11812                                  * The wrapped offset is less than the offset.
11813                                  * This can happen if we allocated buffer space
11814                                  * that induced a wrap, but then we didn't
11815                                  * subsequently take the space due to an error
11816                                  * or false predicate evaluation.  This is
11817                                  * okay; we know that _this_ allocation isn't
11818                                  * going to induce a wrap.  We still can't
11819                                  * reset the wrapped offset to be zero,
11820                                  * however: the space may have been trashed in
11821                                  * the previous failed probe attempt.  But at
11822                                  * least the wrapped offset doesn't need to
11823                                  * be adjusted at all...
11824                                  */
11825                                 goto out;
11826                         }
11827                 }
11828 
11829                 while (offs + total > woffs) {
11830                         dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
11831                         size_t size;
11832 
11833                         if (epid == DTRACE_EPIDNONE) {
11834                                 size = sizeof (uint32_t);
11835                         } else {
11836                                 ASSERT3U(epid, <=, state->dts_necbs);
11837                                 ASSERT(state->dts_ecbs[epid - 1] != NULL);
11838 
11839                                 size = state->dts_ecbs[epid - 1]->dte_size;
11840                         }
11841 
11842                         ASSERT(woffs + size <= buf->dtb_size);
11843                         ASSERT(size != 0);
11844 
11845                         if (woffs + size == buf->dtb_size) {
11846                                 /*
11847                                  * We've reached the end of the buffer; we want
11848                                  * to set the wrapped offset to 0 and break
11849                                  * out.  However, if the offs is 0, then we're
11850                                  * in a strange edge-condition:  the amount of
11851                                  * space that we want to reserve plus the size
11852                                  * of the record that we're overwriting is
11853                                  * greater than the size of the buffer.  This
11854                                  * is problematic because if we reserve the
11855                                  * space but subsequently don't consume it (due
11856                                  * to a failed predicate or error) the wrapped
11857                                  * offset will be 0 -- yet the EPID at offset 0
11858                                  * will not be committed.  This situation is
11859                                  * relatively easy to deal with:  if we're in
11860                                  * this case, the buffer is indistinguishable
11861                                  * from one that hasn't wrapped; we need only
11862                                  * finish the job by clearing the wrapped bit,
11863                                  * explicitly setting the offset to be 0, and
11864                                  * zero'ing out the old data in the buffer.
11865                                  */
11866                                 if (offs == 0) {
11867                                         buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
11868                                         buf->dtb_offset = 0;
11869                                         woffs = total;
11870 
11871                                         while (woffs < buf->dtb_size)
11872                                                 tomax[woffs++] = 0;
11873                                 }
11874 
11875                                 woffs = 0;
11876                                 break;
11877                         }
11878 
11879                         woffs += size;
11880                 }
11881 
11882                 /*
11883                  * We have a wrapped offset.  It may be that the wrapped offset
11884                  * has become zero -- that's okay.
11885                  */
11886                 buf->dtb_xamot_offset = woffs;
11887         }
11888 
11889 out:
11890         /*
11891          * Now we can plow the buffer with any necessary padding.
11892          */
11893         while (offs & (align - 1)) {
11894                 /*
11895                  * Assert that our alignment is off by a number which
11896                  * is itself sizeof (uint32_t) aligned.
11897                  */
11898                 ASSERT(!((align - (offs & (align - 1))) &
11899                     (sizeof (uint32_t) - 1)));
11900                 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
11901                 offs += sizeof (uint32_t);
11902         }
11903 
11904         if (buf->dtb_flags & DTRACEBUF_FILL) {
11905                 if (offs + needed > buf->dtb_size - state->dts_reserve) {
11906                         buf->dtb_flags |= DTRACEBUF_FULL;
11907                         return (-1);
11908                 }
11909         }
11910 
11911         if (mstate == NULL)
11912                 return (offs);
11913 
11914         /*
11915          * For ring buffers and fill buffers, the scratch space is always
11916          * the inactive buffer.
11917          */
11918         mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
11919         mstate->dtms_scratch_size = buf->dtb_size;
11920         mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
11921 
11922         return (offs);
11923 }
11924 
11925 static void
11926 dtrace_buffer_polish(dtrace_buffer_t *buf)
11927 {
11928         ASSERT(buf->dtb_flags & DTRACEBUF_RING);
11929         ASSERT(MUTEX_HELD(&dtrace_lock));
11930 
11931         if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
11932                 return;
11933 
11934         /*
11935          * We need to polish the ring buffer.  There are three cases:
11936          *
11937          * - The first (and presumably most common) is that there is no gap
11938          *   between the buffer offset and the wrapped offset.  In this case,
11939          *   there is nothing in the buffer that isn't valid data; we can
11940          *   mark the buffer as polished and return.
11941          *
11942          * - The second (less common than the first but still more common
11943          *   than the third) is that there is a gap between the buffer offset
11944          *   and the wrapped offset, and the wrapped offset is larger than the
11945          *   buffer offset.  This can happen because of an alignment issue, or
11946          *   can happen because of a call to dtrace_buffer_reserve() that
11947          *   didn't subsequently consume the buffer space.  In this case,
11948          *   we need to zero the data from the buffer offset to the wrapped
11949          *   offset.
11950          *
11951          * - The third (and least common) is that there is a gap between the
11952          *   buffer offset and the wrapped offset, but the wrapped offset is
11953          *   _less_ than the buffer offset.  This can only happen because a
11954          *   call to dtrace_buffer_reserve() induced a wrap, but the space
11955          *   was not subsequently consumed.  In this case, we need to zero the
11956          *   space from the offset to the end of the buffer _and_ from the
11957          *   top of the buffer to the wrapped offset.
11958          */
11959         if (buf->dtb_offset < buf->dtb_xamot_offset) {
11960                 bzero(buf->dtb_tomax + buf->dtb_offset,
11961                     buf->dtb_xamot_offset - buf->dtb_offset);
11962         }
11963 
11964         if (buf->dtb_offset > buf->dtb_xamot_offset) {
11965                 bzero(buf->dtb_tomax + buf->dtb_offset,
11966                     buf->dtb_size - buf->dtb_offset);
11967                 bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
11968         }
11969 }
11970 
11971 /*
11972  * This routine determines if data generated at the specified time has likely
11973  * been entirely consumed at user-level.  This routine is called to determine
11974  * if an ECB on a defunct probe (but for an active enabling) can be safely
11975  * disabled and destroyed.
11976  */
11977 static int
11978 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
11979 {
11980         int i;
11981 
11982         for (i = 0; i < NCPU; i++) {
11983                 dtrace_buffer_t *buf = &bufs[i];
11984 
11985                 if (buf->dtb_size == 0)
11986                         continue;
11987 
11988                 if (buf->dtb_flags & DTRACEBUF_RING)
11989                         return (0);
11990 
11991                 if (!buf->dtb_switched && buf->dtb_offset != 0)
11992                         return (0);
11993 
11994                 if (buf->dtb_switched - buf->dtb_interval < when)
11995                         return (0);
11996         }
11997 
11998         return (1);
11999 }
12000 
12001 static void
12002 dtrace_buffer_free(dtrace_buffer_t *bufs)
12003 {
12004         int i;
12005 
12006         for (i = 0; i < NCPU; i++) {
12007                 dtrace_buffer_t *buf = &bufs[i];
12008 
12009                 if (buf->dtb_tomax == NULL) {
12010                         ASSERT(buf->dtb_xamot == NULL);
12011                         ASSERT(buf->dtb_size == 0);
12012                         continue;
12013                 }
12014 
12015                 if (buf->dtb_xamot != NULL) {
12016                         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
12017                         kmem_free(buf->dtb_xamot, buf->dtb_size);
12018                 }
12019 
12020                 kmem_free(buf->dtb_tomax, buf->dtb_size);
12021                 buf->dtb_size = 0;
12022                 buf->dtb_tomax = NULL;
12023                 buf->dtb_xamot = NULL;
12024         }
12025 }
12026 
12027 /*
12028  * DTrace Enabling Functions
12029  */
12030 static dtrace_enabling_t *
12031 dtrace_enabling_create(dtrace_vstate_t *vstate)
12032 {
12033         dtrace_enabling_t *enab;
12034 
12035         enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
12036         enab->dten_vstate = vstate;
12037 
12038         return (enab);
12039 }
12040 
12041 static void
12042 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
12043 {
12044         dtrace_ecbdesc_t **ndesc;
12045         size_t osize, nsize;
12046 
12047         /*
12048          * We can't add to enablings after we've enabled them, or after we've
12049          * retained them.
12050          */
12051         ASSERT(enab->dten_probegen == 0);
12052         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
12053 
12054         if (enab->dten_ndesc < enab->dten_maxdesc) {
12055                 enab->dten_desc[enab->dten_ndesc++] = ecb;
12056                 return;
12057         }
12058 
12059         osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
12060 
12061         if (enab->dten_maxdesc == 0) {
12062                 enab->dten_maxdesc = 1;
12063         } else {
12064                 enab->dten_maxdesc <<= 1;
12065         }
12066 
12067         ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
12068 
12069         nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
12070         ndesc = kmem_zalloc(nsize, KM_SLEEP);
12071         bcopy(enab->dten_desc, ndesc, osize);
12072         kmem_free(enab->dten_desc, osize);
12073 
12074         enab->dten_desc = ndesc;
12075         enab->dten_desc[enab->dten_ndesc++] = ecb;
12076 }
12077 
12078 static void
12079 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
12080     dtrace_probedesc_t *pd)
12081 {
12082         dtrace_ecbdesc_t *new;
12083         dtrace_predicate_t *pred;
12084         dtrace_actdesc_t *act;
12085 
12086         /*
12087          * We're going to create a new ECB description that matches the
12088          * specified ECB in every way, but has the specified probe description.
12089          */
12090         new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
12091 
12092         if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
12093                 dtrace_predicate_hold(pred);
12094 
12095         for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
12096                 dtrace_actdesc_hold(act);
12097 
12098         new->dted_action = ecb->dted_action;
12099         new->dted_pred = ecb->dted_pred;
12100         new->dted_probe = *pd;
12101         new->dted_uarg = ecb->dted_uarg;
12102 
12103         dtrace_enabling_add(enab, new);
12104 }
12105 
12106 static void
12107 dtrace_enabling_dump(dtrace_enabling_t *enab)
12108 {
12109         int i;
12110 
12111         for (i = 0; i < enab->dten_ndesc; i++) {
12112                 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
12113 
12114                 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
12115                     desc->dtpd_provider, desc->dtpd_mod,
12116                     desc->dtpd_func, desc->dtpd_name);
12117         }
12118 }
12119 
12120 static void
12121 dtrace_enabling_destroy(dtrace_enabling_t *enab)
12122 {
12123         int i;
12124         dtrace_ecbdesc_t *ep;
12125         dtrace_vstate_t *vstate = enab->dten_vstate;
12126 
12127         ASSERT(MUTEX_HELD(&dtrace_lock));
12128 
12129         for (i = 0; i < enab->dten_ndesc; i++) {
12130                 dtrace_actdesc_t *act, *next;
12131                 dtrace_predicate_t *pred;
12132 
12133                 ep = enab->dten_desc[i];
12134 
12135                 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
12136                         dtrace_predicate_release(pred, vstate);
12137 
12138                 for (act = ep->dted_action; act != NULL; act = next) {
12139                         next = act->dtad_next;
12140                         dtrace_actdesc_release(act, vstate);
12141                 }
12142 
12143                 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
12144         }
12145 
12146         kmem_free(enab->dten_desc,
12147             enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
12148 
12149         /*
12150          * If this was a retained enabling, decrement the dts_nretained count
12151          * and take it off of the dtrace_retained list.
12152          */
12153         if (enab->dten_prev != NULL || enab->dten_next != NULL ||
12154             dtrace_retained == enab) {
12155                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
12156                 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
12157                 enab->dten_vstate->dtvs_state->dts_nretained--;
12158                 dtrace_retained_gen++;
12159         }
12160 
12161         if (enab->dten_prev == NULL) {
12162                 if (dtrace_retained == enab) {
12163                         dtrace_retained = enab->dten_next;
12164 
12165                         if (dtrace_retained != NULL)
12166                                 dtrace_retained->dten_prev = NULL;
12167                 }
12168         } else {
12169                 ASSERT(enab != dtrace_retained);
12170                 ASSERT(dtrace_retained != NULL);
12171                 enab->dten_prev->dten_next = enab->dten_next;
12172         }
12173 
12174         if (enab->dten_next != NULL) {
12175                 ASSERT(dtrace_retained != NULL);
12176                 enab->dten_next->dten_prev = enab->dten_prev;
12177         }
12178 
12179         kmem_free(enab, sizeof (dtrace_enabling_t));
12180 }
12181 
12182 static int
12183 dtrace_enabling_retain(dtrace_enabling_t *enab)
12184 {
12185         dtrace_state_t *state;
12186 
12187         ASSERT(MUTEX_HELD(&dtrace_lock));
12188         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
12189         ASSERT(enab->dten_vstate != NULL);
12190 
12191         state = enab->dten_vstate->dtvs_state;
12192         ASSERT(state != NULL);
12193 
12194         /*
12195          * We only allow each state to retain dtrace_retain_max enablings.
12196          */
12197         if (state->dts_nretained >= dtrace_retain_max)
12198                 return (ENOSPC);
12199 
12200         state->dts_nretained++;
12201         dtrace_retained_gen++;
12202 
12203         if (dtrace_retained == NULL) {
12204                 dtrace_retained = enab;
12205                 return (0);
12206         }
12207 
12208         enab->dten_next = dtrace_retained;
12209         dtrace_retained->dten_prev = enab;
12210         dtrace_retained = enab;
12211 
12212         return (0);
12213 }
12214 
12215 static int
12216 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
12217     dtrace_probedesc_t *create)
12218 {
12219         dtrace_enabling_t *new, *enab;
12220         int found = 0, err = ENOENT;
12221 
12222         ASSERT(MUTEX_HELD(&dtrace_lock));
12223         ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
12224         ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
12225         ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
12226         ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
12227 
12228         new = dtrace_enabling_create(&state->dts_vstate);
12229 
12230         /*
12231          * Iterate over all retained enablings, looking for enablings that
12232          * match the specified state.
12233          */
12234         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
12235                 int i;
12236 
12237                 /*
12238                  * dtvs_state can only be NULL for helper enablings -- and
12239                  * helper enablings can't be retained.
12240                  */
12241                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
12242 
12243                 if (enab->dten_vstate->dtvs_state != state)
12244                         continue;
12245 
12246                 /*
12247                  * Now iterate over each probe description; we're looking for
12248                  * an exact match to the specified probe description.
12249                  */
12250                 for (i = 0; i < enab->dten_ndesc; i++) {
12251                         dtrace_ecbdesc_t *ep = enab->dten_desc[i];
12252                         dtrace_probedesc_t *pd = &ep->dted_probe;
12253 
12254                         if (strcmp(pd->dtpd_provider, match->dtpd_provider))
12255                                 continue;
12256 
12257                         if (strcmp(pd->dtpd_mod, match->dtpd_mod))
12258                                 continue;
12259 
12260                         if (strcmp(pd->dtpd_func, match->dtpd_func))
12261                                 continue;
12262 
12263                         if (strcmp(pd->dtpd_name, match->dtpd_name))
12264                                 continue;
12265 
12266                         /*
12267                          * We have a winning probe!  Add it to our growing
12268                          * enabling.
12269                          */
12270                         found = 1;
12271                         dtrace_enabling_addlike(new, ep, create);
12272                 }
12273         }
12274 
12275         if (!found || (err = dtrace_enabling_retain(new)) != 0) {
12276                 dtrace_enabling_destroy(new);
12277                 return (err);
12278         }
12279 
12280         return (0);
12281 }
12282 
12283 static void
12284 dtrace_enabling_retract(dtrace_state_t *state)
12285 {
12286         dtrace_enabling_t *enab, *next;
12287 
12288         ASSERT(MUTEX_HELD(&dtrace_lock));
12289 
12290         /*
12291          * Iterate over all retained enablings, destroy the enablings retained
12292          * for the specified state.
12293          */
12294         for (enab = dtrace_retained; enab != NULL; enab = next) {
12295                 next = enab->dten_next;
12296 
12297                 /*
12298                  * dtvs_state can only be NULL for helper enablings -- and
12299                  * helper enablings can't be retained.
12300                  */
12301                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
12302 
12303                 if (enab->dten_vstate->dtvs_state == state) {
12304                         ASSERT(state->dts_nretained > 0);
12305                         dtrace_enabling_destroy(enab);
12306                 }
12307         }
12308 
12309         ASSERT(state->dts_nretained == 0);
12310 }
12311 
12312 static int
12313 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
12314 {
12315         int i = 0;
12316         int total_matched = 0, matched = 0;
12317 
12318         ASSERT(MUTEX_HELD(&cpu_lock));
12319         ASSERT(MUTEX_HELD(&dtrace_lock));
12320 
12321         for (i = 0; i < enab->dten_ndesc; i++) {
12322                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
12323 
12324                 enab->dten_current = ep;
12325                 enab->dten_error = 0;
12326 
12327                 /*
12328                  * If a provider failed to enable a probe then get out and
12329                  * let the consumer know we failed.
12330                  */
12331                 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
12332                         return (EBUSY);
12333 
12334                 total_matched += matched;
12335 
12336                 if (enab->dten_error != 0) {
12337                         /*
12338                          * If we get an error half-way through enabling the
12339                          * probes, we kick out -- perhaps with some number of
12340                          * them enabled.  Leaving enabled probes enabled may
12341                          * be slightly confusing for user-level, but we expect
12342                          * that no one will attempt to actually drive on in
12343                          * the face of such errors.  If this is an anonymous
12344                          * enabling (indicated with a NULL nmatched pointer),
12345                          * we cmn_err() a message.  We aren't expecting to
12346                          * get such an error -- such as it can exist at all,
12347                          * it would be a result of corrupted DOF in the driver
12348                          * properties.
12349                          */
12350                         if (nmatched == NULL) {
12351                                 cmn_err(CE_WARN, "dtrace_enabling_match() "
12352                                     "error on %p: %d", (void *)ep,
12353                                     enab->dten_error);
12354                         }
12355 
12356                         return (enab->dten_error);
12357                 }
12358         }
12359 
12360         enab->dten_probegen = dtrace_probegen;
12361         if (nmatched != NULL)
12362                 *nmatched = total_matched;
12363 
12364         return (0);
12365 }
12366 
12367 static void
12368 dtrace_enabling_matchall(void)
12369 {
12370         dtrace_enabling_t *enab;
12371 
12372         mutex_enter(&cpu_lock);
12373         mutex_enter(&dtrace_lock);
12374 
12375         /*
12376          * Iterate over all retained enablings to see if any probes match
12377          * against them.  We only perform this operation on enablings for which
12378          * we have sufficient permissions by virtue of being in the global zone
12379          * or in the same zone as the DTrace client.  Because we can be called
12380          * after dtrace_detach() has been called, we cannot assert that there
12381          * are retained enablings.  We can safely load from dtrace_retained,
12382          * however:  the taskq_destroy() at the end of dtrace_detach() will
12383          * block pending our completion.
12384          */
12385         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
12386                 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred;
12387                 cred_t *cr = dcr->dcr_cred;
12388                 zoneid_t zone = cr != NULL ? crgetzonedid(cr) : 0;
12389 
12390                 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL &&
12391                     (zone == GLOBAL_ZONEID || getzonedid() == zone)))
12392                         (void) dtrace_enabling_match(enab, NULL);
12393         }
12394 
12395         mutex_exit(&dtrace_lock);
12396         mutex_exit(&cpu_lock);
12397 }
12398 
12399 /*
12400  * If an enabling is to be enabled without having matched probes (that is, if
12401  * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
12402  * enabling must be _primed_ by creating an ECB for every ECB description.
12403  * This must be done to assure that we know the number of speculations, the
12404  * number of aggregations, the minimum buffer size needed, etc. before we
12405  * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
12406  * enabling any probes, we create ECBs for every ECB decription, but with a
12407  * NULL probe -- which is exactly what this function does.
12408  */
12409 static void
12410 dtrace_enabling_prime(dtrace_state_t *state)
12411 {
12412         dtrace_enabling_t *enab;
12413         int i;
12414 
12415         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
12416                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
12417 
12418                 if (enab->dten_vstate->dtvs_state != state)
12419                         continue;
12420 
12421                 /*
12422                  * We don't want to prime an enabling more than once, lest
12423                  * we allow a malicious user to induce resource exhaustion.
12424                  * (The ECBs that result from priming an enabling aren't
12425                  * leaked -- but they also aren't deallocated until the
12426                  * consumer state is destroyed.)
12427                  */
12428                 if (enab->dten_primed)
12429                         continue;
12430 
12431                 for (i = 0; i < enab->dten_ndesc; i++) {
12432                         enab->dten_current = enab->dten_desc[i];
12433                         (void) dtrace_probe_enable(NULL, enab);
12434                 }
12435 
12436                 enab->dten_primed = 1;
12437         }
12438 }
12439 
12440 /*
12441  * Called to indicate that probes should be provided due to retained
12442  * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
12443  * must take an initial lap through the enabling calling the dtps_provide()
12444  * entry point explicitly to allow for autocreated probes.
12445  */
12446 static void
12447 dtrace_enabling_provide(dtrace_provider_t *prv)
12448 {
12449         int i, all = 0;
12450         dtrace_probedesc_t desc;
12451         dtrace_genid_t gen;
12452 
12453         ASSERT(MUTEX_HELD(&dtrace_lock));
12454         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
12455 
12456         if (prv == NULL) {
12457                 all = 1;
12458                 prv = dtrace_provider;
12459         }
12460 
12461         do {
12462                 dtrace_enabling_t *enab;
12463                 void *parg = prv->dtpv_arg;
12464 
12465 retry:
12466                 gen = dtrace_retained_gen;
12467                 for (enab = dtrace_retained; enab != NULL;
12468                     enab = enab->dten_next) {
12469                         for (i = 0; i < enab->dten_ndesc; i++) {
12470                                 desc = enab->dten_desc[i]->dted_probe;
12471                                 mutex_exit(&dtrace_lock);
12472                                 prv->dtpv_pops.dtps_provide(parg, &desc);
12473                                 mutex_enter(&dtrace_lock);
12474                                 /*
12475                                  * Process the retained enablings again if
12476                                  * they have changed while we weren't holding
12477                                  * dtrace_lock.
12478                                  */
12479                                 if (gen != dtrace_retained_gen)
12480                                         goto retry;
12481                         }
12482                 }
12483         } while (all && (prv = prv->dtpv_next) != NULL);
12484 
12485         mutex_exit(&dtrace_lock);
12486         dtrace_probe_provide(NULL, all ? NULL : prv);
12487         mutex_enter(&dtrace_lock);
12488 }
12489 
12490 /*
12491  * Called to reap ECBs that are attached to probes from defunct providers.
12492  */
12493 static void
12494 dtrace_enabling_reap(void)
12495 {
12496         dtrace_provider_t *prov;
12497         dtrace_probe_t *probe;
12498         dtrace_ecb_t *ecb;
12499         hrtime_t when;
12500         int i;
12501 
12502         mutex_enter(&cpu_lock);
12503         mutex_enter(&dtrace_lock);
12504 
12505         for (i = 0; i < dtrace_nprobes; i++) {
12506                 if ((probe = dtrace_probes[i]) == NULL)
12507                         continue;
12508 
12509                 if (probe->dtpr_ecb == NULL)
12510                         continue;
12511 
12512                 prov = probe->dtpr_provider;
12513 
12514                 if ((when = prov->dtpv_defunct) == 0)
12515                         continue;
12516 
12517                 /*
12518                  * We have ECBs on a defunct provider:  we want to reap these
12519                  * ECBs to allow the provider to unregister.  The destruction
12520                  * of these ECBs must be done carefully:  if we destroy the ECB
12521                  * and the consumer later wishes to consume an EPID that
12522                  * corresponds to the destroyed ECB (and if the EPID metadata
12523                  * has not been previously consumed), the consumer will abort
12524                  * processing on the unknown EPID.  To reduce (but not, sadly,
12525                  * eliminate) the possibility of this, we will only destroy an
12526                  * ECB for a defunct provider if, for the state that
12527                  * corresponds to the ECB:
12528                  *
12529                  *  (a) There is no speculative tracing (which can effectively
12530                  *      cache an EPID for an arbitrary amount of time).
12531                  *
12532                  *  (b) The principal buffers have been switched twice since the
12533                  *      provider became defunct.
12534                  *
12535                  *  (c) The aggregation buffers are of zero size or have been
12536                  *      switched twice since the provider became defunct.
12537                  *
12538                  * We use dts_speculates to determine (a) and call a function
12539                  * (dtrace_buffer_consumed()) to determine (b) and (c).  Note
12540                  * that as soon as we've been unable to destroy one of the ECBs
12541                  * associated with the probe, we quit trying -- reaping is only
12542                  * fruitful in as much as we can destroy all ECBs associated
12543                  * with the defunct provider's probes.
12544                  */
12545                 while ((ecb = probe->dtpr_ecb) != NULL) {
12546                         dtrace_state_t *state = ecb->dte_state;
12547                         dtrace_buffer_t *buf = state->dts_buffer;
12548                         dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
12549 
12550                         if (state->dts_speculates)
12551                                 break;
12552 
12553                         if (!dtrace_buffer_consumed(buf, when))
12554                                 break;
12555 
12556                         if (!dtrace_buffer_consumed(aggbuf, when))
12557                                 break;
12558 
12559                         dtrace_ecb_disable(ecb);
12560                         ASSERT(probe->dtpr_ecb != ecb);
12561                         dtrace_ecb_destroy(ecb);
12562                 }
12563         }
12564 
12565         mutex_exit(&dtrace_lock);
12566         mutex_exit(&cpu_lock);
12567 }
12568 
12569 /*
12570  * DTrace DOF Functions
12571  */
12572 /*ARGSUSED*/
12573 static void
12574 dtrace_dof_error(dof_hdr_t *dof, const char *str)
12575 {
12576         if (dtrace_err_verbose)
12577                 cmn_err(CE_WARN, "failed to process DOF: %s", str);
12578 
12579 #ifdef DTRACE_ERRDEBUG
12580         dtrace_errdebug(str);
12581 #endif
12582 }
12583 
12584 /*
12585  * Create DOF out of a currently enabled state.  Right now, we only create
12586  * DOF containing the run-time options -- but this could be expanded to create
12587  * complete DOF representing the enabled state.
12588  */
12589 static dof_hdr_t *
12590 dtrace_dof_create(dtrace_state_t *state)
12591 {
12592         dof_hdr_t *dof;
12593         dof_sec_t *sec;
12594         dof_optdesc_t *opt;
12595         int i, len = sizeof (dof_hdr_t) +
12596             roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
12597             sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12598 
12599         ASSERT(MUTEX_HELD(&dtrace_lock));
12600 
12601         dof = kmem_zalloc(len, KM_SLEEP);
12602         dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
12603         dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
12604         dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
12605         dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
12606 
12607         dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
12608         dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
12609         dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
12610         dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
12611         dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
12612         dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
12613 
12614         dof->dofh_flags = 0;
12615         dof->dofh_hdrsize = sizeof (dof_hdr_t);
12616         dof->dofh_secsize = sizeof (dof_sec_t);
12617         dof->dofh_secnum = 1;        /* only DOF_SECT_OPTDESC */
12618         dof->dofh_secoff = sizeof (dof_hdr_t);
12619         dof->dofh_loadsz = len;
12620         dof->dofh_filesz = len;
12621         dof->dofh_pad = 0;
12622 
12623         /*
12624          * Fill in the option section header...
12625          */
12626         sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
12627         sec->dofs_type = DOF_SECT_OPTDESC;
12628         sec->dofs_align = sizeof (uint64_t);
12629         sec->dofs_flags = DOF_SECF_LOAD;
12630         sec->dofs_entsize = sizeof (dof_optdesc_t);
12631 
12632         opt = (dof_optdesc_t *)((uintptr_t)sec +
12633             roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
12634 
12635         sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
12636         sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
12637 
12638         for (i = 0; i < DTRACEOPT_MAX; i++) {
12639                 opt[i].dofo_option = i;
12640                 opt[i].dofo_strtab = DOF_SECIDX_NONE;
12641                 opt[i].dofo_value = state->dts_options[i];
12642         }
12643 
12644         return (dof);
12645 }
12646 
12647 static dof_hdr_t *
12648 dtrace_dof_copyin(uintptr_t uarg, int *errp)
12649 {
12650         dof_hdr_t hdr, *dof;
12651 
12652         ASSERT(!MUTEX_HELD(&dtrace_lock));
12653 
12654         /*
12655          * First, we're going to copyin() the sizeof (dof_hdr_t).
12656          */
12657         if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
12658                 dtrace_dof_error(NULL, "failed to copyin DOF header");
12659                 *errp = EFAULT;
12660                 return (NULL);
12661         }
12662 
12663         /*
12664          * Now we'll allocate the entire DOF and copy it in -- provided
12665          * that the length isn't outrageous.
12666          */
12667         if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
12668                 dtrace_dof_error(&hdr, "load size exceeds maximum");
12669                 *errp = E2BIG;
12670                 return (NULL);
12671         }
12672 
12673         if (hdr.dofh_loadsz < sizeof (hdr)) {
12674                 dtrace_dof_error(&hdr, "invalid load size");
12675                 *errp = EINVAL;
12676                 return (NULL);
12677         }
12678 
12679         dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
12680 
12681         if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
12682             dof->dofh_loadsz != hdr.dofh_loadsz) {
12683                 kmem_free(dof, hdr.dofh_loadsz);
12684                 *errp = EFAULT;
12685                 return (NULL);
12686         }
12687 
12688         return (dof);
12689 }
12690 
12691 static dof_hdr_t *
12692 dtrace_dof_property(const char *name)
12693 {
12694         uchar_t *buf;
12695         uint64_t loadsz;
12696         unsigned int len, i;
12697         dof_hdr_t *dof;
12698 
12699         /*
12700          * Unfortunately, array of values in .conf files are always (and
12701          * only) interpreted to be integer arrays.  We must read our DOF
12702          * as an integer array, and then squeeze it into a byte array.
12703          */
12704         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
12705             (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
12706                 return (NULL);
12707 
12708         for (i = 0; i < len; i++)
12709                 buf[i] = (uchar_t)(((int *)buf)[i]);
12710 
12711         if (len < sizeof (dof_hdr_t)) {
12712                 ddi_prop_free(buf);
12713                 dtrace_dof_error(NULL, "truncated header");
12714                 return (NULL);
12715         }
12716 
12717         if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
12718                 ddi_prop_free(buf);
12719                 dtrace_dof_error(NULL, "truncated DOF");
12720                 return (NULL);
12721         }
12722 
12723         if (loadsz >= dtrace_dof_maxsize) {
12724                 ddi_prop_free(buf);
12725                 dtrace_dof_error(NULL, "oversized DOF");
12726                 return (NULL);
12727         }
12728 
12729         dof = kmem_alloc(loadsz, KM_SLEEP);
12730         bcopy(buf, dof, loadsz);
12731         ddi_prop_free(buf);
12732 
12733         return (dof);
12734 }
12735 
12736 static void
12737 dtrace_dof_destroy(dof_hdr_t *dof)
12738 {
12739         kmem_free(dof, dof->dofh_loadsz);
12740 }
12741 
12742 /*
12743  * Return the dof_sec_t pointer corresponding to a given section index.  If the
12744  * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
12745  * a type other than DOF_SECT_NONE is specified, the header is checked against
12746  * this type and NULL is returned if the types do not match.
12747  */
12748 static dof_sec_t *
12749 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
12750 {
12751         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
12752             ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
12753 
12754         if (i >= dof->dofh_secnum) {
12755                 dtrace_dof_error(dof, "referenced section index is invalid");
12756                 return (NULL);
12757         }
12758 
12759         if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
12760                 dtrace_dof_error(dof, "referenced section is not loadable");
12761                 return (NULL);
12762         }
12763 
12764         if (type != DOF_SECT_NONE && type != sec->dofs_type) {
12765                 dtrace_dof_error(dof, "referenced section is the wrong type");
12766                 return (NULL);
12767         }
12768 
12769         return (sec);
12770 }
12771 
12772 static dtrace_probedesc_t *
12773 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
12774 {
12775         dof_probedesc_t *probe;
12776         dof_sec_t *strtab;
12777         uintptr_t daddr = (uintptr_t)dof;
12778         uintptr_t str;
12779         size_t size;
12780 
12781         if (sec->dofs_type != DOF_SECT_PROBEDESC) {
12782                 dtrace_dof_error(dof, "invalid probe section");
12783                 return (NULL);
12784         }
12785 
12786         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12787                 dtrace_dof_error(dof, "bad alignment in probe description");
12788                 return (NULL);
12789         }
12790 
12791         if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
12792                 dtrace_dof_error(dof, "truncated probe description");
12793                 return (NULL);
12794         }
12795 
12796         probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
12797         strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
12798 
12799         if (strtab == NULL)
12800                 return (NULL);
12801 
12802         str = daddr + strtab->dofs_offset;
12803         size = strtab->dofs_size;
12804 
12805         if (probe->dofp_provider >= strtab->dofs_size) {
12806                 dtrace_dof_error(dof, "corrupt probe provider");
12807                 return (NULL);
12808         }
12809 
12810         (void) strncpy(desc->dtpd_provider,
12811             (char *)(str + probe->dofp_provider),
12812             MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
12813 
12814         if (probe->dofp_mod >= strtab->dofs_size) {
12815                 dtrace_dof_error(dof, "corrupt probe module");
12816                 return (NULL);
12817         }
12818 
12819         (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
12820             MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
12821 
12822         if (probe->dofp_func >= strtab->dofs_size) {
12823                 dtrace_dof_error(dof, "corrupt probe function");
12824                 return (NULL);
12825         }
12826 
12827         (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
12828             MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
12829 
12830         if (probe->dofp_name >= strtab->dofs_size) {
12831                 dtrace_dof_error(dof, "corrupt probe name");
12832                 return (NULL);
12833         }
12834 
12835         (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
12836             MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
12837 
12838         return (desc);
12839 }
12840 
12841 static dtrace_difo_t *
12842 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
12843     cred_t *cr)
12844 {
12845         dtrace_difo_t *dp;
12846         size_t ttl = 0;
12847         dof_difohdr_t *dofd;
12848         uintptr_t daddr = (uintptr_t)dof;
12849         size_t max = dtrace_difo_maxsize;
12850         int i, l, n;
12851 
12852         static const struct {
12853                 int section;
12854                 int bufoffs;
12855                 int lenoffs;
12856                 int entsize;
12857                 int align;
12858                 const char *msg;
12859         } difo[] = {
12860                 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
12861                 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
12862                 sizeof (dif_instr_t), "multiple DIF sections" },
12863 
12864                 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
12865                 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
12866                 sizeof (uint64_t), "multiple integer tables" },
12867 
12868                 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
12869                 offsetof(dtrace_difo_t, dtdo_strlen), 0,
12870                 sizeof (char), "multiple string tables" },
12871 
12872                 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
12873                 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
12874                 sizeof (uint_t), "multiple variable tables" },
12875 
12876                 { DOF_SECT_NONE, 0, 0, 0, 0, NULL }
12877         };
12878 
12879         if (sec->dofs_type != DOF_SECT_DIFOHDR) {
12880                 dtrace_dof_error(dof, "invalid DIFO header section");
12881                 return (NULL);
12882         }
12883 
12884         if (sec->dofs_align != sizeof (dof_secidx_t)) {
12885                 dtrace_dof_error(dof, "bad alignment in DIFO header");
12886                 return (NULL);
12887         }
12888 
12889         if (sec->dofs_size < sizeof (dof_difohdr_t) ||
12890             sec->dofs_size % sizeof (dof_secidx_t)) {
12891                 dtrace_dof_error(dof, "bad size in DIFO header");
12892                 return (NULL);
12893         }
12894 
12895         dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12896         n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
12897 
12898         dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
12899         dp->dtdo_rtype = dofd->dofd_rtype;
12900 
12901         for (l = 0; l < n; l++) {
12902                 dof_sec_t *subsec;
12903                 void **bufp;
12904                 uint32_t *lenp;
12905 
12906                 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
12907                     dofd->dofd_links[l])) == NULL)
12908                         goto err; /* invalid section link */
12909 
12910                 if (ttl + subsec->dofs_size > max) {
12911                         dtrace_dof_error(dof, "exceeds maximum size");
12912                         goto err;
12913                 }
12914 
12915                 ttl += subsec->dofs_size;
12916 
12917                 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
12918                         if (subsec->dofs_type != difo[i].section)
12919                                 continue;
12920 
12921                         if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
12922                                 dtrace_dof_error(dof, "section not loaded");
12923                                 goto err;
12924                         }
12925 
12926                         if (subsec->dofs_align != difo[i].align) {
12927                                 dtrace_dof_error(dof, "bad alignment");
12928                                 goto err;
12929                         }
12930 
12931                         bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
12932                         lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
12933 
12934                         if (*bufp != NULL) {
12935                                 dtrace_dof_error(dof, difo[i].msg);
12936                                 goto err;
12937                         }
12938 
12939                         if (difo[i].entsize != subsec->dofs_entsize) {
12940                                 dtrace_dof_error(dof, "entry size mismatch");
12941                                 goto err;
12942                         }
12943 
12944                         if (subsec->dofs_entsize != 0 &&
12945                             (subsec->dofs_size % subsec->dofs_entsize) != 0) {
12946                                 dtrace_dof_error(dof, "corrupt entry size");
12947                                 goto err;
12948                         }
12949 
12950                         *lenp = subsec->dofs_size;
12951                         *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
12952                         bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
12953                             *bufp, subsec->dofs_size);
12954 
12955                         if (subsec->dofs_entsize != 0)
12956                                 *lenp /= subsec->dofs_entsize;
12957 
12958                         break;
12959                 }
12960 
12961                 /*
12962                  * If we encounter a loadable DIFO sub-section that is not
12963                  * known to us, assume this is a broken program and fail.
12964                  */
12965                 if (difo[i].section == DOF_SECT_NONE &&
12966                     (subsec->dofs_flags & DOF_SECF_LOAD)) {
12967                         dtrace_dof_error(dof, "unrecognized DIFO subsection");
12968                         goto err;
12969                 }
12970         }
12971 
12972         if (dp->dtdo_buf == NULL) {
12973                 /*
12974                  * We can't have a DIF object without DIF text.
12975                  */
12976                 dtrace_dof_error(dof, "missing DIF text");
12977                 goto err;
12978         }
12979 
12980         /*
12981          * Before we validate the DIF object, run through the variable table
12982          * looking for the strings -- if any of their size are under, we'll set
12983          * their size to be the system-wide default string size.  Note that
12984          * this should _not_ happen if the "strsize" option has been set --
12985          * in this case, the compiler should have set the size to reflect the
12986          * setting of the option.
12987          */
12988         for (i = 0; i < dp->dtdo_varlen; i++) {
12989                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
12990                 dtrace_diftype_t *t = &v->dtdv_type;
12991 
12992                 if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
12993                         continue;
12994 
12995                 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
12996                         t->dtdt_size = dtrace_strsize_default;
12997         }
12998 
12999         if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
13000                 goto err;
13001 
13002         dtrace_difo_init(dp, vstate);
13003         return (dp);
13004 
13005 err:
13006         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
13007         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
13008         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
13009         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
13010 
13011         kmem_free(dp, sizeof (dtrace_difo_t));
13012         return (NULL);
13013 }
13014 
13015 static dtrace_predicate_t *
13016 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
13017     cred_t *cr)
13018 {
13019         dtrace_difo_t *dp;
13020 
13021         if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
13022                 return (NULL);
13023 
13024         return (dtrace_predicate_create(dp));
13025 }
13026 
13027 static dtrace_actdesc_t *
13028 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
13029     cred_t *cr)
13030 {
13031         dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
13032         dof_actdesc_t *desc;
13033         dof_sec_t *difosec;
13034         size_t offs;
13035         uintptr_t daddr = (uintptr_t)dof;
13036         uint64_t arg;
13037         dtrace_actkind_t kind;
13038 
13039         if (sec->dofs_type != DOF_SECT_ACTDESC) {
13040                 dtrace_dof_error(dof, "invalid action section");
13041                 return (NULL);
13042         }
13043 
13044         if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
13045                 dtrace_dof_error(dof, "truncated action description");
13046                 return (NULL);
13047         }
13048 
13049         if (sec->dofs_align != sizeof (uint64_t)) {
13050                 dtrace_dof_error(dof, "bad alignment in action description");
13051                 return (NULL);
13052         }
13053 
13054         if (sec->dofs_size < sec->dofs_entsize) {
13055                 dtrace_dof_error(dof, "section entry size exceeds total size");
13056                 return (NULL);
13057         }
13058 
13059         if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
13060                 dtrace_dof_error(dof, "bad entry size in action description");
13061                 return (NULL);
13062         }
13063 
13064         if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
13065                 dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
13066                 return (NULL);
13067         }
13068 
13069         for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
13070                 desc = (dof_actdesc_t *)(daddr +
13071                     (uintptr_t)sec->dofs_offset + offs);
13072                 kind = (dtrace_actkind_t)desc->dofa_kind;
13073 
13074                 if ((DTRACEACT_ISPRINTFLIKE(kind) &&
13075                     (kind != DTRACEACT_PRINTA ||
13076                     desc->dofa_strtab != DOF_SECIDX_NONE)) ||
13077                     (kind == DTRACEACT_DIFEXPR &&
13078                     desc->dofa_strtab != DOF_SECIDX_NONE)) {
13079                         dof_sec_t *strtab;
13080                         char *str, *fmt;
13081                         uint64_t i;
13082 
13083                         /*
13084                          * The argument to these actions is an index into the
13085                          * DOF string table.  For printf()-like actions, this
13086                          * is the format string.  For print(), this is the
13087                          * CTF type of the expression result.
13088                          */
13089                         if ((strtab = dtrace_dof_sect(dof,
13090                             DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
13091                                 goto err;
13092 
13093                         str = (char *)((uintptr_t)dof +
13094                             (uintptr_t)strtab->dofs_offset);
13095 
13096                         for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
13097                                 if (str[i] == '\0')
13098                                         break;
13099                         }
13100 
13101                         if (i >= strtab->dofs_size) {
13102                                 dtrace_dof_error(dof, "bogus format string");
13103                                 goto err;
13104                         }
13105 
13106                         if (i == desc->dofa_arg) {
13107                                 dtrace_dof_error(dof, "empty format string");
13108                                 goto err;
13109                         }
13110 
13111                         i -= desc->dofa_arg;
13112                         fmt = kmem_alloc(i + 1, KM_SLEEP);
13113                         bcopy(&str[desc->dofa_arg], fmt, i + 1);
13114                         arg = (uint64_t)(uintptr_t)fmt;
13115                 } else {
13116                         if (kind == DTRACEACT_PRINTA) {
13117                                 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
13118                                 arg = 0;
13119                         } else {
13120                                 arg = desc->dofa_arg;
13121                         }
13122                 }
13123 
13124                 act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
13125                     desc->dofa_uarg, arg);
13126 
13127                 if (last != NULL) {
13128                         last->dtad_next = act;
13129                 } else {
13130                         first = act;
13131                 }
13132 
13133                 last = act;
13134 
13135                 if (desc->dofa_difo == DOF_SECIDX_NONE)
13136                         continue;
13137 
13138                 if ((difosec = dtrace_dof_sect(dof,
13139                     DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
13140                         goto err;
13141 
13142                 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
13143 
13144                 if (act->dtad_difo == NULL)
13145                         goto err;
13146         }
13147 
13148         ASSERT(first != NULL);
13149         return (first);
13150 
13151 err:
13152         for (act = first; act != NULL; act = next) {
13153                 next = act->dtad_next;
13154                 dtrace_actdesc_release(act, vstate);
13155         }
13156 
13157         return (NULL);
13158 }
13159 
13160 static dtrace_ecbdesc_t *
13161 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
13162     cred_t *cr)
13163 {
13164         dtrace_ecbdesc_t *ep;
13165         dof_ecbdesc_t *ecb;
13166         dtrace_probedesc_t *desc;
13167         dtrace_predicate_t *pred = NULL;
13168 
13169         if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
13170                 dtrace_dof_error(dof, "truncated ECB description");
13171                 return (NULL);
13172         }
13173 
13174         if (sec->dofs_align != sizeof (uint64_t)) {
13175                 dtrace_dof_error(dof, "bad alignment in ECB description");
13176                 return (NULL);
13177         }
13178 
13179         ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
13180         sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
13181 
13182         if (sec == NULL)
13183                 return (NULL);
13184 
13185         ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
13186         ep->dted_uarg = ecb->dofe_uarg;
13187         desc = &ep->dted_probe;
13188 
13189         if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
13190                 goto err;
13191 
13192         if (ecb->dofe_pred != DOF_SECIDX_NONE) {
13193                 if ((sec = dtrace_dof_sect(dof,
13194                     DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
13195                         goto err;
13196 
13197                 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
13198                         goto err;
13199 
13200                 ep->dted_pred.dtpdd_predicate = pred;
13201         }
13202 
13203         if (ecb->dofe_actions != DOF_SECIDX_NONE) {
13204                 if ((sec = dtrace_dof_sect(dof,
13205                     DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
13206                         goto err;
13207 
13208                 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
13209 
13210                 if (ep->dted_action == NULL)
13211                         goto err;
13212         }
13213 
13214         return (ep);
13215 
13216 err:
13217         if (pred != NULL)
13218                 dtrace_predicate_release(pred, vstate);
13219         kmem_free(ep, sizeof (dtrace_ecbdesc_t));
13220         return (NULL);
13221 }
13222 
13223 /*
13224  * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
13225  * specified DOF.  At present, this amounts to simply adding 'ubase' to the
13226  * site of any user SETX relocations to account for load object base address.
13227  * In the future, if we need other relocations, this function can be extended.
13228  */
13229 static int
13230 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
13231 {
13232         uintptr_t daddr = (uintptr_t)dof;
13233         uintptr_t ts_end;
13234         dof_relohdr_t *dofr =
13235             (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
13236         dof_sec_t *ss, *rs, *ts;
13237         dof_relodesc_t *r;
13238         uint_t i, n;
13239 
13240         if (sec->dofs_size < sizeof (dof_relohdr_t) ||
13241             sec->dofs_align != sizeof (dof_secidx_t)) {
13242                 dtrace_dof_error(dof, "invalid relocation header");
13243                 return (-1);
13244         }
13245 
13246         ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
13247         rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
13248         ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
13249         ts_end = (uintptr_t)ts + sizeof (dof_sec_t);
13250 
13251         if (ss == NULL || rs == NULL || ts == NULL)
13252                 return (-1); /* dtrace_dof_error() has been called already */
13253 
13254         if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
13255             rs->dofs_align != sizeof (uint64_t)) {
13256                 dtrace_dof_error(dof, "invalid relocation section");
13257                 return (-1);
13258         }
13259 
13260         r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
13261         n = rs->dofs_size / rs->dofs_entsize;
13262 
13263         for (i = 0; i < n; i++) {
13264                 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
13265 
13266                 switch (r->dofr_type) {
13267                 case DOF_RELO_NONE:
13268                         break;
13269                 case DOF_RELO_SETX:
13270                         if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
13271                             sizeof (uint64_t) > ts->dofs_size) {
13272                                 dtrace_dof_error(dof, "bad relocation offset");
13273                                 return (-1);
13274                         }
13275 
13276                         if (taddr >= (uintptr_t)ts && taddr < ts_end) {
13277                                 dtrace_dof_error(dof, "bad relocation offset");
13278                                 return (-1);
13279                         }
13280 
13281                         if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
13282                                 dtrace_dof_error(dof, "misaligned setx relo");
13283                                 return (-1);
13284                         }
13285 
13286                         *(uint64_t *)taddr += ubase;
13287                         break;
13288                 default:
13289                         dtrace_dof_error(dof, "invalid relocation type");
13290                         return (-1);
13291                 }
13292 
13293                 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
13294         }
13295 
13296         return (0);
13297 }
13298 
13299 /*
13300  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
13301  * header:  it should be at the front of a memory region that is at least
13302  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
13303  * size.  It need not be validated in any other way.
13304  */
13305 static int
13306 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
13307     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
13308 {
13309         uint64_t len = dof->dofh_loadsz, seclen;
13310         uintptr_t daddr = (uintptr_t)dof;
13311         dtrace_ecbdesc_t *ep;
13312         dtrace_enabling_t *enab;
13313         uint_t i;
13314 
13315         ASSERT(MUTEX_HELD(&dtrace_lock));
13316         ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
13317 
13318         /*
13319          * Check the DOF header identification bytes.  In addition to checking
13320          * valid settings, we also verify that unused bits/bytes are zeroed so
13321          * we can use them later without fear of regressing existing binaries.
13322          */
13323         if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
13324             DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
13325                 dtrace_dof_error(dof, "DOF magic string mismatch");
13326                 return (-1);
13327         }
13328 
13329         if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
13330             dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
13331                 dtrace_dof_error(dof, "DOF has invalid data model");
13332                 return (-1);
13333         }
13334 
13335         if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
13336                 dtrace_dof_error(dof, "DOF encoding mismatch");
13337                 return (-1);
13338         }
13339 
13340         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
13341             dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
13342                 dtrace_dof_error(dof, "DOF version mismatch");
13343                 return (-1);
13344         }
13345 
13346         if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
13347                 dtrace_dof_error(dof, "DOF uses unsupported instruction set");
13348                 return (-1);
13349         }
13350 
13351         if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
13352                 dtrace_dof_error(dof, "DOF uses too many integer registers");
13353                 return (-1);
13354         }
13355 
13356         if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
13357                 dtrace_dof_error(dof, "DOF uses too many tuple registers");
13358                 return (-1);
13359         }
13360 
13361         for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
13362                 if (dof->dofh_ident[i] != 0) {
13363                         dtrace_dof_error(dof, "DOF has invalid ident byte set");
13364                         return (-1);
13365                 }
13366         }
13367 
13368         if (dof->dofh_flags & ~DOF_FL_VALID) {
13369                 dtrace_dof_error(dof, "DOF has invalid flag bits set");
13370                 return (-1);
13371         }
13372 
13373         if (dof->dofh_secsize == 0) {
13374                 dtrace_dof_error(dof, "zero section header size");
13375                 return (-1);
13376         }
13377 
13378         /*
13379          * Check that the section headers don't exceed the amount of DOF
13380          * data.  Note that we cast the section size and number of sections
13381          * to uint64_t's to prevent possible overflow in the multiplication.
13382          */
13383         seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
13384 
13385         if (dof->dofh_secoff > len || seclen > len ||
13386             dof->dofh_secoff + seclen > len) {
13387                 dtrace_dof_error(dof, "truncated section headers");
13388                 return (-1);
13389         }
13390 
13391         if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
13392                 dtrace_dof_error(dof, "misaligned section headers");
13393                 return (-1);
13394         }
13395 
13396         if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
13397                 dtrace_dof_error(dof, "misaligned section size");
13398                 return (-1);
13399         }
13400 
13401         /*
13402          * Take an initial pass through the section headers to be sure that
13403          * the headers don't have stray offsets.  If the 'noprobes' flag is
13404          * set, do not permit sections relating to providers, probes, or args.
13405          */
13406         for (i = 0; i < dof->dofh_secnum; i++) {
13407                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13408                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13409 
13410                 if (noprobes) {
13411                         switch (sec->dofs_type) {
13412                         case DOF_SECT_PROVIDER:
13413                         case DOF_SECT_PROBES:
13414                         case DOF_SECT_PRARGS:
13415                         case DOF_SECT_PROFFS:
13416                                 dtrace_dof_error(dof, "illegal sections "
13417                                     "for enabling");
13418                                 return (-1);
13419                         }
13420                 }
13421 
13422                 if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
13423                     !(sec->dofs_flags & DOF_SECF_LOAD)) {
13424                         dtrace_dof_error(dof, "loadable section with load "
13425                             "flag unset");
13426                         return (-1);
13427                 }
13428 
13429                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
13430                         continue; /* just ignore non-loadable sections */
13431 
13432                 if (!ISP2(sec->dofs_align)) {
13433                         dtrace_dof_error(dof, "bad section alignment");
13434                         return (-1);
13435                 }
13436 
13437                 if (sec->dofs_offset & (sec->dofs_align - 1)) {
13438                         dtrace_dof_error(dof, "misaligned section");
13439                         return (-1);
13440                 }
13441 
13442                 if (sec->dofs_offset > len || sec->dofs_size > len ||
13443                     sec->dofs_offset + sec->dofs_size > len) {
13444                         dtrace_dof_error(dof, "corrupt section header");
13445                         return (-1);
13446                 }
13447 
13448                 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
13449                     sec->dofs_offset + sec->dofs_size - 1) != '\0') {
13450                         dtrace_dof_error(dof, "non-terminating string table");
13451                         return (-1);
13452                 }
13453         }
13454 
13455         /*
13456          * Take a second pass through the sections and locate and perform any
13457          * relocations that are present.  We do this after the first pass to
13458          * be sure that all sections have had their headers validated.
13459          */
13460         for (i = 0; i < dof->dofh_secnum; i++) {
13461                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13462                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13463 
13464                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
13465                         continue; /* skip sections that are not loadable */
13466 
13467                 switch (sec->dofs_type) {
13468                 case DOF_SECT_URELHDR:
13469                         if (dtrace_dof_relocate(dof, sec, ubase) != 0)
13470                                 return (-1);
13471                         break;
13472                 }
13473         }
13474 
13475         if ((enab = *enabp) == NULL)
13476                 enab = *enabp = dtrace_enabling_create(vstate);
13477 
13478         for (i = 0; i < dof->dofh_secnum; i++) {
13479                 dof_sec_t *sec = (dof_sec_t *)(daddr +
13480                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13481 
13482                 if (sec->dofs_type != DOF_SECT_ECBDESC)
13483                         continue;
13484 
13485                 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
13486                         dtrace_enabling_destroy(enab);
13487                         *enabp = NULL;
13488                         return (-1);
13489                 }
13490 
13491                 dtrace_enabling_add(enab, ep);
13492         }
13493 
13494         return (0);
13495 }
13496 
13497 /*
13498  * Process DOF for any options.  This routine assumes that the DOF has been
13499  * at least processed by dtrace_dof_slurp().
13500  */
13501 static int
13502 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
13503 {
13504         int i, rval;
13505         uint32_t entsize;
13506         size_t offs;
13507         dof_optdesc_t *desc;
13508 
13509         for (i = 0; i < dof->dofh_secnum; i++) {
13510                 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
13511                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
13512 
13513                 if (sec->dofs_type != DOF_SECT_OPTDESC)
13514                         continue;
13515 
13516                 if (sec->dofs_align != sizeof (uint64_t)) {
13517                         dtrace_dof_error(dof, "bad alignment in "
13518                             "option description");
13519                         return (EINVAL);
13520                 }
13521 
13522                 if ((entsize = sec->dofs_entsize) == 0) {
13523                         dtrace_dof_error(dof, "zeroed option entry size");
13524                         return (EINVAL);
13525                 }
13526 
13527                 if (entsize < sizeof (dof_optdesc_t)) {
13528                         dtrace_dof_error(dof, "bad option entry size");
13529                         return (EINVAL);
13530                 }
13531 
13532                 for (offs = 0; offs < sec->dofs_size; offs += entsize) {
13533                         desc = (dof_optdesc_t *)((uintptr_t)dof +
13534                             (uintptr_t)sec->dofs_offset + offs);
13535 
13536                         if (desc->dofo_strtab != DOF_SECIDX_NONE) {
13537                                 dtrace_dof_error(dof, "non-zero option string");
13538                                 return (EINVAL);
13539                         }
13540 
13541                         if (desc->dofo_value == DTRACEOPT_UNSET) {
13542                                 dtrace_dof_error(dof, "unset option");
13543                                 return (EINVAL);
13544                         }
13545 
13546                         if ((rval = dtrace_state_option(state,
13547                             desc->dofo_option, desc->dofo_value)) != 0) {
13548                                 dtrace_dof_error(dof, "rejected option");
13549                                 return (rval);
13550                         }
13551                 }
13552         }
13553 
13554         return (0);
13555 }
13556 
13557 /*
13558  * DTrace Consumer State Functions
13559  */
13560 int
13561 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
13562 {
13563         size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
13564         void *base;
13565         uintptr_t limit;
13566         dtrace_dynvar_t *dvar, *next, *start;
13567         int i;
13568 
13569         ASSERT(MUTEX_HELD(&dtrace_lock));
13570         ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
13571 
13572         bzero(dstate, sizeof (dtrace_dstate_t));
13573 
13574         if ((dstate->dtds_chunksize = chunksize) == 0)
13575                 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
13576 
13577         VERIFY(dstate->dtds_chunksize < LONG_MAX);
13578 
13579         if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
13580                 size = min;
13581 
13582         if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
13583                 return (ENOMEM);
13584 
13585         dstate->dtds_size = size;
13586         dstate->dtds_base = base;
13587         dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
13588         bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
13589 
13590         hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
13591 
13592         if (hashsize != 1 && (hashsize & 1))
13593                 hashsize--;
13594 
13595         dstate->dtds_hashsize = hashsize;
13596         dstate->dtds_hash = dstate->dtds_base;
13597 
13598         /*
13599          * Set all of our hash buckets to point to the single sink, and (if
13600          * it hasn't already been set), set the sink's hash value to be the
13601          * sink sentinel value.  The sink is needed for dynamic variable
13602          * lookups to know that they have iterated over an entire, valid hash
13603          * chain.
13604          */
13605         for (i = 0; i < hashsize; i++)
13606                 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
13607 
13608         if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
13609                 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
13610 
13611         /*
13612          * Determine number of active CPUs.  Divide free list evenly among
13613          * active CPUs.
13614          */
13615         start = (dtrace_dynvar_t *)
13616             ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
13617         limit = (uintptr_t)base + size;
13618 
13619         VERIFY((uintptr_t)start < limit);
13620         VERIFY((uintptr_t)start >= (uintptr_t)base);
13621 
13622         maxper = (limit - (uintptr_t)start) / NCPU;
13623         maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
13624 
13625         for (i = 0; i < NCPU; i++) {
13626                 dstate->dtds_percpu[i].dtdsc_free = dvar = start;
13627 
13628                 /*
13629                  * If we don't even have enough chunks to make it once through
13630                  * NCPUs, we're just going to allocate everything to the first
13631                  * CPU.  And if we're on the last CPU, we're going to allocate
13632                  * whatever is left over.  In either case, we set the limit to
13633                  * be the limit of the dynamic variable space.
13634                  */
13635                 if (maxper == 0 || i == NCPU - 1) {
13636                         limit = (uintptr_t)base + size;
13637                         start = NULL;
13638                 } else {
13639                         limit = (uintptr_t)start + maxper;
13640                         start = (dtrace_dynvar_t *)limit;
13641                 }
13642 
13643                 VERIFY(limit <= (uintptr_t)base + size);
13644 
13645                 for (;;) {
13646                         next = (dtrace_dynvar_t *)((uintptr_t)dvar +
13647                             dstate->dtds_chunksize);
13648 
13649                         if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
13650                                 break;
13651 
13652                         VERIFY((uintptr_t)dvar >= (uintptr_t)base &&
13653                             (uintptr_t)dvar <= (uintptr_t)base + size);
13654                         dvar->dtdv_next = next;
13655                         dvar = next;
13656                 }
13657 
13658                 if (maxper == 0)
13659                         break;
13660         }
13661 
13662         return (0);
13663 }
13664 
13665 void
13666 dtrace_dstate_fini(dtrace_dstate_t *dstate)
13667 {
13668         ASSERT(MUTEX_HELD(&cpu_lock));
13669 
13670         if (dstate->dtds_base == NULL)
13671                 return;
13672 
13673         kmem_free(dstate->dtds_base, dstate->dtds_size);
13674         kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
13675 }
13676 
13677 static void
13678 dtrace_vstate_fini(dtrace_vstate_t *vstate)
13679 {
13680         /*
13681          * Logical XOR, where are you?
13682          */
13683         ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
13684 
13685         if (vstate->dtvs_nglobals > 0) {
13686                 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
13687                     sizeof (dtrace_statvar_t *));
13688         }
13689 
13690         if (vstate->dtvs_ntlocals > 0) {
13691                 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
13692                     sizeof (dtrace_difv_t));
13693         }
13694 
13695         ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
13696 
13697         if (vstate->dtvs_nlocals > 0) {
13698                 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
13699                     sizeof (dtrace_statvar_t *));
13700         }
13701 }
13702 
13703 static void
13704 dtrace_state_clean(dtrace_state_t *state)
13705 {
13706         if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
13707                 return;
13708 
13709         dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
13710         dtrace_speculation_clean(state);
13711 }
13712 
13713 static void
13714 dtrace_state_deadman(dtrace_state_t *state)
13715 {
13716         hrtime_t now;
13717 
13718         dtrace_sync();
13719 
13720         now = dtrace_gethrtime();
13721 
13722         if (state != dtrace_anon.dta_state &&
13723             now - state->dts_laststatus >= dtrace_deadman_user)
13724                 return;
13725 
13726         /*
13727          * We must be sure that dts_alive never appears to be less than the
13728          * value upon entry to dtrace_state_deadman(), and because we lack a
13729          * dtrace_cas64(), we cannot store to it atomically.  We thus instead
13730          * store INT64_MAX to it, followed by a memory barrier, followed by
13731          * the new value.  This assures that dts_alive never appears to be
13732          * less than its true value, regardless of the order in which the
13733          * stores to the underlying storage are issued.
13734          */
13735         state->dts_alive = INT64_MAX;
13736         dtrace_membar_producer();
13737         state->dts_alive = now;
13738 }
13739 
13740 dtrace_state_t *
13741 dtrace_state_create(dev_t *devp, cred_t *cr)
13742 {
13743         minor_t minor;
13744         major_t major;
13745         char c[30];
13746         dtrace_state_t *state;
13747         dtrace_optval_t *opt;
13748         int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
13749 
13750         ASSERT(MUTEX_HELD(&dtrace_lock));
13751         ASSERT(MUTEX_HELD(&cpu_lock));
13752 
13753         minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
13754             VM_BESTFIT | VM_SLEEP);
13755 
13756         if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
13757                 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
13758                 return (NULL);
13759         }
13760 
13761         state = ddi_get_soft_state(dtrace_softstate, minor);
13762         state->dts_epid = DTRACE_EPIDNONE + 1;
13763 
13764         (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
13765         state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
13766             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
13767 
13768         if (devp != NULL) {
13769                 major = getemajor(*devp);
13770         } else {
13771                 major = ddi_driver_major(dtrace_devi);
13772         }
13773 
13774         state->dts_dev = makedevice(major, minor);
13775 
13776         if (devp != NULL)
13777                 *devp = state->dts_dev;
13778 
13779         /*
13780          * We allocate NCPU buffers.  On the one hand, this can be quite
13781          * a bit of memory per instance (nearly 36K on a Starcat).  On the
13782          * other hand, it saves an additional memory reference in the probe
13783          * path.
13784          */
13785         state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
13786         state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
13787         state->dts_cleaner = CYCLIC_NONE;
13788         state->dts_deadman = CYCLIC_NONE;
13789         state->dts_vstate.dtvs_state = state;
13790 
13791         for (i = 0; i < DTRACEOPT_MAX; i++)
13792                 state->dts_options[i] = DTRACEOPT_UNSET;
13793 
13794         /*
13795          * Set the default options.
13796          */
13797         opt = state->dts_options;
13798         opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
13799         opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
13800         opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
13801         opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
13802         opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
13803         opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
13804         opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
13805         opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
13806         opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
13807         opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
13808         opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
13809         opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
13810         opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
13811         opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
13812 
13813         state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
13814 
13815         /*
13816          * Depending on the user credentials, we set flag bits which alter probe
13817          * visibility or the amount of destructiveness allowed.  In the case of
13818          * actual anonymous tracing, or the possession of all privileges, all of
13819          * the normal checks are bypassed.
13820          */
13821         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
13822                 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
13823                 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
13824         } else {
13825                 /*
13826                  * Set up the credentials for this instantiation.  We take a
13827                  * hold on the credential to prevent it from disappearing on
13828                  * us; this in turn prevents the zone_t referenced by this
13829                  * credential from disappearing.  This means that we can
13830                  * examine the credential and the zone from probe context.
13831                  */
13832                 crhold(cr);
13833                 state->dts_cred.dcr_cred = cr;
13834 
13835                 /*
13836                  * CRA_PROC means "we have *some* privilege for dtrace" and
13837                  * unlocks the use of variables like pid, zonename, etc.
13838                  */
13839                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
13840                     PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13841                         state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
13842                 }
13843 
13844                 /*
13845                  * dtrace_user allows use of syscall and profile providers.
13846                  * If the user also has proc_owner and/or proc_zone, we
13847                  * extend the scope to include additional visibility and
13848                  * destructive power.
13849                  */
13850                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
13851                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
13852                                 state->dts_cred.dcr_visible |=
13853                                     DTRACE_CRV_ALLPROC;
13854 
13855                                 state->dts_cred.dcr_action |=
13856                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13857                         }
13858 
13859                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
13860                                 state->dts_cred.dcr_visible |=
13861                                     DTRACE_CRV_ALLZONE;
13862 
13863                                 state->dts_cred.dcr_action |=
13864                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13865                         }
13866 
13867                         /*
13868                          * If we have all privs in whatever zone this is,
13869                          * we can do destructive things to processes which
13870                          * have altered credentials.
13871                          */
13872                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13873                             cr->cr_zone->zone_privset)) {
13874                                 state->dts_cred.dcr_action |=
13875                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13876                         }
13877                 }
13878 
13879                 /*
13880                  * Holding the dtrace_kernel privilege also implies that
13881                  * the user has the dtrace_user privilege from a visibility
13882                  * perspective.  But without further privileges, some
13883                  * destructive actions are not available.
13884                  */
13885                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
13886                         /*
13887                          * Make all probes in all zones visible.  However,
13888                          * this doesn't mean that all actions become available
13889                          * to all zones.
13890                          */
13891                         state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
13892                             DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
13893 
13894                         state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
13895                             DTRACE_CRA_PROC;
13896                         /*
13897                          * Holding proc_owner means that destructive actions
13898                          * for *this* zone are allowed.
13899                          */
13900                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13901                                 state->dts_cred.dcr_action |=
13902                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13903 
13904                         /*
13905                          * Holding proc_zone means that destructive actions
13906                          * for this user/group ID in all zones is allowed.
13907                          */
13908                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13909                                 state->dts_cred.dcr_action |=
13910                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13911 
13912                         /*
13913                          * If we have all privs in whatever zone this is,
13914                          * we can do destructive things to processes which
13915                          * have altered credentials.
13916                          */
13917                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
13918                             cr->cr_zone->zone_privset)) {
13919                                 state->dts_cred.dcr_action |=
13920                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
13921                         }
13922                 }
13923 
13924                 /*
13925                  * Holding the dtrace_proc privilege gives control over fasttrap
13926                  * and pid providers.  We need to grant wider destructive
13927                  * privileges in the event that the user has proc_owner and/or
13928                  * proc_zone.
13929                  */
13930                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
13931                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
13932                                 state->dts_cred.dcr_action |=
13933                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
13934 
13935                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
13936                                 state->dts_cred.dcr_action |=
13937                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
13938                 }
13939         }
13940 
13941         return (state);
13942 }
13943 
13944 static int
13945 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
13946 {
13947         dtrace_optval_t *opt = state->dts_options, size;
13948         processorid_t cpu;
13949         int flags = 0, rval, factor, divisor = 1;
13950 
13951         ASSERT(MUTEX_HELD(&dtrace_lock));
13952         ASSERT(MUTEX_HELD(&cpu_lock));
13953         ASSERT(which < DTRACEOPT_MAX);
13954         ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
13955             (state == dtrace_anon.dta_state &&
13956             state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
13957 
13958         if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
13959                 return (0);
13960 
13961         if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
13962                 cpu = opt[DTRACEOPT_CPU];
13963 
13964         if (which == DTRACEOPT_SPECSIZE)
13965                 flags |= DTRACEBUF_NOSWITCH;
13966 
13967         if (which == DTRACEOPT_BUFSIZE) {
13968                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
13969                         flags |= DTRACEBUF_RING;
13970 
13971                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
13972                         flags |= DTRACEBUF_FILL;
13973 
13974                 if (state != dtrace_anon.dta_state ||
13975                     state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
13976                         flags |= DTRACEBUF_INACTIVE;
13977         }
13978 
13979         for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
13980                 /*
13981                  * The size must be 8-byte aligned.  If the size is not 8-byte
13982                  * aligned, drop it down by the difference.
13983                  */
13984                 if (size & (sizeof (uint64_t) - 1))
13985                         size -= size & (sizeof (uint64_t) - 1);
13986 
13987                 if (size < state->dts_reserve) {
13988                         /*
13989                          * Buffers always must be large enough to accommodate
13990                          * their prereserved space.  We return E2BIG instead
13991                          * of ENOMEM in this case to allow for user-level
13992                          * software to differentiate the cases.
13993                          */
13994                         return (E2BIG);
13995                 }
13996 
13997                 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
13998 
13999                 if (rval != ENOMEM) {
14000                         opt[which] = size;
14001                         return (rval);
14002                 }
14003 
14004                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
14005                         return (rval);
14006 
14007                 for (divisor = 2; divisor < factor; divisor <<= 1)
14008                         continue;
14009         }
14010 
14011         return (ENOMEM);
14012 }
14013 
14014 static int
14015 dtrace_state_buffers(dtrace_state_t *state)
14016 {
14017         dtrace_speculation_t *spec = state->dts_speculations;
14018         int rval, i;
14019 
14020         if ((rval = dtrace_state_buffer(state, state->dts_buffer,
14021             DTRACEOPT_BUFSIZE)) != 0)
14022                 return (rval);
14023 
14024         if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
14025             DTRACEOPT_AGGSIZE)) != 0)
14026                 return (rval);
14027 
14028         for (i = 0; i < state->dts_nspeculations; i++) {
14029                 if ((rval = dtrace_state_buffer(state,
14030                     spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
14031                         return (rval);
14032         }
14033 
14034         return (0);
14035 }
14036 
14037 static void
14038 dtrace_state_prereserve(dtrace_state_t *state)
14039 {
14040         dtrace_ecb_t *ecb;
14041         dtrace_probe_t *probe;
14042 
14043         state->dts_reserve = 0;
14044 
14045         if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
14046                 return;
14047 
14048         /*
14049          * If our buffer policy is a "fill" buffer policy, we need to set the
14050          * prereserved space to be the space required by the END probes.
14051          */
14052         probe = dtrace_probes[dtrace_probeid_end - 1];
14053         ASSERT(probe != NULL);
14054 
14055         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
14056                 if (ecb->dte_state != state)
14057                         continue;
14058 
14059                 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
14060         }
14061 }
14062 
14063 static int
14064 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
14065 {
14066         dtrace_optval_t *opt = state->dts_options, sz, nspec;
14067         dtrace_speculation_t *spec;
14068         dtrace_buffer_t *buf;
14069         cyc_handler_t hdlr;
14070         cyc_time_t when;
14071         int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
14072         dtrace_icookie_t cookie;
14073 
14074         mutex_enter(&cpu_lock);
14075         mutex_enter(&dtrace_lock);
14076 
14077         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
14078                 rval = EBUSY;
14079                 goto out;
14080         }
14081 
14082         /*
14083          * Before we can perform any checks, we must prime all of the
14084          * retained enablings that correspond to this state.
14085          */
14086         dtrace_enabling_prime(state);
14087 
14088         if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
14089                 rval = EACCES;
14090                 goto out;
14091         }
14092 
14093         dtrace_state_prereserve(state);
14094 
14095         /*
14096          * Now we want to do is try to allocate our speculations.
14097          * We do not automatically resize the number of speculations; if
14098          * this fails, we will fail the operation.
14099          */
14100         nspec = opt[DTRACEOPT_NSPEC];
14101         ASSERT(nspec != DTRACEOPT_UNSET);
14102 
14103         if (nspec > INT_MAX) {
14104                 rval = ENOMEM;
14105                 goto out;
14106         }
14107 
14108         spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
14109             KM_NOSLEEP | KM_NORMALPRI);
14110 
14111         if (spec == NULL) {
14112                 rval = ENOMEM;
14113                 goto out;
14114         }
14115 
14116         state->dts_speculations = spec;
14117         state->dts_nspeculations = (int)nspec;
14118 
14119         for (i = 0; i < nspec; i++) {
14120                 if ((buf = kmem_zalloc(bufsize,
14121                     KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
14122                         rval = ENOMEM;
14123                         goto err;
14124                 }
14125 
14126                 spec[i].dtsp_buffer = buf;
14127         }
14128 
14129         if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
14130                 if (dtrace_anon.dta_state == NULL) {
14131                         rval = ENOENT;
14132                         goto out;
14133                 }
14134 
14135                 if (state->dts_necbs != 0) {
14136                         rval = EALREADY;
14137                         goto out;
14138                 }
14139 
14140                 state->dts_anon = dtrace_anon_grab();
14141                 ASSERT(state->dts_anon != NULL);
14142                 state = state->dts_anon;
14143 
14144                 /*
14145                  * We want "grabanon" to be set in the grabbed state, so we'll
14146                  * copy that option value from the grabbing state into the
14147                  * grabbed state.
14148                  */
14149                 state->dts_options[DTRACEOPT_GRABANON] =
14150                     opt[DTRACEOPT_GRABANON];
14151 
14152                 *cpu = dtrace_anon.dta_beganon;
14153 
14154                 /*
14155                  * If the anonymous state is active (as it almost certainly
14156                  * is if the anonymous enabling ultimately matched anything),
14157                  * we don't allow any further option processing -- but we
14158                  * don't return failure.
14159                  */
14160                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
14161                         goto out;
14162         }
14163 
14164         if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
14165             opt[DTRACEOPT_AGGSIZE] != 0) {
14166                 if (state->dts_aggregations == NULL) {
14167                         /*
14168                          * We're not going to create an aggregation buffer
14169                          * because we don't have any ECBs that contain
14170                          * aggregations -- set this option to 0.
14171                          */
14172                         opt[DTRACEOPT_AGGSIZE] = 0;
14173                 } else {
14174                         /*
14175                          * If we have an aggregation buffer, we must also have
14176                          * a buffer to use as scratch.
14177                          */
14178                         if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
14179                             opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
14180                                 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
14181                         }
14182                 }
14183         }
14184 
14185         if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
14186             opt[DTRACEOPT_SPECSIZE] != 0) {
14187                 if (!state->dts_speculates) {
14188                         /*
14189                          * We're not going to create speculation buffers
14190                          * because we don't have any ECBs that actually
14191                          * speculate -- set the speculation size to 0.
14192                          */
14193                         opt[DTRACEOPT_SPECSIZE] = 0;
14194                 }
14195         }
14196 
14197         /*
14198          * The bare minimum size for any buffer that we're actually going to
14199          * do anything to is sizeof (uint64_t).
14200          */
14201         sz = sizeof (uint64_t);
14202 
14203         if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
14204             (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
14205             (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
14206                 /*
14207                  * A buffer size has been explicitly set to 0 (or to a size
14208                  * that will be adjusted to 0) and we need the space -- we
14209                  * need to return failure.  We return ENOSPC to differentiate
14210                  * it from failing to allocate a buffer due to failure to meet
14211                  * the reserve (for which we return E2BIG).
14212                  */
14213                 rval = ENOSPC;
14214                 goto out;
14215         }
14216 
14217         if ((rval = dtrace_state_buffers(state)) != 0)
14218                 goto err;
14219 
14220         if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
14221                 sz = dtrace_dstate_defsize;
14222 
14223         do {
14224                 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
14225 
14226                 if (rval == 0)
14227                         break;
14228 
14229                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
14230                         goto err;
14231         } while (sz >>= 1);
14232 
14233         opt[DTRACEOPT_DYNVARSIZE] = sz;
14234 
14235         if (rval != 0)
14236                 goto err;
14237 
14238         if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
14239                 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
14240 
14241         if (opt[DTRACEOPT_CLEANRATE] == 0)
14242                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
14243 
14244         if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
14245                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
14246 
14247         if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
14248                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
14249 
14250         hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
14251         hdlr.cyh_arg = state;
14252         hdlr.cyh_level = CY_LOW_LEVEL;
14253 
14254         when.cyt_when = 0;
14255         when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
14256 
14257         state->dts_cleaner = cyclic_add(&hdlr, &when);
14258 
14259         hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
14260         hdlr.cyh_arg = state;
14261         hdlr.cyh_level = CY_LOW_LEVEL;
14262 
14263         when.cyt_when = 0;
14264         when.cyt_interval = dtrace_deadman_interval;
14265 
14266         state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
14267         state->dts_deadman = cyclic_add(&hdlr, &when);
14268 
14269         state->dts_activity = DTRACE_ACTIVITY_WARMUP;
14270 
14271         if (state->dts_getf != 0 &&
14272             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
14273                 /*
14274                  * We don't have kernel privs but we have at least one call
14275                  * to getf(); we need to bump our zone's count, and (if
14276                  * this is the first enabling to have an unprivileged call
14277                  * to getf()) we need to hook into closef().
14278                  */
14279                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++;
14280 
14281                 if (dtrace_getf++ == 0) {
14282                         ASSERT(dtrace_closef == NULL);
14283                         dtrace_closef = dtrace_getf_barrier;
14284                 }
14285         }
14286 
14287         /*
14288          * Now it's time to actually fire the BEGIN probe.  We need to disable
14289          * interrupts here both to record the CPU on which we fired the BEGIN
14290          * probe (the data from this CPU will be processed first at user
14291          * level) and to manually activate the buffer for this CPU.
14292          */
14293         cookie = dtrace_interrupt_disable();
14294         *cpu = CPU->cpu_id;
14295         ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
14296         state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
14297 
14298         dtrace_probe(dtrace_probeid_begin,
14299             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
14300         dtrace_interrupt_enable(cookie);
14301         /*
14302          * We may have had an exit action from a BEGIN probe; only change our
14303          * state to ACTIVE if we're still in WARMUP.
14304          */
14305         ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
14306             state->dts_activity == DTRACE_ACTIVITY_DRAINING);
14307 
14308         if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
14309                 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
14310 
14311         /*
14312          * Regardless of whether or not now we're in ACTIVE or DRAINING, we
14313          * want each CPU to transition its principal buffer out of the
14314          * INACTIVE state.  Doing this assures that no CPU will suddenly begin
14315          * processing an ECB halfway down a probe's ECB chain; all CPUs will
14316          * atomically transition from processing none of a state's ECBs to
14317          * processing all of them.
14318          */
14319         dtrace_xcall(DTRACE_CPUALL,
14320             (dtrace_xcall_t)dtrace_buffer_activate, state);
14321         goto out;
14322 
14323 err:
14324         dtrace_buffer_free(state->dts_buffer);
14325         dtrace_buffer_free(state->dts_aggbuffer);
14326 
14327         if ((nspec = state->dts_nspeculations) == 0) {
14328                 ASSERT(state->dts_speculations == NULL);
14329                 goto out;
14330         }
14331 
14332         spec = state->dts_speculations;
14333         ASSERT(spec != NULL);
14334 
14335         for (i = 0; i < state->dts_nspeculations; i++) {
14336                 if ((buf = spec[i].dtsp_buffer) == NULL)
14337                         break;
14338 
14339                 dtrace_buffer_free(buf);
14340                 kmem_free(buf, bufsize);
14341         }
14342 
14343         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
14344         state->dts_nspeculations = 0;
14345         state->dts_speculations = NULL;
14346 
14347 out:
14348         mutex_exit(&dtrace_lock);
14349         mutex_exit(&cpu_lock);
14350 
14351         return (rval);
14352 }
14353 
14354 static int
14355 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
14356 {
14357         dtrace_icookie_t cookie;
14358 
14359         ASSERT(MUTEX_HELD(&dtrace_lock));
14360 
14361         if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
14362             state->dts_activity != DTRACE_ACTIVITY_DRAINING)
14363                 return (EINVAL);
14364 
14365         /*
14366          * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
14367          * to be sure that every CPU has seen it.  See below for the details
14368          * on why this is done.
14369          */
14370         state->dts_activity = DTRACE_ACTIVITY_DRAINING;
14371         dtrace_sync();
14372 
14373         /*
14374          * By this point, it is impossible for any CPU to be still processing
14375          * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
14376          * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
14377          * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
14378          * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
14379          * iff we're in the END probe.
14380          */
14381         state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
14382         dtrace_sync();
14383         ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
14384 
14385         /*
14386          * Finally, we can release the reserve and call the END probe.  We
14387          * disable interrupts across calling the END probe to allow us to
14388          * return the CPU on which we actually called the END probe.  This
14389          * allows user-land to be sure that this CPU's principal buffer is
14390          * processed last.
14391          */
14392         state->dts_reserve = 0;
14393 
14394         cookie = dtrace_interrupt_disable();
14395         *cpu = CPU->cpu_id;
14396         dtrace_probe(dtrace_probeid_end,
14397             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
14398         dtrace_interrupt_enable(cookie);
14399 
14400         state->dts_activity = DTRACE_ACTIVITY_STOPPED;
14401         dtrace_sync();
14402 
14403         if (state->dts_getf != 0 &&
14404             !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
14405                 /*
14406                  * We don't have kernel privs but we have at least one call
14407                  * to getf(); we need to lower our zone's count, and (if
14408                  * this is the last enabling to have an unprivileged call
14409                  * to getf()) we need to clear the closef() hook.
14410                  */
14411                 ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0);
14412                 ASSERT(dtrace_closef == dtrace_getf_barrier);
14413                 ASSERT(dtrace_getf > 0);
14414 
14415                 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--;
14416 
14417                 if (--dtrace_getf == 0)
14418                         dtrace_closef = NULL;
14419         }
14420 
14421         return (0);
14422 }
14423 
14424 static int
14425 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
14426     dtrace_optval_t val)
14427 {
14428         ASSERT(MUTEX_HELD(&dtrace_lock));
14429 
14430         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
14431                 return (EBUSY);
14432 
14433         if (option >= DTRACEOPT_MAX)
14434                 return (EINVAL);
14435 
14436         if (option != DTRACEOPT_CPU && val < 0)
14437                 return (EINVAL);
14438 
14439         switch (option) {
14440         case DTRACEOPT_DESTRUCTIVE:
14441                 if (dtrace_destructive_disallow)
14442                         return (EACCES);
14443 
14444                 state->dts_cred.dcr_destructive = 1;
14445                 break;
14446 
14447         case DTRACEOPT_BUFSIZE:
14448         case DTRACEOPT_DYNVARSIZE:
14449         case DTRACEOPT_AGGSIZE:
14450         case DTRACEOPT_SPECSIZE:
14451         case DTRACEOPT_STRSIZE:
14452                 if (val < 0)
14453                         return (EINVAL);
14454 
14455                 if (val >= LONG_MAX) {
14456                         /*
14457                          * If this is an otherwise negative value, set it to
14458                          * the highest multiple of 128m less than LONG_MAX.
14459                          * Technically, we're adjusting the size without
14460                          * regard to the buffer resizing policy, but in fact,
14461                          * this has no effect -- if we set the buffer size to
14462                          * ~LONG_MAX and the buffer policy is ultimately set to
14463                          * be "manual", the buffer allocation is guaranteed to
14464                          * fail, if only because the allocation requires two
14465                          * buffers.  (We set the the size to the highest
14466                          * multiple of 128m because it ensures that the size
14467                          * will remain a multiple of a megabyte when
14468                          * repeatedly halved -- all the way down to 15m.)
14469                          */
14470                         val = LONG_MAX - (1 << 27) + 1;
14471                 }
14472         }
14473 
14474         state->dts_options[option] = val;
14475 
14476         return (0);
14477 }
14478 
14479 static void
14480 dtrace_state_destroy(dtrace_state_t *state)
14481 {
14482         dtrace_ecb_t *ecb;
14483         dtrace_vstate_t *vstate = &state->dts_vstate;
14484         minor_t minor = getminor(state->dts_dev);
14485         int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
14486         dtrace_speculation_t *spec = state->dts_speculations;
14487         int nspec = state->dts_nspeculations;
14488         uint32_t match;
14489 
14490         ASSERT(MUTEX_HELD(&dtrace_lock));
14491         ASSERT(MUTEX_HELD(&cpu_lock));
14492 
14493         /*
14494          * First, retract any retained enablings for this state.
14495          */
14496         dtrace_enabling_retract(state);
14497         ASSERT(state->dts_nretained == 0);
14498 
14499         if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
14500             state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
14501                 /*
14502                  * We have managed to come into dtrace_state_destroy() on a
14503                  * hot enabling -- almost certainly because of a disorderly
14504                  * shutdown of a consumer.  (That is, a consumer that is
14505                  * exiting without having called dtrace_stop().) In this case,
14506                  * we're going to set our activity to be KILLED, and then
14507                  * issue a sync to be sure that everyone is out of probe
14508                  * context before we start blowing away ECBs.
14509                  */
14510                 state->dts_activity = DTRACE_ACTIVITY_KILLED;
14511                 dtrace_sync();
14512         }
14513 
14514         /*
14515          * Release the credential hold we took in dtrace_state_create().
14516          */
14517         if (state->dts_cred.dcr_cred != NULL)
14518                 crfree(state->dts_cred.dcr_cred);
14519 
14520         /*
14521          * Now we can safely disable and destroy any enabled probes.  Because
14522          * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
14523          * (especially if they're all enabled), we take two passes through the
14524          * ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes, and
14525          * in the second we disable whatever is left over.
14526          */
14527         for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
14528                 for (i = 0; i < state->dts_necbs; i++) {
14529                         if ((ecb = state->dts_ecbs[i]) == NULL)
14530                                 continue;
14531 
14532                         if (match && ecb->dte_probe != NULL) {
14533                                 dtrace_probe_t *probe = ecb->dte_probe;
14534                                 dtrace_provider_t *prov = probe->dtpr_provider;
14535 
14536                                 if (!(prov->dtpv_priv.dtpp_flags & match))
14537                                         continue;
14538                         }
14539 
14540                         dtrace_ecb_disable(ecb);
14541                         dtrace_ecb_destroy(ecb);
14542                 }
14543 
14544                 if (!match)
14545                         break;
14546         }
14547 
14548         /*
14549          * Before we free the buffers, perform one more sync to assure that
14550          * every CPU is out of probe context.
14551          */
14552         dtrace_sync();
14553 
14554         dtrace_buffer_free(state->dts_buffer);
14555         dtrace_buffer_free(state->dts_aggbuffer);
14556 
14557         for (i = 0; i < nspec; i++)
14558                 dtrace_buffer_free(spec[i].dtsp_buffer);
14559 
14560         if (state->dts_cleaner != CYCLIC_NONE)
14561                 cyclic_remove(state->dts_cleaner);
14562 
14563         if (state->dts_deadman != CYCLIC_NONE)
14564                 cyclic_remove(state->dts_deadman);
14565 
14566         dtrace_dstate_fini(&vstate->dtvs_dynvars);
14567         dtrace_vstate_fini(vstate);
14568         kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
14569 
14570         if (state->dts_aggregations != NULL) {
14571 #ifdef DEBUG
14572                 for (i = 0; i < state->dts_naggregations; i++)
14573                         ASSERT(state->dts_aggregations[i] == NULL);
14574 #endif
14575                 ASSERT(state->dts_naggregations > 0);
14576                 kmem_free(state->dts_aggregations,
14577                     state->dts_naggregations * sizeof (dtrace_aggregation_t *));
14578         }
14579 
14580         kmem_free(state->dts_buffer, bufsize);
14581         kmem_free(state->dts_aggbuffer, bufsize);
14582 
14583         for (i = 0; i < nspec; i++)
14584                 kmem_free(spec[i].dtsp_buffer, bufsize);
14585 
14586         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
14587 
14588         dtrace_format_destroy(state);
14589 
14590         vmem_destroy(state->dts_aggid_arena);
14591         ddi_soft_state_free(dtrace_softstate, minor);
14592         vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
14593 }
14594 
14595 /*
14596  * DTrace Anonymous Enabling Functions
14597  */
14598 static dtrace_state_t *
14599 dtrace_anon_grab(void)
14600 {
14601         dtrace_state_t *state;
14602 
14603         ASSERT(MUTEX_HELD(&dtrace_lock));
14604 
14605         if ((state = dtrace_anon.dta_state) == NULL) {
14606                 ASSERT(dtrace_anon.dta_enabling == NULL);
14607                 return (NULL);
14608         }
14609 
14610         ASSERT(dtrace_anon.dta_enabling != NULL);
14611         ASSERT(dtrace_retained != NULL);
14612 
14613         dtrace_enabling_destroy(dtrace_anon.dta_enabling);
14614         dtrace_anon.dta_enabling = NULL;
14615         dtrace_anon.dta_state = NULL;
14616 
14617         return (state);
14618 }
14619 
14620 static void
14621 dtrace_anon_property(void)
14622 {
14623         int i, rv;
14624         dtrace_state_t *state;
14625         dof_hdr_t *dof;
14626         char c[32];             /* enough for "dof-data-" + digits */
14627 
14628         ASSERT(MUTEX_HELD(&dtrace_lock));
14629         ASSERT(MUTEX_HELD(&cpu_lock));
14630 
14631         for (i = 0; ; i++) {
14632                 (void) snprintf(c, sizeof (c), "dof-data-%d", i);
14633 
14634                 dtrace_err_verbose = 1;
14635 
14636                 if ((dof = dtrace_dof_property(c)) == NULL) {
14637                         dtrace_err_verbose = 0;
14638                         break;
14639                 }
14640 
14641                 /*
14642                  * We want to create anonymous state, so we need to transition
14643                  * the kernel debugger to indicate that DTrace is active.  If
14644                  * this fails (e.g. because the debugger has modified text in
14645                  * some way), we won't continue with the processing.
14646                  */
14647                 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
14648                         cmn_err(CE_NOTE, "kernel debugger active; anonymous "
14649                             "enabling ignored.");
14650                         dtrace_dof_destroy(dof);
14651                         break;
14652                 }
14653 
14654                 /*
14655                  * If we haven't allocated an anonymous state, we'll do so now.
14656                  */
14657                 if ((state = dtrace_anon.dta_state) == NULL) {
14658                         state = dtrace_state_create(NULL, NULL);
14659                         dtrace_anon.dta_state = state;
14660 
14661                         if (state == NULL) {
14662                                 /*
14663                                  * This basically shouldn't happen:  the only
14664                                  * failure mode from dtrace_state_create() is a
14665                                  * failure of ddi_soft_state_zalloc() that
14666                                  * itself should never happen.  Still, the
14667                                  * interface allows for a failure mode, and
14668                                  * we want to fail as gracefully as possible:
14669                                  * we'll emit an error message and cease
14670                                  * processing anonymous state in this case.
14671                                  */
14672                                 cmn_err(CE_WARN, "failed to create "
14673                                     "anonymous state");
14674                                 dtrace_dof_destroy(dof);
14675                                 break;
14676                         }
14677                 }
14678 
14679                 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
14680                     &dtrace_anon.dta_enabling, 0, B_TRUE);
14681 
14682                 if (rv == 0)
14683                         rv = dtrace_dof_options(dof, state);
14684 
14685                 dtrace_err_verbose = 0;
14686                 dtrace_dof_destroy(dof);
14687 
14688                 if (rv != 0) {
14689                         /*
14690                          * This is malformed DOF; chuck any anonymous state
14691                          * that we created.
14692                          */
14693                         ASSERT(dtrace_anon.dta_enabling == NULL);
14694                         dtrace_state_destroy(state);
14695                         dtrace_anon.dta_state = NULL;
14696                         break;
14697                 }
14698 
14699                 ASSERT(dtrace_anon.dta_enabling != NULL);
14700         }
14701 
14702         if (dtrace_anon.dta_enabling != NULL) {
14703                 int rval;
14704 
14705                 /*
14706                  * dtrace_enabling_retain() can only fail because we are
14707                  * trying to retain more enablings than are allowed -- but
14708                  * we only have one anonymous enabling, and we are guaranteed
14709                  * to be allowed at least one retained enabling; we assert
14710                  * that dtrace_enabling_retain() returns success.
14711                  */
14712                 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
14713                 ASSERT(rval == 0);
14714 
14715                 dtrace_enabling_dump(dtrace_anon.dta_enabling);
14716         }
14717 }
14718 
14719 /*
14720  * DTrace Helper Functions
14721  */
14722 static void
14723 dtrace_helper_trace(dtrace_helper_action_t *helper,
14724     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
14725 {
14726         uint32_t size, next, nnext, i;
14727         dtrace_helptrace_t *ent, *buffer;
14728         uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14729 
14730         if ((buffer = dtrace_helptrace_buffer) == NULL)
14731                 return;
14732 
14733         ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
14734 
14735         /*
14736          * What would a tracing framework be without its own tracing
14737          * framework?  (Well, a hell of a lot simpler, for starters...)
14738          */
14739         size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
14740             sizeof (uint64_t) - sizeof (uint64_t);
14741 
14742         /*
14743          * Iterate until we can allocate a slot in the trace buffer.
14744          */
14745         do {
14746                 next = dtrace_helptrace_next;
14747 
14748                 if (next + size < dtrace_helptrace_bufsize) {
14749                         nnext = next + size;
14750                 } else {
14751                         nnext = size;
14752                 }
14753         } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
14754 
14755         /*
14756          * We have our slot; fill it in.
14757          */
14758         if (nnext == size) {
14759                 dtrace_helptrace_wrapped++;
14760                 next = 0;
14761         }
14762 
14763         ent = (dtrace_helptrace_t *)((uintptr_t)buffer + next);
14764         ent->dtht_helper = helper;
14765         ent->dtht_where = where;
14766         ent->dtht_nlocals = vstate->dtvs_nlocals;
14767 
14768         ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
14769             mstate->dtms_fltoffs : -1;
14770         ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
14771         ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
14772 
14773         for (i = 0; i < vstate->dtvs_nlocals; i++) {
14774                 dtrace_statvar_t *svar;
14775 
14776                 if ((svar = vstate->dtvs_locals[i]) == NULL)
14777                         continue;
14778 
14779                 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
14780                 ent->dtht_locals[i] =
14781                     ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
14782         }
14783 }
14784 
14785 static uint64_t
14786 dtrace_helper(int which, dtrace_mstate_t *mstate,
14787     dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
14788 {
14789         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
14790         uint64_t sarg0 = mstate->dtms_arg[0];
14791         uint64_t sarg1 = mstate->dtms_arg[1];
14792         uint64_t rval;
14793         dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
14794         dtrace_helper_action_t *helper;
14795         dtrace_vstate_t *vstate;
14796         dtrace_difo_t *pred;
14797         int i, trace = dtrace_helptrace_buffer != NULL;
14798 
14799         ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
14800 
14801         if (helpers == NULL)
14802                 return (0);
14803 
14804         if ((helper = helpers->dthps_actions[which]) == NULL)
14805                 return (0);
14806 
14807         vstate = &helpers->dthps_vstate;
14808         mstate->dtms_arg[0] = arg0;
14809         mstate->dtms_arg[1] = arg1;
14810 
14811         /*
14812          * Now iterate over each helper.  If its predicate evaluates to 'true',
14813          * we'll call the corresponding actions.  Note that the below calls
14814          * to dtrace_dif_emulate() may set faults in machine state.  This is
14815          * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
14816          * the stored DIF offset with its own (which is the desired behavior).
14817          * Also, note the calls to dtrace_dif_emulate() may allocate scratch
14818          * from machine state; this is okay, too.
14819          */
14820         for (; helper != NULL; helper = helper->dtha_next) {
14821                 if ((pred = helper->dtha_predicate) != NULL) {
14822                         if (trace)
14823                                 dtrace_helper_trace(helper, mstate, vstate, 0);
14824 
14825                         if (!dtrace_dif_emulate(pred, mstate, vstate, state))
14826                                 goto next;
14827 
14828                         if (*flags & CPU_DTRACE_FAULT)
14829                                 goto err;
14830                 }
14831 
14832                 for (i = 0; i < helper->dtha_nactions; i++) {
14833                         if (trace)
14834                                 dtrace_helper_trace(helper,
14835                                     mstate, vstate, i + 1);
14836 
14837                         rval = dtrace_dif_emulate(helper->dtha_actions[i],
14838                             mstate, vstate, state);
14839 
14840                         if (*flags & CPU_DTRACE_FAULT)
14841                                 goto err;
14842                 }
14843 
14844 next:
14845                 if (trace)
14846                         dtrace_helper_trace(helper, mstate, vstate,
14847                             DTRACE_HELPTRACE_NEXT);
14848         }
14849 
14850         if (trace)
14851                 dtrace_helper_trace(helper, mstate, vstate,
14852                     DTRACE_HELPTRACE_DONE);
14853 
14854         /*
14855          * Restore the arg0 that we saved upon entry.
14856          */
14857         mstate->dtms_arg[0] = sarg0;
14858         mstate->dtms_arg[1] = sarg1;
14859 
14860         return (rval);
14861 
14862 err:
14863         if (trace)
14864                 dtrace_helper_trace(helper, mstate, vstate,
14865                     DTRACE_HELPTRACE_ERR);
14866 
14867         /*
14868          * Restore the arg0 that we saved upon entry.
14869          */
14870         mstate->dtms_arg[0] = sarg0;
14871         mstate->dtms_arg[1] = sarg1;
14872 
14873         return (0);
14874 }
14875 
14876 static void
14877 dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
14878     dtrace_vstate_t *vstate)
14879 {
14880         int i;
14881 
14882         if (helper->dtha_predicate != NULL)
14883                 dtrace_difo_release(helper->dtha_predicate, vstate);
14884 
14885         for (i = 0; i < helper->dtha_nactions; i++) {
14886                 ASSERT(helper->dtha_actions[i] != NULL);
14887                 dtrace_difo_release(helper->dtha_actions[i], vstate);
14888         }
14889 
14890         kmem_free(helper->dtha_actions,
14891             helper->dtha_nactions * sizeof (dtrace_difo_t *));
14892         kmem_free(helper, sizeof (dtrace_helper_action_t));
14893 }
14894 
14895 static int
14896 dtrace_helper_destroygen(int gen)
14897 {
14898         proc_t *p = curproc;
14899         dtrace_helpers_t *help = p->p_dtrace_helpers;
14900         dtrace_vstate_t *vstate;
14901         int i;
14902 
14903         ASSERT(MUTEX_HELD(&dtrace_lock));
14904 
14905         if (help == NULL || gen > help->dthps_generation)
14906                 return (EINVAL);
14907 
14908         vstate = &help->dthps_vstate;
14909 
14910         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14911                 dtrace_helper_action_t *last = NULL, *h, *next;
14912 
14913                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
14914                         next = h->dtha_next;
14915 
14916                         if (h->dtha_generation == gen) {
14917                                 if (last != NULL) {
14918                                         last->dtha_next = next;
14919                                 } else {
14920                                         help->dthps_actions[i] = next;
14921                                 }
14922 
14923                                 dtrace_helper_action_destroy(h, vstate);
14924                         } else {
14925                                 last = h;
14926                         }
14927                 }
14928         }
14929 
14930         /*
14931          * Interate until we've cleared out all helper providers with the
14932          * given generation number.
14933          */
14934         for (;;) {
14935                 dtrace_helper_provider_t *prov;
14936 
14937                 /*
14938                  * Look for a helper provider with the right generation. We
14939                  * have to start back at the beginning of the list each time
14940                  * because we drop dtrace_lock. It's unlikely that we'll make
14941                  * more than two passes.
14942                  */
14943                 for (i = 0; i < help->dthps_nprovs; i++) {
14944                         prov = help->dthps_provs[i];
14945 
14946                         if (prov->dthp_generation == gen)
14947                                 break;
14948                 }
14949 
14950                 /*
14951                  * If there were no matches, we're done.
14952                  */
14953                 if (i == help->dthps_nprovs)
14954                         break;
14955 
14956                 /*
14957                  * Move the last helper provider into this slot.
14958                  */
14959                 help->dthps_nprovs--;
14960                 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
14961                 help->dthps_provs[help->dthps_nprovs] = NULL;
14962 
14963                 mutex_exit(&dtrace_lock);
14964 
14965                 /*
14966                  * If we have a meta provider, remove this helper provider.
14967                  */
14968                 mutex_enter(&dtrace_meta_lock);
14969                 if (dtrace_meta_pid != NULL) {
14970                         ASSERT(dtrace_deferred_pid == NULL);
14971                         dtrace_helper_provider_remove(&prov->dthp_prov,
14972                             p->p_pid);
14973                 }
14974                 mutex_exit(&dtrace_meta_lock);
14975 
14976                 dtrace_helper_provider_destroy(prov);
14977 
14978                 mutex_enter(&dtrace_lock);
14979         }
14980 
14981         return (0);
14982 }
14983 
14984 static int
14985 dtrace_helper_validate(dtrace_helper_action_t *helper)
14986 {
14987         int err = 0, i;
14988         dtrace_difo_t *dp;
14989 
14990         if ((dp = helper->dtha_predicate) != NULL)
14991                 err += dtrace_difo_validate_helper(dp);
14992 
14993         for (i = 0; i < helper->dtha_nactions; i++)
14994                 err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
14995 
14996         return (err == 0);
14997 }
14998 
14999 static int
15000 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
15001 {
15002         dtrace_helpers_t *help;
15003         dtrace_helper_action_t *helper, *last;
15004         dtrace_actdesc_t *act;
15005         dtrace_vstate_t *vstate;
15006         dtrace_predicate_t *pred;
15007         int count = 0, nactions = 0, i;
15008 
15009         if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
15010                 return (EINVAL);
15011 
15012         help = curproc->p_dtrace_helpers;
15013         last = help->dthps_actions[which];
15014         vstate = &help->dthps_vstate;
15015 
15016         for (count = 0; last != NULL; last = last->dtha_next) {
15017                 count++;
15018                 if (last->dtha_next == NULL)
15019                         break;
15020         }
15021 
15022         /*
15023          * If we already have dtrace_helper_actions_max helper actions for this
15024          * helper action type, we'll refuse to add a new one.
15025          */
15026         if (count >= dtrace_helper_actions_max)
15027                 return (ENOSPC);
15028 
15029         helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
15030         helper->dtha_generation = help->dthps_generation;
15031 
15032         if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
15033                 ASSERT(pred->dtp_difo != NULL);
15034                 dtrace_difo_hold(pred->dtp_difo);
15035                 helper->dtha_predicate = pred->dtp_difo;
15036         }
15037 
15038         for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
15039                 if (act->dtad_kind != DTRACEACT_DIFEXPR)
15040                         goto err;
15041 
15042                 if (act->dtad_difo == NULL)
15043                         goto err;
15044 
15045                 nactions++;
15046         }
15047 
15048         helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
15049             (helper->dtha_nactions = nactions), KM_SLEEP);
15050 
15051         for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
15052                 dtrace_difo_hold(act->dtad_difo);
15053                 helper->dtha_actions[i++] = act->dtad_difo;
15054         }
15055 
15056         if (!dtrace_helper_validate(helper))
15057                 goto err;
15058 
15059         if (last == NULL) {
15060                 help->dthps_actions[which] = helper;
15061         } else {
15062                 last->dtha_next = helper;
15063         }
15064 
15065         if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
15066                 dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
15067                 dtrace_helptrace_next = 0;
15068         }
15069 
15070         return (0);
15071 err:
15072         dtrace_helper_action_destroy(helper, vstate);
15073         return (EINVAL);
15074 }
15075 
15076 static void
15077 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
15078     dof_helper_t *dofhp)
15079 {
15080         ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
15081 
15082         mutex_enter(&dtrace_meta_lock);
15083         mutex_enter(&dtrace_lock);
15084 
15085         if (!dtrace_attached() || dtrace_meta_pid == NULL) {
15086                 /*
15087                  * If the dtrace module is loaded but not attached, or if
15088                  * there aren't isn't a meta provider registered to deal with
15089                  * these provider descriptions, we need to postpone creating
15090                  * the actual providers until later.
15091                  */
15092 
15093                 if (help->dthps_next == NULL && help->dthps_prev == NULL &&
15094                     dtrace_deferred_pid != help) {
15095                         help->dthps_deferred = 1;
15096                         help->dthps_pid = p->p_pid;
15097                         help->dthps_next = dtrace_deferred_pid;
15098                         help->dthps_prev = NULL;
15099                         if (dtrace_deferred_pid != NULL)
15100                                 dtrace_deferred_pid->dthps_prev = help;
15101                         dtrace_deferred_pid = help;
15102                 }
15103 
15104                 mutex_exit(&dtrace_lock);
15105 
15106         } else if (dofhp != NULL) {
15107                 /*
15108                  * If the dtrace module is loaded and we have a particular
15109                  * helper provider description, pass that off to the
15110                  * meta provider.
15111                  */
15112 
15113                 mutex_exit(&dtrace_lock);
15114 
15115                 dtrace_helper_provide(dofhp, p->p_pid);
15116 
15117         } else {
15118                 /*
15119                  * Otherwise, just pass all the helper provider descriptions
15120                  * off to the meta provider.
15121                  */
15122 
15123                 int i;
15124                 mutex_exit(&dtrace_lock);
15125 
15126                 for (i = 0; i < help->dthps_nprovs; i++) {
15127                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
15128                             p->p_pid);
15129                 }
15130         }
15131 
15132         mutex_exit(&dtrace_meta_lock);
15133 }
15134 
15135 static int
15136 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
15137 {
15138         dtrace_helpers_t *help;
15139         dtrace_helper_provider_t *hprov, **tmp_provs;
15140         uint_t tmp_maxprovs, i;
15141 
15142         ASSERT(MUTEX_HELD(&dtrace_lock));
15143 
15144         help = curproc->p_dtrace_helpers;
15145         ASSERT(help != NULL);
15146 
15147         /*
15148          * If we already have dtrace_helper_providers_max helper providers,
15149          * we're refuse to add a new one.
15150          */
15151         if (help->dthps_nprovs >= dtrace_helper_providers_max)
15152                 return (ENOSPC);
15153 
15154         /*
15155          * Check to make sure this isn't a duplicate.
15156          */
15157         for (i = 0; i < help->dthps_nprovs; i++) {
15158                 if (dofhp->dofhp_addr ==
15159                     help->dthps_provs[i]->dthp_prov.dofhp_addr)
15160                         return (EALREADY);
15161         }
15162 
15163         hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
15164         hprov->dthp_prov = *dofhp;
15165         hprov->dthp_ref = 1;
15166         hprov->dthp_generation = gen;
15167 
15168         /*
15169          * Allocate a bigger table for helper providers if it's already full.
15170          */
15171         if (help->dthps_maxprovs == help->dthps_nprovs) {
15172                 tmp_maxprovs = help->dthps_maxprovs;
15173                 tmp_provs = help->dthps_provs;
15174 
15175                 if (help->dthps_maxprovs == 0)
15176                         help->dthps_maxprovs = 2;
15177                 else
15178                         help->dthps_maxprovs *= 2;
15179                 if (help->dthps_maxprovs > dtrace_helper_providers_max)
15180                         help->dthps_maxprovs = dtrace_helper_providers_max;
15181 
15182                 ASSERT(tmp_maxprovs < help->dthps_maxprovs);
15183 
15184                 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
15185                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
15186 
15187                 if (tmp_provs != NULL) {
15188                         bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
15189                             sizeof (dtrace_helper_provider_t *));
15190                         kmem_free(tmp_provs, tmp_maxprovs *
15191                             sizeof (dtrace_helper_provider_t *));
15192                 }
15193         }
15194 
15195         help->dthps_provs[help->dthps_nprovs] = hprov;
15196         help->dthps_nprovs++;
15197 
15198         return (0);
15199 }
15200 
15201 static void
15202 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
15203 {
15204         mutex_enter(&dtrace_lock);
15205 
15206         if (--hprov->dthp_ref == 0) {
15207                 dof_hdr_t *dof;
15208                 mutex_exit(&dtrace_lock);
15209                 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
15210                 dtrace_dof_destroy(dof);
15211                 kmem_free(hprov, sizeof (dtrace_helper_provider_t));
15212         } else {
15213                 mutex_exit(&dtrace_lock);
15214         }
15215 }
15216 
15217 static int
15218 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
15219 {
15220         uintptr_t daddr = (uintptr_t)dof;
15221         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
15222         dof_provider_t *provider;
15223         dof_probe_t *probe;
15224         uint8_t *arg;
15225         char *strtab, *typestr;
15226         dof_stridx_t typeidx;
15227         size_t typesz;
15228         uint_t nprobes, j, k;
15229 
15230         ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
15231 
15232         if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
15233                 dtrace_dof_error(dof, "misaligned section offset");
15234                 return (-1);
15235         }
15236 
15237         /*
15238          * The section needs to be large enough to contain the DOF provider
15239          * structure appropriate for the given version.
15240          */
15241         if (sec->dofs_size <
15242             ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
15243             offsetof(dof_provider_t, dofpv_prenoffs) :
15244             sizeof (dof_provider_t))) {
15245                 dtrace_dof_error(dof, "provider section too small");
15246                 return (-1);
15247         }
15248 
15249         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
15250         str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
15251         prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
15252         arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
15253         off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
15254 
15255         if (str_sec == NULL || prb_sec == NULL ||
15256             arg_sec == NULL || off_sec == NULL)
15257                 return (-1);
15258 
15259         enoff_sec = NULL;
15260 
15261         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
15262             provider->dofpv_prenoffs != DOF_SECT_NONE &&
15263             (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
15264             provider->dofpv_prenoffs)) == NULL)
15265                 return (-1);
15266 
15267         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
15268 
15269         if (provider->dofpv_name >= str_sec->dofs_size ||
15270             strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
15271                 dtrace_dof_error(dof, "invalid provider name");
15272                 return (-1);
15273         }
15274 
15275         if (prb_sec->dofs_entsize == 0 ||
15276             prb_sec->dofs_entsize > prb_sec->dofs_size) {
15277                 dtrace_dof_error(dof, "invalid entry size");
15278                 return (-1);
15279         }
15280 
15281         if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
15282                 dtrace_dof_error(dof, "misaligned entry size");
15283                 return (-1);
15284         }
15285 
15286         if (off_sec->dofs_entsize != sizeof (uint32_t)) {
15287                 dtrace_dof_error(dof, "invalid entry size");
15288                 return (-1);
15289         }
15290 
15291         if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
15292                 dtrace_dof_error(dof, "misaligned section offset");
15293                 return (-1);
15294         }
15295 
15296         if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
15297                 dtrace_dof_error(dof, "invalid entry size");
15298                 return (-1);
15299         }
15300 
15301         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
15302 
15303         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
15304 
15305         /*
15306          * Take a pass through the probes to check for errors.
15307          */
15308         for (j = 0; j < nprobes; j++) {
15309                 probe = (dof_probe_t *)(uintptr_t)(daddr +
15310                     prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
15311 
15312                 if (probe->dofpr_func >= str_sec->dofs_size) {
15313                         dtrace_dof_error(dof, "invalid function name");
15314                         return (-1);
15315                 }
15316 
15317                 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
15318                         dtrace_dof_error(dof, "function name too long");
15319                         return (-1);
15320                 }
15321 
15322                 if (probe->dofpr_name >= str_sec->dofs_size ||
15323                     strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
15324                         dtrace_dof_error(dof, "invalid probe name");
15325                         return (-1);
15326                 }
15327 
15328                 /*
15329                  * The offset count must not wrap the index, and the offsets
15330                  * must also not overflow the section's data.
15331                  */
15332                 if (probe->dofpr_offidx + probe->dofpr_noffs <
15333                     probe->dofpr_offidx ||
15334                     (probe->dofpr_offidx + probe->dofpr_noffs) *
15335                     off_sec->dofs_entsize > off_sec->dofs_size) {
15336                         dtrace_dof_error(dof, "invalid probe offset");
15337                         return (-1);
15338                 }
15339 
15340                 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
15341                         /*
15342                          * If there's no is-enabled offset section, make sure
15343                          * there aren't any is-enabled offsets. Otherwise
15344                          * perform the same checks as for probe offsets
15345                          * (immediately above).
15346                          */
15347                         if (enoff_sec == NULL) {
15348                                 if (probe->dofpr_enoffidx != 0 ||
15349                                     probe->dofpr_nenoffs != 0) {
15350                                         dtrace_dof_error(dof, "is-enabled "
15351                                             "offsets with null section");
15352                                         return (-1);
15353                                 }
15354                         } else if (probe->dofpr_enoffidx +
15355                             probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
15356                             (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
15357                             enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
15358                                 dtrace_dof_error(dof, "invalid is-enabled "
15359                                     "offset");
15360                                 return (-1);
15361                         }
15362 
15363                         if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
15364                                 dtrace_dof_error(dof, "zero probe and "
15365                                     "is-enabled offsets");
15366                                 return (-1);
15367                         }
15368                 } else if (probe->dofpr_noffs == 0) {
15369                         dtrace_dof_error(dof, "zero probe offsets");
15370                         return (-1);
15371                 }
15372 
15373                 if (probe->dofpr_argidx + probe->dofpr_xargc <
15374                     probe->dofpr_argidx ||
15375                     (probe->dofpr_argidx + probe->dofpr_xargc) *
15376                     arg_sec->dofs_entsize > arg_sec->dofs_size) {
15377                         dtrace_dof_error(dof, "invalid args");
15378                         return (-1);
15379                 }
15380 
15381                 typeidx = probe->dofpr_nargv;
15382                 typestr = strtab + probe->dofpr_nargv;
15383                 for (k = 0; k < probe->dofpr_nargc; k++) {
15384                         if (typeidx >= str_sec->dofs_size) {
15385                                 dtrace_dof_error(dof, "bad "
15386                                     "native argument type");
15387                                 return (-1);
15388                         }
15389 
15390                         typesz = strlen(typestr) + 1;
15391                         if (typesz > DTRACE_ARGTYPELEN) {
15392                                 dtrace_dof_error(dof, "native "
15393                                     "argument type too long");
15394                                 return (-1);
15395                         }
15396                         typeidx += typesz;
15397                         typestr += typesz;
15398                 }
15399 
15400                 typeidx = probe->dofpr_xargv;
15401                 typestr = strtab + probe->dofpr_xargv;
15402                 for (k = 0; k < probe->dofpr_xargc; k++) {
15403                         if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
15404                                 dtrace_dof_error(dof, "bad "
15405                                     "native argument index");
15406                                 return (-1);
15407                         }
15408 
15409                         if (typeidx >= str_sec->dofs_size) {
15410                                 dtrace_dof_error(dof, "bad "
15411                                     "translated argument type");
15412                                 return (-1);
15413                         }
15414 
15415                         typesz = strlen(typestr) + 1;
15416                         if (typesz > DTRACE_ARGTYPELEN) {
15417                                 dtrace_dof_error(dof, "translated argument "
15418                                     "type too long");
15419                                 return (-1);
15420                         }
15421 
15422                         typeidx += typesz;
15423                         typestr += typesz;
15424                 }
15425         }
15426 
15427         return (0);
15428 }
15429 
15430 static int
15431 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
15432 {
15433         dtrace_helpers_t *help;
15434         dtrace_vstate_t *vstate;
15435         dtrace_enabling_t *enab = NULL;
15436         int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
15437         uintptr_t daddr = (uintptr_t)dof;
15438 
15439         ASSERT(MUTEX_HELD(&dtrace_lock));
15440 
15441         if ((help = curproc->p_dtrace_helpers) == NULL)
15442                 help = dtrace_helpers_create(curproc);
15443 
15444         vstate = &help->dthps_vstate;
15445 
15446         if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
15447             dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
15448                 dtrace_dof_destroy(dof);
15449                 return (rv);
15450         }
15451 
15452         /*
15453          * Look for helper providers and validate their descriptions.
15454          */
15455         if (dhp != NULL) {
15456                 for (i = 0; i < dof->dofh_secnum; i++) {
15457                         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
15458                             dof->dofh_secoff + i * dof->dofh_secsize);
15459 
15460                         if (sec->dofs_type != DOF_SECT_PROVIDER)
15461                                 continue;
15462 
15463                         if (dtrace_helper_provider_validate(dof, sec) != 0) {
15464                                 dtrace_enabling_destroy(enab);
15465                                 dtrace_dof_destroy(dof);
15466                                 return (-1);
15467                         }
15468 
15469                         nprovs++;
15470                 }
15471         }
15472 
15473         /*
15474          * Now we need to walk through the ECB descriptions in the enabling.
15475          */
15476         for (i = 0; i < enab->dten_ndesc; i++) {
15477                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
15478                 dtrace_probedesc_t *desc = &ep->dted_probe;
15479 
15480                 if (strcmp(desc->dtpd_provider, "dtrace") != 0)
15481                         continue;
15482 
15483                 if (strcmp(desc->dtpd_mod, "helper") != 0)
15484                         continue;
15485 
15486                 if (strcmp(desc->dtpd_func, "ustack") != 0)
15487                         continue;
15488 
15489                 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
15490                     ep)) != 0) {
15491                         /*
15492                          * Adding this helper action failed -- we are now going
15493                          * to rip out the entire generation and return failure.
15494                          */
15495                         (void) dtrace_helper_destroygen(help->dthps_generation);
15496                         dtrace_enabling_destroy(enab);
15497                         dtrace_dof_destroy(dof);
15498                         return (-1);
15499                 }
15500 
15501                 nhelpers++;
15502         }
15503 
15504         if (nhelpers < enab->dten_ndesc)
15505                 dtrace_dof_error(dof, "unmatched helpers");
15506 
15507         gen = help->dthps_generation++;
15508         dtrace_enabling_destroy(enab);
15509 
15510         if (dhp != NULL && nprovs > 0) {
15511                 /*
15512                  * Now that this is in-kernel, we change the sense of the
15513                  * members:  dofhp_dof denotes the in-kernel copy of the DOF
15514                  * and dofhp_addr denotes the address at user-level.
15515                  */
15516                 dhp->dofhp_addr = dhp->dofhp_dof;
15517                 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
15518 
15519                 if (dtrace_helper_provider_add(dhp, gen) == 0) {
15520                         mutex_exit(&dtrace_lock);
15521                         dtrace_helper_provider_register(curproc, help, dhp);
15522                         mutex_enter(&dtrace_lock);
15523 
15524                         destroy = 0;
15525                 }
15526         }
15527 
15528         if (destroy)
15529                 dtrace_dof_destroy(dof);
15530 
15531         return (gen);
15532 }
15533 
15534 static dtrace_helpers_t *
15535 dtrace_helpers_create(proc_t *p)
15536 {
15537         dtrace_helpers_t *help;
15538 
15539         ASSERT(MUTEX_HELD(&dtrace_lock));
15540         ASSERT(p->p_dtrace_helpers == NULL);
15541 
15542         help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
15543         help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
15544             DTRACE_NHELPER_ACTIONS, KM_SLEEP);
15545 
15546         p->p_dtrace_helpers = help;
15547         dtrace_helpers++;
15548 
15549         return (help);
15550 }
15551 
15552 static void
15553 dtrace_helpers_destroy(proc_t *p)
15554 {
15555         dtrace_helpers_t *help;
15556         dtrace_vstate_t *vstate;
15557         int i;
15558 
15559         mutex_enter(&dtrace_lock);
15560 
15561         ASSERT(p->p_dtrace_helpers != NULL);
15562         ASSERT(dtrace_helpers > 0);
15563 
15564         help = p->p_dtrace_helpers;
15565         vstate = &help->dthps_vstate;
15566 
15567         /*
15568          * We're now going to lose the help from this process.
15569          */
15570         p->p_dtrace_helpers = NULL;
15571         if (p == curproc) {
15572                 dtrace_sync();
15573         } else {
15574                 /*
15575                  * It is sometimes necessary to clean up dtrace helpers from a
15576                  * an incomplete child process as part of a failed fork
15577                  * operation.  In such situations, a dtrace_sync() call should
15578                  * be unnecessary as the process should be devoid of threads,
15579                  * much less any in probe context.
15580                  */
15581                 VERIFY(p->p_stat == SIDL);
15582         }
15583 
15584         /*
15585          * Destroy the helper actions.
15586          */
15587         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15588                 dtrace_helper_action_t *h, *next;
15589 
15590                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
15591                         next = h->dtha_next;
15592                         dtrace_helper_action_destroy(h, vstate);
15593                         h = next;
15594                 }
15595         }
15596 
15597         mutex_exit(&dtrace_lock);
15598 
15599         /*
15600          * Destroy the helper providers.
15601          */
15602         if (help->dthps_maxprovs > 0) {
15603                 mutex_enter(&dtrace_meta_lock);
15604                 if (dtrace_meta_pid != NULL) {
15605                         ASSERT(dtrace_deferred_pid == NULL);
15606 
15607                         for (i = 0; i < help->dthps_nprovs; i++) {
15608                                 dtrace_helper_provider_remove(
15609                                     &help->dthps_provs[i]->dthp_prov, p->p_pid);
15610                         }
15611                 } else {
15612                         mutex_enter(&dtrace_lock);
15613                         ASSERT(help->dthps_deferred == 0 ||
15614                             help->dthps_next != NULL ||
15615                             help->dthps_prev != NULL ||
15616                             help == dtrace_deferred_pid);
15617 
15618                         /*
15619                          * Remove the helper from the deferred list.
15620                          */
15621                         if (help->dthps_next != NULL)
15622                                 help->dthps_next->dthps_prev = help->dthps_prev;
15623                         if (help->dthps_prev != NULL)
15624                                 help->dthps_prev->dthps_next = help->dthps_next;
15625                         if (dtrace_deferred_pid == help) {
15626                                 dtrace_deferred_pid = help->dthps_next;
15627                                 ASSERT(help->dthps_prev == NULL);
15628                         }
15629 
15630                         mutex_exit(&dtrace_lock);
15631                 }
15632 
15633                 mutex_exit(&dtrace_meta_lock);
15634 
15635                 for (i = 0; i < help->dthps_nprovs; i++) {
15636                         dtrace_helper_provider_destroy(help->dthps_provs[i]);
15637                 }
15638 
15639                 kmem_free(help->dthps_provs, help->dthps_maxprovs *
15640                     sizeof (dtrace_helper_provider_t *));
15641         }
15642 
15643         mutex_enter(&dtrace_lock);
15644 
15645         dtrace_vstate_fini(&help->dthps_vstate);
15646         kmem_free(help->dthps_actions,
15647             sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
15648         kmem_free(help, sizeof (dtrace_helpers_t));
15649 
15650         --dtrace_helpers;
15651         mutex_exit(&dtrace_lock);
15652 }
15653 
15654 static void
15655 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
15656 {
15657         dtrace_helpers_t *help, *newhelp;
15658         dtrace_helper_action_t *helper, *new, *last;
15659         dtrace_difo_t *dp;
15660         dtrace_vstate_t *vstate;
15661         int i, j, sz, hasprovs = 0;
15662 
15663         mutex_enter(&dtrace_lock);
15664         ASSERT(from->p_dtrace_helpers != NULL);
15665         ASSERT(dtrace_helpers > 0);
15666 
15667         help = from->p_dtrace_helpers;
15668         newhelp = dtrace_helpers_create(to);
15669         ASSERT(to->p_dtrace_helpers != NULL);
15670 
15671         newhelp->dthps_generation = help->dthps_generation;
15672         vstate = &newhelp->dthps_vstate;
15673 
15674         /*
15675          * Duplicate the helper actions.
15676          */
15677         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
15678                 if ((helper = help->dthps_actions[i]) == NULL)
15679                         continue;
15680 
15681                 for (last = NULL; helper != NULL; helper = helper->dtha_next) {
15682                         new = kmem_zalloc(sizeof (dtrace_helper_action_t),
15683                             KM_SLEEP);
15684                         new->dtha_generation = helper->dtha_generation;
15685 
15686                         if ((dp = helper->dtha_predicate) != NULL) {
15687                                 dp = dtrace_difo_duplicate(dp, vstate);
15688                                 new->dtha_predicate = dp;
15689                         }
15690 
15691                         new->dtha_nactions = helper->dtha_nactions;
15692                         sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
15693                         new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
15694 
15695                         for (j = 0; j < new->dtha_nactions; j++) {
15696                                 dtrace_difo_t *dp = helper->dtha_actions[j];
15697 
15698                                 ASSERT(dp != NULL);
15699                                 dp = dtrace_difo_duplicate(dp, vstate);
15700                                 new->dtha_actions[j] = dp;
15701                         }
15702 
15703                         if (last != NULL) {
15704                                 last->dtha_next = new;
15705                         } else {
15706                                 newhelp->dthps_actions[i] = new;
15707                         }
15708 
15709                         last = new;
15710                 }
15711         }
15712 
15713         /*
15714          * Duplicate the helper providers and register them with the
15715          * DTrace framework.
15716          */
15717         if (help->dthps_nprovs > 0) {
15718                 newhelp->dthps_nprovs = help->dthps_nprovs;
15719                 newhelp->dthps_maxprovs = help->dthps_nprovs;
15720                 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
15721                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
15722                 for (i = 0; i < newhelp->dthps_nprovs; i++) {
15723                         newhelp->dthps_provs[i] = help->dthps_provs[i];
15724                         newhelp->dthps_provs[i]->dthp_ref++;
15725                 }
15726 
15727                 hasprovs = 1;
15728         }
15729 
15730         mutex_exit(&dtrace_lock);
15731 
15732         if (hasprovs)
15733                 dtrace_helper_provider_register(to, newhelp, NULL);
15734 }
15735 
15736 /*
15737  * DTrace Hook Functions
15738  */
15739 static void
15740 dtrace_module_loaded(struct modctl *ctl)
15741 {
15742         dtrace_provider_t *prv;
15743 
15744         mutex_enter(&dtrace_provider_lock);
15745         mutex_enter(&mod_lock);
15746 
15747         ASSERT(ctl->mod_busy);
15748 
15749         /*
15750          * We're going to call each providers per-module provide operation
15751          * specifying only this module.
15752          */
15753         for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
15754                 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
15755 
15756         mutex_exit(&mod_lock);
15757         mutex_exit(&dtrace_provider_lock);
15758 
15759         /*
15760          * If we have any retained enablings, we need to match against them.
15761          * Enabling probes requires that cpu_lock be held, and we cannot hold
15762          * cpu_lock here -- it is legal for cpu_lock to be held when loading a
15763          * module.  (In particular, this happens when loading scheduling
15764          * classes.)  So if we have any retained enablings, we need to dispatch
15765          * our task queue to do the match for us.
15766          */
15767         mutex_enter(&dtrace_lock);
15768 
15769         if (dtrace_retained == NULL) {
15770                 mutex_exit(&dtrace_lock);
15771                 return;
15772         }
15773 
15774         (void) taskq_dispatch(dtrace_taskq,
15775             (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
15776 
15777         mutex_exit(&dtrace_lock);
15778 
15779         /*
15780          * And now, for a little heuristic sleaze:  in general, we want to
15781          * match modules as soon as they load.  However, we cannot guarantee
15782          * this, because it would lead us to the lock ordering violation
15783          * outlined above.  The common case, of course, is that cpu_lock is
15784          * _not_ held -- so we delay here for a clock tick, hoping that that's
15785          * long enough for the task queue to do its work.  If it's not, it's
15786          * not a serious problem -- it just means that the module that we
15787          * just loaded may not be immediately instrumentable.
15788          */
15789         delay(1);
15790 }
15791 
15792 static void
15793 dtrace_module_unloaded(struct modctl *ctl)
15794 {
15795         dtrace_probe_t template, *probe, *first, *next;
15796         dtrace_provider_t *prov;
15797 
15798         template.dtpr_mod = ctl->mod_modname;
15799 
15800         mutex_enter(&dtrace_provider_lock);
15801         mutex_enter(&mod_lock);
15802         mutex_enter(&dtrace_lock);
15803 
15804         if (dtrace_bymod == NULL) {
15805                 /*
15806                  * The DTrace module is loaded (obviously) but not attached;
15807                  * we don't have any work to do.
15808                  */
15809                 mutex_exit(&dtrace_provider_lock);
15810                 mutex_exit(&mod_lock);
15811                 mutex_exit(&dtrace_lock);
15812                 return;
15813         }
15814 
15815         for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
15816             probe != NULL; probe = probe->dtpr_nextmod) {
15817                 if (probe->dtpr_ecb != NULL) {
15818                         mutex_exit(&dtrace_provider_lock);
15819                         mutex_exit(&mod_lock);
15820                         mutex_exit(&dtrace_lock);
15821 
15822                         /*
15823                          * This shouldn't _actually_ be possible -- we're
15824                          * unloading a module that has an enabled probe in it.
15825                          * (It's normally up to the provider to make sure that
15826                          * this can't happen.)  However, because dtps_enable()
15827                          * doesn't have a failure mode, there can be an
15828                          * enable/unload race.  Upshot:  we don't want to
15829                          * assert, but we're not going to disable the
15830                          * probe, either.
15831                          */
15832                         if (dtrace_err_verbose) {
15833                                 cmn_err(CE_WARN, "unloaded module '%s' had "
15834                                     "enabled probes", ctl->mod_modname);
15835                         }
15836 
15837                         return;
15838                 }
15839         }
15840 
15841         probe = first;
15842 
15843         for (first = NULL; probe != NULL; probe = next) {
15844                 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
15845 
15846                 dtrace_probes[probe->dtpr_id - 1] = NULL;
15847 
15848                 next = probe->dtpr_nextmod;
15849                 dtrace_hash_remove(dtrace_bymod, probe);
15850                 dtrace_hash_remove(dtrace_byfunc, probe);
15851                 dtrace_hash_remove(dtrace_byname, probe);
15852 
15853                 if (first == NULL) {
15854                         first = probe;
15855                         probe->dtpr_nextmod = NULL;
15856                 } else {
15857                         probe->dtpr_nextmod = first;
15858                         first = probe;
15859                 }
15860         }
15861 
15862         /*
15863          * We've removed all of the module's probes from the hash chains and
15864          * from the probe array.  Now issue a dtrace_sync() to be sure that
15865          * everyone has cleared out from any probe array processing.
15866          */
15867         dtrace_sync();
15868 
15869         for (probe = first; probe != NULL; probe = first) {
15870                 first = probe->dtpr_nextmod;
15871                 prov = probe->dtpr_provider;
15872                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
15873                     probe->dtpr_arg);
15874                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
15875                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
15876                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
15877                 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
15878                 kmem_free(probe, sizeof (dtrace_probe_t));
15879         }
15880 
15881         mutex_exit(&dtrace_lock);
15882         mutex_exit(&mod_lock);
15883         mutex_exit(&dtrace_provider_lock);
15884 }
15885 
15886 void
15887 dtrace_suspend(void)
15888 {
15889         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
15890 }
15891 
15892 void
15893 dtrace_resume(void)
15894 {
15895         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
15896 }
15897 
15898 static int
15899 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu, void *ptr __unused)
15900 {
15901         ASSERT(MUTEX_HELD(&cpu_lock));
15902         mutex_enter(&dtrace_lock);
15903 
15904         switch (what) {
15905         case CPU_CONFIG: {
15906                 dtrace_state_t *state;
15907                 dtrace_optval_t *opt, rs, c;
15908 
15909                 /*
15910                  * For now, we only allocate a new buffer for anonymous state.
15911                  */
15912                 if ((state = dtrace_anon.dta_state) == NULL)
15913                         break;
15914 
15915                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
15916                         break;
15917 
15918                 opt = state->dts_options;
15919                 c = opt[DTRACEOPT_CPU];
15920 
15921                 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
15922                         break;
15923 
15924                 /*
15925                  * Regardless of what the actual policy is, we're going to
15926                  * temporarily set our resize policy to be manual.  We're
15927                  * also going to temporarily set our CPU option to denote
15928                  * the newly configured CPU.
15929                  */
15930                 rs = opt[DTRACEOPT_BUFRESIZE];
15931                 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
15932                 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
15933 
15934                 (void) dtrace_state_buffers(state);
15935 
15936                 opt[DTRACEOPT_BUFRESIZE] = rs;
15937                 opt[DTRACEOPT_CPU] = c;
15938 
15939                 break;
15940         }
15941 
15942         case CPU_UNCONFIG:
15943                 /*
15944                  * We don't free the buffer in the CPU_UNCONFIG case.  (The
15945                  * buffer will be freed when the consumer exits.)
15946                  */
15947                 break;
15948 
15949         default:
15950                 break;
15951         }
15952 
15953         mutex_exit(&dtrace_lock);
15954         return (0);
15955 }
15956 
15957 static void
15958 dtrace_cpu_setup_initial(processorid_t cpu)
15959 {
15960         (void) dtrace_cpu_setup(CPU_CONFIG, cpu, NULL);
15961 }
15962 
15963 static void
15964 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
15965 {
15966         if (dtrace_toxranges >= dtrace_toxranges_max) {
15967                 int osize, nsize;
15968                 dtrace_toxrange_t *range;
15969 
15970                 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15971 
15972                 if (osize == 0) {
15973                         ASSERT(dtrace_toxrange == NULL);
15974                         ASSERT(dtrace_toxranges_max == 0);
15975                         dtrace_toxranges_max = 1;
15976                 } else {
15977                         dtrace_toxranges_max <<= 1;
15978                 }
15979 
15980                 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
15981                 range = kmem_zalloc(nsize, KM_SLEEP);
15982 
15983                 if (dtrace_toxrange != NULL) {
15984                         ASSERT(osize != 0);
15985                         bcopy(dtrace_toxrange, range, osize);
15986                         kmem_free(dtrace_toxrange, osize);
15987                 }
15988 
15989                 dtrace_toxrange = range;
15990         }
15991 
15992         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == (uintptr_t)NULL);
15993         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == (uintptr_t)NULL);
15994 
15995         dtrace_toxrange[dtrace_toxranges].dtt_base = base;
15996         dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
15997         dtrace_toxranges++;
15998 }
15999 
16000 static void
16001 dtrace_getf_barrier()
16002 {
16003         /*
16004          * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
16005          * that contain calls to getf(), this routine will be called on every
16006          * closef() before either the underlying vnode is released or the
16007          * file_t itself is freed.  By the time we are here, it is essential
16008          * that the file_t can no longer be accessed from a call to getf()
16009          * in probe context -- that assures that a dtrace_sync() can be used
16010          * to clear out any enablings referring to the old structures.
16011          */
16012         if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 ||
16013             kcred->cr_zone->zone_dtrace_getf != 0)
16014                 dtrace_sync();
16015 }
16016 
16017 /*
16018  * DTrace Driver Cookbook Functions
16019  */
16020 /*ARGSUSED*/
16021 static int
16022 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
16023 {
16024         dtrace_provider_id_t id;
16025         dtrace_state_t *state = NULL;
16026         dtrace_enabling_t *enab;
16027 
16028         mutex_enter(&cpu_lock);
16029         mutex_enter(&dtrace_provider_lock);
16030         mutex_enter(&dtrace_lock);
16031 
16032         if (ddi_soft_state_init(&dtrace_softstate,
16033             sizeof (dtrace_state_t), 0) != 0) {
16034                 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
16035                 mutex_exit(&cpu_lock);
16036                 mutex_exit(&dtrace_provider_lock);
16037                 mutex_exit(&dtrace_lock);
16038                 return (DDI_FAILURE);
16039         }
16040 
16041         if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
16042             DTRACEMNRN_DTRACE, DDI_PSEUDO, 0) == DDI_FAILURE ||
16043             ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
16044             DTRACEMNRN_HELPER, DDI_PSEUDO, 0) == DDI_FAILURE) {
16045                 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
16046                 ddi_remove_minor_node(devi, NULL);
16047                 ddi_soft_state_fini(&dtrace_softstate);
16048                 mutex_exit(&cpu_lock);
16049                 mutex_exit(&dtrace_provider_lock);
16050                 mutex_exit(&dtrace_lock);
16051                 return (DDI_FAILURE);
16052         }
16053 
16054         ddi_report_dev(devi);
16055         dtrace_devi = devi;
16056 
16057         dtrace_modload = dtrace_module_loaded;
16058         dtrace_modunload = dtrace_module_unloaded;
16059         dtrace_cpu_init = dtrace_cpu_setup_initial;
16060         dtrace_helpers_cleanup = dtrace_helpers_destroy;
16061         dtrace_helpers_fork = dtrace_helpers_duplicate;
16062         dtrace_cpustart_init = dtrace_suspend;
16063         dtrace_cpustart_fini = dtrace_resume;
16064         dtrace_debugger_init = dtrace_suspend;
16065         dtrace_debugger_fini = dtrace_resume;
16066 
16067         register_cpu_setup_func(dtrace_cpu_setup, NULL);
16068 
16069         ASSERT(MUTEX_HELD(&cpu_lock));
16070 
16071         dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
16072             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
16073         dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
16074             UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
16075             VM_SLEEP | VMC_IDENTIFIER);
16076         dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
16077             1, INT_MAX, 0);
16078 
16079         dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
16080             sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
16081             NULL, NULL, NULL, NULL, NULL, 0);
16082 
16083         ASSERT(MUTEX_HELD(&cpu_lock));
16084         dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
16085             offsetof(dtrace_probe_t, dtpr_nextmod),
16086             offsetof(dtrace_probe_t, dtpr_prevmod));
16087 
16088         dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
16089             offsetof(dtrace_probe_t, dtpr_nextfunc),
16090             offsetof(dtrace_probe_t, dtpr_prevfunc));
16091 
16092         dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
16093             offsetof(dtrace_probe_t, dtpr_nextname),
16094             offsetof(dtrace_probe_t, dtpr_prevname));
16095 
16096         if (dtrace_retain_max < 1) {
16097                 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
16098                     "setting to 1", dtrace_retain_max);
16099                 dtrace_retain_max = 1;
16100         }
16101 
16102         /*
16103          * Now discover our toxic ranges.
16104          */
16105         dtrace_toxic_ranges(dtrace_toxrange_add);
16106 
16107         /*
16108          * Before we register ourselves as a provider to our own framework,
16109          * we would like to assert that dtrace_provider is NULL -- but that's
16110          * not true if we were loaded as a dependency of a DTrace provider.
16111          * Once we've registered, we can assert that dtrace_provider is our
16112          * pseudo provider.
16113          */
16114         (void) dtrace_register("dtrace", &dtrace_provider_attr,
16115             DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
16116 
16117         ASSERT(dtrace_provider != NULL);
16118         ASSERT((dtrace_provider_id_t)dtrace_provider == id);
16119 
16120         dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
16121             dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
16122         dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
16123             dtrace_provider, NULL, NULL, "END", 0, NULL);
16124         dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
16125             dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
16126 
16127         dtrace_anon_property();
16128         mutex_exit(&cpu_lock);
16129 
16130         /*
16131          * If there are already providers, we must ask them to provide their
16132          * probes, and then match any anonymous enabling against them.  Note
16133          * that there should be no other retained enablings at this time:
16134          * the only retained enablings at this time should be the anonymous
16135          * enabling.
16136          */
16137         if (dtrace_anon.dta_enabling != NULL) {
16138                 ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
16139 
16140                 dtrace_enabling_provide(NULL);
16141                 state = dtrace_anon.dta_state;
16142 
16143                 /*
16144                  * We couldn't hold cpu_lock across the above call to
16145                  * dtrace_enabling_provide(), but we must hold it to actually
16146                  * enable the probes.  We have to drop all of our locks, pick
16147                  * up cpu_lock, and regain our locks before matching the
16148                  * retained anonymous enabling.
16149                  */
16150                 mutex_exit(&dtrace_lock);
16151                 mutex_exit(&dtrace_provider_lock);
16152 
16153                 mutex_enter(&cpu_lock);
16154                 mutex_enter(&dtrace_provider_lock);
16155                 mutex_enter(&dtrace_lock);
16156 
16157                 if ((enab = dtrace_anon.dta_enabling) != NULL)
16158                         (void) dtrace_enabling_match(enab, NULL);
16159 
16160                 mutex_exit(&cpu_lock);
16161         }
16162 
16163         mutex_exit(&dtrace_lock);
16164         mutex_exit(&dtrace_provider_lock);
16165 
16166         if (state != NULL) {
16167                 /*
16168                  * If we created any anonymous state, set it going now.
16169                  */
16170                 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
16171         }
16172 
16173         return (DDI_SUCCESS);
16174 }
16175 
16176 /*ARGSUSED*/
16177 static int
16178 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
16179 {
16180         dtrace_state_t *state;
16181         uint32_t priv;
16182         uid_t uid;
16183         zoneid_t zoneid;
16184 
16185         if (getminor(*devp) == DTRACEMNRN_HELPER)
16186                 return (0);
16187 
16188         /*
16189          * If this wasn't an open with the "helper" minor, then it must be
16190          * the "dtrace" minor.
16191          */
16192         if (getminor(*devp) != DTRACEMNRN_DTRACE)
16193                 return (ENXIO);
16194 
16195         /*
16196          * If no DTRACE_PRIV_* bits are set in the credential, then the
16197          * caller lacks sufficient permission to do anything with DTrace.
16198          */
16199         dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
16200         if (priv == DTRACE_PRIV_NONE)
16201                 return (EACCES);
16202 
16203         /*
16204          * Ask all providers to provide all their probes.
16205          */
16206         mutex_enter(&dtrace_provider_lock);
16207         dtrace_probe_provide(NULL, NULL);
16208         mutex_exit(&dtrace_provider_lock);
16209 
16210         mutex_enter(&cpu_lock);
16211         mutex_enter(&dtrace_lock);
16212         dtrace_opens++;
16213         dtrace_membar_producer();
16214 
16215         /*
16216          * If the kernel debugger is active (that is, if the kernel debugger
16217          * modified text in some way), we won't allow the open.
16218          */
16219         if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
16220                 dtrace_opens--;
16221                 mutex_exit(&cpu_lock);
16222                 mutex_exit(&dtrace_lock);
16223                 return (EBUSY);
16224         }
16225 
16226         if (dtrace_helptrace_enable && dtrace_helptrace_buffer == NULL) {
16227                 /*
16228                  * If DTrace helper tracing is enabled, we need to allocate the
16229                  * trace buffer and initialize the values.
16230                  */
16231                 dtrace_helptrace_buffer =
16232                     kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
16233                 dtrace_helptrace_next = 0;
16234                 dtrace_helptrace_wrapped = 0;
16235                 dtrace_helptrace_enable = 0;
16236         }
16237 
16238         state = dtrace_state_create(devp, cred_p);
16239         mutex_exit(&cpu_lock);
16240 
16241         if (state == NULL) {
16242                 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
16243                         (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
16244                 mutex_exit(&dtrace_lock);
16245                 return (EAGAIN);
16246         }
16247 
16248         mutex_exit(&dtrace_lock);
16249 
16250         return (0);
16251 }
16252 
16253 /*ARGSUSED*/
16254 static int
16255 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
16256 {
16257         minor_t minor = getminor(dev);
16258         dtrace_state_t *state;
16259         dtrace_helptrace_t *buf = NULL;
16260 
16261         if (minor == DTRACEMNRN_HELPER)
16262                 return (0);
16263 
16264         state = ddi_get_soft_state(dtrace_softstate, minor);
16265 
16266         mutex_enter(&cpu_lock);
16267         mutex_enter(&dtrace_lock);
16268 
16269         if (state->dts_anon) {
16270                 /*
16271                  * There is anonymous state. Destroy that first.
16272                  */
16273                 ASSERT(dtrace_anon.dta_state == NULL);
16274                 dtrace_state_destroy(state->dts_anon);
16275         }
16276 
16277         if (dtrace_helptrace_disable) {
16278                 /*
16279                  * If we have been told to disable helper tracing, set the
16280                  * buffer to NULL before calling into dtrace_state_destroy();
16281                  * we take advantage of its dtrace_sync() to know that no
16282                  * CPU is in probe context with enabled helper tracing
16283                  * after it returns.
16284                  */
16285                 buf = dtrace_helptrace_buffer;
16286                 dtrace_helptrace_buffer = NULL;
16287         }
16288 
16289         dtrace_state_destroy(state);
16290         ASSERT(dtrace_opens > 0);
16291 
16292         /*
16293          * Only relinquish control of the kernel debugger interface when there
16294          * are no consumers and no anonymous enablings.
16295          */
16296         if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
16297                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
16298 
16299         if (buf != NULL) {
16300                 kmem_free(buf, dtrace_helptrace_bufsize);
16301                 dtrace_helptrace_disable = 0;
16302         }
16303 
16304         mutex_exit(&dtrace_lock);
16305         mutex_exit(&cpu_lock);
16306 
16307         return (0);
16308 }
16309 
16310 /*ARGSUSED*/
16311 static int
16312 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
16313 {
16314         int rval;
16315         dof_helper_t help, *dhp = NULL;
16316 
16317         switch (cmd) {
16318         case DTRACEHIOC_ADDDOF:
16319                 if (copyin((void *)arg, &help, sizeof (help)) != 0) {
16320                         dtrace_dof_error(NULL, "failed to copyin DOF helper");
16321                         return (EFAULT);
16322                 }
16323 
16324                 dhp = &help;
16325                 arg = (intptr_t)help.dofhp_dof;
16326                 /*FALLTHROUGH*/
16327 
16328         case DTRACEHIOC_ADD: {
16329                 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
16330 
16331                 if (dof == NULL)
16332                         return (rval);
16333 
16334                 mutex_enter(&dtrace_lock);
16335 
16336                 /*
16337                  * dtrace_helper_slurp() takes responsibility for the dof --
16338                  * it may free it now or it may save it and free it later.
16339                  */
16340                 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
16341                         *rv = rval;
16342                         rval = 0;
16343                 } else {
16344                         rval = EINVAL;
16345                 }
16346 
16347                 mutex_exit(&dtrace_lock);
16348                 return (rval);
16349         }
16350 
16351         case DTRACEHIOC_REMOVE: {
16352                 mutex_enter(&dtrace_lock);
16353                 rval = dtrace_helper_destroygen(arg);
16354                 mutex_exit(&dtrace_lock);
16355 
16356                 return (rval);
16357         }
16358 
16359         default:
16360                 break;
16361         }
16362 
16363         return (ENOTTY);
16364 }
16365 
16366 /*ARGSUSED*/
16367 static int
16368 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
16369 {
16370         minor_t minor = getminor(dev);
16371         dtrace_state_t *state;
16372         int rval;
16373 
16374         if (minor == DTRACEMNRN_HELPER)
16375                 return (dtrace_ioctl_helper(cmd, arg, rv));
16376 
16377         state = ddi_get_soft_state(dtrace_softstate, minor);
16378 
16379         if (state->dts_anon) {
16380                 ASSERT(dtrace_anon.dta_state == NULL);
16381                 state = state->dts_anon;
16382         }
16383 
16384         switch (cmd) {
16385         case DTRACEIOC_PROVIDER: {
16386                 dtrace_providerdesc_t pvd;
16387                 dtrace_provider_t *pvp;
16388 
16389                 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
16390                         return (EFAULT);
16391 
16392                 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
16393                 mutex_enter(&dtrace_provider_lock);
16394 
16395                 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
16396                         if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
16397                                 break;
16398                 }
16399 
16400                 mutex_exit(&dtrace_provider_lock);
16401 
16402                 if (pvp == NULL)
16403                         return (ESRCH);
16404 
16405                 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
16406                 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
16407                 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
16408                         return (EFAULT);
16409 
16410                 return (0);
16411         }
16412 
16413         case DTRACEIOC_EPROBE: {
16414                 dtrace_eprobedesc_t epdesc;
16415                 dtrace_ecb_t *ecb;
16416                 dtrace_action_t *act;
16417                 void *buf;
16418                 size_t size;
16419                 uintptr_t dest;
16420                 int nrecs;
16421 
16422                 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
16423                         return (EFAULT);
16424 
16425                 mutex_enter(&dtrace_lock);
16426 
16427                 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
16428                         mutex_exit(&dtrace_lock);
16429                         return (EINVAL);
16430                 }
16431 
16432                 if (ecb->dte_probe == NULL) {
16433                         mutex_exit(&dtrace_lock);
16434                         return (EINVAL);
16435                 }
16436 
16437                 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
16438                 epdesc.dtepd_uarg = ecb->dte_uarg;
16439                 epdesc.dtepd_size = ecb->dte_size;
16440 
16441                 nrecs = epdesc.dtepd_nrecs;
16442                 epdesc.dtepd_nrecs = 0;
16443                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
16444                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
16445                                 continue;
16446 
16447                         epdesc.dtepd_nrecs++;
16448                 }
16449 
16450                 /*
16451                  * Now that we have the size, we need to allocate a temporary
16452                  * buffer in which to store the complete description.  We need
16453                  * the temporary buffer to be able to drop dtrace_lock()
16454                  * across the copyout(), below.
16455                  */
16456                 size = sizeof (dtrace_eprobedesc_t) +
16457                     (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
16458 
16459                 buf = kmem_alloc(size, KM_SLEEP);
16460                 dest = (uintptr_t)buf;
16461 
16462                 bcopy(&epdesc, (void *)dest, sizeof (epdesc));
16463                 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
16464 
16465                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
16466                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
16467                                 continue;
16468 
16469                         if (nrecs-- == 0)
16470                                 break;
16471 
16472                         bcopy(&act->dta_rec, (void *)dest,
16473                             sizeof (dtrace_recdesc_t));
16474                         dest += sizeof (dtrace_recdesc_t);
16475                 }
16476 
16477                 mutex_exit(&dtrace_lock);
16478 
16479                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
16480                         kmem_free(buf, size);
16481                         return (EFAULT);
16482                 }
16483 
16484                 kmem_free(buf, size);
16485                 return (0);
16486         }
16487 
16488         case DTRACEIOC_AGGDESC: {
16489                 dtrace_aggdesc_t aggdesc;
16490                 dtrace_action_t *act;
16491                 dtrace_aggregation_t *agg;
16492                 int nrecs;
16493                 uint32_t offs;
16494                 dtrace_recdesc_t *lrec;
16495                 void *buf;
16496                 size_t size;
16497                 uintptr_t dest;
16498 
16499                 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
16500                         return (EFAULT);
16501 
16502                 mutex_enter(&dtrace_lock);
16503 
16504                 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
16505                         mutex_exit(&dtrace_lock);
16506                         return (EINVAL);
16507                 }
16508 
16509                 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
16510 
16511                 nrecs = aggdesc.dtagd_nrecs;
16512                 aggdesc.dtagd_nrecs = 0;
16513 
16514                 offs = agg->dtag_base;
16515                 lrec = &agg->dtag_action.dta_rec;
16516                 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
16517 
16518                 for (act = agg->dtag_first; ; act = act->dta_next) {
16519                         ASSERT(act->dta_intuple ||
16520                             DTRACEACT_ISAGG(act->dta_kind));
16521 
16522                         /*
16523                          * If this action has a record size of zero, it
16524                          * denotes an argument to the aggregating action.
16525                          * Because the presence of this record doesn't (or
16526                          * shouldn't) affect the way the data is interpreted,
16527                          * we don't copy it out to save user-level the
16528                          * confusion of dealing with a zero-length record.
16529                          */
16530                         if (act->dta_rec.dtrd_size == 0) {
16531                                 ASSERT(agg->dtag_hasarg);
16532                                 continue;
16533                         }
16534 
16535                         aggdesc.dtagd_nrecs++;
16536 
16537                         if (act == &agg->dtag_action)
16538                                 break;
16539                 }
16540 
16541                 /*
16542                  * Now that we have the size, we need to allocate a temporary
16543                  * buffer in which to store the complete description.  We need
16544                  * the temporary buffer to be able to drop dtrace_lock()
16545                  * across the copyout(), below.
16546                  */
16547                 size = sizeof (dtrace_aggdesc_t) +
16548                     (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
16549 
16550                 buf = kmem_alloc(size, KM_SLEEP);
16551                 dest = (uintptr_t)buf;
16552 
16553                 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
16554                 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
16555 
16556                 for (act = agg->dtag_first; ; act = act->dta_next) {
16557                         dtrace_recdesc_t rec = act->dta_rec;
16558 
16559                         /*
16560                          * See the comment in the above loop for why we pass
16561                          * over zero-length records.
16562                          */
16563                         if (rec.dtrd_size == 0) {
16564                                 ASSERT(agg->dtag_hasarg);
16565                                 continue;
16566                         }
16567 
16568                         if (nrecs-- == 0)
16569                                 break;
16570 
16571                         rec.dtrd_offset -= offs;
16572                         bcopy(&rec, (void *)dest, sizeof (rec));
16573                         dest += sizeof (dtrace_recdesc_t);
16574 
16575                         if (act == &agg->dtag_action)
16576                                 break;
16577                 }
16578 
16579                 mutex_exit(&dtrace_lock);
16580 
16581                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
16582                         kmem_free(buf, size);
16583                         return (EFAULT);
16584                 }
16585 
16586                 kmem_free(buf, size);
16587                 return (0);
16588         }
16589 
16590         case DTRACEIOC_ENABLE: {
16591                 dof_hdr_t *dof;
16592                 dtrace_enabling_t *enab = NULL;
16593                 dtrace_vstate_t *vstate;
16594                 int err = 0;
16595 
16596                 *rv = 0;
16597 
16598                 /*
16599                  * If a NULL argument has been passed, we take this as our
16600                  * cue to reevaluate our enablings.
16601                  */
16602                 if (arg == 0) {
16603                         dtrace_enabling_matchall();
16604 
16605                         return (0);
16606                 }
16607 
16608                 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
16609                         return (rval);
16610 
16611                 mutex_enter(&cpu_lock);
16612                 mutex_enter(&dtrace_lock);
16613                 vstate = &state->dts_vstate;
16614 
16615                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
16616                         mutex_exit(&dtrace_lock);
16617                         mutex_exit(&cpu_lock);
16618                         dtrace_dof_destroy(dof);
16619                         return (EBUSY);
16620                 }
16621 
16622                 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
16623                         mutex_exit(&dtrace_lock);
16624                         mutex_exit(&cpu_lock);
16625                         dtrace_dof_destroy(dof);
16626                         return (EINVAL);
16627                 }
16628 
16629                 if ((rval = dtrace_dof_options(dof, state)) != 0) {
16630                         dtrace_enabling_destroy(enab);
16631                         mutex_exit(&dtrace_lock);
16632                         mutex_exit(&cpu_lock);
16633                         dtrace_dof_destroy(dof);
16634                         return (rval);
16635                 }
16636 
16637                 if ((err = dtrace_enabling_match(enab, rv)) == 0) {
16638                         err = dtrace_enabling_retain(enab);
16639                 } else {
16640                         dtrace_enabling_destroy(enab);
16641                 }
16642 
16643                 mutex_exit(&cpu_lock);
16644                 mutex_exit(&dtrace_lock);
16645                 dtrace_dof_destroy(dof);
16646 
16647                 return (err);
16648         }
16649 
16650         case DTRACEIOC_REPLICATE: {
16651                 dtrace_repldesc_t desc;
16652                 dtrace_probedesc_t *match = &desc.dtrpd_match;
16653                 dtrace_probedesc_t *create = &desc.dtrpd_create;
16654                 int err;
16655 
16656                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16657                         return (EFAULT);
16658 
16659                 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16660                 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16661                 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16662                 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16663 
16664                 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16665                 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16666                 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16667                 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16668 
16669                 mutex_enter(&dtrace_lock);
16670                 err = dtrace_enabling_replicate(state, match, create);
16671                 mutex_exit(&dtrace_lock);
16672 
16673                 return (err);
16674         }
16675 
16676         case DTRACEIOC_PROBEMATCH:
16677         case DTRACEIOC_PROBES: {
16678                 dtrace_probe_t *probe = NULL;
16679                 dtrace_probedesc_t desc;
16680                 dtrace_probekey_t pkey;
16681                 dtrace_id_t i;
16682                 int m = 0;
16683                 uint32_t priv;
16684                 uid_t uid;
16685                 zoneid_t zoneid;
16686 
16687                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16688                         return (EFAULT);
16689 
16690                 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
16691                 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
16692                 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
16693                 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
16694 
16695                 /*
16696                  * Before we attempt to match this probe, we want to give
16697                  * all providers the opportunity to provide it.
16698                  */
16699                 if (desc.dtpd_id == DTRACE_IDNONE) {
16700                         mutex_enter(&dtrace_provider_lock);
16701                         dtrace_probe_provide(&desc, NULL);
16702                         mutex_exit(&dtrace_provider_lock);
16703                         desc.dtpd_id++;
16704                 }
16705 
16706                 if (cmd == DTRACEIOC_PROBEMATCH)  {
16707                         dtrace_probekey(&desc, &pkey);
16708                         pkey.dtpk_id = DTRACE_IDNONE;
16709                 }
16710 
16711                 dtrace_cred2priv(cr, &priv, &uid, &zoneid);
16712 
16713                 mutex_enter(&dtrace_lock);
16714 
16715                 if (cmd == DTRACEIOC_PROBEMATCH) {
16716                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16717                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16718                                     (m = dtrace_match_probe(probe, &pkey,
16719                                     priv, uid, zoneid)) != 0)
16720                                         break;
16721                         }
16722 
16723                         if (m < 0) {
16724                                 mutex_exit(&dtrace_lock);
16725                                 return (EINVAL);
16726                         }
16727 
16728                 } else {
16729                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
16730                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
16731                                     dtrace_match_priv(probe, priv, uid, zoneid))
16732                                         break;
16733                         }
16734                 }
16735 
16736                 if (probe == NULL) {
16737                         mutex_exit(&dtrace_lock);
16738                         return (ESRCH);
16739                 }
16740 
16741                 dtrace_probe_description(probe, &desc);
16742                 mutex_exit(&dtrace_lock);
16743 
16744                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16745                         return (EFAULT);
16746 
16747                 return (0);
16748         }
16749 
16750         case DTRACEIOC_PROBEARG: {
16751                 dtrace_argdesc_t desc;
16752                 dtrace_probe_t *probe;
16753                 dtrace_provider_t *prov;
16754 
16755                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16756                         return (EFAULT);
16757 
16758                 if (desc.dtargd_id == DTRACE_IDNONE)
16759                         return (EINVAL);
16760 
16761                 if (desc.dtargd_ndx == DTRACE_ARGNONE)
16762                         return (EINVAL);
16763 
16764                 mutex_enter(&dtrace_provider_lock);
16765                 mutex_enter(&mod_lock);
16766                 mutex_enter(&dtrace_lock);
16767 
16768                 if (desc.dtargd_id > dtrace_nprobes) {
16769                         mutex_exit(&dtrace_lock);
16770                         mutex_exit(&mod_lock);
16771                         mutex_exit(&dtrace_provider_lock);
16772                         return (EINVAL);
16773                 }
16774 
16775                 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
16776                         mutex_exit(&dtrace_lock);
16777                         mutex_exit(&mod_lock);
16778                         mutex_exit(&dtrace_provider_lock);
16779                         return (EINVAL);
16780                 }
16781 
16782                 mutex_exit(&dtrace_lock);
16783 
16784                 prov = probe->dtpr_provider;
16785 
16786                 if (prov->dtpv_pops.dtps_getargdesc == NULL) {
16787                         /*
16788                          * There isn't any typed information for this probe.
16789                          * Set the argument number to DTRACE_ARGNONE.
16790                          */
16791                         desc.dtargd_ndx = DTRACE_ARGNONE;
16792                 } else {
16793                         desc.dtargd_native[0] = '\0';
16794                         desc.dtargd_xlate[0] = '\0';
16795                         desc.dtargd_mapping = desc.dtargd_ndx;
16796 
16797                         prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
16798                             probe->dtpr_id, probe->dtpr_arg, &desc);
16799                 }
16800 
16801                 mutex_exit(&mod_lock);
16802                 mutex_exit(&dtrace_provider_lock);
16803 
16804                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16805                         return (EFAULT);
16806 
16807                 return (0);
16808         }
16809 
16810         case DTRACEIOC_GO: {
16811                 processorid_t cpuid;
16812                 rval = dtrace_state_go(state, &cpuid);
16813 
16814                 if (rval != 0)
16815                         return (rval);
16816 
16817                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16818                         return (EFAULT);
16819 
16820                 return (0);
16821         }
16822 
16823         case DTRACEIOC_STOP: {
16824                 processorid_t cpuid;
16825 
16826                 mutex_enter(&dtrace_lock);
16827                 rval = dtrace_state_stop(state, &cpuid);
16828                 mutex_exit(&dtrace_lock);
16829 
16830                 if (rval != 0)
16831                         return (rval);
16832 
16833                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
16834                         return (EFAULT);
16835 
16836                 return (0);
16837         }
16838 
16839         case DTRACEIOC_DOFGET: {
16840                 dof_hdr_t hdr, *dof;
16841                 uint64_t len;
16842 
16843                 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
16844                         return (EFAULT);
16845 
16846                 mutex_enter(&dtrace_lock);
16847                 dof = dtrace_dof_create(state);
16848                 mutex_exit(&dtrace_lock);
16849 
16850                 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
16851                 rval = copyout(dof, (void *)arg, len);
16852                 dtrace_dof_destroy(dof);
16853 
16854                 return (rval == 0 ? 0 : EFAULT);
16855         }
16856 
16857         case DTRACEIOC_AGGSNAP:
16858         case DTRACEIOC_BUFSNAP: {
16859                 dtrace_bufdesc_t desc;
16860                 caddr_t cached;
16861                 dtrace_buffer_t *buf;
16862 
16863                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
16864                         return (EFAULT);
16865 
16866                 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
16867                         return (EINVAL);
16868 
16869                 mutex_enter(&dtrace_lock);
16870 
16871                 if (cmd == DTRACEIOC_BUFSNAP) {
16872                         buf = &state->dts_buffer[desc.dtbd_cpu];
16873                 } else {
16874                         buf = &state->dts_aggbuffer[desc.dtbd_cpu];
16875                 }
16876 
16877                 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
16878                         size_t sz = buf->dtb_offset;
16879 
16880                         if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
16881                                 mutex_exit(&dtrace_lock);
16882                                 return (EBUSY);
16883                         }
16884 
16885                         /*
16886                          * If this buffer has already been consumed, we're
16887                          * going to indicate that there's nothing left here
16888                          * to consume.
16889                          */
16890                         if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
16891                                 mutex_exit(&dtrace_lock);
16892 
16893                                 desc.dtbd_size = 0;
16894                                 desc.dtbd_drops = 0;
16895                                 desc.dtbd_errors = 0;
16896                                 desc.dtbd_oldest = 0;
16897                                 sz = sizeof (desc);
16898 
16899                                 if (copyout(&desc, (void *)arg, sz) != 0)
16900                                         return (EFAULT);
16901 
16902                                 return (0);
16903                         }
16904 
16905                         /*
16906                          * If this is a ring buffer that has wrapped, we want
16907                          * to copy the whole thing out.
16908                          */
16909                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
16910                                 dtrace_buffer_polish(buf);
16911                                 sz = buf->dtb_size;
16912                         }
16913 
16914                         if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
16915                                 mutex_exit(&dtrace_lock);
16916                                 return (EFAULT);
16917                         }
16918 
16919                         desc.dtbd_size = sz;
16920                         desc.dtbd_drops = buf->dtb_drops;
16921                         desc.dtbd_errors = buf->dtb_errors;
16922                         desc.dtbd_oldest = buf->dtb_xamot_offset;
16923                         desc.dtbd_timestamp = dtrace_gethrtime();
16924 
16925                         mutex_exit(&dtrace_lock);
16926 
16927                         if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16928                                 return (EFAULT);
16929 
16930                         buf->dtb_flags |= DTRACEBUF_CONSUMED;
16931 
16932                         return (0);
16933                 }
16934 
16935                 if (buf->dtb_tomax == NULL) {
16936                         ASSERT(buf->dtb_xamot == NULL);
16937                         mutex_exit(&dtrace_lock);
16938                         return (ENOENT);
16939                 }
16940 
16941                 cached = buf->dtb_tomax;
16942                 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
16943 
16944                 dtrace_xcall(desc.dtbd_cpu,
16945                     (dtrace_xcall_t)dtrace_buffer_switch, buf);
16946 
16947                 state->dts_errors += buf->dtb_xamot_errors;
16948 
16949                 /*
16950                  * If the buffers did not actually switch, then the cross call
16951                  * did not take place -- presumably because the given CPU is
16952                  * not in the ready set.  If this is the case, we'll return
16953                  * ENOENT.
16954                  */
16955                 if (buf->dtb_tomax == cached) {
16956                         ASSERT(buf->dtb_xamot != cached);
16957                         mutex_exit(&dtrace_lock);
16958                         return (ENOENT);
16959                 }
16960 
16961                 ASSERT(cached == buf->dtb_xamot);
16962 
16963                 /*
16964                  * We have our snapshot; now copy it out.
16965                  */
16966                 if (copyout(buf->dtb_xamot, desc.dtbd_data,
16967                     buf->dtb_xamot_offset) != 0) {
16968                         mutex_exit(&dtrace_lock);
16969                         return (EFAULT);
16970                 }
16971 
16972                 desc.dtbd_size = buf->dtb_xamot_offset;
16973                 desc.dtbd_drops = buf->dtb_xamot_drops;
16974                 desc.dtbd_errors = buf->dtb_xamot_errors;
16975                 desc.dtbd_oldest = 0;
16976                 desc.dtbd_timestamp = buf->dtb_switched;
16977 
16978                 mutex_exit(&dtrace_lock);
16979 
16980                 /*
16981                  * Finally, copy out the buffer description.
16982                  */
16983                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
16984                         return (EFAULT);
16985 
16986                 return (0);
16987         }
16988 
16989         case DTRACEIOC_CONF: {
16990                 dtrace_conf_t conf;
16991 
16992                 bzero(&conf, sizeof (conf));
16993                 conf.dtc_difversion = DIF_VERSION;
16994                 conf.dtc_difintregs = DIF_DIR_NREGS;
16995                 conf.dtc_diftupregs = DIF_DTR_NREGS;
16996                 conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
16997 
16998                 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
16999                         return (EFAULT);
17000 
17001                 return (0);
17002         }
17003 
17004         case DTRACEIOC_STATUS: {
17005                 dtrace_status_t stat;
17006                 dtrace_dstate_t *dstate;
17007                 int i, j;
17008                 uint64_t nerrs;
17009 
17010                 /*
17011                  * See the comment in dtrace_state_deadman() for the reason
17012                  * for setting dts_laststatus to INT64_MAX before setting
17013                  * it to the correct value.
17014                  */
17015                 state->dts_laststatus = INT64_MAX;
17016                 dtrace_membar_producer();
17017                 state->dts_laststatus = dtrace_gethrtime();
17018 
17019                 bzero(&stat, sizeof (stat));
17020 
17021                 mutex_enter(&dtrace_lock);
17022 
17023                 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
17024                         mutex_exit(&dtrace_lock);
17025                         return (ENOENT);
17026                 }
17027 
17028                 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
17029                         stat.dtst_exiting = 1;
17030 
17031                 nerrs = state->dts_errors;
17032                 dstate = &state->dts_vstate.dtvs_dynvars;
17033 
17034                 for (i = 0; i < NCPU; i++) {
17035                         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
17036 
17037                         stat.dtst_dyndrops += dcpu->dtdsc_drops;
17038                         stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
17039                         stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
17040 
17041                         if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
17042                                 stat.dtst_filled++;
17043 
17044                         nerrs += state->dts_buffer[i].dtb_errors;
17045 
17046                         for (j = 0; j < state->dts_nspeculations; j++) {
17047                                 dtrace_speculation_t *spec;
17048                                 dtrace_buffer_t *buf;
17049 
17050                                 spec = &state->dts_speculations[j];
17051                                 buf = &spec->dtsp_buffer[i];
17052                                 stat.dtst_specdrops += buf->dtb_xamot_drops;
17053                         }
17054                 }
17055 
17056                 stat.dtst_specdrops_busy = state->dts_speculations_busy;
17057                 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
17058                 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
17059                 stat.dtst_dblerrors = state->dts_dblerrors;
17060                 stat.dtst_killed =
17061                     (state->dts_activity == DTRACE_ACTIVITY_KILLED);
17062                 stat.dtst_errors = nerrs;
17063 
17064                 mutex_exit(&dtrace_lock);
17065 
17066                 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
17067                         return (EFAULT);
17068 
17069                 return (0);
17070         }
17071 
17072         case DTRACEIOC_FORMAT: {
17073                 dtrace_fmtdesc_t fmt;
17074                 char *str;
17075                 int len;
17076 
17077                 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
17078                         return (EFAULT);
17079 
17080                 mutex_enter(&dtrace_lock);
17081 
17082                 if (fmt.dtfd_format == 0 ||
17083                     fmt.dtfd_format > state->dts_nformats) {
17084                         mutex_exit(&dtrace_lock);
17085                         return (EINVAL);
17086                 }
17087 
17088                 /*
17089                  * Format strings are allocated contiguously and they are
17090                  * never freed; if a format index is less than the number
17091                  * of formats, we can assert that the format map is non-NULL
17092                  * and that the format for the specified index is non-NULL.
17093                  */
17094                 ASSERT(state->dts_formats != NULL);
17095                 str = state->dts_formats[fmt.dtfd_format - 1];
17096                 ASSERT(str != NULL);
17097 
17098                 len = strlen(str) + 1;
17099 
17100                 if (len > fmt.dtfd_length) {
17101                         fmt.dtfd_length = len;
17102 
17103                         if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
17104                                 mutex_exit(&dtrace_lock);
17105                                 return (EINVAL);
17106                         }
17107                 } else {
17108                         if (copyout(str, fmt.dtfd_string, len) != 0) {
17109                                 mutex_exit(&dtrace_lock);
17110                                 return (EINVAL);
17111                         }
17112                 }
17113 
17114                 mutex_exit(&dtrace_lock);
17115                 return (0);
17116         }
17117 
17118         default:
17119                 break;
17120         }
17121 
17122         return (ENOTTY);
17123 }
17124 
17125 /*ARGSUSED*/
17126 static int
17127 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
17128 {
17129         dtrace_state_t *state;
17130 
17131         switch (cmd) {
17132         case DDI_DETACH:
17133                 break;
17134 
17135         case DDI_SUSPEND:
17136                 return (DDI_SUCCESS);
17137 
17138         default:
17139                 return (DDI_FAILURE);
17140         }
17141 
17142         mutex_enter(&cpu_lock);
17143         mutex_enter(&dtrace_provider_lock);
17144         mutex_enter(&dtrace_lock);
17145 
17146         ASSERT(dtrace_opens == 0);
17147 
17148         if (dtrace_helpers > 0) {
17149                 mutex_exit(&dtrace_provider_lock);
17150                 mutex_exit(&dtrace_lock);
17151                 mutex_exit(&cpu_lock);
17152                 return (DDI_FAILURE);
17153         }
17154 
17155         if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
17156                 mutex_exit(&dtrace_provider_lock);
17157                 mutex_exit(&dtrace_lock);
17158                 mutex_exit(&cpu_lock);
17159                 return (DDI_FAILURE);
17160         }
17161 
17162         dtrace_provider = NULL;
17163 
17164         if ((state = dtrace_anon_grab()) != NULL) {
17165                 /*
17166                  * If there were ECBs on this state, the provider should
17167                  * have not been allowed to detach; assert that there is
17168                  * none.
17169                  */
17170                 ASSERT(state->dts_necbs == 0);
17171                 dtrace_state_destroy(state);
17172 
17173                 /*
17174                  * If we're being detached with anonymous state, we need to
17175                  * indicate to the kernel debugger that DTrace is now inactive.
17176                  */
17177                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
17178         }
17179 
17180         bzero(&dtrace_anon, sizeof (dtrace_anon_t));
17181         unregister_cpu_setup_func(dtrace_cpu_setup, NULL);
17182         dtrace_cpu_init = NULL;
17183         dtrace_helpers_cleanup = NULL;
17184         dtrace_helpers_fork = NULL;
17185         dtrace_cpustart_init = NULL;
17186         dtrace_cpustart_fini = NULL;
17187         dtrace_debugger_init = NULL;
17188         dtrace_debugger_fini = NULL;
17189         dtrace_modload = NULL;
17190         dtrace_modunload = NULL;
17191 
17192         ASSERT(dtrace_getf == 0);
17193         ASSERT(dtrace_closef == NULL);
17194 
17195         mutex_exit(&cpu_lock);
17196 
17197         kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
17198         dtrace_probes = NULL;
17199         dtrace_nprobes = 0;
17200 
17201         dtrace_hash_destroy(dtrace_bymod);
17202         dtrace_hash_destroy(dtrace_byfunc);
17203         dtrace_hash_destroy(dtrace_byname);
17204         dtrace_bymod = NULL;
17205         dtrace_byfunc = NULL;
17206         dtrace_byname = NULL;
17207 
17208         kmem_cache_destroy(dtrace_state_cache);
17209         vmem_destroy(dtrace_minor);
17210         vmem_destroy(dtrace_arena);
17211 
17212         if (dtrace_toxrange != NULL) {
17213                 kmem_free(dtrace_toxrange,
17214                     dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
17215                 dtrace_toxrange = NULL;
17216                 dtrace_toxranges = 0;
17217                 dtrace_toxranges_max = 0;
17218         }
17219 
17220         ddi_remove_minor_node(dtrace_devi, NULL);
17221         dtrace_devi = NULL;
17222 
17223         ddi_soft_state_fini(&dtrace_softstate);
17224 
17225         ASSERT(dtrace_vtime_references == 0);
17226         ASSERT(dtrace_opens == 0);
17227         ASSERT(dtrace_retained == NULL);
17228 
17229         mutex_exit(&dtrace_lock);
17230         mutex_exit(&dtrace_provider_lock);
17231 
17232         /*
17233          * We don't destroy the task queue until after we have dropped our
17234          * locks (taskq_destroy() may block on running tasks).  To prevent
17235          * attempting to do work after we have effectively detached but before
17236          * the task queue has been destroyed, all tasks dispatched via the
17237          * task queue must check that DTrace is still attached before
17238          * performing any operation.
17239          */
17240         taskq_destroy(dtrace_taskq);
17241         dtrace_taskq = NULL;
17242 
17243         return (DDI_SUCCESS);
17244 }
17245 
17246 /*ARGSUSED*/
17247 static int
17248 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
17249 {
17250         int error;
17251 
17252         switch (infocmd) {
17253         case DDI_INFO_DEVT2DEVINFO:
17254                 *result = (void *)dtrace_devi;
17255                 error = DDI_SUCCESS;
17256                 break;
17257         case DDI_INFO_DEVT2INSTANCE:
17258                 *result = (void *)0;
17259                 error = DDI_SUCCESS;
17260                 break;
17261         default:
17262                 error = DDI_FAILURE;
17263         }
17264         return (error);
17265 }
17266 
17267 static struct cb_ops dtrace_cb_ops = {
17268         dtrace_open,            /* open */
17269         dtrace_close,           /* close */
17270         nulldev,                /* strategy */
17271         nulldev,                /* print */
17272         nodev,                  /* dump */
17273         nodev,                  /* read */
17274         nodev,                  /* write */
17275         dtrace_ioctl,           /* ioctl */
17276         nodev,                  /* devmap */
17277         nodev,                  /* mmap */
17278         nodev,                  /* segmap */
17279         nochpoll,               /* poll */
17280         ddi_prop_op,            /* cb_prop_op */
17281         0,                      /* streamtab  */
17282         D_NEW | D_MP            /* Driver compatibility flag */
17283 };
17284 
17285 static struct dev_ops dtrace_ops = {
17286         DEVO_REV,               /* devo_rev */
17287         0,                      /* refcnt */
17288         dtrace_info,            /* get_dev_info */
17289         nulldev,                /* identify */
17290         nulldev,                /* probe */
17291         dtrace_attach,          /* attach */
17292         dtrace_detach,          /* detach */
17293         nodev,                  /* reset */
17294         &dtrace_cb_ops,             /* driver operations */
17295         NULL,                   /* bus operations */
17296         nodev,                  /* dev power */
17297         ddi_quiesce_not_needed,         /* quiesce */
17298 };
17299 
17300 static struct modldrv modldrv = {
17301         &mod_driverops,             /* module type (this is a pseudo driver) */
17302         "Dynamic Tracing",      /* name of module */
17303         &dtrace_ops,                /* driver ops */
17304 };
17305 
17306 static struct modlinkage modlinkage = {
17307         MODREV_1,
17308         (void *)&modldrv,
17309         NULL
17310 };
17311 
17312 int
17313 _init(void)
17314 {
17315         return (mod_install(&modlinkage));
17316 }
17317 
17318 int
17319 _info(struct modinfo *modinfop)
17320 {
17321         return (mod_info(&modlinkage, modinfop));
17322 }
17323 
17324 int
17325 _fini(void)
17326 {
17327         return (mod_remove(&modlinkage));
17328 }