1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  25  * Copyright (c) 2012 by Delphix. All rights reserved.
  26  */
  27 
  28 /*
  29  * DTrace - Dynamic Tracing for Solaris
  30  *
  31  * This is the implementation of the Solaris Dynamic Tracing framework
  32  * (DTrace).  The user-visible interface to DTrace is described at length in
  33  * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
  34  * library, the in-kernel DTrace framework, and the DTrace providers are
  35  * described in the block comments in the <sys/dtrace.h> header file.  The
  36  * internal architecture of DTrace is described in the block comments in the
  37  * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
  38  * implementation very much assume mastery of all of these sources; if one has
  39  * an unanswered question about the implementation, one should consult them
  40  * first.
  41  *
  42  * The functions here are ordered roughly as follows:
  43  *
  44  *   - Probe context functions
  45  *   - Probe hashing functions
  46  *   - Non-probe context utility functions
  47  *   - Matching functions
  48  *   - Provider-to-Framework API functions
  49  *   - Probe management functions
  50  *   - DIF object functions
  51  *   - Format functions
  52  *   - Predicate functions
  53  *   - ECB functions
  54  *   - Buffer functions
  55  *   - Enabling functions
  56  *   - DOF functions
  57  *   - Anonymous enabling functions
  58  *   - Consumer state functions
  59  *   - Helper functions
  60  *   - Hook functions
  61  *   - Driver cookbook functions
  62  *
  63  * Each group of functions begins with a block comment labelled the "DTrace
  64  * [Group] Functions", allowing one to find each block by searching forward
  65  * on capital-f functions.
  66  */
  67 #include <sys/errno.h>
  68 #include <sys/stat.h>
  69 #include <sys/modctl.h>
  70 #include <sys/conf.h>
  71 #include <sys/systm.h>
  72 #include <sys/ddi.h>
  73 #include <sys/sunddi.h>
  74 #include <sys/cpuvar.h>
  75 #include <sys/kmem.h>
  76 #include <sys/strsubr.h>
  77 #include <sys/sysmacros.h>
  78 #include <sys/dtrace_impl.h>
  79 #include <sys/atomic.h>
  80 #include <sys/cmn_err.h>
  81 #include <sys/mutex_impl.h>
  82 #include <sys/rwlock_impl.h>
  83 #include <sys/ctf_api.h>
  84 #include <sys/panic.h>
  85 #include <sys/priv_impl.h>
  86 #include <sys/policy.h>
  87 #include <sys/cred_impl.h>
  88 #include <sys/procfs_isa.h>
  89 #include <sys/taskq.h>
  90 #include <sys/mkdev.h>
  91 #include <sys/kdi.h>
  92 #include <sys/zone.h>
  93 #include <sys/socket.h>
  94 #include <netinet/in.h>
  95 
  96 /*
  97  * DTrace Tunable Variables
  98  *
  99  * The following variables may be tuned by adding a line to /etc/system that
 100  * includes both the name of the DTrace module ("dtrace") and the name of the
 101  * variable.  For example:
 102  *
 103  *   set dtrace:dtrace_destructive_disallow = 1
 104  *
 105  * In general, the only variables that one should be tuning this way are those
 106  * that affect system-wide DTrace behavior, and for which the default behavior
 107  * is undesirable.  Most of these variables are tunable on a per-consumer
 108  * basis using DTrace options, and need not be tuned on a system-wide basis.
 109  * When tuning these variables, avoid pathological values; while some attempt
 110  * is made to verify the integrity of these variables, they are not considered
 111  * part of the supported interface to DTrace, and they are therefore not
 112  * checked comprehensively.  Further, these variables should not be tuned
 113  * dynamically via "mdb -kw" or other means; they should only be tuned via
 114  * /etc/system.
 115  */
 116 int             dtrace_destructive_disallow = 0;
 117 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 118 size_t          dtrace_difo_maxsize = (256 * 1024);
 119 dtrace_optval_t dtrace_dof_maxsize = (256 * 1024);
 120 size_t          dtrace_global_maxsize = (16 * 1024);
 121 size_t          dtrace_actions_max = (16 * 1024);
 122 size_t          dtrace_retain_max = 1024;
 123 dtrace_optval_t dtrace_helper_actions_max = 1024;
 124 dtrace_optval_t dtrace_helper_providers_max = 32;
 125 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
 126 size_t          dtrace_strsize_default = 256;
 127 dtrace_optval_t dtrace_cleanrate_default = 9900990;             /* 101 hz */
 128 dtrace_optval_t dtrace_cleanrate_min = 200000;                  /* 5000 hz */
 129 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;  /* 1/minute */
 130 dtrace_optval_t dtrace_aggrate_default = NANOSEC;               /* 1 hz */
 131 dtrace_optval_t dtrace_statusrate_default = NANOSEC;            /* 1 hz */
 132 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;  /* 6/minute */
 133 dtrace_optval_t dtrace_switchrate_default = NANOSEC;            /* 1 hz */
 134 dtrace_optval_t dtrace_nspec_default = 1;
 135 dtrace_optval_t dtrace_specsize_default = 32 * 1024;
 136 dtrace_optval_t dtrace_stackframes_default = 20;
 137 dtrace_optval_t dtrace_ustackframes_default = 20;
 138 dtrace_optval_t dtrace_jstackframes_default = 50;
 139 dtrace_optval_t dtrace_jstackstrsize_default = 512;
 140 int             dtrace_msgdsize_max = 128;
 141 hrtime_t        dtrace_chill_max = 500 * (NANOSEC / MILLISEC);  /* 500 ms */
 142 hrtime_t        dtrace_chill_interval = NANOSEC;                /* 1000 ms */
 143 int             dtrace_devdepth_max = 32;
 144 int             dtrace_err_verbose;
 145 hrtime_t        dtrace_deadman_interval = NANOSEC;
 146 hrtime_t        dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
 147 hrtime_t        dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
 148 hrtime_t        dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
 149 
 150 /*
 151  * DTrace External Variables
 152  *
 153  * As dtrace(7D) is a kernel module, any DTrace variables are obviously
 154  * available to DTrace consumers via the backtick (`) syntax.  One of these,
 155  * dtrace_zero, is made deliberately so:  it is provided as a source of
 156  * well-known, zero-filled memory.  While this variable is not documented,
 157  * it is used by some translators as an implementation detail.
 158  */
 159 const char      dtrace_zero[256] = { 0 };       /* zero-filled memory */
 160 
 161 /*
 162  * DTrace Internal Variables
 163  */
 164 static dev_info_t       *dtrace_devi;           /* device info */
 165 static vmem_t           *dtrace_arena;          /* probe ID arena */
 166 static vmem_t           *dtrace_minor;          /* minor number arena */
 167 static taskq_t          *dtrace_taskq;          /* task queue */
 168 static dtrace_probe_t   **dtrace_probes;        /* array of all probes */
 169 static int              dtrace_nprobes;         /* number of probes */
 170 static dtrace_provider_t *dtrace_provider;      /* provider list */
 171 static dtrace_meta_t    *dtrace_meta_pid;       /* user-land meta provider */
 172 static int              dtrace_opens;           /* number of opens */
 173 static int              dtrace_helpers;         /* number of helpers */
 174 static void             *dtrace_softstate;      /* softstate pointer */
 175 static dtrace_hash_t    *dtrace_bymod;          /* probes hashed by module */
 176 static dtrace_hash_t    *dtrace_byfunc;         /* probes hashed by function */
 177 static dtrace_hash_t    *dtrace_byname;         /* probes hashed by name */
 178 static dtrace_toxrange_t *dtrace_toxrange;      /* toxic range array */
 179 static int              dtrace_toxranges;       /* number of toxic ranges */
 180 static int              dtrace_toxranges_max;   /* size of toxic range array */
 181 static dtrace_anon_t    dtrace_anon;            /* anonymous enabling */
 182 static kmem_cache_t     *dtrace_state_cache;    /* cache for dynamic state */
 183 static uint64_t         dtrace_vtime_references; /* number of vtimestamp refs */
 184 static kthread_t        *dtrace_panicked;       /* panicking thread */
 185 static dtrace_ecb_t     *dtrace_ecb_create_cache; /* cached created ECB */
 186 static dtrace_genid_t   dtrace_probegen;        /* current probe generation */
 187 static dtrace_helpers_t *dtrace_deferred_pid;   /* deferred helper list */
 188 static dtrace_enabling_t *dtrace_retained;      /* list of retained enablings */
 189 static dtrace_genid_t   dtrace_retained_gen;    /* current retained enab gen */
 190 static dtrace_dynvar_t  dtrace_dynhash_sink;    /* end of dynamic hash chains */
 191 static int              dtrace_dynvar_failclean; /* dynvars failed to clean */
 192 
 193 /*
 194  * DTrace Locking
 195  * DTrace is protected by three (relatively coarse-grained) locks:
 196  *
 197  * (1) dtrace_lock is required to manipulate essentially any DTrace state,
 198  *     including enabling state, probes, ECBs, consumer state, helper state,
 199  *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
 200  *     probe context is lock-free -- synchronization is handled via the
 201  *     dtrace_sync() cross call mechanism.
 202  *
 203  * (2) dtrace_provider_lock is required when manipulating provider state, or
 204  *     when provider state must be held constant.
 205  *
 206  * (3) dtrace_meta_lock is required when manipulating meta provider state, or
 207  *     when meta provider state must be held constant.
 208  *
 209  * The lock ordering between these three locks is dtrace_meta_lock before
 210  * dtrace_provider_lock before dtrace_lock.  (In particular, there are
 211  * several places where dtrace_provider_lock is held by the framework as it
 212  * calls into the providers -- which then call back into the framework,
 213  * grabbing dtrace_lock.)
 214  *
 215  * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
 216  * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
 217  * role as a coarse-grained lock; it is acquired before both of these locks.
 218  * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
 219  * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
 220  * mod_lock is similar with respect to dtrace_provider_lock in that it must be
 221  * acquired _between_ dtrace_provider_lock and dtrace_lock.
 222  */
 223 static kmutex_t         dtrace_lock;            /* probe state lock */
 224 static kmutex_t         dtrace_provider_lock;   /* provider state lock */
 225 static kmutex_t         dtrace_meta_lock;       /* meta-provider state lock */
 226 
 227 /*
 228  * DTrace Provider Variables
 229  *
 230  * These are the variables relating to DTrace as a provider (that is, the
 231  * provider of the BEGIN, END, and ERROR probes).
 232  */
 233 static dtrace_pattr_t   dtrace_provider_attr = {
 234 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 235 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 236 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
 237 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 238 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 239 };
 240 
 241 static void
 242 dtrace_nullop(void)
 243 {}
 244 
 245 static int
 246 dtrace_enable_nullop(void)
 247 {
 248         return (0);
 249 }
 250 
 251 static dtrace_pops_t    dtrace_provider_ops = {
 252         (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
 253         (void (*)(void *, struct modctl *))dtrace_nullop,
 254         (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
 255         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 256         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 257         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 258         NULL,
 259         NULL,
 260         NULL,
 261         (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
 262 };
 263 
 264 static dtrace_id_t      dtrace_probeid_begin;   /* special BEGIN probe */
 265 static dtrace_id_t      dtrace_probeid_end;     /* special END probe */
 266 dtrace_id_t             dtrace_probeid_error;   /* special ERROR probe */
 267 
 268 /*
 269  * DTrace Helper Tracing Variables
 270  */
 271 uint32_t dtrace_helptrace_next = 0;
 272 uint32_t dtrace_helptrace_nlocals;
 273 char    *dtrace_helptrace_buffer;
 274 int     dtrace_helptrace_bufsize = 512 * 1024;
 275 
 276 #ifdef DEBUG
 277 int     dtrace_helptrace_enabled = 1;
 278 #else
 279 int     dtrace_helptrace_enabled = 0;
 280 #endif
 281 
 282 /*
 283  * DTrace Error Hashing
 284  *
 285  * On DEBUG kernels, DTrace will track the errors that has seen in a hash
 286  * table.  This is very useful for checking coverage of tests that are
 287  * expected to induce DIF or DOF processing errors, and may be useful for
 288  * debugging problems in the DIF code generator or in DOF generation .  The
 289  * error hash may be examined with the ::dtrace_errhash MDB dcmd.
 290  */
 291 #ifdef DEBUG
 292 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
 293 static const char *dtrace_errlast;
 294 static kthread_t *dtrace_errthread;
 295 static kmutex_t dtrace_errlock;
 296 #endif
 297 
 298 /*
 299  * DTrace Macros and Constants
 300  *
 301  * These are various macros that are useful in various spots in the
 302  * implementation, along with a few random constants that have no meaning
 303  * outside of the implementation.  There is no real structure to this cpp
 304  * mishmash -- but is there ever?
 305  */
 306 #define DTRACE_HASHSTR(hash, probe)     \
 307         dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
 308 
 309 #define DTRACE_HASHNEXT(hash, probe)    \
 310         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
 311 
 312 #define DTRACE_HASHPREV(hash, probe)    \
 313         (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
 314 
 315 #define DTRACE_HASHEQ(hash, lhs, rhs)   \
 316         (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
 317             *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
 318 
 319 #define DTRACE_AGGHASHSIZE_SLEW         17
 320 
 321 #define DTRACE_V4MAPPED_OFFSET          (sizeof (uint32_t) * 3)
 322 
 323 /*
 324  * The key for a thread-local variable consists of the lower 61 bits of the
 325  * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
 326  * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
 327  * equal to a variable identifier.  This is necessary (but not sufficient) to
 328  * assure that global associative arrays never collide with thread-local
 329  * variables.  To guarantee that they cannot collide, we must also define the
 330  * order for keying dynamic variables.  That order is:
 331  *
 332  *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
 333  *
 334  * Because the variable-key and the tls-key are in orthogonal spaces, there is
 335  * no way for a global variable key signature to match a thread-local key
 336  * signature.
 337  */
 338 #define DTRACE_TLS_THRKEY(where) { \
 339         uint_t intr = 0; \
 340         uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
 341         for (; actv; actv >>= 1) \
 342                 intr++; \
 343         ASSERT(intr < (1 << 3)); \
 344         (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
 345             (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
 346 }
 347 
 348 #define DT_BSWAP_8(x)   ((x) & 0xff)
 349 #define DT_BSWAP_16(x)  ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
 350 #define DT_BSWAP_32(x)  ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
 351 #define DT_BSWAP_64(x)  ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
 352 
 353 #define DT_MASK_LO 0x00000000FFFFFFFFULL
 354 
 355 #define DTRACE_STORE(type, tomax, offset, what) \
 356         *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
 357 
 358 #ifndef __x86
 359 #define DTRACE_ALIGNCHECK(addr, size, flags)                            \
 360         if (addr & (size - 1)) {                                    \
 361                 *flags |= CPU_DTRACE_BADALIGN;                          \
 362                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 363                 return (0);                                             \
 364         }
 365 #else
 366 #define DTRACE_ALIGNCHECK(addr, size, flags)
 367 #endif
 368 
 369 /*
 370  * Test whether a range of memory starting at testaddr of size testsz falls
 371  * within the range of memory described by addr, sz.  We take care to avoid
 372  * problems with overflow and underflow of the unsigned quantities, and
 373  * disallow all negative sizes.  Ranges of size 0 are allowed.
 374  */
 375 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
 376         ((testaddr) - (baseaddr) < (basesz) && \
 377         (testaddr) + (testsz) - (baseaddr) <= (basesz) && \
 378         (testaddr) + (testsz) >= (testaddr))
 379 
 380 /*
 381  * Test whether alloc_sz bytes will fit in the scratch region.  We isolate
 382  * alloc_sz on the righthand side of the comparison in order to avoid overflow
 383  * or underflow in the comparison with it.  This is simpler than the INRANGE
 384  * check above, because we know that the dtms_scratch_ptr is valid in the
 385  * range.  Allocations of size zero are allowed.
 386  */
 387 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
 388         ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
 389         (mstate)->dtms_scratch_ptr >= (alloc_sz))
 390 
 391 #define DTRACE_LOADFUNC(bits)                                           \
 392 /*CSTYLED*/                                                             \
 393 uint##bits##_t                                                          \
 394 dtrace_load##bits(uintptr_t addr)                                       \
 395 {                                                                       \
 396         size_t size = bits / NBBY;                                      \
 397         /*CSTYLED*/                                                     \
 398         uint##bits##_t rval;                                            \
 399         int i;                                                          \
 400         volatile uint16_t *flags = (volatile uint16_t *)                \
 401             &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;                    \
 402                                                                         \
 403         DTRACE_ALIGNCHECK(addr, size, flags);                           \
 404                                                                         \
 405         for (i = 0; i < dtrace_toxranges; i++) {                     \
 406                 if (addr >= dtrace_toxrange[i].dtt_limit)            \
 407                         continue;                                       \
 408                                                                         \
 409                 if (addr + size <= dtrace_toxrange[i].dtt_base)              \
 410                         continue;                                       \
 411                                                                         \
 412                 /*                                                      \
 413                  * This address falls within a toxic region; return 0.  \
 414                  */                                                     \
 415                 *flags |= CPU_DTRACE_BADADDR;                           \
 416                 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;     \
 417                 return (0);                                             \
 418         }                                                               \
 419                                                                         \
 420         *flags |= CPU_DTRACE_NOFAULT;                                   \
 421         /*CSTYLED*/                                                     \
 422         rval = *((volatile uint##bits##_t *)addr);                      \
 423         *flags &= ~CPU_DTRACE_NOFAULT;                                      \
 424                                                                         \
 425         return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0);           \
 426 }
 427 
 428 #ifdef _LP64
 429 #define dtrace_loadptr  dtrace_load64
 430 #else
 431 #define dtrace_loadptr  dtrace_load32
 432 #endif
 433 
 434 #define DTRACE_DYNHASH_FREE     0
 435 #define DTRACE_DYNHASH_SINK     1
 436 #define DTRACE_DYNHASH_VALID    2
 437 
 438 #define DTRACE_MATCH_FAIL       -1
 439 #define DTRACE_MATCH_NEXT       0
 440 #define DTRACE_MATCH_DONE       1
 441 #define DTRACE_ANCHORED(probe)  ((probe)->dtpr_func[0] != '\0')
 442 #define DTRACE_STATE_ALIGN      64
 443 
 444 #define DTRACE_FLAGS2FLT(flags)                                         \
 445         (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :               \
 446         ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :            \
 447         ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :                \
 448         ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :            \
 449         ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :            \
 450         ((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :             \
 451         ((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :             \
 452         ((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :   \
 453         ((flags) & CPU_DTRACE_BADSTACK) ?  DTRACEFLT_BADSTACK :             \
 454         DTRACEFLT_UNKNOWN)
 455 
 456 #define DTRACEACT_ISSTRING(act)                                         \
 457         ((act)->dta_kind == DTRACEACT_DIFEXPR &&                     \
 458         (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
 459 
 460 static size_t dtrace_strlen(const char *, size_t);
 461 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
 462 static void dtrace_enabling_provide(dtrace_provider_t *);
 463 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
 464 static void dtrace_enabling_matchall(void);
 465 static void dtrace_enabling_reap(void);
 466 static dtrace_state_t *dtrace_anon_grab(void);
 467 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
 468     dtrace_state_t *, uint64_t, uint64_t);
 469 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
 470 static void dtrace_buffer_drop(dtrace_buffer_t *);
 471 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
 472 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
 473     dtrace_state_t *, dtrace_mstate_t *);
 474 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
 475     dtrace_optval_t);
 476 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
 477 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
 478 
 479 /*
 480  * DTrace Probe Context Functions
 481  *
 482  * These functions are called from probe context.  Because probe context is
 483  * any context in which C may be called, arbitrarily locks may be held,
 484  * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
 485  * As a result, functions called from probe context may only call other DTrace
 486  * support functions -- they may not interact at all with the system at large.
 487  * (Note that the ASSERT macro is made probe-context safe by redefining it in
 488  * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
 489  * loads are to be performed from probe context, they _must_ be in terms of
 490  * the safe dtrace_load*() variants.
 491  *
 492  * Some functions in this block are not actually called from probe context;
 493  * for these functions, there will be a comment above the function reading
 494  * "Note:  not called from probe context."
 495  */
 496 void
 497 dtrace_panic(const char *format, ...)
 498 {
 499         va_list alist;
 500 
 501         va_start(alist, format);
 502         dtrace_vpanic(format, alist);
 503         va_end(alist);
 504 }
 505 
 506 int
 507 dtrace_assfail(const char *a, const char *f, int l)
 508 {
 509         dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
 510 
 511         /*
 512          * We just need something here that even the most clever compiler
 513          * cannot optimize away.
 514          */
 515         return (a[(uintptr_t)f]);
 516 }
 517 
 518 /*
 519  * Atomically increment a specified error counter from probe context.
 520  */
 521 static void
 522 dtrace_error(uint32_t *counter)
 523 {
 524         /*
 525          * Most counters stored to in probe context are per-CPU counters.
 526          * However, there are some error conditions that are sufficiently
 527          * arcane that they don't merit per-CPU storage.  If these counters
 528          * are incremented concurrently on different CPUs, scalability will be
 529          * adversely affected -- but we don't expect them to be white-hot in a
 530          * correctly constructed enabling...
 531          */
 532         uint32_t oval, nval;
 533 
 534         do {
 535                 oval = *counter;
 536 
 537                 if ((nval = oval + 1) == 0) {
 538                         /*
 539                          * If the counter would wrap, set it to 1 -- assuring
 540                          * that the counter is never zero when we have seen
 541                          * errors.  (The counter must be 32-bits because we
 542                          * aren't guaranteed a 64-bit compare&swap operation.)
 543                          * To save this code both the infamy of being fingered
 544                          * by a priggish news story and the indignity of being
 545                          * the target of a neo-puritan witch trial, we're
 546                          * carefully avoiding any colorful description of the
 547                          * likelihood of this condition -- but suffice it to
 548                          * say that it is only slightly more likely than the
 549                          * overflow of predicate cache IDs, as discussed in
 550                          * dtrace_predicate_create().
 551                          */
 552                         nval = 1;
 553                 }
 554         } while (dtrace_cas32(counter, oval, nval) != oval);
 555 }
 556 
 557 /*
 558  * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
 559  * uint8_t, a uint16_t, a uint32_t and a uint64_t.
 560  */
 561 DTRACE_LOADFUNC(8)
 562 DTRACE_LOADFUNC(16)
 563 DTRACE_LOADFUNC(32)
 564 DTRACE_LOADFUNC(64)
 565 
 566 static int
 567 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
 568 {
 569         if (dest < mstate->dtms_scratch_base)
 570                 return (0);
 571 
 572         if (dest + size < dest)
 573                 return (0);
 574 
 575         if (dest + size > mstate->dtms_scratch_ptr)
 576                 return (0);
 577 
 578         return (1);
 579 }
 580 
 581 static int
 582 dtrace_canstore_statvar(uint64_t addr, size_t sz,
 583     dtrace_statvar_t **svars, int nsvars)
 584 {
 585         int i;
 586 
 587         for (i = 0; i < nsvars; i++) {
 588                 dtrace_statvar_t *svar = svars[i];
 589 
 590                 if (svar == NULL || svar->dtsv_size == 0)
 591                         continue;
 592 
 593                 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
 594                         return (1);
 595         }
 596 
 597         return (0);
 598 }
 599 
 600 /*
 601  * Check to see if the address is within a memory region to which a store may
 602  * be issued.  This includes the DTrace scratch areas, and any DTrace variable
 603  * region.  The caller of dtrace_canstore() is responsible for performing any
 604  * alignment checks that are needed before stores are actually executed.
 605  */
 606 static int
 607 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 608     dtrace_vstate_t *vstate)
 609 {
 610         /*
 611          * First, check to see if the address is in scratch space...
 612          */
 613         if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
 614             mstate->dtms_scratch_size))
 615                 return (1);
 616 
 617         /*
 618          * Now check to see if it's a dynamic variable.  This check will pick
 619          * up both thread-local variables and any global dynamically-allocated
 620          * variables.
 621          */
 622         if (DTRACE_INRANGE(addr, sz, (uintptr_t)vstate->dtvs_dynvars.dtds_base,
 623             vstate->dtvs_dynvars.dtds_size)) {
 624                 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
 625                 uintptr_t base = (uintptr_t)dstate->dtds_base +
 626                     (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
 627                 uintptr_t chunkoffs;
 628 
 629                 /*
 630                  * Before we assume that we can store here, we need to make
 631                  * sure that it isn't in our metadata -- storing to our
 632                  * dynamic variable metadata would corrupt our state.  For
 633                  * the range to not include any dynamic variable metadata,
 634                  * it must:
 635                  *
 636                  *      (1) Start above the hash table that is at the base of
 637                  *      the dynamic variable space
 638                  *
 639                  *      (2) Have a starting chunk offset that is beyond the
 640                  *      dtrace_dynvar_t that is at the base of every chunk
 641                  *
 642                  *      (3) Not span a chunk boundary
 643                  *
 644                  */
 645                 if (addr < base)
 646                         return (0);
 647 
 648                 chunkoffs = (addr - base) % dstate->dtds_chunksize;
 649 
 650                 if (chunkoffs < sizeof (dtrace_dynvar_t))
 651                         return (0);
 652 
 653                 if (chunkoffs + sz > dstate->dtds_chunksize)
 654                         return (0);
 655 
 656                 return (1);
 657         }
 658 
 659         /*
 660          * Finally, check the static local and global variables.  These checks
 661          * take the longest, so we perform them last.
 662          */
 663         if (dtrace_canstore_statvar(addr, sz,
 664             vstate->dtvs_locals, vstate->dtvs_nlocals))
 665                 return (1);
 666 
 667         if (dtrace_canstore_statvar(addr, sz,
 668             vstate->dtvs_globals, vstate->dtvs_nglobals))
 669                 return (1);
 670 
 671         return (0);
 672 }
 673 
 674 
 675 /*
 676  * Convenience routine to check to see if the address is within a memory
 677  * region in which a load may be issued given the user's privilege level;
 678  * if not, it sets the appropriate error flags and loads 'addr' into the
 679  * illegal value slot.
 680  *
 681  * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
 682  * appropriate memory access protection.
 683  */
 684 static int
 685 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 686     dtrace_vstate_t *vstate)
 687 {
 688         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
 689 
 690         /*
 691          * If we hold the privilege to read from kernel memory, then
 692          * everything is readable.
 693          */
 694         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 695                 return (1);
 696 
 697         /*
 698          * You can obviously read that which you can store.
 699          */
 700         if (dtrace_canstore(addr, sz, mstate, vstate))
 701                 return (1);
 702 
 703         /*
 704          * We're allowed to read from our own string table.
 705          */
 706         if (DTRACE_INRANGE(addr, sz, (uintptr_t)mstate->dtms_difo->dtdo_strtab,
 707             mstate->dtms_difo->dtdo_strlen))
 708                 return (1);
 709 
 710         DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
 711         *illval = addr;
 712         return (0);
 713 }
 714 
 715 /*
 716  * Convenience routine to check to see if a given string is within a memory
 717  * region in which a load may be issued given the user's privilege level;
 718  * this exists so that we don't need to issue unnecessary dtrace_strlen()
 719  * calls in the event that the user has all privileges.
 720  */
 721 static int
 722 dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
 723     dtrace_vstate_t *vstate)
 724 {
 725         size_t strsz;
 726 
 727         /*
 728          * If we hold the privilege to read from kernel memory, then
 729          * everything is readable.
 730          */
 731         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 732                 return (1);
 733 
 734         strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
 735         if (dtrace_canload(addr, strsz, mstate, vstate))
 736                 return (1);
 737 
 738         return (0);
 739 }
 740 
 741 /*
 742  * Convenience routine to check to see if a given variable is within a memory
 743  * region in which a load may be issued given the user's privilege level.
 744  */
 745 static int
 746 dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
 747     dtrace_vstate_t *vstate)
 748 {
 749         size_t sz;
 750         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
 751 
 752         /*
 753          * If we hold the privilege to read from kernel memory, then
 754          * everything is readable.
 755          */
 756         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
 757                 return (1);
 758 
 759         if (type->dtdt_kind == DIF_TYPE_STRING)
 760                 sz = dtrace_strlen(src,
 761                     vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
 762         else
 763                 sz = type->dtdt_size;
 764 
 765         return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
 766 }
 767 
 768 /*
 769  * Compare two strings using safe loads.
 770  */
 771 static int
 772 dtrace_strncmp(char *s1, char *s2, size_t limit)
 773 {
 774         uint8_t c1, c2;
 775         volatile uint16_t *flags;
 776 
 777         if (s1 == s2 || limit == 0)
 778                 return (0);
 779 
 780         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 781 
 782         do {
 783                 if (s1 == NULL) {
 784                         c1 = '\0';
 785                 } else {
 786                         c1 = dtrace_load8((uintptr_t)s1++);
 787                 }
 788 
 789                 if (s2 == NULL) {
 790                         c2 = '\0';
 791                 } else {
 792                         c2 = dtrace_load8((uintptr_t)s2++);
 793                 }
 794 
 795                 if (c1 != c2)
 796                         return (c1 - c2);
 797         } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
 798 
 799         return (0);
 800 }
 801 
 802 /*
 803  * Compute strlen(s) for a string using safe memory accesses.  The additional
 804  * len parameter is used to specify a maximum length to ensure completion.
 805  */
 806 static size_t
 807 dtrace_strlen(const char *s, size_t lim)
 808 {
 809         uint_t len;
 810 
 811         for (len = 0; len != lim; len++) {
 812                 if (dtrace_load8((uintptr_t)s++) == '\0')
 813                         break;
 814         }
 815 
 816         return (len);
 817 }
 818 
 819 /*
 820  * Check if an address falls within a toxic region.
 821  */
 822 static int
 823 dtrace_istoxic(uintptr_t kaddr, size_t size)
 824 {
 825         uintptr_t taddr, tsize;
 826         int i;
 827 
 828         for (i = 0; i < dtrace_toxranges; i++) {
 829                 taddr = dtrace_toxrange[i].dtt_base;
 830                 tsize = dtrace_toxrange[i].dtt_limit - taddr;
 831 
 832                 if (kaddr - taddr < tsize) {
 833                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 834                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
 835                         return (1);
 836                 }
 837 
 838                 if (taddr - kaddr < size) {
 839                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
 840                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
 841                         return (1);
 842                 }
 843         }
 844 
 845         return (0);
 846 }
 847 
 848 /*
 849  * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
 850  * memory specified by the DIF program.  The dst is assumed to be safe memory
 851  * that we can store to directly because it is managed by DTrace.  As with
 852  * standard bcopy, overlapping copies are handled properly.
 853  */
 854 static void
 855 dtrace_bcopy(const void *src, void *dst, size_t len)
 856 {
 857         if (len != 0) {
 858                 uint8_t *s1 = dst;
 859                 const uint8_t *s2 = src;
 860 
 861                 if (s1 <= s2) {
 862                         do {
 863                                 *s1++ = dtrace_load8((uintptr_t)s2++);
 864                         } while (--len != 0);
 865                 } else {
 866                         s2 += len;
 867                         s1 += len;
 868 
 869                         do {
 870                                 *--s1 = dtrace_load8((uintptr_t)--s2);
 871                         } while (--len != 0);
 872                 }
 873         }
 874 }
 875 
 876 /*
 877  * Copy src to dst using safe memory accesses, up to either the specified
 878  * length, or the point that a nul byte is encountered.  The src is assumed to
 879  * be unsafe memory specified by the DIF program.  The dst is assumed to be
 880  * safe memory that we can store to directly because it is managed by DTrace.
 881  * Unlike dtrace_bcopy(), overlapping regions are not handled.
 882  */
 883 static void
 884 dtrace_strcpy(const void *src, void *dst, size_t len)
 885 {
 886         if (len != 0) {
 887                 uint8_t *s1 = dst, c;
 888                 const uint8_t *s2 = src;
 889 
 890                 do {
 891                         *s1++ = c = dtrace_load8((uintptr_t)s2++);
 892                 } while (--len != 0 && c != '\0');
 893         }
 894 }
 895 
 896 /*
 897  * Copy src to dst, deriving the size and type from the specified (BYREF)
 898  * variable type.  The src is assumed to be unsafe memory specified by the DIF
 899  * program.  The dst is assumed to be DTrace variable memory that is of the
 900  * specified type; we assume that we can store to directly.
 901  */
 902 static void
 903 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
 904 {
 905         ASSERT(type->dtdt_flags & DIF_TF_BYREF);
 906 
 907         if (type->dtdt_kind == DIF_TYPE_STRING) {
 908                 dtrace_strcpy(src, dst, type->dtdt_size);
 909         } else {
 910                 dtrace_bcopy(src, dst, type->dtdt_size);
 911         }
 912 }
 913 
 914 /*
 915  * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
 916  * unsafe memory specified by the DIF program.  The s2 data is assumed to be
 917  * safe memory that we can access directly because it is managed by DTrace.
 918  */
 919 static int
 920 dtrace_bcmp(const void *s1, const void *s2, size_t len)
 921 {
 922         volatile uint16_t *flags;
 923 
 924         flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
 925 
 926         if (s1 == s2)
 927                 return (0);
 928 
 929         if (s1 == NULL || s2 == NULL)
 930                 return (1);
 931 
 932         if (s1 != s2 && len != 0) {
 933                 const uint8_t *ps1 = s1;
 934                 const uint8_t *ps2 = s2;
 935 
 936                 do {
 937                         if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
 938                                 return (1);
 939                 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
 940         }
 941         return (0);
 942 }
 943 
 944 /*
 945  * Zero the specified region using a simple byte-by-byte loop.  Note that this
 946  * is for safe DTrace-managed memory only.
 947  */
 948 static void
 949 dtrace_bzero(void *dst, size_t len)
 950 {
 951         uchar_t *cp;
 952 
 953         for (cp = dst; len != 0; len--)
 954                 *cp++ = 0;
 955 }
 956 
 957 static void
 958 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
 959 {
 960         uint64_t result[2];
 961 
 962         result[0] = addend1[0] + addend2[0];
 963         result[1] = addend1[1] + addend2[1] +
 964             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
 965 
 966         sum[0] = result[0];
 967         sum[1] = result[1];
 968 }
 969 
 970 /*
 971  * Shift the 128-bit value in a by b. If b is positive, shift left.
 972  * If b is negative, shift right.
 973  */
 974 static void
 975 dtrace_shift_128(uint64_t *a, int b)
 976 {
 977         uint64_t mask;
 978 
 979         if (b == 0)
 980                 return;
 981 
 982         if (b < 0) {
 983                 b = -b;
 984                 if (b >= 64) {
 985                         a[0] = a[1] >> (b - 64);
 986                         a[1] = 0;
 987                 } else {
 988                         a[0] >>= b;
 989                         mask = 1LL << (64 - b);
 990                         mask -= 1;
 991                         a[0] |= ((a[1] & mask) << (64 - b));
 992                         a[1] >>= b;
 993                 }
 994         } else {
 995                 if (b >= 64) {
 996                         a[1] = a[0] << (b - 64);
 997                         a[0] = 0;
 998                 } else {
 999                         a[1] <<= b;
1000                         mask = a[0] >> (64 - b);
1001                         a[1] |= mask;
1002                         a[0] <<= b;
1003                 }
1004         }
1005 }
1006 
1007 /*
1008  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1009  * use native multiplication on those, and then re-combine into the
1010  * resulting 128-bit value.
1011  *
1012  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1013  *     hi1 * hi2 << 64 +
1014  *     hi1 * lo2 << 32 +
1015  *     hi2 * lo1 << 32 +
1016  *     lo1 * lo2
1017  */
1018 static void
1019 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
1020 {
1021         uint64_t hi1, hi2, lo1, lo2;
1022         uint64_t tmp[2];
1023 
1024         hi1 = factor1 >> 32;
1025         hi2 = factor2 >> 32;
1026 
1027         lo1 = factor1 & DT_MASK_LO;
1028         lo2 = factor2 & DT_MASK_LO;
1029 
1030         product[0] = lo1 * lo2;
1031         product[1] = hi1 * hi2;
1032 
1033         tmp[0] = hi1 * lo2;
1034         tmp[1] = 0;
1035         dtrace_shift_128(tmp, 32);
1036         dtrace_add_128(product, tmp, product);
1037 
1038         tmp[0] = hi2 * lo1;
1039         tmp[1] = 0;
1040         dtrace_shift_128(tmp, 32);
1041         dtrace_add_128(product, tmp, product);
1042 }
1043 
1044 /*
1045  * This privilege check should be used by actions and subroutines to
1046  * verify that the user credentials of the process that enabled the
1047  * invoking ECB match the target credentials
1048  */
1049 static int
1050 dtrace_priv_proc_common_user(dtrace_state_t *state)
1051 {
1052         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1053 
1054         /*
1055          * We should always have a non-NULL state cred here, since if cred
1056          * is null (anonymous tracing), we fast-path bypass this routine.
1057          */
1058         ASSERT(s_cr != NULL);
1059 
1060         if ((cr = CRED()) != NULL &&
1061             s_cr->cr_uid == cr->cr_uid &&
1062             s_cr->cr_uid == cr->cr_ruid &&
1063             s_cr->cr_uid == cr->cr_suid &&
1064             s_cr->cr_gid == cr->cr_gid &&
1065             s_cr->cr_gid == cr->cr_rgid &&
1066             s_cr->cr_gid == cr->cr_sgid)
1067                 return (1);
1068 
1069         return (0);
1070 }
1071 
1072 /*
1073  * This privilege check should be used by actions and subroutines to
1074  * verify that the zone of the process that enabled the invoking ECB
1075  * matches the target credentials
1076  */
1077 static int
1078 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1079 {
1080         cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1081 
1082         /*
1083          * We should always have a non-NULL state cred here, since if cred
1084          * is null (anonymous tracing), we fast-path bypass this routine.
1085          */
1086         ASSERT(s_cr != NULL);
1087 
1088         if ((cr = CRED()) != NULL &&
1089             s_cr->cr_zone == cr->cr_zone)
1090                 return (1);
1091 
1092         return (0);
1093 }
1094 
1095 /*
1096  * This privilege check should be used by actions and subroutines to
1097  * verify that the process has not setuid or changed credentials.
1098  */
1099 static int
1100 dtrace_priv_proc_common_nocd()
1101 {
1102         proc_t *proc;
1103 
1104         if ((proc = ttoproc(curthread)) != NULL &&
1105             !(proc->p_flag & SNOCD))
1106                 return (1);
1107 
1108         return (0);
1109 }
1110 
1111 static int
1112 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate)
1113 {
1114         int action = state->dts_cred.dcr_action;
1115 
1116         if (!(mstate->dtms_access & DTRACE_ACCESS_PROC))
1117                 goto bad;
1118 
1119         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
1120             dtrace_priv_proc_common_zone(state) == 0)
1121                 goto bad;
1122 
1123         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
1124             dtrace_priv_proc_common_user(state) == 0)
1125                 goto bad;
1126 
1127         if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
1128             dtrace_priv_proc_common_nocd() == 0)
1129                 goto bad;
1130 
1131         return (1);
1132 
1133 bad:
1134         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1135 
1136         return (0);
1137 }
1138 
1139 static int
1140 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate)
1141 {
1142         if (mstate->dtms_access & DTRACE_ACCESS_PROC) {
1143                 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1144                         return (1);
1145 
1146                 if (dtrace_priv_proc_common_zone(state) &&
1147                     dtrace_priv_proc_common_user(state) &&
1148                     dtrace_priv_proc_common_nocd())
1149                         return (1);
1150         }
1151 
1152         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1153 
1154         return (0);
1155 }
1156 
1157 static int
1158 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate)
1159 {
1160         if ((mstate->dtms_access & DTRACE_ACCESS_PROC) &&
1161             (state->dts_cred.dcr_action & DTRACE_CRA_PROC))
1162                 return (1);
1163 
1164         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1165 
1166         return (0);
1167 }
1168 
1169 static int
1170 dtrace_priv_kernel(dtrace_state_t *state)
1171 {
1172         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1173                 return (1);
1174 
1175         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1176 
1177         return (0);
1178 }
1179 
1180 static int
1181 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1182 {
1183         if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1184                 return (1);
1185 
1186         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1187 
1188         return (0);
1189 }
1190 
1191 /*
1192  * Determine if the dte_cond of the specified ECB allows for processing of
1193  * the current probe to continue.  Note that this routine may allow continued
1194  * processing, but with access(es) stripped from the mstate's dtms_access
1195  * field.
1196  */
1197 static int
1198 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
1199     dtrace_ecb_t *ecb)
1200 {
1201         dtrace_probe_t *probe = ecb->dte_probe;
1202         dtrace_provider_t *prov = probe->dtpr_provider;
1203         dtrace_pops_t *pops = &prov->dtpv_pops;
1204         int mode = DTRACE_MODE_NOPRIV_DROP;
1205 
1206         ASSERT(ecb->dte_cond);
1207 
1208         if (pops->dtps_mode != NULL) {
1209                 mode = pops->dtps_mode(prov->dtpv_arg,
1210                     probe->dtpr_id, probe->dtpr_arg);
1211 
1212                 ASSERT((mode & DTRACE_MODE_USER) ||
1213                     (mode & DTRACE_MODE_KERNEL));
1214                 ASSERT((mode & DTRACE_MODE_NOPRIV_RESTRICT) ||
1215                     (mode & DTRACE_MODE_NOPRIV_DROP));
1216         }
1217 
1218         /*
1219          * If the dte_cond bits indicate that this consumer is only allowed to
1220          * see user-mode firings of this probe, call the provider's dtps_mode()
1221          * entry point to check that the probe was fired while in a user
1222          * context.  If that's not the case, use the policy specified by the
1223          * provider to determine if we drop the probe or merely restrict
1224          * operation.
1225          */
1226         if (ecb->dte_cond & DTRACE_COND_USERMODE) {
1227                 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
1228 
1229                 if (!(mode & DTRACE_MODE_USER)) {
1230                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1231                                 return (0);
1232 
1233                         mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1234                 }
1235         }
1236 
1237         /*
1238          * This is more subtle than it looks. We have to be absolutely certain
1239          * that CRED() isn't going to change out from under us so it's only
1240          * legit to examine that structure if we're in constrained situations.
1241          * Currently, the only times we'll this check is if a non-super-user
1242          * has enabled the profile or syscall providers -- providers that
1243          * allow visibility of all processes. For the profile case, the check
1244          * above will ensure that we're examining a user context.
1245          */
1246         if (ecb->dte_cond & DTRACE_COND_OWNER) {
1247                 cred_t *cr;
1248                 cred_t *s_cr = state->dts_cred.dcr_cred;
1249                 proc_t *proc;
1250 
1251                 ASSERT(s_cr != NULL);
1252 
1253                 if ((cr = CRED()) == NULL ||
1254                     s_cr->cr_uid != cr->cr_uid ||
1255                     s_cr->cr_uid != cr->cr_ruid ||
1256                     s_cr->cr_uid != cr->cr_suid ||
1257                     s_cr->cr_gid != cr->cr_gid ||
1258                     s_cr->cr_gid != cr->cr_rgid ||
1259                     s_cr->cr_gid != cr->cr_sgid ||
1260                     (proc = ttoproc(curthread)) == NULL ||
1261                     (proc->p_flag & SNOCD)) {
1262                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1263                                 return (0);
1264 
1265                         mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
1266                 }
1267         }
1268 
1269         /*
1270          * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1271          * in our zone, check to see if our mode policy is to restrict rather
1272          * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1273          * and DTRACE_ACCESS_ARGS
1274          */
1275         if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
1276                 cred_t *cr;
1277                 cred_t *s_cr = state->dts_cred.dcr_cred;
1278 
1279                 ASSERT(s_cr != NULL);
1280 
1281                 if ((cr = CRED()) == NULL ||
1282                     s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
1283                         if (mode & DTRACE_MODE_NOPRIV_DROP)
1284                                 return (0);
1285 
1286                         mstate->dtms_access &=
1287                             ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
1288                 }
1289         }
1290 
1291         return (1);
1292 }
1293 
1294 /*
1295  * Note:  not called from probe context.  This function is called
1296  * asynchronously (and at a regular interval) from outside of probe context to
1297  * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
1298  * cleaning is explained in detail in <sys/dtrace_impl.h>.
1299  */
1300 void
1301 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
1302 {
1303         dtrace_dynvar_t *dirty;
1304         dtrace_dstate_percpu_t *dcpu;
1305         dtrace_dynvar_t **rinsep;
1306         int i, j, work = 0;
1307 
1308         for (i = 0; i < NCPU; i++) {
1309                 dcpu = &dstate->dtds_percpu[i];
1310                 rinsep = &dcpu->dtdsc_rinsing;
1311 
1312                 /*
1313                  * If the dirty list is NULL, there is no dirty work to do.
1314                  */
1315                 if (dcpu->dtdsc_dirty == NULL)
1316                         continue;
1317 
1318                 if (dcpu->dtdsc_rinsing != NULL) {
1319                         /*
1320                          * If the rinsing list is non-NULL, then it is because
1321                          * this CPU was selected to accept another CPU's
1322                          * dirty list -- and since that time, dirty buffers
1323                          * have accumulated.  This is a highly unlikely
1324                          * condition, but we choose to ignore the dirty
1325                          * buffers -- they'll be picked up a future cleanse.
1326                          */
1327                         continue;
1328                 }
1329 
1330                 if (dcpu->dtdsc_clean != NULL) {
1331                         /*
1332                          * If the clean list is non-NULL, then we're in a
1333                          * situation where a CPU has done deallocations (we
1334                          * have a non-NULL dirty list) but no allocations (we
1335                          * also have a non-NULL clean list).  We can't simply
1336                          * move the dirty list into the clean list on this
1337                          * CPU, yet we also don't want to allow this condition
1338                          * to persist, lest a short clean list prevent a
1339                          * massive dirty list from being cleaned (which in
1340                          * turn could lead to otherwise avoidable dynamic
1341                          * drops).  To deal with this, we look for some CPU
1342                          * with a NULL clean list, NULL dirty list, and NULL
1343                          * rinsing list -- and then we borrow this CPU to
1344                          * rinse our dirty list.
1345                          */
1346                         for (j = 0; j < NCPU; j++) {
1347                                 dtrace_dstate_percpu_t *rinser;
1348 
1349                                 rinser = &dstate->dtds_percpu[j];
1350 
1351                                 if (rinser->dtdsc_rinsing != NULL)
1352                                         continue;
1353 
1354                                 if (rinser->dtdsc_dirty != NULL)
1355                                         continue;
1356 
1357                                 if (rinser->dtdsc_clean != NULL)
1358                                         continue;
1359 
1360                                 rinsep = &rinser->dtdsc_rinsing;
1361                                 break;
1362                         }
1363 
1364                         if (j == NCPU) {
1365                                 /*
1366                                  * We were unable to find another CPU that
1367                                  * could accept this dirty list -- we are
1368                                  * therefore unable to clean it now.
1369                                  */
1370                                 dtrace_dynvar_failclean++;
1371                                 continue;
1372                         }
1373                 }
1374 
1375                 work = 1;
1376 
1377                 /*
1378                  * Atomically move the dirty list aside.
1379                  */
1380                 do {
1381                         dirty = dcpu->dtdsc_dirty;
1382 
1383                         /*
1384                          * Before we zap the dirty list, set the rinsing list.
1385                          * (This allows for a potential assertion in
1386                          * dtrace_dynvar():  if a free dynamic variable appears
1387                          * on a hash chain, either the dirty list or the
1388                          * rinsing list for some CPU must be non-NULL.)
1389                          */
1390                         *rinsep = dirty;
1391                         dtrace_membar_producer();
1392                 } while (dtrace_casptr(&dcpu->dtdsc_dirty,
1393                     dirty, NULL) != dirty);
1394         }
1395 
1396         if (!work) {
1397                 /*
1398                  * We have no work to do; we can simply return.
1399                  */
1400                 return;
1401         }
1402 
1403         dtrace_sync();
1404 
1405         for (i = 0; i < NCPU; i++) {
1406                 dcpu = &dstate->dtds_percpu[i];
1407 
1408                 if (dcpu->dtdsc_rinsing == NULL)
1409                         continue;
1410 
1411                 /*
1412                  * We are now guaranteed that no hash chain contains a pointer
1413                  * into this dirty list; we can make it clean.
1414                  */
1415                 ASSERT(dcpu->dtdsc_clean == NULL);
1416                 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
1417                 dcpu->dtdsc_rinsing = NULL;
1418         }
1419 
1420         /*
1421          * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1422          * sure that all CPUs have seen all of the dtdsc_clean pointers.
1423          * This prevents a race whereby a CPU incorrectly decides that
1424          * the state should be something other than DTRACE_DSTATE_CLEAN
1425          * after dtrace_dynvar_clean() has completed.
1426          */
1427         dtrace_sync();
1428 
1429         dstate->dtds_state = DTRACE_DSTATE_CLEAN;
1430 }
1431 
1432 /*
1433  * Depending on the value of the op parameter, this function looks-up,
1434  * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
1435  * allocation is requested, this function will return a pointer to a
1436  * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1437  * variable can be allocated.  If NULL is returned, the appropriate counter
1438  * will be incremented.
1439  */
1440 dtrace_dynvar_t *
1441 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
1442     dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
1443     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
1444 {
1445         uint64_t hashval = DTRACE_DYNHASH_VALID;
1446         dtrace_dynhash_t *hash = dstate->dtds_hash;
1447         dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
1448         processorid_t me = CPU->cpu_id, cpu = me;
1449         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
1450         size_t bucket, ksize;
1451         size_t chunksize = dstate->dtds_chunksize;
1452         uintptr_t kdata, lock, nstate;
1453         uint_t i;
1454 
1455         ASSERT(nkeys != 0);
1456 
1457         /*
1458          * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
1459          * algorithm.  For the by-value portions, we perform the algorithm in
1460          * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
1461          * bit, and seems to have only a minute effect on distribution.  For
1462          * the by-reference data, we perform "One-at-a-time" iterating (safely)
1463          * over each referenced byte.  It's painful to do this, but it's much
1464          * better than pathological hash distribution.  The efficacy of the
1465          * hashing algorithm (and a comparison with other algorithms) may be
1466          * found by running the ::dtrace_dynstat MDB dcmd.
1467          */
1468         for (i = 0; i < nkeys; i++) {
1469                 if (key[i].dttk_size == 0) {
1470                         uint64_t val = key[i].dttk_value;
1471 
1472                         hashval += (val >> 48) & 0xffff;
1473                         hashval += (hashval << 10);
1474                         hashval ^= (hashval >> 6);
1475 
1476                         hashval += (val >> 32) & 0xffff;
1477                         hashval += (hashval << 10);
1478                         hashval ^= (hashval >> 6);
1479 
1480                         hashval += (val >> 16) & 0xffff;
1481                         hashval += (hashval << 10);
1482                         hashval ^= (hashval >> 6);
1483 
1484                         hashval += val & 0xffff;
1485                         hashval += (hashval << 10);
1486                         hashval ^= (hashval >> 6);
1487                 } else {
1488                         /*
1489                          * This is incredibly painful, but it beats the hell
1490                          * out of the alternative.
1491                          */
1492                         uint64_t j, size = key[i].dttk_size;
1493                         uintptr_t base = (uintptr_t)key[i].dttk_value;
1494 
1495                         if (!dtrace_canload(base, size, mstate, vstate))
1496                                 break;
1497 
1498                         for (j = 0; j < size; j++) {
1499                                 hashval += dtrace_load8(base + j);
1500                                 hashval += (hashval << 10);
1501                                 hashval ^= (hashval >> 6);
1502                         }
1503                 }
1504         }
1505 
1506         if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
1507                 return (NULL);
1508 
1509         hashval += (hashval << 3);
1510         hashval ^= (hashval >> 11);
1511         hashval += (hashval << 15);
1512 
1513         /*
1514          * There is a remote chance (ideally, 1 in 2^31) that our hashval
1515          * comes out to be one of our two sentinel hash values.  If this
1516          * actually happens, we set the hashval to be a value known to be a
1517          * non-sentinel value.
1518          */
1519         if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
1520                 hashval = DTRACE_DYNHASH_VALID;
1521 
1522         /*
1523          * Yes, it's painful to do a divide here.  If the cycle count becomes
1524          * important here, tricks can be pulled to reduce it.  (However, it's
1525          * critical that hash collisions be kept to an absolute minimum;
1526          * they're much more painful than a divide.)  It's better to have a
1527          * solution that generates few collisions and still keeps things
1528          * relatively simple.
1529          */
1530         bucket = hashval % dstate->dtds_hashsize;
1531 
1532         if (op == DTRACE_DYNVAR_DEALLOC) {
1533                 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1534 
1535                 for (;;) {
1536                         while ((lock = *lockp) & 1)
1537                                 continue;
1538 
1539                         if (dtrace_casptr((void *)lockp,
1540                             (void *)lock, (void *)(lock + 1)) == (void *)lock)
1541                                 break;
1542                 }
1543 
1544                 dtrace_membar_producer();
1545         }
1546 
1547 top:
1548         prev = NULL;
1549         lock = hash[bucket].dtdh_lock;
1550 
1551         dtrace_membar_consumer();
1552 
1553         start = hash[bucket].dtdh_chain;
1554         ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
1555             start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
1556             op != DTRACE_DYNVAR_DEALLOC));
1557 
1558         for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1559                 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1560                 dtrace_key_t *dkey = &dtuple->dtt_key[0];
1561 
1562                 if (dvar->dtdv_hashval != hashval) {
1563                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
1564                                 /*
1565                                  * We've reached the sink, and therefore the
1566                                  * end of the hash chain; we can kick out of
1567                                  * the loop knowing that we have seen a valid
1568                                  * snapshot of state.
1569                                  */
1570                                 ASSERT(dvar->dtdv_next == NULL);
1571                                 ASSERT(dvar == &dtrace_dynhash_sink);
1572                                 break;
1573                         }
1574 
1575                         if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
1576                                 /*
1577                                  * We've gone off the rails:  somewhere along
1578                                  * the line, one of the members of this hash
1579                                  * chain was deleted.  Note that we could also
1580                                  * detect this by simply letting this loop run
1581                                  * to completion, as we would eventually hit
1582                                  * the end of the dirty list.  However, we
1583                                  * want to avoid running the length of the
1584                                  * dirty list unnecessarily (it might be quite
1585                                  * long), so we catch this as early as
1586                                  * possible by detecting the hash marker.  In
1587                                  * this case, we simply set dvar to NULL and
1588                                  * break; the conditional after the loop will
1589                                  * send us back to top.
1590                                  */
1591                                 dvar = NULL;
1592                                 break;
1593                         }
1594 
1595                         goto next;
1596                 }
1597 
1598                 if (dtuple->dtt_nkeys != nkeys)
1599                         goto next;
1600 
1601                 for (i = 0; i < nkeys; i++, dkey++) {
1602                         if (dkey->dttk_size != key[i].dttk_size)
1603                                 goto next; /* size or type mismatch */
1604 
1605                         if (dkey->dttk_size != 0) {
1606                                 if (dtrace_bcmp(
1607                                     (void *)(uintptr_t)key[i].dttk_value,
1608                                     (void *)(uintptr_t)dkey->dttk_value,
1609                                     dkey->dttk_size))
1610                                         goto next;
1611                         } else {
1612                                 if (dkey->dttk_value != key[i].dttk_value)
1613                                         goto next;
1614                         }
1615                 }
1616 
1617                 if (op != DTRACE_DYNVAR_DEALLOC)
1618                         return (dvar);
1619 
1620                 ASSERT(dvar->dtdv_next == NULL ||
1621                     dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
1622 
1623                 if (prev != NULL) {
1624                         ASSERT(hash[bucket].dtdh_chain != dvar);
1625                         ASSERT(start != dvar);
1626                         ASSERT(prev->dtdv_next == dvar);
1627                         prev->dtdv_next = dvar->dtdv_next;
1628                 } else {
1629                         if (dtrace_casptr(&hash[bucket].dtdh_chain,
1630                             start, dvar->dtdv_next) != start) {
1631                                 /*
1632                                  * We have failed to atomically swing the
1633                                  * hash table head pointer, presumably because
1634                                  * of a conflicting allocation on another CPU.
1635                                  * We need to reread the hash chain and try
1636                                  * again.
1637                                  */
1638                                 goto top;
1639                         }
1640                 }
1641 
1642                 dtrace_membar_producer();
1643 
1644                 /*
1645                  * Now set the hash value to indicate that it's free.
1646                  */
1647                 ASSERT(hash[bucket].dtdh_chain != dvar);
1648                 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1649 
1650                 dtrace_membar_producer();
1651 
1652                 /*
1653                  * Set the next pointer to point at the dirty list, and
1654                  * atomically swing the dirty pointer to the newly freed dvar.
1655                  */
1656                 do {
1657                         next = dcpu->dtdsc_dirty;
1658                         dvar->dtdv_next = next;
1659                 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1660 
1661                 /*
1662                  * Finally, unlock this hash bucket.
1663                  */
1664                 ASSERT(hash[bucket].dtdh_lock == lock);
1665                 ASSERT(lock & 1);
1666                 hash[bucket].dtdh_lock++;
1667 
1668                 return (NULL);
1669 next:
1670                 prev = dvar;
1671                 continue;
1672         }
1673 
1674         if (dvar == NULL) {
1675                 /*
1676                  * If dvar is NULL, it is because we went off the rails:
1677                  * one of the elements that we traversed in the hash chain
1678                  * was deleted while we were traversing it.  In this case,
1679                  * we assert that we aren't doing a dealloc (deallocs lock
1680                  * the hash bucket to prevent themselves from racing with
1681                  * one another), and retry the hash chain traversal.
1682                  */
1683                 ASSERT(op != DTRACE_DYNVAR_DEALLOC);
1684                 goto top;
1685         }
1686 
1687         if (op != DTRACE_DYNVAR_ALLOC) {
1688                 /*
1689                  * If we are not to allocate a new variable, we want to
1690                  * return NULL now.  Before we return, check that the value
1691                  * of the lock word hasn't changed.  If it has, we may have
1692                  * seen an inconsistent snapshot.
1693                  */
1694                 if (op == DTRACE_DYNVAR_NOALLOC) {
1695                         if (hash[bucket].dtdh_lock != lock)
1696                                 goto top;
1697                 } else {
1698                         ASSERT(op == DTRACE_DYNVAR_DEALLOC);
1699                         ASSERT(hash[bucket].dtdh_lock == lock);
1700                         ASSERT(lock & 1);
1701                         hash[bucket].dtdh_lock++;
1702                 }
1703 
1704                 return (NULL);
1705         }
1706 
1707         /*
1708          * We need to allocate a new dynamic variable.  The size we need is the
1709          * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
1710          * size of any auxiliary key data (rounded up to 8-byte alignment) plus
1711          * the size of any referred-to data (dsize).  We then round the final
1712          * size up to the chunksize for allocation.
1713          */
1714         for (ksize = 0, i = 0; i < nkeys; i++)
1715                 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
1716 
1717         /*
1718          * This should be pretty much impossible, but could happen if, say,
1719          * strange DIF specified the tuple.  Ideally, this should be an
1720          * assertion and not an error condition -- but that requires that the
1721          * chunksize calculation in dtrace_difo_chunksize() be absolutely
1722          * bullet-proof.  (That is, it must not be able to be fooled by
1723          * malicious DIF.)  Given the lack of backwards branches in DIF,
1724          * solving this would presumably not amount to solving the Halting
1725          * Problem -- but it still seems awfully hard.
1726          */
1727         if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
1728             ksize + dsize > chunksize) {
1729                 dcpu->dtdsc_drops++;
1730                 return (NULL);
1731         }
1732 
1733         nstate = DTRACE_DSTATE_EMPTY;
1734 
1735         do {
1736 retry:
1737                 free = dcpu->dtdsc_free;
1738 
1739                 if (free == NULL) {
1740                         dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
1741                         void *rval;
1742 
1743                         if (clean == NULL) {
1744                                 /*
1745                                  * We're out of dynamic variable space on
1746                                  * this CPU.  Unless we have tried all CPUs,
1747                                  * we'll try to allocate from a different
1748                                  * CPU.
1749                                  */
1750                                 switch (dstate->dtds_state) {
1751                                 case DTRACE_DSTATE_CLEAN: {
1752                                         void *sp = &dstate->dtds_state;
1753 
1754                                         if (++cpu >= NCPU)
1755                                                 cpu = 0;
1756 
1757                                         if (dcpu->dtdsc_dirty != NULL &&
1758                                             nstate == DTRACE_DSTATE_EMPTY)
1759                                                 nstate = DTRACE_DSTATE_DIRTY;
1760 
1761                                         if (dcpu->dtdsc_rinsing != NULL)
1762                                                 nstate = DTRACE_DSTATE_RINSING;
1763 
1764                                         dcpu = &dstate->dtds_percpu[cpu];
1765 
1766                                         if (cpu != me)
1767                                                 goto retry;
1768 
1769                                         (void) dtrace_cas32(sp,
1770                                             DTRACE_DSTATE_CLEAN, nstate);
1771 
1772                                         /*
1773                                          * To increment the correct bean
1774                                          * counter, take another lap.
1775                                          */
1776                                         goto retry;
1777                                 }
1778 
1779                                 case DTRACE_DSTATE_DIRTY:
1780                                         dcpu->dtdsc_dirty_drops++;
1781                                         break;
1782 
1783                                 case DTRACE_DSTATE_RINSING:
1784                                         dcpu->dtdsc_rinsing_drops++;
1785                                         break;
1786 
1787                                 case DTRACE_DSTATE_EMPTY:
1788                                         dcpu->dtdsc_drops++;
1789                                         break;
1790                                 }
1791 
1792                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
1793                                 return (NULL);
1794                         }
1795 
1796                         /*
1797                          * The clean list appears to be non-empty.  We want to
1798                          * move the clean list to the free list; we start by
1799                          * moving the clean pointer aside.
1800                          */
1801                         if (dtrace_casptr(&dcpu->dtdsc_clean,
1802                             clean, NULL) != clean) {
1803                                 /*
1804                                  * We are in one of two situations:
1805                                  *
1806                                  *  (a) The clean list was switched to the
1807                                  *      free list by another CPU.
1808                                  *
1809                                  *  (b) The clean list was added to by the
1810                                  *      cleansing cyclic.
1811                                  *
1812                                  * In either of these situations, we can
1813                                  * just reattempt the free list allocation.
1814                                  */
1815                                 goto retry;
1816                         }
1817 
1818                         ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
1819 
1820                         /*
1821                          * Now we'll move the clean list to our free list.
1822                          * It's impossible for this to fail:  the only way
1823                          * the free list can be updated is through this
1824                          * code path, and only one CPU can own the clean list.
1825                          * Thus, it would only be possible for this to fail if
1826                          * this code were racing with dtrace_dynvar_clean().
1827                          * (That is, if dtrace_dynvar_clean() updated the clean
1828                          * list, and we ended up racing to update the free
1829                          * list.)  This race is prevented by the dtrace_sync()
1830                          * in dtrace_dynvar_clean() -- which flushes the
1831                          * owners of the clean lists out before resetting
1832                          * the clean lists.
1833                          */
1834                         dcpu = &dstate->dtds_percpu[me];
1835                         rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
1836                         ASSERT(rval == NULL);
1837                         goto retry;
1838                 }
1839 
1840                 dvar = free;
1841                 new_free = dvar->dtdv_next;
1842         } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
1843 
1844         /*
1845          * We have now allocated a new chunk.  We copy the tuple keys into the
1846          * tuple array and copy any referenced key data into the data space
1847          * following the tuple array.  As we do this, we relocate dttk_value
1848          * in the final tuple to point to the key data address in the chunk.
1849          */
1850         kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
1851         dvar->dtdv_data = (void *)(kdata + ksize);
1852         dvar->dtdv_tuple.dtt_nkeys = nkeys;
1853 
1854         for (i = 0; i < nkeys; i++) {
1855                 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
1856                 size_t kesize = key[i].dttk_size;
1857 
1858                 if (kesize != 0) {
1859                         dtrace_bcopy(
1860                             (const void *)(uintptr_t)key[i].dttk_value,
1861                             (void *)kdata, kesize);
1862                         dkey->dttk_value = kdata;
1863                         kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
1864                 } else {
1865                         dkey->dttk_value = key[i].dttk_value;
1866                 }
1867 
1868                 dkey->dttk_size = kesize;
1869         }
1870 
1871         ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
1872         dvar->dtdv_hashval = hashval;
1873         dvar->dtdv_next = start;
1874 
1875         if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
1876                 return (dvar);
1877 
1878         /*
1879          * The cas has failed.  Either another CPU is adding an element to
1880          * this hash chain, or another CPU is deleting an element from this
1881          * hash chain.  The simplest way to deal with both of these cases
1882          * (though not necessarily the most efficient) is to free our
1883          * allocated block and tail-call ourselves.  Note that the free is
1884          * to the dirty list and _not_ to the free list.  This is to prevent
1885          * races with allocators, above.
1886          */
1887         dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1888 
1889         dtrace_membar_producer();
1890 
1891         do {
1892                 free = dcpu->dtdsc_dirty;
1893                 dvar->dtdv_next = free;
1894         } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
1895 
1896         return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
1897 }
1898 
1899 /*ARGSUSED*/
1900 static void
1901 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
1902 {
1903         if ((int64_t)nval < (int64_t)*oval)
1904                 *oval = nval;
1905 }
1906 
1907 /*ARGSUSED*/
1908 static void
1909 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
1910 {
1911         if ((int64_t)nval > (int64_t)*oval)
1912                 *oval = nval;
1913 }
1914 
1915 static void
1916 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
1917 {
1918         int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
1919         int64_t val = (int64_t)nval;
1920 
1921         if (val < 0) {
1922                 for (i = 0; i < zero; i++) {
1923                         if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
1924                                 quanta[i] += incr;
1925                                 return;
1926                         }
1927                 }
1928         } else {
1929                 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
1930                         if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
1931                                 quanta[i - 1] += incr;
1932                                 return;
1933                         }
1934                 }
1935 
1936                 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
1937                 return;
1938         }
1939 
1940         ASSERT(0);
1941 }
1942 
1943 static void
1944 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
1945 {
1946         uint64_t arg = *lquanta++;
1947         int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1948         uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1949         uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
1950         int32_t val = (int32_t)nval, level;
1951 
1952         ASSERT(step != 0);
1953         ASSERT(levels != 0);
1954 
1955         if (val < base) {
1956                 /*
1957                  * This is an underflow.
1958                  */
1959                 lquanta[0] += incr;
1960                 return;
1961         }
1962 
1963         level = (val - base) / step;
1964 
1965         if (level < levels) {
1966                 lquanta[level + 1] += incr;
1967                 return;
1968         }
1969 
1970         /*
1971          * This is an overflow.
1972          */
1973         lquanta[levels + 1] += incr;
1974 }
1975 
1976 static int
1977 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
1978     uint16_t high, uint16_t nsteps, int64_t value)
1979 {
1980         int64_t this = 1, last, next;
1981         int base = 1, order;
1982 
1983         ASSERT(factor <= nsteps);
1984         ASSERT(nsteps % factor == 0);
1985 
1986         for (order = 0; order < low; order++)
1987                 this *= factor;
1988 
1989         /*
1990          * If our value is less than our factor taken to the power of the
1991          * low order of magnitude, it goes into the zeroth bucket.
1992          */
1993         if (value < (last = this))
1994                 return (0);
1995 
1996         for (this *= factor; order <= high; order++) {
1997                 int nbuckets = this > nsteps ? nsteps : this;
1998 
1999                 if ((next = this * factor) < this) {
2000                         /*
2001                          * We should not generally get log/linear quantizations
2002                          * with a high magnitude that allows 64-bits to
2003                          * overflow, but we nonetheless protect against this
2004                          * by explicitly checking for overflow, and clamping
2005                          * our value accordingly.
2006                          */
2007                         value = this - 1;
2008                 }
2009 
2010                 if (value < this) {
2011                         /*
2012                          * If our value lies within this order of magnitude,
2013                          * determine its position by taking the offset within
2014                          * the order of magnitude, dividing by the bucket
2015                          * width, and adding to our (accumulated) base.
2016                          */
2017                         return (base + (value - last) / (this / nbuckets));
2018                 }
2019 
2020                 base += nbuckets - (nbuckets / factor);
2021                 last = this;
2022                 this = next;
2023         }
2024 
2025         /*
2026          * Our value is greater than or equal to our factor taken to the
2027          * power of one plus the high magnitude -- return the top bucket.
2028          */
2029         return (base);
2030 }
2031 
2032 static void
2033 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
2034 {
2035         uint64_t arg = *llquanta++;
2036         uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2037         uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2038         uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2039         uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2040 
2041         llquanta[dtrace_aggregate_llquantize_bucket(factor,
2042             low, high, nsteps, nval)] += incr;
2043 }
2044 
2045 /*ARGSUSED*/
2046 static void
2047 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
2048 {
2049         data[0]++;
2050         data[1] += nval;
2051 }
2052 
2053 /*ARGSUSED*/
2054 static void
2055 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
2056 {
2057         int64_t snval = (int64_t)nval;
2058         uint64_t tmp[2];
2059 
2060         data[0]++;
2061         data[1] += nval;
2062 
2063         /*
2064          * What we want to say here is:
2065          *
2066          * data[2] += nval * nval;
2067          *
2068          * But given that nval is 64-bit, we could easily overflow, so
2069          * we do this as 128-bit arithmetic.
2070          */
2071         if (snval < 0)
2072                 snval = -snval;
2073 
2074         dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
2075         dtrace_add_128(data + 2, tmp, data + 2);
2076 }
2077 
2078 /*ARGSUSED*/
2079 static void
2080 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
2081 {
2082         *oval = *oval + 1;
2083 }
2084 
2085 /*ARGSUSED*/
2086 static void
2087 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
2088 {
2089         *oval += nval;
2090 }
2091 
2092 /*
2093  * Aggregate given the tuple in the principal data buffer, and the aggregating
2094  * action denoted by the specified dtrace_aggregation_t.  The aggregation
2095  * buffer is specified as the buf parameter.  This routine does not return
2096  * failure; if there is no space in the aggregation buffer, the data will be
2097  * dropped, and a corresponding counter incremented.
2098  */
2099 static void
2100 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
2101     intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
2102 {
2103         dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
2104         uint32_t i, ndx, size, fsize;
2105         uint32_t align = sizeof (uint64_t) - 1;
2106         dtrace_aggbuffer_t *agb;
2107         dtrace_aggkey_t *key;
2108         uint32_t hashval = 0, limit, isstr;
2109         caddr_t tomax, data, kdata;
2110         dtrace_actkind_t action;
2111         dtrace_action_t *act;
2112         uintptr_t offs;
2113 
2114         if (buf == NULL)
2115                 return;
2116 
2117         if (!agg->dtag_hasarg) {
2118                 /*
2119                  * Currently, only quantize() and lquantize() take additional
2120                  * arguments, and they have the same semantics:  an increment
2121                  * value that defaults to 1 when not present.  If additional
2122                  * aggregating actions take arguments, the setting of the
2123                  * default argument value will presumably have to become more
2124                  * sophisticated...
2125                  */
2126                 arg = 1;
2127         }
2128 
2129         action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
2130         size = rec->dtrd_offset - agg->dtag_base;
2131         fsize = size + rec->dtrd_size;
2132 
2133         ASSERT(dbuf->dtb_tomax != NULL);
2134         data = dbuf->dtb_tomax + offset + agg->dtag_base;
2135 
2136         if ((tomax = buf->dtb_tomax) == NULL) {
2137                 dtrace_buffer_drop(buf);
2138                 return;
2139         }
2140 
2141         /*
2142          * The metastructure is always at the bottom of the buffer.
2143          */
2144         agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
2145             sizeof (dtrace_aggbuffer_t));
2146 
2147         if (buf->dtb_offset == 0) {
2148                 /*
2149                  * We just kludge up approximately 1/8th of the size to be
2150                  * buckets.  If this guess ends up being routinely
2151                  * off-the-mark, we may need to dynamically readjust this
2152                  * based on past performance.
2153                  */
2154                 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
2155 
2156                 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
2157                     (uintptr_t)tomax || hashsize == 0) {
2158                         /*
2159                          * We've been given a ludicrously small buffer;
2160                          * increment our drop count and leave.
2161                          */
2162                         dtrace_buffer_drop(buf);
2163                         return;
2164                 }
2165 
2166                 /*
2167                  * And now, a pathetic attempt to try to get a an odd (or
2168                  * perchance, a prime) hash size for better hash distribution.
2169                  */
2170                 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
2171                         hashsize -= DTRACE_AGGHASHSIZE_SLEW;
2172 
2173                 agb->dtagb_hashsize = hashsize;
2174                 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
2175                     agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
2176                 agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
2177 
2178                 for (i = 0; i < agb->dtagb_hashsize; i++)
2179                         agb->dtagb_hash[i] = NULL;
2180         }
2181 
2182         ASSERT(agg->dtag_first != NULL);
2183         ASSERT(agg->dtag_first->dta_intuple);
2184 
2185         /*
2186          * Calculate the hash value based on the key.  Note that we _don't_
2187          * include the aggid in the hashing (but we will store it as part of
2188          * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
2189          * algorithm: a simple, quick algorithm that has no known funnels, and
2190          * gets good distribution in practice.  The efficacy of the hashing
2191          * algorithm (and a comparison with other algorithms) may be found by
2192          * running the ::dtrace_aggstat MDB dcmd.
2193          */
2194         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2195                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2196                 limit = i + act->dta_rec.dtrd_size;
2197                 ASSERT(limit <= size);
2198                 isstr = DTRACEACT_ISSTRING(act);
2199 
2200                 for (; i < limit; i++) {
2201                         hashval += data[i];
2202                         hashval += (hashval << 10);
2203                         hashval ^= (hashval >> 6);
2204 
2205                         if (isstr && data[i] == '\0')
2206                                 break;
2207                 }
2208         }
2209 
2210         hashval += (hashval << 3);
2211         hashval ^= (hashval >> 11);
2212         hashval += (hashval << 15);
2213 
2214         /*
2215          * Yes, the divide here is expensive -- but it's generally the least
2216          * of the performance issues given the amount of data that we iterate
2217          * over to compute hash values, compare data, etc.
2218          */
2219         ndx = hashval % agb->dtagb_hashsize;
2220 
2221         for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
2222                 ASSERT((caddr_t)key >= tomax);
2223                 ASSERT((caddr_t)key < tomax + buf->dtb_size);
2224 
2225                 if (hashval != key->dtak_hashval || key->dtak_size != size)
2226                         continue;
2227 
2228                 kdata = key->dtak_data;
2229                 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
2230 
2231                 for (act = agg->dtag_first; act->dta_intuple;
2232                     act = act->dta_next) {
2233                         i = act->dta_rec.dtrd_offset - agg->dtag_base;
2234                         limit = i + act->dta_rec.dtrd_size;
2235                         ASSERT(limit <= size);
2236                         isstr = DTRACEACT_ISSTRING(act);
2237 
2238                         for (; i < limit; i++) {
2239                                 if (kdata[i] != data[i])
2240                                         goto next;
2241 
2242                                 if (isstr && data[i] == '\0')
2243                                         break;
2244                         }
2245                 }
2246 
2247                 if (action != key->dtak_action) {
2248                         /*
2249                          * We are aggregating on the same value in the same
2250                          * aggregation with two different aggregating actions.
2251                          * (This should have been picked up in the compiler,
2252                          * so we may be dealing with errant or devious DIF.)
2253                          * This is an error condition; we indicate as much,
2254                          * and return.
2255                          */
2256                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2257                         return;
2258                 }
2259 
2260                 /*
2261                  * This is a hit:  we need to apply the aggregator to
2262                  * the value at this key.
2263                  */
2264                 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
2265                 return;
2266 next:
2267                 continue;
2268         }
2269 
2270         /*
2271          * We didn't find it.  We need to allocate some zero-filled space,
2272          * link it into the hash table appropriately, and apply the aggregator
2273          * to the (zero-filled) value.
2274          */
2275         offs = buf->dtb_offset;
2276         while (offs & (align - 1))
2277                 offs += sizeof (uint32_t);
2278 
2279         /*
2280          * If we don't have enough room to both allocate a new key _and_
2281          * its associated data, increment the drop count and return.
2282          */
2283         if ((uintptr_t)tomax + offs + fsize >
2284             agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
2285                 dtrace_buffer_drop(buf);
2286                 return;
2287         }
2288 
2289         /*CONSTCOND*/
2290         ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
2291         key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
2292         agb->dtagb_free -= sizeof (dtrace_aggkey_t);
2293 
2294         key->dtak_data = kdata = tomax + offs;
2295         buf->dtb_offset = offs + fsize;
2296 
2297         /*
2298          * Now copy the data across.
2299          */
2300         *((dtrace_aggid_t *)kdata) = agg->dtag_id;
2301 
2302         for (i = sizeof (dtrace_aggid_t); i < size; i++)
2303                 kdata[i] = data[i];
2304 
2305         /*
2306          * Because strings are not zeroed out by default, we need to iterate
2307          * looking for actions that store strings, and we need to explicitly
2308          * pad these strings out with zeroes.
2309          */
2310         for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2311                 int nul;
2312 
2313                 if (!DTRACEACT_ISSTRING(act))
2314                         continue;
2315 
2316                 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2317                 limit = i + act->dta_rec.dtrd_size;
2318                 ASSERT(limit <= size);
2319 
2320                 for (nul = 0; i < limit; i++) {
2321                         if (nul) {
2322                                 kdata[i] = '\0';
2323                                 continue;
2324                         }
2325 
2326                         if (data[i] != '\0')
2327                                 continue;
2328 
2329                         nul = 1;
2330                 }
2331         }
2332 
2333         for (i = size; i < fsize; i++)
2334                 kdata[i] = 0;
2335 
2336         key->dtak_hashval = hashval;
2337         key->dtak_size = size;
2338         key->dtak_action = action;
2339         key->dtak_next = agb->dtagb_hash[ndx];
2340         agb->dtagb_hash[ndx] = key;
2341 
2342         /*
2343          * Finally, apply the aggregator.
2344          */
2345         *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
2346         agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
2347 }
2348 
2349 /*
2350  * Given consumer state, this routine finds a speculation in the INACTIVE
2351  * state and transitions it into the ACTIVE state.  If there is no speculation
2352  * in the INACTIVE state, 0 is returned.  In this case, no error counter is
2353  * incremented -- it is up to the caller to take appropriate action.
2354  */
2355 static int
2356 dtrace_speculation(dtrace_state_t *state)
2357 {
2358         int i = 0;
2359         dtrace_speculation_state_t current;
2360         uint32_t *stat = &state->dts_speculations_unavail, count;
2361 
2362         while (i < state->dts_nspeculations) {
2363                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2364 
2365                 current = spec->dtsp_state;
2366 
2367                 if (current != DTRACESPEC_INACTIVE) {
2368                         if (current == DTRACESPEC_COMMITTINGMANY ||
2369                             current == DTRACESPEC_COMMITTING ||
2370                             current == DTRACESPEC_DISCARDING)
2371                                 stat = &state->dts_speculations_busy;
2372                         i++;
2373                         continue;
2374                 }
2375 
2376                 if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2377                     current, DTRACESPEC_ACTIVE) == current)
2378                         return (i + 1);
2379         }
2380 
2381         /*
2382          * We couldn't find a speculation.  If we found as much as a single
2383          * busy speculation buffer, we'll attribute this failure as "busy"
2384          * instead of "unavail".
2385          */
2386         do {
2387                 count = *stat;
2388         } while (dtrace_cas32(stat, count, count + 1) != count);
2389 
2390         return (0);
2391 }
2392 
2393 /*
2394  * This routine commits an active speculation.  If the specified speculation
2395  * is not in a valid state to perform a commit(), this routine will silently do
2396  * nothing.  The state of the specified speculation is transitioned according
2397  * to the state transition diagram outlined in <sys/dtrace_impl.h>
2398  */
2399 static void
2400 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2401     dtrace_specid_t which)
2402 {
2403         dtrace_speculation_t *spec;
2404         dtrace_buffer_t *src, *dest;
2405         uintptr_t daddr, saddr, dlimit, slimit;
2406         dtrace_speculation_state_t current, new;
2407         intptr_t offs;
2408         uint64_t timestamp;
2409 
2410         if (which == 0)
2411                 return;
2412 
2413         if (which > state->dts_nspeculations) {
2414                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2415                 return;
2416         }
2417 
2418         spec = &state->dts_speculations[which - 1];
2419         src = &spec->dtsp_buffer[cpu];
2420         dest = &state->dts_buffer[cpu];
2421 
2422         do {
2423                 current = spec->dtsp_state;
2424 
2425                 if (current == DTRACESPEC_COMMITTINGMANY)
2426                         break;
2427 
2428                 switch (current) {
2429                 case DTRACESPEC_INACTIVE:
2430                 case DTRACESPEC_DISCARDING:
2431                         return;
2432 
2433                 case DTRACESPEC_COMMITTING:
2434                         /*
2435                          * This is only possible if we are (a) commit()'ing
2436                          * without having done a prior speculate() on this CPU
2437                          * and (b) racing with another commit() on a different
2438                          * CPU.  There's nothing to do -- we just assert that
2439                          * our offset is 0.
2440                          */
2441                         ASSERT(src->dtb_offset == 0);
2442                         return;
2443 
2444                 case DTRACESPEC_ACTIVE:
2445                         new = DTRACESPEC_COMMITTING;
2446                         break;
2447 
2448                 case DTRACESPEC_ACTIVEONE:
2449                         /*
2450                          * This speculation is active on one CPU.  If our
2451                          * buffer offset is non-zero, we know that the one CPU
2452                          * must be us.  Otherwise, we are committing on a
2453                          * different CPU from the speculate(), and we must
2454                          * rely on being asynchronously cleaned.
2455                          */
2456                         if (src->dtb_offset != 0) {
2457                                 new = DTRACESPEC_COMMITTING;
2458                                 break;
2459                         }
2460                         /*FALLTHROUGH*/
2461 
2462                 case DTRACESPEC_ACTIVEMANY:
2463                         new = DTRACESPEC_COMMITTINGMANY;
2464                         break;
2465 
2466                 default:
2467                         ASSERT(0);
2468                 }
2469         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2470             current, new) != current);
2471 
2472         /*
2473          * We have set the state to indicate that we are committing this
2474          * speculation.  Now reserve the necessary space in the destination
2475          * buffer.
2476          */
2477         if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
2478             sizeof (uint64_t), state, NULL)) < 0) {
2479                 dtrace_buffer_drop(dest);
2480                 goto out;
2481         }
2482 
2483         /*
2484          * We have sufficient space to copy the speculative buffer into the
2485          * primary buffer.  First, modify the speculative buffer, filling
2486          * in the timestamp of all entries with the current time.  The data
2487          * must have the commit() time rather than the time it was traced,
2488          * so that all entries in the primary buffer are in timestamp order.
2489          */
2490         timestamp = dtrace_gethrtime();
2491         saddr = (uintptr_t)src->dtb_tomax;
2492         slimit = saddr + src->dtb_offset;
2493         while (saddr < slimit) {
2494                 size_t size;
2495                 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr;
2496 
2497                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2498                         saddr += sizeof (dtrace_epid_t);
2499                         continue;
2500                 }
2501                 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs);
2502                 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size;
2503 
2504                 ASSERT3U(saddr + size, <=, slimit);
2505                 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t));
2506                 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX);
2507 
2508                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp);
2509 
2510                 saddr += size;
2511         }
2512 
2513         /*
2514          * Copy the buffer across.  (Note that this is a
2515          * highly subobtimal bcopy(); in the unlikely event that this becomes
2516          * a serious performance issue, a high-performance DTrace-specific
2517          * bcopy() should obviously be invented.)
2518          */
2519         daddr = (uintptr_t)dest->dtb_tomax + offs;
2520         dlimit = daddr + src->dtb_offset;
2521         saddr = (uintptr_t)src->dtb_tomax;
2522 
2523         /*
2524          * First, the aligned portion.
2525          */
2526         while (dlimit - daddr >= sizeof (uint64_t)) {
2527                 *((uint64_t *)daddr) = *((uint64_t *)saddr);
2528 
2529                 daddr += sizeof (uint64_t);
2530                 saddr += sizeof (uint64_t);
2531         }
2532 
2533         /*
2534          * Now any left-over bit...
2535          */
2536         while (dlimit - daddr)
2537                 *((uint8_t *)daddr++) = *((uint8_t *)saddr++);
2538 
2539         /*
2540          * Finally, commit the reserved space in the destination buffer.
2541          */
2542         dest->dtb_offset = offs + src->dtb_offset;
2543 
2544 out:
2545         /*
2546          * If we're lucky enough to be the only active CPU on this speculation
2547          * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2548          */
2549         if (current == DTRACESPEC_ACTIVE ||
2550             (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
2551                 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
2552                     DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
2553 
2554                 ASSERT(rval == DTRACESPEC_COMMITTING);
2555         }
2556 
2557         src->dtb_offset = 0;
2558         src->dtb_xamot_drops += src->dtb_drops;
2559         src->dtb_drops = 0;
2560 }
2561 
2562 /*
2563  * This routine discards an active speculation.  If the specified speculation
2564  * is not in a valid state to perform a discard(), this routine will silently
2565  * do nothing.  The state of the specified speculation is transitioned
2566  * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2567  */
2568 static void
2569 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2570     dtrace_specid_t which)
2571 {
2572         dtrace_speculation_t *spec;
2573         dtrace_speculation_state_t current, new;
2574         dtrace_buffer_t *buf;
2575 
2576         if (which == 0)
2577                 return;
2578 
2579         if (which > state->dts_nspeculations) {
2580                 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2581                 return;
2582         }
2583 
2584         spec = &state->dts_speculations[which - 1];
2585         buf = &spec->dtsp_buffer[cpu];
2586 
2587         do {
2588                 current = spec->dtsp_state;
2589 
2590                 switch (current) {
2591                 case DTRACESPEC_INACTIVE:
2592                 case DTRACESPEC_COMMITTINGMANY:
2593                 case DTRACESPEC_COMMITTING:
2594                 case DTRACESPEC_DISCARDING:
2595                         return;
2596 
2597                 case DTRACESPEC_ACTIVE:
2598                 case DTRACESPEC_ACTIVEMANY:
2599                         new = DTRACESPEC_DISCARDING;
2600                         break;
2601 
2602                 case DTRACESPEC_ACTIVEONE:
2603                         if (buf->dtb_offset != 0) {
2604                                 new = DTRACESPEC_INACTIVE;
2605                         } else {
2606                                 new = DTRACESPEC_DISCARDING;
2607                         }
2608                         break;
2609 
2610                 default:
2611                         ASSERT(0);
2612                 }
2613         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2614             current, new) != current);
2615 
2616         buf->dtb_offset = 0;
2617         buf->dtb_drops = 0;
2618 }
2619 
2620 /*
2621  * Note:  not called from probe context.  This function is called
2622  * asynchronously from cross call context to clean any speculations that are
2623  * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
2624  * transitioned back to the INACTIVE state until all CPUs have cleaned the
2625  * speculation.
2626  */
2627 static void
2628 dtrace_speculation_clean_here(dtrace_state_t *state)
2629 {
2630         dtrace_icookie_t cookie;
2631         processorid_t cpu = CPU->cpu_id;
2632         dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2633         dtrace_specid_t i;
2634 
2635         cookie = dtrace_interrupt_disable();
2636 
2637         if (dest->dtb_tomax == NULL) {
2638                 dtrace_interrupt_enable(cookie);
2639                 return;
2640         }
2641 
2642         for (i = 0; i < state->dts_nspeculations; i++) {
2643                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2644                 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
2645 
2646                 if (src->dtb_tomax == NULL)
2647                         continue;
2648 
2649                 if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
2650                         src->dtb_offset = 0;
2651                         continue;
2652                 }
2653 
2654                 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2655                         continue;
2656 
2657                 if (src->dtb_offset == 0)
2658                         continue;
2659 
2660                 dtrace_speculation_commit(state, cpu, i + 1);
2661         }
2662 
2663         dtrace_interrupt_enable(cookie);
2664 }
2665 
2666 /*
2667  * Note:  not called from probe context.  This function is called
2668  * asynchronously (and at a regular interval) to clean any speculations that
2669  * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
2670  * is work to be done, it cross calls all CPUs to perform that work;
2671  * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2672  * INACTIVE state until they have been cleaned by all CPUs.
2673  */
2674 static void
2675 dtrace_speculation_clean(dtrace_state_t *state)
2676 {
2677         int work = 0, rv;
2678         dtrace_specid_t i;
2679 
2680         for (i = 0; i < state->dts_nspeculations; i++) {
2681                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2682 
2683                 ASSERT(!spec->dtsp_cleaning);
2684 
2685                 if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
2686                     spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2687                         continue;
2688 
2689                 work++;
2690                 spec->dtsp_cleaning = 1;
2691         }
2692 
2693         if (!work)
2694                 return;
2695 
2696         dtrace_xcall(DTRACE_CPUALL,
2697             (dtrace_xcall_t)dtrace_speculation_clean_here, state);
2698 
2699         /*
2700          * We now know that all CPUs have committed or discarded their
2701          * speculation buffers, as appropriate.  We can now set the state
2702          * to inactive.
2703          */
2704         for (i = 0; i < state->dts_nspeculations; i++) {
2705                 dtrace_speculation_t *spec = &state->dts_speculations[i];
2706                 dtrace_speculation_state_t current, new;
2707 
2708                 if (!spec->dtsp_cleaning)
2709                         continue;
2710 
2711                 current = spec->dtsp_state;
2712                 ASSERT(current == DTRACESPEC_DISCARDING ||
2713                     current == DTRACESPEC_COMMITTINGMANY);
2714 
2715                 new = DTRACESPEC_INACTIVE;
2716 
2717                 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
2718                 ASSERT(rv == current);
2719                 spec->dtsp_cleaning = 0;
2720         }
2721 }
2722 
2723 /*
2724  * Called as part of a speculate() to get the speculative buffer associated
2725  * with a given speculation.  Returns NULL if the specified speculation is not
2726  * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
2727  * the active CPU is not the specified CPU -- the speculation will be
2728  * atomically transitioned into the ACTIVEMANY state.
2729  */
2730 static dtrace_buffer_t *
2731 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2732     dtrace_specid_t which)
2733 {
2734         dtrace_speculation_t *spec;
2735         dtrace_speculation_state_t current, new;
2736         dtrace_buffer_t *buf;
2737 
2738         if (which == 0)
2739                 return (NULL);
2740 
2741         if (which > state->dts_nspeculations) {
2742                 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2743                 return (NULL);
2744         }
2745 
2746         spec = &state->dts_speculations[which - 1];
2747         buf = &spec->dtsp_buffer[cpuid];
2748 
2749         do {
2750                 current = spec->dtsp_state;
2751 
2752                 switch (current) {
2753                 case DTRACESPEC_INACTIVE:
2754                 case DTRACESPEC_COMMITTINGMANY:
2755                 case DTRACESPEC_DISCARDING:
2756                         return (NULL);
2757 
2758                 case DTRACESPEC_COMMITTING:
2759                         ASSERT(buf->dtb_offset == 0);
2760                         return (NULL);
2761 
2762                 case DTRACESPEC_ACTIVEONE:
2763                         /*
2764                          * This speculation is currently active on one CPU.
2765                          * Check the offset in the buffer; if it's non-zero,
2766                          * that CPU must be us (and we leave the state alone).
2767                          * If it's zero, assume that we're starting on a new
2768                          * CPU -- and change the state to indicate that the
2769                          * speculation is active on more than one CPU.
2770                          */
2771                         if (buf->dtb_offset != 0)
2772                                 return (buf);
2773 
2774                         new = DTRACESPEC_ACTIVEMANY;
2775                         break;
2776 
2777                 case DTRACESPEC_ACTIVEMANY:
2778                         return (buf);
2779 
2780                 case DTRACESPEC_ACTIVE:
2781                         new = DTRACESPEC_ACTIVEONE;
2782                         break;
2783 
2784                 default:
2785                         ASSERT(0);
2786                 }
2787         } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2788             current, new) != current);
2789 
2790         ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
2791         return (buf);
2792 }
2793 
2794 /*
2795  * Return a string.  In the event that the user lacks the privilege to access
2796  * arbitrary kernel memory, we copy the string out to scratch memory so that we
2797  * don't fail access checking.
2798  *
2799  * dtrace_dif_variable() uses this routine as a helper for various
2800  * builtin values such as 'execname' and 'probefunc.'
2801  */
2802 uintptr_t
2803 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
2804     dtrace_mstate_t *mstate)
2805 {
2806         uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2807         uintptr_t ret;
2808         size_t strsz;
2809 
2810         /*
2811          * The easy case: this probe is allowed to read all of memory, so
2812          * we can just return this as a vanilla pointer.
2813          */
2814         if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
2815                 return (addr);
2816 
2817         /*
2818          * This is the tougher case: we copy the string in question from
2819          * kernel memory into scratch memory and return it that way: this
2820          * ensures that we won't trip up when access checking tests the
2821          * BYREF return value.
2822          */
2823         strsz = dtrace_strlen((char *)addr, size) + 1;
2824 
2825         if (mstate->dtms_scratch_ptr + strsz >
2826             mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2827                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2828                 return (NULL);
2829         }
2830 
2831         dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
2832             strsz);
2833         ret = mstate->dtms_scratch_ptr;
2834         mstate->dtms_scratch_ptr += strsz;
2835         return (ret);
2836 }
2837 
2838 /*
2839  * This function implements the DIF emulator's variable lookups.  The emulator
2840  * passes a reserved variable identifier and optional built-in array index.
2841  */
2842 static uint64_t
2843 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
2844     uint64_t ndx)
2845 {
2846         /*
2847          * If we're accessing one of the uncached arguments, we'll turn this
2848          * into a reference in the args array.
2849          */
2850         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
2851                 ndx = v - DIF_VAR_ARG0;
2852                 v = DIF_VAR_ARGS;
2853         }
2854 
2855         switch (v) {
2856         case DIF_VAR_ARGS:
2857                 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) {
2858                         cpu_core[CPU->cpu_id].cpuc_dtrace_flags |=
2859                             CPU_DTRACE_KPRIV;
2860                         return (0);
2861                 }
2862 
2863                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
2864                 if (ndx >= sizeof (mstate->dtms_arg) /
2865                     sizeof (mstate->dtms_arg[0])) {
2866                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2867                         dtrace_provider_t *pv;
2868                         uint64_t val;
2869 
2870                         pv = mstate->dtms_probe->dtpr_provider;
2871                         if (pv->dtpv_pops.dtps_getargval != NULL)
2872                                 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
2873                                     mstate->dtms_probe->dtpr_id,
2874                                     mstate->dtms_probe->dtpr_arg, ndx, aframes);
2875                         else
2876                                 val = dtrace_getarg(ndx, aframes);
2877 
2878                         /*
2879                          * This is regrettably required to keep the compiler
2880                          * from tail-optimizing the call to dtrace_getarg().
2881                          * The condition always evaluates to true, but the
2882                          * compiler has no way of figuring that out a priori.
2883                          * (None of this would be necessary if the compiler
2884                          * could be relied upon to _always_ tail-optimize
2885                          * the call to dtrace_getarg() -- but it can't.)
2886                          */
2887                         if (mstate->dtms_probe != NULL)
2888                                 return (val);
2889 
2890                         ASSERT(0);
2891                 }
2892 
2893                 return (mstate->dtms_arg[ndx]);
2894 
2895         case DIF_VAR_UREGS: {
2896                 klwp_t *lwp;
2897 
2898                 if (!dtrace_priv_proc(state, mstate))
2899                         return (0);
2900 
2901                 if ((lwp = curthread->t_lwp) == NULL) {
2902                         DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
2903                         cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
2904                         return (0);
2905                 }
2906 
2907                 return (dtrace_getreg(lwp->lwp_regs, ndx));
2908         }
2909 
2910         case DIF_VAR_VMREGS: {
2911                 uint64_t rval;
2912 
2913                 if (!dtrace_priv_kernel(state))
2914                         return (0);
2915 
2916                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2917 
2918                 rval = dtrace_getvmreg(ndx,
2919                     &cpu_core[CPU->cpu_id].cpuc_dtrace_flags);
2920 
2921                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2922 
2923                 return (rval);
2924         }
2925 
2926         case DIF_VAR_CURTHREAD:
2927                 if (!dtrace_priv_kernel(state))
2928                         return (0);
2929                 return ((uint64_t)(uintptr_t)curthread);
2930 
2931         case DIF_VAR_TIMESTAMP:
2932                 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
2933                         mstate->dtms_timestamp = dtrace_gethrtime();
2934                         mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
2935                 }
2936                 return (mstate->dtms_timestamp);
2937 
2938         case DIF_VAR_VTIMESTAMP:
2939                 ASSERT(dtrace_vtime_references != 0);
2940                 return (curthread->t_dtrace_vtime);
2941 
2942         case DIF_VAR_WALLTIMESTAMP:
2943                 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
2944                         mstate->dtms_walltimestamp = dtrace_gethrestime();
2945                         mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
2946                 }
2947                 return (mstate->dtms_walltimestamp);
2948 
2949         case DIF_VAR_IPL:
2950                 if (!dtrace_priv_kernel(state))
2951                         return (0);
2952                 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
2953                         mstate->dtms_ipl = dtrace_getipl();
2954                         mstate->dtms_present |= DTRACE_MSTATE_IPL;
2955                 }
2956                 return (mstate->dtms_ipl);
2957 
2958         case DIF_VAR_EPID:
2959                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
2960                 return (mstate->dtms_epid);
2961 
2962         case DIF_VAR_ID:
2963                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2964                 return (mstate->dtms_probe->dtpr_id);
2965 
2966         case DIF_VAR_STACKDEPTH:
2967                 if (!dtrace_priv_kernel(state))
2968                         return (0);
2969                 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
2970                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2971 
2972                         mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
2973                         mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
2974                 }
2975                 return (mstate->dtms_stackdepth);
2976 
2977         case DIF_VAR_USTACKDEPTH:
2978                 if (!dtrace_priv_proc(state, mstate))
2979                         return (0);
2980                 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
2981                         /*
2982                          * See comment in DIF_VAR_PID.
2983                          */
2984                         if (DTRACE_ANCHORED(mstate->dtms_probe) &&
2985                             CPU_ON_INTR(CPU)) {
2986                                 mstate->dtms_ustackdepth = 0;
2987                         } else {
2988                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2989                                 mstate->dtms_ustackdepth =
2990                                     dtrace_getustackdepth();
2991                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2992                         }
2993                         mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
2994                 }
2995                 return (mstate->dtms_ustackdepth);
2996 
2997         case DIF_VAR_CALLER:
2998                 if (!dtrace_priv_kernel(state))
2999                         return (0);
3000                 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
3001                         int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3002 
3003                         if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
3004                                 /*
3005                                  * If this is an unanchored probe, we are
3006                                  * required to go through the slow path:
3007                                  * dtrace_caller() only guarantees correct
3008                                  * results for anchored probes.
3009                                  */
3010                                 pc_t caller[2];
3011 
3012                                 dtrace_getpcstack(caller, 2, aframes,
3013                                     (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
3014                                 mstate->dtms_caller = caller[1];
3015                         } else if ((mstate->dtms_caller =
3016                             dtrace_caller(aframes)) == -1) {
3017                                 /*
3018                                  * We have failed to do this the quick way;
3019                                  * we must resort to the slower approach of
3020                                  * calling dtrace_getpcstack().
3021                                  */
3022                                 pc_t caller;
3023 
3024                                 dtrace_getpcstack(&caller, 1, aframes, NULL);
3025                                 mstate->dtms_caller = caller;
3026                         }
3027 
3028                         mstate->dtms_present |= DTRACE_MSTATE_CALLER;
3029                 }
3030                 return (mstate->dtms_caller);
3031 
3032         case DIF_VAR_UCALLER:
3033                 if (!dtrace_priv_proc(state, mstate))
3034                         return (0);
3035 
3036                 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
3037                         uint64_t ustack[3];
3038 
3039                         /*
3040                          * dtrace_getupcstack() fills in the first uint64_t
3041                          * with the current PID.  The second uint64_t will
3042                          * be the program counter at user-level.  The third
3043                          * uint64_t will contain the caller, which is what
3044                          * we're after.
3045                          */
3046                         ustack[2] = NULL;
3047                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3048                         dtrace_getupcstack(ustack, 3);
3049                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3050                         mstate->dtms_ucaller = ustack[2];
3051                         mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
3052                 }
3053 
3054                 return (mstate->dtms_ucaller);
3055 
3056         case DIF_VAR_PROBEPROV:
3057                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3058                 return (dtrace_dif_varstr(
3059                     (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
3060                     state, mstate));
3061 
3062         case DIF_VAR_PROBEMOD:
3063                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3064                 return (dtrace_dif_varstr(
3065                     (uintptr_t)mstate->dtms_probe->dtpr_mod,
3066                     state, mstate));
3067 
3068         case DIF_VAR_PROBEFUNC:
3069                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3070                 return (dtrace_dif_varstr(
3071                     (uintptr_t)mstate->dtms_probe->dtpr_func,
3072                     state, mstate));
3073 
3074         case DIF_VAR_PROBENAME:
3075                 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3076                 return (dtrace_dif_varstr(
3077                     (uintptr_t)mstate->dtms_probe->dtpr_name,
3078                     state, mstate));
3079 
3080         case DIF_VAR_PID:
3081                 if (!dtrace_priv_proc(state, mstate))
3082                         return (0);
3083 
3084                 /*
3085                  * Note that we are assuming that an unanchored probe is
3086                  * always due to a high-level interrupt.  (And we're assuming
3087                  * that there is only a single high level interrupt.)
3088                  */
3089                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3090                         return (pid0.pid_id);
3091 
3092                 /*
3093                  * It is always safe to dereference one's own t_procp pointer:
3094                  * it always points to a valid, allocated proc structure.
3095                  * Further, it is always safe to dereference the p_pidp member
3096                  * of one's own proc structure.  (These are truisms becuase
3097                  * threads and processes don't clean up their own state --
3098                  * they leave that task to whomever reaps them.)
3099                  */
3100                 return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
3101 
3102         case DIF_VAR_PPID:
3103                 if (!dtrace_priv_proc(state, mstate))
3104                         return (0);
3105 
3106                 /*
3107                  * See comment in DIF_VAR_PID.
3108                  */
3109                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3110                         return (pid0.pid_id);
3111 
3112                 /*
3113                  * It is always safe to dereference one's own t_procp pointer:
3114                  * it always points to a valid, allocated proc structure.
3115                  * (This is true because threads don't clean up their own
3116                  * state -- they leave that task to whomever reaps them.)
3117                  */
3118                 return ((uint64_t)curthread->t_procp->p_ppid);
3119 
3120         case DIF_VAR_TID:
3121                 /*
3122                  * See comment in DIF_VAR_PID.
3123                  */
3124                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3125                         return (0);
3126 
3127                 return ((uint64_t)curthread->t_tid);
3128 
3129         case DIF_VAR_EXECNAME:
3130                 if (!dtrace_priv_proc(state, mstate))
3131                         return (0);
3132 
3133                 /*
3134                  * See comment in DIF_VAR_PID.
3135                  */
3136                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3137                         return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
3138 
3139                 /*
3140                  * It is always safe to dereference one's own t_procp pointer:
3141                  * it always points to a valid, allocated proc structure.
3142                  * (This is true because threads don't clean up their own
3143                  * state -- they leave that task to whomever reaps them.)
3144                  */
3145                 return (dtrace_dif_varstr(
3146                     (uintptr_t)curthread->t_procp->p_user.u_comm,
3147                     state, mstate));
3148 
3149         case DIF_VAR_ZONENAME:
3150                 if (!dtrace_priv_proc(state, mstate))
3151                         return (0);
3152 
3153                 /*
3154                  * See comment in DIF_VAR_PID.
3155                  */
3156                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3157                         return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
3158 
3159                 /*
3160                  * It is always safe to dereference one's own t_procp pointer:
3161                  * it always points to a valid, allocated proc structure.
3162                  * (This is true because threads don't clean up their own
3163                  * state -- they leave that task to whomever reaps them.)
3164                  */
3165                 return (dtrace_dif_varstr(
3166                     (uintptr_t)curthread->t_procp->p_zone->zone_name,
3167                     state, mstate));
3168 
3169         case DIF_VAR_UID:
3170                 if (!dtrace_priv_proc(state, mstate))
3171                         return (0);
3172 
3173                 /*
3174                  * See comment in DIF_VAR_PID.
3175                  */
3176                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3177                         return ((uint64_t)p0.p_cred->cr_uid);
3178 
3179                 /*
3180                  * It is always safe to dereference one's own t_procp pointer:
3181                  * it always points to a valid, allocated proc structure.
3182                  * (This is true because threads don't clean up their own
3183                  * state -- they leave that task to whomever reaps them.)
3184                  *
3185                  * Additionally, it is safe to dereference one's own process
3186                  * credential, since this is never NULL after process birth.
3187                  */
3188                 return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
3189 
3190         case DIF_VAR_GID:
3191                 if (!dtrace_priv_proc(state, mstate))
3192                         return (0);
3193 
3194                 /*
3195                  * See comment in DIF_VAR_PID.
3196                  */
3197                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3198                         return ((uint64_t)p0.p_cred->cr_gid);
3199 
3200                 /*
3201                  * It is always safe to dereference one's own t_procp pointer:
3202                  * it always points to a valid, allocated proc structure.
3203                  * (This is true because threads don't clean up their own
3204                  * state -- they leave that task to whomever reaps them.)
3205                  *
3206                  * Additionally, it is safe to dereference one's own process
3207                  * credential, since this is never NULL after process birth.
3208                  */
3209                 return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
3210 
3211         case DIF_VAR_ERRNO: {
3212                 klwp_t *lwp;
3213                 if (!dtrace_priv_proc(state, mstate))
3214                         return (0);
3215 
3216                 /*
3217                  * See comment in DIF_VAR_PID.
3218                  */
3219                 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3220                         return (0);
3221 
3222                 /*
3223                  * It is always safe to dereference one's own t_lwp pointer in
3224                  * the event that this pointer is non-NULL.  (This is true
3225                  * because threads and lwps don't clean up their own state --
3226                  * they leave that task to whomever reaps them.)
3227                  */
3228                 if ((lwp = curthread->t_lwp) == NULL)
3229                         return (0);
3230 
3231                 return ((uint64_t)lwp->lwp_errno);
3232         }
3233         default:
3234                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3235                 return (0);
3236         }
3237 }
3238 
3239 /*
3240  * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
3241  * Notice that we don't bother validating the proper number of arguments or
3242  * their types in the tuple stack.  This isn't needed because all argument
3243  * interpretation is safe because of our load safety -- the worst that can
3244  * happen is that a bogus program can obtain bogus results.
3245  */
3246 static void
3247 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
3248     dtrace_key_t *tupregs, int nargs,
3249     dtrace_mstate_t *mstate, dtrace_state_t *state)
3250 {
3251         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
3252         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
3253         dtrace_vstate_t *vstate = &state->dts_vstate;
3254 
3255         union {
3256                 mutex_impl_t mi;
3257                 uint64_t mx;
3258         } m;
3259 
3260         union {
3261                 krwlock_t ri;
3262                 uintptr_t rw;
3263         } r;
3264 
3265         switch (subr) {
3266         case DIF_SUBR_RAND:
3267                 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
3268                 break;
3269 
3270         case DIF_SUBR_MUTEX_OWNED:
3271                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3272                     mstate, vstate)) {
3273                         regs[rd] = NULL;
3274                         break;
3275                 }
3276 
3277                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3278                 if (MUTEX_TYPE_ADAPTIVE(&m.mi))
3279                         regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
3280                 else
3281                         regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
3282                 break;
3283 
3284         case DIF_SUBR_MUTEX_OWNER:
3285                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3286                     mstate, vstate)) {
3287                         regs[rd] = NULL;
3288                         break;
3289                 }
3290 
3291                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3292                 if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
3293                     MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
3294                         regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
3295                 else
3296                         regs[rd] = 0;
3297                 break;
3298 
3299         case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
3300                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3301                     mstate, vstate)) {
3302                         regs[rd] = NULL;
3303                         break;
3304                 }
3305 
3306                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3307                 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
3308                 break;
3309 
3310         case DIF_SUBR_MUTEX_TYPE_SPIN:
3311                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3312                     mstate, vstate)) {
3313                         regs[rd] = NULL;
3314                         break;
3315                 }
3316 
3317                 m.mx = dtrace_load64(tupregs[0].dttk_value);
3318                 regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
3319                 break;
3320 
3321         case DIF_SUBR_RW_READ_HELD: {
3322                 uintptr_t tmp;
3323 
3324                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
3325                     mstate, vstate)) {
3326                         regs[rd] = NULL;
3327                         break;
3328                 }
3329 
3330                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3331                 regs[rd] = _RW_READ_HELD(&r.ri, tmp);
3332                 break;
3333         }
3334 
3335         case DIF_SUBR_RW_WRITE_HELD:
3336                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3337                     mstate, vstate)) {
3338                         regs[rd] = NULL;
3339                         break;
3340                 }
3341 
3342                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3343                 regs[rd] = _RW_WRITE_HELD(&r.ri);
3344                 break;
3345 
3346         case DIF_SUBR_RW_ISWRITER:
3347                 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3348                     mstate, vstate)) {
3349                         regs[rd] = NULL;
3350                         break;
3351                 }
3352 
3353                 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3354                 regs[rd] = _RW_ISWRITER(&r.ri);
3355                 break;
3356 
3357         case DIF_SUBR_BCOPY: {
3358                 /*
3359                  * We need to be sure that the destination is in the scratch
3360                  * region -- no other region is allowed.
3361                  */
3362                 uintptr_t src = tupregs[0].dttk_value;
3363                 uintptr_t dest = tupregs[1].dttk_value;
3364                 size_t size = tupregs[2].dttk_value;
3365 
3366                 if (!dtrace_inscratch(dest, size, mstate)) {
3367                         *flags |= CPU_DTRACE_BADADDR;
3368                         *illval = regs[rd];
3369                         break;
3370                 }
3371 
3372                 if (!dtrace_canload(src, size, mstate, vstate)) {
3373                         regs[rd] = NULL;
3374                         break;
3375                 }
3376 
3377                 dtrace_bcopy((void *)src, (void *)dest, size);
3378                 break;
3379         }
3380 
3381         case DIF_SUBR_ALLOCA:
3382         case DIF_SUBR_COPYIN: {
3383                 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
3384                 uint64_t size =
3385                     tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
3386                 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
3387 
3388                 /*
3389                  * This action doesn't require any credential checks since
3390                  * probes will not activate in user contexts to which the
3391                  * enabling user does not have permissions.
3392                  */
3393 
3394                 /*
3395                  * Rounding up the user allocation size could have overflowed
3396                  * a large, bogus allocation (like -1ULL) to 0.
3397                  */
3398                 if (scratch_size < size ||
3399                     !DTRACE_INSCRATCH(mstate, scratch_size)) {
3400                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3401                         regs[rd] = NULL;
3402                         break;
3403                 }
3404 
3405                 if (subr == DIF_SUBR_COPYIN) {
3406                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3407                         dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
3408                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3409                 }
3410 
3411                 mstate->dtms_scratch_ptr += scratch_size;
3412                 regs[rd] = dest;
3413                 break;
3414         }
3415 
3416         case DIF_SUBR_COPYINTO: {
3417                 uint64_t size = tupregs[1].dttk_value;
3418                 uintptr_t dest = tupregs[2].dttk_value;
3419 
3420                 /*
3421                  * This action doesn't require any credential checks since
3422                  * probes will not activate in user contexts to which the
3423                  * enabling user does not have permissions.
3424                  */
3425                 if (!dtrace_inscratch(dest, size, mstate)) {
3426                         *flags |= CPU_DTRACE_BADADDR;
3427                         *illval = regs[rd];
3428                         break;
3429                 }
3430 
3431                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3432                 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
3433                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3434                 break;
3435         }
3436 
3437         case DIF_SUBR_COPYINSTR: {
3438                 uintptr_t dest = mstate->dtms_scratch_ptr;
3439                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3440 
3441                 if (nargs > 1 && tupregs[1].dttk_value < size)
3442                         size = tupregs[1].dttk_value + 1;
3443 
3444                 /*
3445                  * This action doesn't require any credential checks since
3446                  * probes will not activate in user contexts to which the
3447                  * enabling user does not have permissions.
3448                  */
3449                 if (!DTRACE_INSCRATCH(mstate, size)) {
3450                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3451                         regs[rd] = NULL;
3452                         break;
3453                 }
3454 
3455                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3456                 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
3457                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3458 
3459                 ((char *)dest)[size - 1] = '\0';
3460                 mstate->dtms_scratch_ptr += size;
3461                 regs[rd] = dest;
3462                 break;
3463         }
3464 
3465         case DIF_SUBR_MSGSIZE:
3466         case DIF_SUBR_MSGDSIZE: {
3467                 uintptr_t baddr = tupregs[0].dttk_value, daddr;
3468                 uintptr_t wptr, rptr;
3469                 size_t count = 0;
3470                 int cont = 0;
3471 
3472                 while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
3473 
3474                         if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
3475                             vstate)) {
3476                                 regs[rd] = NULL;
3477                                 break;
3478                         }
3479 
3480                         wptr = dtrace_loadptr(baddr +
3481                             offsetof(mblk_t, b_wptr));
3482 
3483                         rptr = dtrace_loadptr(baddr +
3484                             offsetof(mblk_t, b_rptr));
3485 
3486                         if (wptr < rptr) {
3487                                 *flags |= CPU_DTRACE_BADADDR;
3488                                 *illval = tupregs[0].dttk_value;
3489                                 break;
3490                         }
3491 
3492                         daddr = dtrace_loadptr(baddr +
3493                             offsetof(mblk_t, b_datap));
3494 
3495                         baddr = dtrace_loadptr(baddr +
3496                             offsetof(mblk_t, b_cont));
3497 
3498                         /*
3499                          * We want to prevent against denial-of-service here,
3500                          * so we're only going to search the list for
3501                          * dtrace_msgdsize_max mblks.
3502                          */
3503                         if (cont++ > dtrace_msgdsize_max) {
3504                                 *flags |= CPU_DTRACE_ILLOP;
3505                                 break;
3506                         }
3507 
3508                         if (subr == DIF_SUBR_MSGDSIZE) {
3509                                 if (dtrace_load8(daddr +
3510                                     offsetof(dblk_t, db_type)) != M_DATA)
3511                                         continue;
3512                         }
3513 
3514                         count += wptr - rptr;
3515                 }
3516 
3517                 if (!(*flags & CPU_DTRACE_FAULT))
3518                         regs[rd] = count;
3519 
3520                 break;
3521         }
3522 
3523         case DIF_SUBR_PROGENYOF: {
3524                 pid_t pid = tupregs[0].dttk_value;
3525                 proc_t *p;
3526                 int rval = 0;
3527 
3528                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3529 
3530                 for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
3531                         if (p->p_pidp->pid_id == pid) {
3532                                 rval = 1;
3533                                 break;
3534                         }
3535                 }
3536 
3537                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3538 
3539                 regs[rd] = rval;
3540                 break;
3541         }
3542 
3543         case DIF_SUBR_SPECULATION:
3544                 regs[rd] = dtrace_speculation(state);
3545                 break;
3546 
3547         case DIF_SUBR_COPYOUT: {
3548                 uintptr_t kaddr = tupregs[0].dttk_value;
3549                 uintptr_t uaddr = tupregs[1].dttk_value;
3550                 uint64_t size = tupregs[2].dttk_value;
3551 
3552                 if (!dtrace_destructive_disallow &&
3553                     dtrace_priv_proc_control(state, mstate) &&
3554                     !dtrace_istoxic(kaddr, size)) {
3555                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3556                         dtrace_copyout(kaddr, uaddr, size, flags);
3557                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3558                 }
3559                 break;
3560         }
3561 
3562         case DIF_SUBR_COPYOUTSTR: {
3563                 uintptr_t kaddr = tupregs[0].dttk_value;
3564                 uintptr_t uaddr = tupregs[1].dttk_value;
3565                 uint64_t size = tupregs[2].dttk_value;
3566 
3567                 if (!dtrace_destructive_disallow &&
3568                     dtrace_priv_proc_control(state, mstate) &&
3569                     !dtrace_istoxic(kaddr, size)) {
3570                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3571                         dtrace_copyoutstr(kaddr, uaddr, size, flags);
3572                         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3573                 }
3574                 break;
3575         }
3576 
3577         case DIF_SUBR_STRLEN: {
3578                 size_t sz;
3579                 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
3580                 sz = dtrace_strlen((char *)addr,
3581                     state->dts_options[DTRACEOPT_STRSIZE]);
3582 
3583                 if (!dtrace_canload(addr, sz + 1, mstate, vstate)) {
3584                         regs[rd] = NULL;
3585                         break;
3586                 }
3587 
3588                 regs[rd] = sz;
3589 
3590                 break;
3591         }
3592 
3593         case DIF_SUBR_STRCHR:
3594         case DIF_SUBR_STRRCHR: {
3595                 /*
3596                  * We're going to iterate over the string looking for the
3597                  * specified character.  We will iterate until we have reached
3598                  * the string length or we have found the character.  If this
3599                  * is DIF_SUBR_STRRCHR, we will look for the last occurrence
3600                  * of the specified character instead of the first.
3601                  */
3602                 uintptr_t saddr = tupregs[0].dttk_value;
3603                 uintptr_t addr = tupregs[0].dttk_value;
3604                 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
3605                 char c, target = (char)tupregs[1].dttk_value;
3606 
3607                 for (regs[rd] = NULL; addr < limit; addr++) {
3608                         if ((c = dtrace_load8(addr)) == target) {
3609                                 regs[rd] = addr;
3610 
3611                                 if (subr == DIF_SUBR_STRCHR)
3612                                         break;
3613                         }
3614 
3615                         if (c == '\0')
3616                                 break;
3617                 }
3618 
3619                 if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) {
3620                         regs[rd] = NULL;
3621                         break;
3622                 }
3623 
3624                 break;
3625         }
3626 
3627         case DIF_SUBR_STRSTR:
3628         case DIF_SUBR_INDEX:
3629         case DIF_SUBR_RINDEX: {
3630                 /*
3631                  * We're going to iterate over the string looking for the
3632                  * specified string.  We will iterate until we have reached
3633                  * the string length or we have found the string.  (Yes, this
3634                  * is done in the most naive way possible -- but considering
3635                  * that the string we're searching for is likely to be
3636                  * relatively short, the complexity of Rabin-Karp or similar
3637                  * hardly seems merited.)
3638                  */
3639                 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
3640                 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
3641                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3642                 size_t len = dtrace_strlen(addr, size);
3643                 size_t sublen = dtrace_strlen(substr, size);
3644                 char *limit = addr + len, *orig = addr;
3645                 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
3646                 int inc = 1;
3647 
3648                 regs[rd] = notfound;
3649 
3650                 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
3651                         regs[rd] = NULL;
3652                         break;
3653                 }
3654 
3655                 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
3656                     vstate)) {
3657                         regs[rd] = NULL;
3658                         break;
3659                 }
3660 
3661                 /*
3662                  * strstr() and index()/rindex() have similar semantics if
3663                  * both strings are the empty string: strstr() returns a
3664                  * pointer to the (empty) string, and index() and rindex()
3665                  * both return index 0 (regardless of any position argument).
3666                  */
3667                 if (sublen == 0 && len == 0) {
3668                         if (subr == DIF_SUBR_STRSTR)
3669                                 regs[rd] = (uintptr_t)addr;
3670                         else
3671                                 regs[rd] = 0;
3672                         break;
3673                 }
3674 
3675                 if (subr != DIF_SUBR_STRSTR) {
3676                         if (subr == DIF_SUBR_RINDEX) {
3677                                 limit = orig - 1;
3678                                 addr += len;
3679                                 inc = -1;
3680                         }
3681 
3682                         /*
3683                          * Both index() and rindex() take an optional position
3684                          * argument that denotes the starting position.
3685                          */
3686                         if (nargs == 3) {
3687                                 int64_t pos = (int64_t)tupregs[2].dttk_value;
3688 
3689                                 /*
3690                                  * If the position argument to index() is
3691                                  * negative, Perl implicitly clamps it at
3692                                  * zero.  This semantic is a little surprising
3693                                  * given the special meaning of negative
3694                                  * positions to similar Perl functions like
3695                                  * substr(), but it appears to reflect a
3696                                  * notion that index() can start from a
3697                                  * negative index and increment its way up to
3698                                  * the string.  Given this notion, Perl's
3699                                  * rindex() is at least self-consistent in
3700                                  * that it implicitly clamps positions greater
3701                                  * than the string length to be the string
3702                                  * length.  Where Perl completely loses
3703                                  * coherence, however, is when the specified
3704                                  * substring is the empty string ("").  In
3705                                  * this case, even if the position is
3706                                  * negative, rindex() returns 0 -- and even if
3707                                  * the position is greater than the length,
3708                                  * index() returns the string length.  These
3709                                  * semantics violate the notion that index()
3710                                  * should never return a value less than the
3711                                  * specified position and that rindex() should
3712                                  * never return a value greater than the
3713                                  * specified position.  (One assumes that
3714                                  * these semantics are artifacts of Perl's
3715                                  * implementation and not the results of
3716                                  * deliberate design -- it beggars belief that
3717                                  * even Larry Wall could desire such oddness.)
3718                                  * While in the abstract one would wish for
3719                                  * consistent position semantics across
3720                                  * substr(), index() and rindex() -- or at the
3721                                  * very least self-consistent position
3722                                  * semantics for index() and rindex() -- we
3723                                  * instead opt to keep with the extant Perl
3724                                  * semantics, in all their broken glory.  (Do
3725                                  * we have more desire to maintain Perl's
3726                                  * semantics than Perl does?  Probably.)
3727                                  */
3728                                 if (subr == DIF_SUBR_RINDEX) {
3729                                         if (pos < 0) {
3730                                                 if (sublen == 0)
3731                                                         regs[rd] = 0;
3732                                                 break;
3733                                         }
3734 
3735                                         if (pos > len)
3736                                                 pos = len;
3737                                 } else {
3738                                         if (pos < 0)
3739                                                 pos = 0;
3740 
3741                                         if (pos >= len) {
3742                                                 if (sublen == 0)
3743                                                         regs[rd] = len;
3744                                                 break;
3745                                         }
3746                                 }
3747 
3748                                 addr = orig + pos;
3749                         }
3750                 }
3751 
3752                 for (regs[rd] = notfound; addr != limit; addr += inc) {
3753                         if (dtrace_strncmp(addr, substr, sublen) == 0) {
3754                                 if (subr != DIF_SUBR_STRSTR) {
3755                                         /*
3756                                          * As D index() and rindex() are
3757                                          * modeled on Perl (and not on awk),
3758                                          * we return a zero-based (and not a
3759                                          * one-based) index.  (For you Perl
3760                                          * weenies: no, we're not going to add
3761                                          * $[ -- and shouldn't you be at a con
3762                                          * or something?)
3763                                          */
3764                                         regs[rd] = (uintptr_t)(addr - orig);
3765                                         break;
3766                                 }
3767 
3768                                 ASSERT(subr == DIF_SUBR_STRSTR);
3769                                 regs[rd] = (uintptr_t)addr;
3770                                 break;
3771                         }
3772                 }
3773 
3774                 break;
3775         }
3776 
3777         case DIF_SUBR_STRTOK: {
3778                 uintptr_t addr = tupregs[0].dttk_value;
3779                 uintptr_t tokaddr = tupregs[1].dttk_value;
3780                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3781                 uintptr_t limit, toklimit = tokaddr + size;
3782                 uint8_t c, tokmap[32];   /* 256 / 8 */
3783                 char *dest = (char *)mstate->dtms_scratch_ptr;
3784                 int i;
3785 
3786                 /*
3787                  * Check both the token buffer and (later) the input buffer,
3788                  * since both could be non-scratch addresses.
3789                  */
3790                 if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) {
3791                         regs[rd] = NULL;
3792                         break;
3793                 }
3794 
3795                 if (!DTRACE_INSCRATCH(mstate, size)) {
3796                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3797                         regs[rd] = NULL;
3798                         break;
3799                 }
3800 
3801                 if (addr == NULL) {
3802                         /*
3803                          * If the address specified is NULL, we use our saved
3804                          * strtok pointer from the mstate.  Note that this
3805                          * means that the saved strtok pointer is _only_
3806                          * valid within multiple enablings of the same probe --
3807                          * it behaves like an implicit clause-local variable.
3808                          */
3809                         addr = mstate->dtms_strtok;
3810                 } else {
3811                         /*
3812                          * If the user-specified address is non-NULL we must
3813                          * access check it.  This is the only time we have
3814                          * a chance to do so, since this address may reside
3815                          * in the string table of this clause-- future calls
3816                          * (when we fetch addr from mstate->dtms_strtok)
3817                          * would fail this access check.
3818                          */
3819                         if (!dtrace_strcanload(addr, size, mstate, vstate)) {
3820                                 regs[rd] = NULL;
3821                                 break;
3822                         }
3823                 }
3824 
3825                 /*
3826                  * First, zero the token map, and then process the token
3827                  * string -- setting a bit in the map for every character
3828                  * found in the token string.
3829                  */
3830                 for (i = 0; i < sizeof (tokmap); i++)
3831                         tokmap[i] = 0;
3832 
3833                 for (; tokaddr < toklimit; tokaddr++) {
3834                         if ((c = dtrace_load8(tokaddr)) == '\0')
3835                                 break;
3836 
3837                         ASSERT((c >> 3) < sizeof (tokmap));
3838                         tokmap[c >> 3] |= (1 << (c & 0x7));
3839                 }
3840 
3841                 for (limit = addr + size; addr < limit; addr++) {
3842                         /*
3843                          * We're looking for a character that is _not_ contained
3844                          * in the token string.
3845                          */
3846                         if ((c = dtrace_load8(addr)) == '\0')
3847                                 break;
3848 
3849                         if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
3850                                 break;
3851                 }
3852 
3853                 if (c == '\0') {
3854                         /*
3855                          * We reached the end of the string without finding
3856                          * any character that was not in the token string.
3857                          * We return NULL in this case, and we set the saved
3858                          * address to NULL as well.
3859                          */
3860                         regs[rd] = NULL;
3861                         mstate->dtms_strtok = NULL;
3862                         break;
3863                 }
3864 
3865                 /*
3866                  * From here on, we're copying into the destination string.
3867                  */
3868                 for (i = 0; addr < limit && i < size - 1; addr++) {
3869                         if ((c = dtrace_load8(addr)) == '\0')
3870                                 break;
3871 
3872                         if (tokmap[c >> 3] & (1 << (c & 0x7)))
3873                                 break;
3874 
3875                         ASSERT(i < size);
3876                         dest[i++] = c;
3877                 }
3878 
3879                 ASSERT(i < size);
3880                 dest[i] = '\0';
3881                 regs[rd] = (uintptr_t)dest;
3882                 mstate->dtms_scratch_ptr += size;
3883                 mstate->dtms_strtok = addr;
3884                 break;
3885         }
3886 
3887         case DIF_SUBR_SUBSTR: {
3888                 uintptr_t s = tupregs[0].dttk_value;
3889                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3890                 char *d = (char *)mstate->dtms_scratch_ptr;
3891                 int64_t index = (int64_t)tupregs[1].dttk_value;
3892                 int64_t remaining = (int64_t)tupregs[2].dttk_value;
3893                 size_t len = dtrace_strlen((char *)s, size);
3894                 int64_t i;
3895 
3896                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
3897                         regs[rd] = NULL;
3898                         break;
3899                 }
3900 
3901                 if (!DTRACE_INSCRATCH(mstate, size)) {
3902                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3903                         regs[rd] = NULL;
3904                         break;
3905                 }
3906 
3907                 if (nargs <= 2)
3908                         remaining = (int64_t)size;
3909 
3910                 if (index < 0) {
3911                         index += len;
3912 
3913                         if (index < 0 && index + remaining > 0) {
3914                                 remaining += index;
3915                                 index = 0;
3916                         }
3917                 }
3918 
3919                 if (index >= len || index < 0) {
3920                         remaining = 0;
3921                 } else if (remaining < 0) {
3922                         remaining += len - index;
3923                 } else if (index + remaining > size) {
3924                         remaining = size - index;
3925                 }
3926 
3927                 for (i = 0; i < remaining; i++) {
3928                         if ((d[i] = dtrace_load8(s + index + i)) == '\0')
3929                                 break;
3930                 }
3931 
3932                 d[i] = '\0';
3933 
3934                 mstate->dtms_scratch_ptr += size;
3935                 regs[rd] = (uintptr_t)d;
3936                 break;
3937         }
3938 
3939         case DIF_SUBR_TOUPPER:
3940         case DIF_SUBR_TOLOWER: {
3941                 uintptr_t s = tupregs[0].dttk_value;
3942                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3943                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
3944                 size_t len = dtrace_strlen((char *)s, size);
3945                 char lower, upper, convert;
3946                 int64_t i;
3947 
3948                 if (subr == DIF_SUBR_TOUPPER) {
3949                         lower = 'a';
3950                         upper = 'z';
3951                         convert = 'A';
3952                 } else {
3953                         lower = 'A';
3954                         upper = 'Z';
3955                         convert = 'a';
3956                 }
3957 
3958                 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
3959                         regs[rd] = NULL;
3960                         break;
3961                 }
3962 
3963                 if (!DTRACE_INSCRATCH(mstate, size)) {
3964                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3965                         regs[rd] = NULL;
3966                         break;
3967                 }
3968 
3969                 for (i = 0; i < size - 1; i++) {
3970                         if ((c = dtrace_load8(s + i)) == '\0')
3971                                 break;
3972 
3973                         if (c >= lower && c <= upper)
3974                                 c = convert + (c - lower);
3975 
3976                         dest[i] = c;
3977                 }
3978 
3979                 ASSERT(i < size);
3980                 dest[i] = '\0';
3981                 regs[rd] = (uintptr_t)dest;
3982                 mstate->dtms_scratch_ptr += size;
3983                 break;
3984         }
3985 
3986 case DIF_SUBR_GETMAJOR:
3987 #ifdef _LP64
3988                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
3989 #else
3990                 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
3991 #endif
3992                 break;
3993 
3994         case DIF_SUBR_GETMINOR:
3995 #ifdef _LP64
3996                 regs[rd] = tupregs[0].dttk_value & MAXMIN64;
3997 #else
3998                 regs[rd] = tupregs[0].dttk_value & MAXMIN;
3999 #endif
4000                 break;
4001 
4002         case DIF_SUBR_DDI_PATHNAME: {
4003                 /*
4004                  * This one is a galactic mess.  We are going to roughly
4005                  * emulate ddi_pathname(), but it's made more complicated
4006                  * by the fact that we (a) want to include the minor name and
4007                  * (b) must proceed iteratively instead of recursively.
4008                  */
4009                 uintptr_t dest = mstate->dtms_scratch_ptr;
4010                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4011                 char *start = (char *)dest, *end = start + size - 1;
4012                 uintptr_t daddr = tupregs[0].dttk_value;
4013                 int64_t minor = (int64_t)tupregs[1].dttk_value;
4014                 char *s;
4015                 int i, len, depth = 0;
4016 
4017                 /*
4018                  * Due to all the pointer jumping we do and context we must
4019                  * rely upon, we just mandate that the user must have kernel
4020                  * read privileges to use this routine.
4021                  */
4022                 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
4023                         *flags |= CPU_DTRACE_KPRIV;
4024                         *illval = daddr;
4025                         regs[rd] = NULL;
4026                 }
4027 
4028                 if (!DTRACE_INSCRATCH(mstate, size)) {
4029                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4030                         regs[rd] = NULL;
4031                         break;
4032                 }
4033 
4034                 *end = '\0';
4035 
4036                 /*
4037                  * We want to have a name for the minor.  In order to do this,
4038                  * we need to walk the minor list from the devinfo.  We want
4039                  * to be sure that we don't infinitely walk a circular list,
4040                  * so we check for circularity by sending a scout pointer
4041                  * ahead two elements for every element that we iterate over;
4042                  * if the list is circular, these will ultimately point to the
4043                  * same element.  You may recognize this little trick as the
4044                  * answer to a stupid interview question -- one that always
4045                  * seems to be asked by those who had to have it laboriously
4046                  * explained to them, and who can't even concisely describe
4047                  * the conditions under which one would be forced to resort to
4048                  * this technique.  Needless to say, those conditions are
4049                  * found here -- and probably only here.  Is this the only use
4050                  * of this infamous trick in shipping, production code?  If it
4051                  * isn't, it probably should be...
4052                  */
4053                 if (minor != -1) {
4054                         uintptr_t maddr = dtrace_loadptr(daddr +
4055                             offsetof(struct dev_info, devi_minor));
4056 
4057                         uintptr_t next = offsetof(struct ddi_minor_data, next);
4058                         uintptr_t name = offsetof(struct ddi_minor_data,
4059                             d_minor) + offsetof(struct ddi_minor, name);
4060                         uintptr_t dev = offsetof(struct ddi_minor_data,
4061                             d_minor) + offsetof(struct ddi_minor, dev);
4062                         uintptr_t scout;
4063 
4064                         if (maddr != NULL)
4065                                 scout = dtrace_loadptr(maddr + next);
4066 
4067                         while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4068                                 uint64_t m;
4069 #ifdef _LP64
4070                                 m = dtrace_load64(maddr + dev) & MAXMIN64;
4071 #else
4072                                 m = dtrace_load32(maddr + dev) & MAXMIN;
4073 #endif
4074                                 if (m != minor) {
4075                                         maddr = dtrace_loadptr(maddr + next);
4076 
4077                                         if (scout == NULL)
4078                                                 continue;
4079 
4080                                         scout = dtrace_loadptr(scout + next);
4081 
4082                                         if (scout == NULL)
4083                                                 continue;
4084 
4085                                         scout = dtrace_loadptr(scout + next);
4086 
4087                                         if (scout == NULL)
4088                                                 continue;
4089 
4090                                         if (scout == maddr) {
4091                                                 *flags |= CPU_DTRACE_ILLOP;
4092                                                 break;
4093                                         }
4094 
4095                                         continue;
4096                                 }
4097 
4098                                 /*
4099                                  * We have the minor data.  Now we need to
4100                                  * copy the minor's name into the end of the
4101                                  * pathname.
4102                                  */
4103                                 s = (char *)dtrace_loadptr(maddr + name);
4104                                 len = dtrace_strlen(s, size);
4105 
4106                                 if (*flags & CPU_DTRACE_FAULT)
4107                                         break;
4108 
4109                                 if (len != 0) {
4110                                         if ((end -= (len + 1)) < start)
4111                                                 break;
4112 
4113                                         *end = ':';
4114                                 }
4115 
4116                                 for (i = 1; i <= len; i++)
4117                                         end[i] = dtrace_load8((uintptr_t)s++);
4118                                 break;
4119                         }
4120                 }
4121 
4122                 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4123                         ddi_node_state_t devi_state;
4124 
4125                         devi_state = dtrace_load32(daddr +
4126                             offsetof(struct dev_info, devi_node_state));
4127 
4128                         if (*flags & CPU_DTRACE_FAULT)
4129                                 break;
4130 
4131                         if (devi_state >= DS_INITIALIZED) {
4132                                 s = (char *)dtrace_loadptr(daddr +
4133                                     offsetof(struct dev_info, devi_addr));
4134                                 len = dtrace_strlen(s, size);
4135 
4136                                 if (*flags & CPU_DTRACE_FAULT)
4137                                         break;
4138 
4139                                 if (len != 0) {
4140                                         if ((end -= (len + 1)) < start)
4141                                                 break;
4142 
4143                                         *end = '@';
4144                                 }
4145 
4146                                 for (i = 1; i <= len; i++)
4147                                         end[i] = dtrace_load8((uintptr_t)s++);
4148                         }
4149 
4150                         /*
4151                          * Now for the node name...
4152                          */
4153                         s = (char *)dtrace_loadptr(daddr +
4154                             offsetof(struct dev_info, devi_node_name));
4155 
4156                         daddr = dtrace_loadptr(daddr +
4157                             offsetof(struct dev_info, devi_parent));
4158 
4159                         /*
4160                          * If our parent is NULL (that is, if we're the root
4161                          * node), we're going to use the special path
4162                          * "devices".
4163                          */
4164                         if (daddr == NULL)
4165                                 s = "devices";
4166 
4167                         len = dtrace_strlen(s, size);
4168                         if (*flags & CPU_DTRACE_FAULT)
4169                                 break;
4170 
4171                         if ((end -= (len + 1)) < start)
4172                                 break;
4173 
4174                         for (i = 1; i <= len; i++)
4175                                 end[i] = dtrace_load8((uintptr_t)s++);
4176                         *end = '/';
4177 
4178                         if (depth++ > dtrace_devdepth_max) {
4179                                 *flags |= CPU_DTRACE_ILLOP;
4180                                 break;
4181                         }
4182                 }
4183 
4184                 if (end < start)
4185                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4186 
4187                 if (daddr == NULL) {
4188                         regs[rd] = (uintptr_t)end;
4189                         mstate->dtms_scratch_ptr += size;
4190                 }
4191 
4192                 break;
4193         }
4194 
4195         case DIF_SUBR_STRJOIN: {
4196                 char *d = (char *)mstate->dtms_scratch_ptr;
4197                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4198                 uintptr_t s1 = tupregs[0].dttk_value;
4199                 uintptr_t s2 = tupregs[1].dttk_value;
4200                 int i = 0;
4201 
4202                 if (!dtrace_strcanload(s1, size, mstate, vstate) ||
4203                     !dtrace_strcanload(s2, size, mstate, vstate)) {
4204                         regs[rd] = NULL;
4205                         break;
4206                 }
4207 
4208                 if (!DTRACE_INSCRATCH(mstate, size)) {
4209                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4210                         regs[rd] = NULL;
4211                         break;
4212                 }
4213 
4214                 for (;;) {
4215                         if (i >= size) {
4216                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4217                                 regs[rd] = NULL;
4218                                 break;
4219                         }
4220 
4221                         if ((d[i++] = dtrace_load8(s1++)) == '\0') {
4222                                 i--;
4223                                 break;
4224                         }
4225                 }
4226 
4227                 for (;;) {
4228                         if (i >= size) {
4229                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4230                                 regs[rd] = NULL;
4231                                 break;
4232                         }
4233 
4234                         if ((d[i++] = dtrace_load8(s2++)) == '\0')
4235                                 break;
4236                 }
4237 
4238                 if (i < size) {
4239                         mstate->dtms_scratch_ptr += i;
4240                         regs[rd] = (uintptr_t)d;
4241                 }
4242 
4243                 break;
4244         }
4245 
4246         case DIF_SUBR_LLTOSTR: {
4247                 int64_t i = (int64_t)tupregs[0].dttk_value;
4248                 uint64_t val, digit;
4249                 uint64_t size = 65;     /* enough room for 2^64 in binary */
4250                 char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
4251                 int base = 10;
4252 
4253                 if (nargs > 1) {
4254                         if ((base = tupregs[1].dttk_value) <= 1 ||
4255                             base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4256                                 *flags |= CPU_DTRACE_ILLOP;
4257                                 break;
4258                         }
4259                 }
4260 
4261                 val = (base == 10 && i < 0) ? i * -1 : i;
4262 
4263                 if (!DTRACE_INSCRATCH(mstate, size)) {
4264                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4265                         regs[rd] = NULL;
4266                         break;
4267                 }
4268 
4269                 for (*end-- = '\0'; val; val /= base) {
4270                         if ((digit = val % base) <= '9' - '0') {
4271                                 *end-- = '0' + digit;
4272                         } else {
4273                                 *end-- = 'a' + (digit - ('9' - '0') - 1);
4274                         }
4275                 }
4276 
4277                 if (i == 0 && base == 16)
4278                         *end-- = '0';
4279 
4280                 if (base == 16)
4281                         *end-- = 'x';
4282 
4283                 if (i == 0 || base == 8 || base == 16)
4284                         *end-- = '0';
4285 
4286                 if (i < 0 && base == 10)
4287                         *end-- = '-';
4288 
4289                 regs[rd] = (uintptr_t)end + 1;
4290                 mstate->dtms_scratch_ptr += size;
4291                 break;
4292         }
4293 
4294         case DIF_SUBR_HTONS:
4295         case DIF_SUBR_NTOHS:
4296 #ifdef _BIG_ENDIAN
4297                 regs[rd] = (uint16_t)tupregs[0].dttk_value;
4298 #else
4299                 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
4300 #endif
4301                 break;
4302 
4303 
4304         case DIF_SUBR_HTONL:
4305         case DIF_SUBR_NTOHL:
4306 #ifdef _BIG_ENDIAN
4307                 regs[rd] = (uint32_t)tupregs[0].dttk_value;
4308 #else
4309                 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
4310 #endif
4311                 break;
4312 
4313 
4314         case DIF_SUBR_HTONLL:
4315         case DIF_SUBR_NTOHLL:
4316 #ifdef _BIG_ENDIAN
4317                 regs[rd] = (uint64_t)tupregs[0].dttk_value;
4318 #else
4319                 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
4320 #endif
4321                 break;
4322 
4323 
4324         case DIF_SUBR_DIRNAME:
4325         case DIF_SUBR_BASENAME: {
4326                 char *dest = (char *)mstate->dtms_scratch_ptr;
4327                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4328                 uintptr_t src = tupregs[0].dttk_value;
4329                 int i, j, len = dtrace_strlen((char *)src, size);
4330                 int lastbase = -1, firstbase = -1, lastdir = -1;
4331                 int start, end;
4332 
4333                 if (!dtrace_canload(src, len + 1, mstate, vstate)) {
4334                         regs[rd] = NULL;
4335                         break;
4336                 }
4337 
4338                 if (!DTRACE_INSCRATCH(mstate, size)) {
4339                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4340                         regs[rd] = NULL;
4341                         break;
4342                 }
4343 
4344                 /*
4345                  * The basename and dirname for a zero-length string is
4346                  * defined to be "."
4347                  */
4348                 if (len == 0) {
4349                         len = 1;
4350                         src = (uintptr_t)".";
4351                 }
4352 
4353                 /*
4354                  * Start from the back of the string, moving back toward the
4355                  * front until we see a character that isn't a slash.  That
4356                  * character is the last character in the basename.
4357                  */
4358                 for (i = len - 1; i >= 0; i--) {
4359                         if (dtrace_load8(src + i) != '/')
4360                                 break;
4361                 }
4362 
4363                 if (i >= 0)
4364                         lastbase = i;
4365 
4366                 /*
4367                  * Starting from the last character in the basename, move
4368                  * towards the front until we find a slash.  The character
4369                  * that we processed immediately before that is the first
4370                  * character in the basename.
4371                  */
4372                 for (; i >= 0; i--) {
4373                         if (dtrace_load8(src + i) == '/')
4374                                 break;
4375                 }
4376 
4377                 if (i >= 0)
4378                         firstbase = i + 1;
4379 
4380                 /*
4381                  * Now keep going until we find a non-slash character.  That
4382                  * character is the last character in the dirname.
4383                  */
4384                 for (; i >= 0; i--) {
4385                         if (dtrace_load8(src + i) != '/')
4386                                 break;
4387                 }
4388 
4389                 if (i >= 0)
4390                         lastdir = i;
4391 
4392                 ASSERT(!(lastbase == -1 && firstbase != -1));
4393                 ASSERT(!(firstbase == -1 && lastdir != -1));
4394 
4395                 if (lastbase == -1) {
4396                         /*
4397                          * We didn't find a non-slash character.  We know that
4398                          * the length is non-zero, so the whole string must be
4399                          * slashes.  In either the dirname or the basename
4400                          * case, we return '/'.
4401                          */
4402                         ASSERT(firstbase == -1);
4403                         firstbase = lastbase = lastdir = 0;
4404                 }
4405 
4406                 if (firstbase == -1) {
4407                         /*
4408                          * The entire string consists only of a basename
4409                          * component.  If we're looking for dirname, we need
4410                          * to change our string to be just "."; if we're
4411                          * looking for a basename, we'll just set the first
4412                          * character of the basename to be 0.
4413                          */
4414                         if (subr == DIF_SUBR_DIRNAME) {
4415                                 ASSERT(lastdir == -1);
4416                                 src = (uintptr_t)".";
4417                                 lastdir = 0;
4418                         } else {
4419                                 firstbase = 0;
4420                         }
4421                 }
4422 
4423                 if (subr == DIF_SUBR_DIRNAME) {
4424                         if (lastdir == -1) {
4425                                 /*
4426                                  * We know that we have a slash in the name --
4427                                  * or lastdir would be set to 0, above.  And
4428                                  * because lastdir is -1, we know that this
4429                                  * slash must be the first character.  (That
4430                                  * is, the full string must be of the form
4431                                  * "/basename".)  In this case, the last
4432                                  * character of the directory name is 0.
4433                                  */
4434                                 lastdir = 0;
4435                         }
4436 
4437                         start = 0;
4438                         end = lastdir;
4439                 } else {
4440                         ASSERT(subr == DIF_SUBR_BASENAME);
4441                         ASSERT(firstbase != -1 && lastbase != -1);
4442                         start = firstbase;
4443                         end = lastbase;
4444                 }
4445 
4446                 for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
4447                         dest[j] = dtrace_load8(src + i);
4448 
4449                 dest[j] = '\0';
4450                 regs[rd] = (uintptr_t)dest;
4451                 mstate->dtms_scratch_ptr += size;
4452                 break;
4453         }
4454 
4455         case DIF_SUBR_CLEANPATH: {
4456                 char *dest = (char *)mstate->dtms_scratch_ptr, c;
4457                 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4458                 uintptr_t src = tupregs[0].dttk_value;
4459                 int i = 0, j = 0;
4460 
4461                 if (!dtrace_strcanload(src, size, mstate, vstate)) {
4462                         regs[rd] = NULL;
4463                         break;
4464                 }
4465 
4466                 if (!DTRACE_INSCRATCH(mstate, size)) {
4467                         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4468                         regs[rd] = NULL;
4469                         break;
4470                 }
4471 
4472                 /*
4473                  * Move forward, loading each character.
4474                  */
4475                 do {
4476                         c = dtrace_load8(src + i++);
4477 next:
4478                         if (j + 5 >= size)   /* 5 = strlen("/..c\0") */
4479                                 break;
4480 
4481                         if (c != '/') {
4482                                 dest[j++] = c;
4483                                 continue;
4484                         }
4485 
4486                         c = dtrace_load8(src + i++);
4487 
4488                         if (c == '/') {
4489                                 /*
4490                                  * We have two slashes -- we can just advance
4491                                  * to the next character.
4492                                  */
4493                                 goto next;
4494                         }
4495 
4496                         if (c != '.') {
4497                                 /*
4498                                  * This is not "." and it's not ".." -- we can
4499                                  * just store the "/" and this character and
4500                                  * drive on.
4501                                  */
4502                                 dest[j++] = '/';
4503                                 dest[j++] = c;
4504                                 continue;
4505                         }
4506 
4507                         c = dtrace_load8(src + i++);
4508 
4509                         if (c == '/') {
4510                                 /*
4511                                  * This is a "/./" component.  We're not going
4512                                  * to store anything in the destination buffer;
4513                                  * we're just going to go to the next component.
4514                                  */
4515                                 goto next;
4516                         }
4517 
4518                         if (c != '.') {
4519                                 /*
4520                                  * This is not ".." -- we can just store the
4521                                  * "/." and this character and continue
4522                                  * processing.
4523                                  */
4524                                 dest[j++] = '/';
4525                                 dest[j++] = '.';
4526                                 dest[j++] = c;
4527                                 continue;
4528                         }
4529 
4530                         c = dtrace_load8(src + i++);
4531 
4532                         if (c != '/' && c != '\0') {
4533                                 /*
4534                                  * This is not ".." -- it's "..[mumble]".
4535                                  * We'll store the "/.." and this character
4536                                  * and continue processing.
4537                                  */
4538                                 dest[j++] = '/';
4539                                 dest[j++] = '.';
4540                                 dest[j++] = '.';
4541                                 dest[j++] = c;
4542                                 continue;
4543                         }
4544 
4545                         /*
4546                          * This is "/../" or "/..\0".  We need to back up
4547                          * our destination pointer until we find a "/".
4548                          */
4549                         i--;
4550                         while (j != 0 && dest[--j] != '/')
4551                                 continue;
4552 
4553                         if (c == '\0')
4554                                 dest[++j] = '/';
4555                 } while (c != '\0');
4556 
4557                 dest[j] = '\0';
4558                 regs[rd] = (uintptr_t)dest;
4559                 mstate->dtms_scratch_ptr += size;
4560                 break;
4561         }
4562 
4563         case DIF_SUBR_INET_NTOA:
4564         case DIF_SUBR_INET_NTOA6:
4565         case DIF_SUBR_INET_NTOP: {
4566                 size_t size;
4567                 int af, argi, i;
4568                 char *base, *end;
4569 
4570                 if (subr == DIF_SUBR_INET_NTOP) {
4571                         af = (int)tupregs[0].dttk_value;
4572                         argi = 1;
4573                 } else {
4574                         af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
4575                         argi = 0;
4576                 }
4577 
4578                 if (af == AF_INET) {
4579                         ipaddr_t ip4;
4580                         uint8_t *ptr8, val;
4581 
4582                         /*
4583                          * Safely load the IPv4 address.
4584                          */
4585                         ip4 = dtrace_load32(tupregs[argi].dttk_value);
4586 
4587                         /*
4588                          * Check an IPv4 string will fit in scratch.
4589                          */
4590                         size = INET_ADDRSTRLEN;
4591                         if (!DTRACE_INSCRATCH(mstate, size)) {
4592                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4593                                 regs[rd] = NULL;
4594                                 break;
4595                         }
4596                         base = (char *)mstate->dtms_scratch_ptr;
4597                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
4598 
4599                         /*
4600                          * Stringify as a dotted decimal quad.
4601                          */
4602                         *end-- = '\0';
4603                         ptr8 = (uint8_t *)&ip4;
4604                         for (i = 3; i >= 0; i--) {
4605                                 val = ptr8[i];
4606 
4607                                 if (val == 0) {
4608                                         *end-- = '0';
4609                                 } else {
4610                                         for (; val; val /= 10) {
4611                                                 *end-- = '0' + (val % 10);
4612                                         }
4613                                 }
4614 
4615                                 if (i > 0)
4616                                         *end-- = '.';
4617                         }
4618                         ASSERT(end + 1 >= base);
4619 
4620                 } else if (af == AF_INET6) {
4621                         struct in6_addr ip6;
4622                         int firstzero, tryzero, numzero, v6end;
4623                         uint16_t val;
4624                         const char digits[] = "0123456789abcdef";
4625 
4626                         /*
4627                          * Stringify using RFC 1884 convention 2 - 16 bit
4628                          * hexadecimal values with a zero-run compression.
4629                          * Lower case hexadecimal digits are used.
4630                          *      eg, fe80::214:4fff:fe0b:76c8.
4631                          * The IPv4 embedded form is returned for inet_ntop,
4632                          * just the IPv4 string is returned for inet_ntoa6.
4633                          */
4634 
4635                         /*
4636                          * Safely load the IPv6 address.
4637                          */
4638                         dtrace_bcopy(
4639                             (void *)(uintptr_t)tupregs[argi].dttk_value,
4640                             (void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
4641 
4642                         /*
4643                          * Check an IPv6 string will fit in scratch.
4644                          */
4645                         size = INET6_ADDRSTRLEN;
4646                         if (!DTRACE_INSCRATCH(mstate, size)) {
4647                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4648                                 regs[rd] = NULL;
4649                                 break;
4650                         }
4651                         base = (char *)mstate->dtms_scratch_ptr;
4652                         end = (char *)mstate->dtms_scratch_ptr + size - 1;
4653                         *end-- = '\0';
4654 
4655                         /*
4656                          * Find the longest run of 16 bit zero values
4657                          * for the single allowed zero compression - "::".
4658                          */
4659                         firstzero = -1;
4660                         tryzero = -1;
4661                         numzero = 1;
4662                         for (i = 0; i < sizeof (struct in6_addr); i++) {
4663                                 if (ip6._S6_un._S6_u8[i] == 0 &&
4664                                     tryzero == -1 && i % 2 == 0) {
4665                                         tryzero = i;
4666                                         continue;
4667                                 }
4668 
4669                                 if (tryzero != -1 &&
4670                                     (ip6._S6_un._S6_u8[i] != 0 ||
4671                                     i == sizeof (struct in6_addr) - 1)) {
4672 
4673                                         if (i - tryzero <= numzero) {
4674                                                 tryzero = -1;
4675                                                 continue;
4676                                         }
4677 
4678                                         firstzero = tryzero;
4679                                         numzero = i - i % 2 - tryzero;
4680                                         tryzero = -1;
4681 
4682                                         if (ip6._S6_un._S6_u8[i] == 0 &&
4683                                             i == sizeof (struct in6_addr) - 1)
4684                                                 numzero += 2;
4685                                 }
4686                         }
4687                         ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
4688 
4689                         /*
4690                          * Check for an IPv4 embedded address.
4691                          */
4692                         v6end = sizeof (struct in6_addr) - 2;
4693                         if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
4694                             IN6_IS_ADDR_V4COMPAT(&ip6)) {
4695                                 for (i = sizeof (struct in6_addr) - 1;
4696                                     i >= DTRACE_V4MAPPED_OFFSET; i--) {
4697                                         ASSERT(end >= base);
4698 
4699                                         val = ip6._S6_un._S6_u8[i];
4700 
4701                                         if (val == 0) {
4702                                                 *end-- = '0';
4703                                         } else {
4704                                                 for (; val; val /= 10) {
4705                                                         *end-- = '0' + val % 10;
4706                                                 }
4707                                         }
4708 
4709                                         if (i > DTRACE_V4MAPPED_OFFSET)
4710                                                 *end-- = '.';
4711                                 }
4712 
4713                                 if (subr == DIF_SUBR_INET_NTOA6)
4714                                         goto inetout;
4715 
4716                                 /*
4717                                  * Set v6end to skip the IPv4 address that
4718                                  * we have already stringified.
4719                                  */
4720                                 v6end = 10;
4721                         }
4722 
4723                         /*
4724                          * Build the IPv6 string by working through the
4725                          * address in reverse.
4726                          */
4727                         for (i = v6end; i >= 0; i -= 2) {
4728                                 ASSERT(end >= base);
4729 
4730                                 if (i == firstzero + numzero - 2) {
4731                                         *end-- = ':';
4732                                         *end-- = ':';
4733                                         i -= numzero - 2;
4734                                         continue;
4735                                 }
4736 
4737                                 if (i < 14 && i != firstzero - 2)
4738                                         *end-- = ':';
4739 
4740                                 val = (ip6._S6_un._S6_u8[i] << 8) +
4741                                     ip6._S6_un._S6_u8[i + 1];
4742 
4743                                 if (val == 0) {
4744                                         *end-- = '0';
4745                                 } else {
4746                                         for (; val; val /= 16) {
4747                                                 *end-- = digits[val % 16];
4748                                         }
4749                                 }
4750                         }
4751                         ASSERT(end + 1 >= base);
4752 
4753                 } else {
4754                         /*
4755                          * The user didn't use AH_INET or AH_INET6.
4756                          */
4757                         DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
4758                         regs[rd] = NULL;
4759                         break;
4760                 }
4761 
4762 inetout:        regs[rd] = (uintptr_t)end + 1;
4763                 mstate->dtms_scratch_ptr += size;
4764                 break;
4765         }
4766 
4767         }
4768 }
4769 
4770 /*
4771  * Emulate the execution of DTrace IR instructions specified by the given
4772  * DIF object.  This function is deliberately void of assertions as all of
4773  * the necessary checks are handled by a call to dtrace_difo_validate().
4774  */
4775 static uint64_t
4776 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
4777     dtrace_vstate_t *vstate, dtrace_state_t *state)
4778 {
4779         const dif_instr_t *text = difo->dtdo_buf;
4780         const uint_t textlen = difo->dtdo_len;
4781         const char *strtab = difo->dtdo_strtab;
4782         const uint64_t *inttab = difo->dtdo_inttab;
4783 
4784         uint64_t rval = 0;
4785         dtrace_statvar_t *svar;
4786         dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
4787         dtrace_difv_t *v;
4788         volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
4789         volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
4790 
4791         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
4792         uint64_t regs[DIF_DIR_NREGS];
4793         uint64_t *tmp;
4794 
4795         uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
4796         int64_t cc_r;
4797         uint_t pc = 0, id, opc;
4798         uint8_t ttop = 0;
4799         dif_instr_t instr;
4800         uint_t r1, r2, rd;
4801 
4802         /*
4803          * We stash the current DIF object into the machine state: we need it
4804          * for subsequent access checking.
4805          */
4806         mstate->dtms_difo = difo;
4807 
4808         regs[DIF_REG_R0] = 0;           /* %r0 is fixed at zero */
4809 
4810         while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
4811                 opc = pc;
4812 
4813                 instr = text[pc++];
4814                 r1 = DIF_INSTR_R1(instr);
4815                 r2 = DIF_INSTR_R2(instr);
4816                 rd = DIF_INSTR_RD(instr);
4817 
4818                 switch (DIF_INSTR_OP(instr)) {
4819                 case DIF_OP_OR:
4820                         regs[rd] = regs[r1] | regs[r2];
4821                         break;
4822                 case DIF_OP_XOR:
4823                         regs[rd] = regs[r1] ^ regs[r2];
4824                         break;
4825                 case DIF_OP_AND:
4826                         regs[rd] = regs[r1] & regs[r2];
4827                         break;
4828                 case DIF_OP_SLL:
4829                         regs[rd] = regs[r1] << regs[r2];
4830                         break;
4831                 case DIF_OP_SRL:
4832                         regs[rd] = regs[r1] >> regs[r2];
4833                         break;
4834                 case DIF_OP_SUB:
4835                         regs[rd] = regs[r1] - regs[r2];
4836                         break;
4837                 case DIF_OP_ADD:
4838                         regs[rd] = regs[r1] + regs[r2];
4839                         break;
4840                 case DIF_OP_MUL:
4841                         regs[rd] = regs[r1] * regs[r2];
4842                         break;
4843                 case DIF_OP_SDIV:
4844                         if (regs[r2] == 0) {
4845                                 regs[rd] = 0;
4846                                 *flags |= CPU_DTRACE_DIVZERO;
4847                         } else {
4848                                 regs[rd] = (int64_t)regs[r1] /
4849                                     (int64_t)regs[r2];
4850                         }
4851                         break;
4852 
4853                 case DIF_OP_UDIV:
4854                         if (regs[r2] == 0) {
4855                                 regs[rd] = 0;
4856                                 *flags |= CPU_DTRACE_DIVZERO;
4857                         } else {
4858                                 regs[rd] = regs[r1] / regs[r2];
4859                         }
4860                         break;
4861 
4862                 case DIF_OP_SREM:
4863                         if (regs[r2] == 0) {
4864                                 regs[rd] = 0;
4865                                 *flags |= CPU_DTRACE_DIVZERO;
4866                         } else {
4867                                 regs[rd] = (int64_t)regs[r1] %
4868                                     (int64_t)regs[r2];
4869                         }
4870                         break;
4871 
4872                 case DIF_OP_UREM:
4873                         if (regs[r2] == 0) {
4874                                 regs[rd] = 0;
4875                                 *flags |= CPU_DTRACE_DIVZERO;
4876                         } else {
4877                                 regs[rd] = regs[r1] % regs[r2];
4878                         }
4879                         break;
4880 
4881                 case DIF_OP_NOT:
4882                         regs[rd] = ~regs[r1];
4883                         break;
4884                 case DIF_OP_MOV:
4885                         regs[rd] = regs[r1];
4886                         break;
4887                 case DIF_OP_CMP:
4888                         cc_r = regs[r1] - regs[r2];
4889                         cc_n = cc_r < 0;
4890                         cc_z = cc_r == 0;
4891                         cc_v = 0;
4892                         cc_c = regs[r1] < regs[r2];
4893                         break;
4894                 case DIF_OP_TST:
4895                         cc_n = cc_v = cc_c = 0;
4896                         cc_z = regs[r1] == 0;
4897                         break;
4898                 case DIF_OP_BA:
4899                         pc = DIF_INSTR_LABEL(instr);
4900                         break;
4901                 case DIF_OP_BE:
4902                         if (cc_z)
4903                                 pc = DIF_INSTR_LABEL(instr);
4904                         break;
4905                 case DIF_OP_BNE:
4906                         if (cc_z == 0)
4907                                 pc = DIF_INSTR_LABEL(instr);
4908                         break;
4909                 case DIF_OP_BG:
4910                         if ((cc_z | (cc_n ^ cc_v)) == 0)
4911                                 pc = DIF_INSTR_LABEL(instr);
4912                         break;
4913                 case DIF_OP_BGU:
4914                         if ((cc_c | cc_z) == 0)
4915                                 pc = DIF_INSTR_LABEL(instr);
4916                         break;
4917                 case DIF_OP_BGE:
4918                         if ((cc_n ^ cc_v) == 0)
4919                                 pc = DIF_INSTR_LABEL(instr);
4920                         break;
4921                 case DIF_OP_BGEU:
4922                         if (cc_c == 0)
4923                                 pc = DIF_INSTR_LABEL(instr);
4924                         break;
4925                 case DIF_OP_BL:
4926                         if (cc_n ^ cc_v)
4927                                 pc = DIF_INSTR_LABEL(instr);
4928                         break;
4929                 case DIF_OP_BLU:
4930                         if (cc_c)
4931                                 pc = DIF_INSTR_LABEL(instr);
4932                         break;
4933                 case DIF_OP_BLE:
4934                         if (cc_z | (cc_n ^ cc_v))
4935                                 pc = DIF_INSTR_LABEL(instr);
4936                         break;
4937                 case DIF_OP_BLEU:
4938                         if (cc_c | cc_z)
4939                                 pc = DIF_INSTR_LABEL(instr);
4940                         break;
4941                 case DIF_OP_RLDSB:
4942                         if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
4943                                 *flags |= CPU_DTRACE_KPRIV;
4944                                 *illval = regs[r1];
4945                                 break;
4946                         }
4947                         /*FALLTHROUGH*/
4948                 case DIF_OP_LDSB:
4949                         regs[rd] = (int8_t)dtrace_load8(regs[r1]);
4950                         break;
4951                 case DIF_OP_RLDSH:
4952                         if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
4953                                 *flags |= CPU_DTRACE_KPRIV;
4954                                 *illval = regs[r1];
4955                                 break;
4956                         }
4957                         /*FALLTHROUGH*/
4958                 case DIF_OP_LDSH:
4959                         regs[rd] = (int16_t)dtrace_load16(regs[r1]);
4960                         break;
4961                 case DIF_OP_RLDSW:
4962                         if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
4963                                 *flags |= CPU_DTRACE_KPRIV;
4964                                 *illval = regs[r1];
4965                                 break;
4966                         }
4967                         /*FALLTHROUGH*/
4968                 case DIF_OP_LDSW:
4969                         regs[rd] = (int32_t)dtrace_load32(regs[r1]);
4970                         break;
4971                 case DIF_OP_RLDUB:
4972                         if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
4973                                 *flags |= CPU_DTRACE_KPRIV;
4974                                 *illval = regs[r1];
4975                                 break;
4976                         }
4977                         /*FALLTHROUGH*/
4978                 case DIF_OP_LDUB:
4979                         regs[rd] = dtrace_load8(regs[r1]);
4980                         break;
4981                 case DIF_OP_RLDUH:
4982                         if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
4983                                 *flags |= CPU_DTRACE_KPRIV;
4984                                 *illval = regs[r1];
4985                                 break;
4986                         }
4987                         /*FALLTHROUGH*/
4988                 case DIF_OP_LDUH:
4989                         regs[rd] = dtrace_load16(regs[r1]);
4990                         break;
4991                 case DIF_OP_RLDUW:
4992                         if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
4993                                 *flags |= CPU_DTRACE_KPRIV;
4994                                 *illval = regs[r1];
4995                                 break;
4996                         }
4997                         /*FALLTHROUGH*/
4998                 case DIF_OP_LDUW:
4999                         regs[rd] = dtrace_load32(regs[r1]);
5000                         break;
5001                 case DIF_OP_RLDX:
5002                         if (!dtrace_canstore(regs[r1], 8, mstate, vstate)) {
5003                                 *flags |= CPU_DTRACE_KPRIV;
5004                                 *illval = regs[r1];
5005                                 break;
5006                         }
5007                         /*FALLTHROUGH*/
5008                 case DIF_OP_LDX:
5009                         regs[rd] = dtrace_load64(regs[r1]);
5010                         break;
5011                 case DIF_OP_ULDSB:
5012                         regs[rd] = (int8_t)
5013                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5014                         break;
5015                 case DIF_OP_ULDSH:
5016                         regs[rd] = (int16_t)
5017                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5018                         break;
5019                 case DIF_OP_ULDSW:
5020                         regs[rd] = (int32_t)
5021                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5022                         break;
5023                 case DIF_OP_ULDUB:
5024                         regs[rd] =
5025                             dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5026                         break;
5027                 case DIF_OP_ULDUH:
5028                         regs[rd] =
5029                             dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5030                         break;
5031                 case DIF_OP_ULDUW:
5032                         regs[rd] =
5033                             dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5034                         break;
5035                 case DIF_OP_ULDX:
5036                         regs[rd] =
5037                             dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5038                         break;
5039                 case DIF_OP_RET:
5040                         rval = regs[rd];
5041                         pc = textlen;
5042                         break;
5043                 case DIF_OP_NOP:
5044                         break;
5045                 case DIF_OP_SETX:
5046                         regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5047                         break;
5048                 case DIF_OP_SETS:
5049                         regs[rd] = (uint64_t)(uintptr_t)
5050                             (strtab + DIF_INSTR_STRING(instr));
5051                         break;
5052                 case DIF_OP_SCMP: {
5053                         size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5054                         uintptr_t s1 = regs[r1];
5055                         uintptr_t s2 = regs[r2];
5056 
5057                         if (s1 != NULL &&
5058                             !dtrace_strcanload(s1, sz, mstate, vstate))
5059                                 break;
5060                         if (s2 != NULL &&
5061                             !dtrace_strcanload(s2, sz, mstate, vstate))
5062                                 break;
5063 
5064                         cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz);
5065 
5066                         cc_n = cc_r < 0;
5067                         cc_z = cc_r == 0;
5068                         cc_v = cc_c = 0;
5069                         break;
5070                 }
5071                 case DIF_OP_LDGA:
5072                         regs[rd] = dtrace_dif_variable(mstate, state,
5073                             r1, regs[r2]);
5074                         break;
5075                 case DIF_OP_LDGS:
5076                         id = DIF_INSTR_VAR(instr);
5077 
5078                         if (id >= DIF_VAR_OTHER_UBASE) {
5079                                 uintptr_t a;
5080 
5081                                 id -= DIF_VAR_OTHER_UBASE;
5082                                 svar = vstate->dtvs_globals[id];
5083                                 ASSERT(svar != NULL);
5084                                 v = &svar->dtsv_var;
5085 
5086                                 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
5087                                         regs[rd] = svar->dtsv_data;
5088                                         break;
5089                                 }
5090 
5091                                 a = (uintptr_t)svar->dtsv_data;
5092 
5093                                 if (*(uint8_t *)a == UINT8_MAX) {
5094                                         /*
5095                                          * If the 0th byte is set to UINT8_MAX
5096                                          * then this is to be treated as a
5097                                          * reference to a NULL variable.
5098                                          */
5099                                         regs[rd] = NULL;
5100                                 } else {
5101                                         regs[rd] = a + sizeof (uint64_t);
5102                                 }
5103 
5104                                 break;
5105                         }
5106 
5107                         regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
5108                         break;
5109 
5110                 case DIF_OP_STGS:
5111                         id = DIF_INSTR_VAR(instr);
5112 
5113                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5114                         id -= DIF_VAR_OTHER_UBASE;
5115 
5116                         svar = vstate->dtvs_globals[id];
5117                         ASSERT(svar != NULL);
5118                         v = &svar->dtsv_var;
5119 
5120                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5121                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5122 
5123                                 ASSERT(a != NULL);
5124                                 ASSERT(svar->dtsv_size != 0);
5125 
5126                                 if (regs[rd] == NULL) {
5127                                         *(uint8_t *)a = UINT8_MAX;
5128                                         break;
5129                                 } else {
5130                                         *(uint8_t *)a = 0;
5131                                         a += sizeof (uint64_t);
5132                                 }
5133                                 if (!dtrace_vcanload(
5134                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5135                                     mstate, vstate))
5136                                         break;
5137 
5138                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5139                                     (void *)a, &v->dtdv_type);
5140                                 break;
5141                         }
5142 
5143                         svar->dtsv_data = regs[rd];
5144                         break;
5145 
5146                 case DIF_OP_LDTA:
5147                         /*
5148                          * There are no DTrace built-in thread-local arrays at
5149                          * present.  This opcode is saved for future work.
5150                          */
5151                         *flags |= CPU_DTRACE_ILLOP;
5152                         regs[rd] = 0;
5153                         break;
5154 
5155                 case DIF_OP_LDLS:
5156                         id = DIF_INSTR_VAR(instr);
5157 
5158                         if (id < DIF_VAR_OTHER_UBASE) {
5159                                 /*
5160                                  * For now, this has no meaning.
5161                                  */
5162                                 regs[rd] = 0;
5163                                 break;
5164                         }
5165 
5166                         id -= DIF_VAR_OTHER_UBASE;
5167 
5168                         ASSERT(id < vstate->dtvs_nlocals);
5169                         ASSERT(vstate->dtvs_locals != NULL);
5170 
5171                         svar = vstate->dtvs_locals[id];
5172                         ASSERT(svar != NULL);
5173                         v = &svar->dtsv_var;
5174 
5175                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5176                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5177                                 size_t sz = v->dtdv_type.dtdt_size;
5178 
5179                                 sz += sizeof (uint64_t);
5180                                 ASSERT(svar->dtsv_size == NCPU * sz);
5181                                 a += CPU->cpu_id * sz;
5182 
5183                                 if (*(uint8_t *)a == UINT8_MAX) {
5184                                         /*
5185                                          * If the 0th byte is set to UINT8_MAX
5186                                          * then this is to be treated as a
5187                                          * reference to a NULL variable.
5188                                          */
5189                                         regs[rd] = NULL;
5190                                 } else {
5191                                         regs[rd] = a + sizeof (uint64_t);
5192                                 }
5193 
5194                                 break;
5195                         }
5196 
5197                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5198                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5199                         regs[rd] = tmp[CPU->cpu_id];
5200                         break;
5201 
5202                 case DIF_OP_STLS:
5203                         id = DIF_INSTR_VAR(instr);
5204 
5205                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5206                         id -= DIF_VAR_OTHER_UBASE;
5207                         ASSERT(id < vstate->dtvs_nlocals);
5208 
5209                         ASSERT(vstate->dtvs_locals != NULL);
5210                         svar = vstate->dtvs_locals[id];
5211                         ASSERT(svar != NULL);
5212                         v = &svar->dtsv_var;
5213 
5214                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5215                                 uintptr_t a = (uintptr_t)svar->dtsv_data;
5216                                 size_t sz = v->dtdv_type.dtdt_size;
5217 
5218                                 sz += sizeof (uint64_t);
5219                                 ASSERT(svar->dtsv_size == NCPU * sz);
5220                                 a += CPU->cpu_id * sz;
5221 
5222                                 if (regs[rd] == NULL) {
5223                                         *(uint8_t *)a = UINT8_MAX;
5224                                         break;
5225                                 } else {
5226                                         *(uint8_t *)a = 0;
5227                                         a += sizeof (uint64_t);
5228                                 }
5229 
5230                                 if (!dtrace_vcanload(
5231                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5232                                     mstate, vstate))
5233                                         break;
5234 
5235                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5236                                     (void *)a, &v->dtdv_type);
5237                                 break;
5238                         }
5239 
5240                         ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5241                         tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5242                         tmp[CPU->cpu_id] = regs[rd];
5243                         break;
5244 
5245                 case DIF_OP_LDTS: {
5246                         dtrace_dynvar_t *dvar;
5247                         dtrace_key_t *key;
5248 
5249                         id = DIF_INSTR_VAR(instr);
5250                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5251                         id -= DIF_VAR_OTHER_UBASE;
5252                         v = &vstate->dtvs_tlocals[id];
5253 
5254                         key = &tupregs[DIF_DTR_NREGS];
5255                         key[0].dttk_value = (uint64_t)id;
5256                         key[0].dttk_size = 0;
5257                         DTRACE_TLS_THRKEY(key[1].dttk_value);
5258                         key[1].dttk_size = 0;
5259 
5260                         dvar = dtrace_dynvar(dstate, 2, key,
5261                             sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
5262                             mstate, vstate);
5263 
5264                         if (dvar == NULL) {
5265                                 regs[rd] = 0;
5266                                 break;
5267                         }
5268 
5269                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5270                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5271                         } else {
5272                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5273                         }
5274 
5275                         break;
5276                 }
5277 
5278                 case DIF_OP_STTS: {
5279                         dtrace_dynvar_t *dvar;
5280                         dtrace_key_t *key;
5281 
5282                         id = DIF_INSTR_VAR(instr);
5283                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5284                         id -= DIF_VAR_OTHER_UBASE;
5285 
5286                         key = &tupregs[DIF_DTR_NREGS];
5287                         key[0].dttk_value = (uint64_t)id;
5288                         key[0].dttk_size = 0;
5289                         DTRACE_TLS_THRKEY(key[1].dttk_value);
5290                         key[1].dttk_size = 0;
5291                         v = &vstate->dtvs_tlocals[id];
5292 
5293                         dvar = dtrace_dynvar(dstate, 2, key,
5294                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5295                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
5296                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
5297                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
5298 
5299                         /*
5300                          * Given that we're storing to thread-local data,
5301                          * we need to flush our predicate cache.
5302                          */
5303                         curthread->t_predcache = NULL;
5304 
5305                         if (dvar == NULL)
5306                                 break;
5307 
5308                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5309                                 if (!dtrace_vcanload(
5310                                     (void *)(uintptr_t)regs[rd],
5311                                     &v->dtdv_type, mstate, vstate))
5312                                         break;
5313 
5314                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5315                                     dvar->dtdv_data, &v->dtdv_type);
5316                         } else {
5317                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
5318                         }
5319 
5320                         break;
5321                 }
5322 
5323                 case DIF_OP_SRA:
5324                         regs[rd] = (int64_t)regs[r1] >> regs[r2];
5325                         break;
5326 
5327                 case DIF_OP_CALL:
5328                         dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
5329                             regs, tupregs, ttop, mstate, state);
5330                         break;
5331 
5332                 case DIF_OP_PUSHTR:
5333                         if (ttop == DIF_DTR_NREGS) {
5334                                 *flags |= CPU_DTRACE_TUPOFLOW;
5335                                 break;
5336                         }
5337 
5338                         if (r1 == DIF_TYPE_STRING) {
5339                                 /*
5340                                  * If this is a string type and the size is 0,
5341                                  * we'll use the system-wide default string
5342                                  * size.  Note that we are _not_ looking at
5343                                  * the value of the DTRACEOPT_STRSIZE option;
5344                                  * had this been set, we would expect to have
5345                                  * a non-zero size value in the "pushtr".
5346                                  */
5347                                 tupregs[ttop].dttk_size =
5348                                     dtrace_strlen((char *)(uintptr_t)regs[rd],
5349                                     regs[r2] ? regs[r2] :
5350                                     dtrace_strsize_default) + 1;
5351                         } else {
5352                                 tupregs[ttop].dttk_size = regs[r2];
5353                         }
5354 
5355                         tupregs[ttop++].dttk_value = regs[rd];
5356                         break;
5357 
5358                 case DIF_OP_PUSHTV:
5359                         if (ttop == DIF_DTR_NREGS) {
5360                                 *flags |= CPU_DTRACE_TUPOFLOW;
5361                                 break;
5362                         }
5363 
5364                         tupregs[ttop].dttk_value = regs[rd];
5365                         tupregs[ttop++].dttk_size = 0;
5366                         break;
5367 
5368                 case DIF_OP_POPTS:
5369                         if (ttop != 0)
5370                                 ttop--;
5371                         break;
5372 
5373                 case DIF_OP_FLUSHTS:
5374                         ttop = 0;
5375                         break;
5376 
5377                 case DIF_OP_LDGAA:
5378                 case DIF_OP_LDTAA: {
5379                         dtrace_dynvar_t *dvar;
5380                         dtrace_key_t *key = tupregs;
5381                         uint_t nkeys = ttop;
5382 
5383                         id = DIF_INSTR_VAR(instr);
5384                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5385                         id -= DIF_VAR_OTHER_UBASE;
5386 
5387                         key[nkeys].dttk_value = (uint64_t)id;
5388                         key[nkeys++].dttk_size = 0;
5389 
5390                         if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
5391                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
5392                                 key[nkeys++].dttk_size = 0;
5393                                 v = &vstate->dtvs_tlocals[id];
5394                         } else {
5395                                 v = &vstate->dtvs_globals[id]->dtsv_var;
5396                         }
5397 
5398                         dvar = dtrace_dynvar(dstate, nkeys, key,
5399                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5400                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
5401                             DTRACE_DYNVAR_NOALLOC, mstate, vstate);
5402 
5403                         if (dvar == NULL) {
5404                                 regs[rd] = 0;
5405                                 break;
5406                         }
5407 
5408                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5409                                 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5410                         } else {
5411                                 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5412                         }
5413 
5414                         break;
5415                 }
5416 
5417                 case DIF_OP_STGAA:
5418                 case DIF_OP_STTAA: {
5419                         dtrace_dynvar_t *dvar;
5420                         dtrace_key_t *key = tupregs;
5421                         uint_t nkeys = ttop;
5422 
5423                         id = DIF_INSTR_VAR(instr);
5424                         ASSERT(id >= DIF_VAR_OTHER_UBASE);
5425                         id -= DIF_VAR_OTHER_UBASE;
5426 
5427                         key[nkeys].dttk_value = (uint64_t)id;
5428                         key[nkeys++].dttk_size = 0;
5429 
5430                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
5431                                 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
5432                                 key[nkeys++].dttk_size = 0;
5433                                 v = &vstate->dtvs_tlocals[id];
5434                         } else {
5435                                 v = &vstate->dtvs_globals[id]->dtsv_var;
5436                         }
5437 
5438                         dvar = dtrace_dynvar(dstate, nkeys, key,
5439                             v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5440                             v->dtdv_type.dtdt_size : sizeof (uint64_t),
5441                             regs[rd] ? DTRACE_DYNVAR_ALLOC :
5442                             DTRACE_DYNVAR_DEALLOC, mstate, vstate);
5443 
5444                         if (dvar == NULL)
5445                                 break;
5446 
5447                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5448                                 if (!dtrace_vcanload(
5449                                     (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5450                                     mstate, vstate))
5451                                         break;
5452 
5453                                 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5454                                     dvar->dtdv_data, &v->dtdv_type);
5455                         } else {
5456                                 *((uint64_t *)dvar->dtdv_data) = regs[rd];
5457                         }
5458 
5459                         break;
5460                 }
5461 
5462                 case DIF_OP_ALLOCS: {
5463                         uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
5464                         size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
5465 
5466                         /*
5467                          * Rounding up the user allocation size could have
5468                          * overflowed large, bogus allocations (like -1ULL) to
5469                          * 0.
5470                          */
5471                         if (size < regs[r1] ||
5472                             !DTRACE_INSCRATCH(mstate, size)) {
5473                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5474                                 regs[rd] = NULL;
5475                                 break;
5476                         }
5477 
5478                         dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
5479                         mstate->dtms_scratch_ptr += size;
5480                         regs[rd] = ptr;
5481                         break;
5482                 }
5483 
5484                 case DIF_OP_COPYS:
5485                         if (!dtrace_canstore(regs[rd], regs[r2],
5486                             mstate, vstate)) {
5487                                 *flags |= CPU_DTRACE_BADADDR;
5488                                 *illval = regs[rd];
5489                                 break;
5490                         }
5491 
5492                         if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
5493                                 break;
5494 
5495                         dtrace_bcopy((void *)(uintptr_t)regs[r1],
5496                             (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
5497                         break;
5498 
5499                 case DIF_OP_STB:
5500                         if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
5501                                 *flags |= CPU_DTRACE_BADADDR;
5502                                 *illval = regs[rd];
5503                                 break;
5504                         }
5505                         *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
5506                         break;
5507 
5508                 case DIF_OP_STH:
5509                         if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
5510                                 *flags |= CPU_DTRACE_BADADDR;
5511                                 *illval = regs[rd];
5512                                 break;
5513                         }
5514                         if (regs[rd] & 1) {
5515                                 *flags |= CPU_DTRACE_BADALIGN;
5516                                 *illval = regs[rd];
5517                                 break;
5518                         }
5519                         *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
5520                         break;
5521 
5522                 case DIF_OP_STW:
5523                         if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
5524                                 *flags |= CPU_DTRACE_BADADDR;
5525                                 *illval = regs[rd];
5526                                 break;
5527                         }
5528                         if (regs[rd] & 3) {
5529                                 *flags |= CPU_DTRACE_BADALIGN;
5530                                 *illval = regs[rd];
5531                                 break;
5532                         }
5533                         *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
5534                         break;
5535 
5536                 case DIF_OP_STX:
5537                         if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
5538                                 *flags |= CPU_DTRACE_BADADDR;
5539                                 *illval = regs[rd];
5540                                 break;
5541                         }
5542                         if (regs[rd] & 7) {
5543                                 *flags |= CPU_DTRACE_BADALIGN;
5544                                 *illval = regs[rd];
5545                                 break;
5546                         }
5547                         *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
5548                         break;
5549                 }
5550         }
5551 
5552         if (!(*flags & CPU_DTRACE_FAULT))
5553                 return (rval);
5554 
5555         mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
5556         mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
5557 
5558         return (0);
5559 }
5560 
5561 static void
5562 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
5563 {
5564         dtrace_probe_t *probe = ecb->dte_probe;
5565         dtrace_provider_t *prov = probe->dtpr_provider;
5566         char c[DTRACE_FULLNAMELEN + 80], *str;
5567         char *msg = "dtrace: breakpoint action at probe ";
5568         char *ecbmsg = " (ecb ";
5569         uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
5570         uintptr_t val = (uintptr_t)ecb;
5571         int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
5572 
5573         if (dtrace_destructive_disallow)
5574                 return;
5575 
5576         /*
5577          * It's impossible to be taking action on the NULL probe.
5578          */
5579         ASSERT(probe != NULL);
5580 
5581         /*
5582          * This is a poor man's (destitute man's?) sprintf():  we want to
5583          * print the provider name, module name, function name and name of
5584          * the probe, along with the hex address of the ECB with the breakpoint
5585          * action -- all of which we must place in the character buffer by
5586          * hand.
5587          */
5588         while (*msg != '\0')
5589                 c[i++] = *msg++;
5590 
5591         for (str = prov->dtpv_name; *str != '\0'; str++)
5592                 c[i++] = *str;
5593         c[i++] = ':';
5594 
5595         for (str = probe->dtpr_mod; *str != '\0'; str++)
5596                 c[i++] = *str;
5597         c[i++] = ':';
5598 
5599         for (str = probe->dtpr_func; *str != '\0'; str++)
5600                 c[i++] = *str;
5601         c[i++] = ':';
5602 
5603         for (str = probe->dtpr_name; *str != '\0'; str++)
5604                 c[i++] = *str;
5605 
5606         while (*ecbmsg != '\0')
5607                 c[i++] = *ecbmsg++;
5608 
5609         while (shift >= 0) {
5610                 mask = (uintptr_t)0xf << shift;
5611 
5612                 if (val >= ((uintptr_t)1 << shift))
5613                         c[i++] = "0123456789abcdef"[(val & mask) >> shift];
5614                 shift -= 4;
5615         }
5616 
5617         c[i++] = ')';
5618         c[i] = '\0';
5619 
5620         debug_enter(c);
5621 }
5622 
5623 static void
5624 dtrace_action_panic(dtrace_ecb_t *ecb)
5625 {
5626         dtrace_probe_t *probe = ecb->dte_probe;
5627 
5628         /*
5629          * It's impossible to be taking action on the NULL probe.
5630          */
5631         ASSERT(probe != NULL);
5632 
5633         if (dtrace_destructive_disallow)
5634                 return;
5635 
5636         if (dtrace_panicked != NULL)
5637                 return;
5638 
5639         if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
5640                 return;
5641 
5642         /*
5643          * We won the right to panic.  (We want to be sure that only one
5644          * thread calls panic() from dtrace_probe(), and that panic() is
5645          * called exactly once.)
5646          */
5647         dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
5648             probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
5649             probe->dtpr_func, probe->dtpr_name, (void *)ecb);
5650 }
5651 
5652 static void
5653 dtrace_action_raise(uint64_t sig)
5654 {
5655         if (dtrace_destructive_disallow)
5656                 return;
5657 
5658         if (sig >= NSIG) {
5659                 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
5660                 return;
5661         }
5662 
5663         /*
5664          * raise() has a queue depth of 1 -- we ignore all subsequent
5665          * invocations of the raise() action.
5666          */
5667         if (curthread->t_dtrace_sig == 0)
5668                 curthread->t_dtrace_sig = (uint8_t)sig;
5669 
5670         curthread->t_sig_check = 1;
5671         aston(curthread);
5672 }
5673 
5674 static void
5675 dtrace_action_stop(void)
5676 {
5677         if (dtrace_destructive_disallow)
5678                 return;
5679 
5680         if (!curthread->t_dtrace_stop) {
5681                 curthread->t_dtrace_stop = 1;
5682                 curthread->t_sig_check = 1;
5683                 aston(curthread);
5684         }
5685 }
5686 
5687 static void
5688 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
5689 {
5690         hrtime_t now;
5691         volatile uint16_t *flags;
5692         cpu_t *cpu = CPU;
5693 
5694         if (dtrace_destructive_disallow)
5695                 return;
5696 
5697         flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
5698 
5699         now = dtrace_gethrtime();
5700 
5701         if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
5702                 /*
5703                  * We need to advance the mark to the current time.
5704                  */
5705                 cpu->cpu_dtrace_chillmark = now;
5706                 cpu->cpu_dtrace_chilled = 0;
5707         }
5708 
5709         /*
5710          * Now check to see if the requested chill time would take us over
5711          * the maximum amount of time allowed in the chill interval.  (Or
5712          * worse, if the calculation itself induces overflow.)
5713          */
5714         if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
5715             cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
5716                 *flags |= CPU_DTRACE_ILLOP;
5717                 return;
5718         }
5719 
5720         while (dtrace_gethrtime() - now < val)
5721                 continue;
5722 
5723         /*
5724          * Normally, we assure that the value of the variable "timestamp" does
5725          * not change within an ECB.  The presence of chill() represents an
5726          * exception to this rule, however.
5727          */
5728         mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
5729         cpu->cpu_dtrace_chilled += val;
5730 }
5731 
5732 static void
5733 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
5734     uint64_t *buf, uint64_t arg)
5735 {
5736         int nframes = DTRACE_USTACK_NFRAMES(arg);
5737         int strsize = DTRACE_USTACK_STRSIZE(arg);
5738         uint64_t *pcs = &buf[1], *fps;
5739         char *str = (char *)&pcs[nframes];
5740         int size, offs = 0, i, j;
5741         uintptr_t old = mstate->dtms_scratch_ptr, saved;
5742         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5743         char *sym;
5744 
5745         /*
5746          * Should be taking a faster path if string space has not been
5747          * allocated.
5748          */
5749         ASSERT(strsize != 0);
5750 
5751         /*
5752          * We will first allocate some temporary space for the frame pointers.
5753          */
5754         fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
5755         size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
5756             (nframes * sizeof (uint64_t));
5757 
5758         if (!DTRACE_INSCRATCH(mstate, size)) {
5759                 /*
5760                  * Not enough room for our frame pointers -- need to indicate
5761                  * that we ran out of scratch space.
5762                  */
5763                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5764                 return;
5765         }
5766 
5767         mstate->dtms_scratch_ptr += size;
5768         saved = mstate->dtms_scratch_ptr;
5769 
5770         /*
5771          * Now get a stack with both program counters and frame pointers.
5772          */
5773         DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5774         dtrace_getufpstack(buf, fps, nframes + 1);
5775         DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5776 
5777         /*
5778          * If that faulted, we're cooked.
5779          */
5780         if (*flags & CPU_DTRACE_FAULT)
5781                 goto out;
5782 
5783         /*
5784          * Now we want to walk up the stack, calling the USTACK helper.  For
5785          * each iteration, we restore the scratch pointer.
5786          */
5787         for (i = 0; i < nframes; i++) {
5788                 mstate->dtms_scratch_ptr = saved;
5789 
5790                 if (offs >= strsize)
5791                         break;
5792 
5793                 sym = (char *)(uintptr_t)dtrace_helper(
5794                     DTRACE_HELPER_ACTION_USTACK,
5795                     mstate, state, pcs[i], fps[i]);
5796 
5797                 /*
5798                  * If we faulted while running the helper, we're going to
5799                  * clear the fault and null out the corresponding string.
5800                  */
5801                 if (*flags & CPU_DTRACE_FAULT) {
5802                         *flags &= ~CPU_DTRACE_FAULT;
5803                         str[offs++] = '\0';
5804                         continue;
5805                 }
5806 
5807                 if (sym == NULL) {
5808                         str[offs++] = '\0';
5809                         continue;
5810                 }
5811 
5812                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5813 
5814                 /*
5815                  * Now copy in the string that the helper returned to us.
5816                  */
5817                 for (j = 0; offs + j < strsize; j++) {
5818                         if ((str[offs + j] = sym[j]) == '\0')
5819                                 break;
5820                 }
5821 
5822                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5823 
5824                 offs += j + 1;
5825         }
5826 
5827         if (offs >= strsize) {
5828                 /*
5829                  * If we didn't have room for all of the strings, we don't
5830                  * abort processing -- this needn't be a fatal error -- but we
5831                  * still want to increment a counter (dts_stkstroverflows) to
5832                  * allow this condition to be warned about.  (If this is from
5833                  * a jstack() action, it is easily tuned via jstackstrsize.)
5834                  */
5835                 dtrace_error(&state->dts_stkstroverflows);
5836         }
5837 
5838         while (offs < strsize)
5839                 str[offs++] = '\0';
5840 
5841 out:
5842         mstate->dtms_scratch_ptr = old;
5843 }
5844 
5845 /*
5846  * If you're looking for the epicenter of DTrace, you just found it.  This
5847  * is the function called by the provider to fire a probe -- from which all
5848  * subsequent probe-context DTrace activity emanates.
5849  */
5850 void
5851 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
5852     uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
5853 {
5854         processorid_t cpuid;
5855         dtrace_icookie_t cookie;
5856         dtrace_probe_t *probe;
5857         dtrace_mstate_t mstate;
5858         dtrace_ecb_t *ecb;
5859         dtrace_action_t *act;
5860         intptr_t offs;
5861         size_t size;
5862         int vtime, onintr;
5863         volatile uint16_t *flags;
5864         hrtime_t now, end;
5865 
5866         /*
5867          * Kick out immediately if this CPU is still being born (in which case
5868          * curthread will be set to -1) or the current thread can't allow
5869          * probes in its current context.
5870          */
5871         if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
5872                 return;
5873 
5874         cookie = dtrace_interrupt_disable();
5875         probe = dtrace_probes[id - 1];
5876         cpuid = CPU->cpu_id;
5877         onintr = CPU_ON_INTR(CPU);
5878 
5879         CPU->cpu_dtrace_probes++;
5880 
5881         if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
5882             probe->dtpr_predcache == curthread->t_predcache) {
5883                 /*
5884                  * We have hit in the predicate cache; we know that
5885                  * this predicate would evaluate to be false.
5886                  */
5887                 dtrace_interrupt_enable(cookie);
5888                 return;
5889         }
5890 
5891         if (panic_quiesce) {
5892                 /*
5893                  * We don't trace anything if we're panicking.
5894                  */
5895                 dtrace_interrupt_enable(cookie);
5896                 return;
5897         }
5898 
5899         now = dtrace_gethrtime();
5900         vtime = dtrace_vtime_references != 0;
5901 
5902         if (vtime && curthread->t_dtrace_start)
5903                 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
5904 
5905         mstate.dtms_difo = NULL;
5906         mstate.dtms_probe = probe;
5907         mstate.dtms_strtok = NULL;
5908         mstate.dtms_arg[0] = arg0;
5909         mstate.dtms_arg[1] = arg1;
5910         mstate.dtms_arg[2] = arg2;
5911         mstate.dtms_arg[3] = arg3;
5912         mstate.dtms_arg[4] = arg4;
5913 
5914         flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
5915 
5916         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
5917                 dtrace_predicate_t *pred = ecb->dte_predicate;
5918                 dtrace_state_t *state = ecb->dte_state;
5919                 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
5920                 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
5921                 dtrace_vstate_t *vstate = &state->dts_vstate;
5922                 dtrace_provider_t *prov = probe->dtpr_provider;
5923                 uint64_t tracememsize = 0;
5924                 int committed = 0;
5925                 caddr_t tomax;
5926 
5927                 /*
5928                  * A little subtlety with the following (seemingly innocuous)
5929                  * declaration of the automatic 'val':  by looking at the
5930                  * code, you might think that it could be declared in the
5931                  * action processing loop, below.  (That is, it's only used in
5932                  * the action processing loop.)  However, it must be declared
5933                  * out of that scope because in the case of DIF expression
5934                  * arguments to aggregating actions, one iteration of the
5935                  * action loop will use the last iteration's value.
5936                  */
5937 #ifdef lint
5938                 uint64_t val = 0;
5939 #else
5940                 uint64_t val;
5941 #endif
5942 
5943                 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
5944                 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC;
5945                 *flags &= ~CPU_DTRACE_ERROR;
5946 
5947                 if (prov == dtrace_provider) {
5948                         /*
5949                          * If dtrace itself is the provider of this probe,
5950                          * we're only going to continue processing the ECB if
5951                          * arg0 (the dtrace_state_t) is equal to the ECB's
5952                          * creating state.  (This prevents disjoint consumers
5953                          * from seeing one another's metaprobes.)
5954                          */
5955                         if (arg0 != (uint64_t)(uintptr_t)state)
5956                                 continue;
5957                 }
5958 
5959                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
5960                         /*
5961                          * We're not currently active.  If our provider isn't
5962                          * the dtrace pseudo provider, we're not interested.
5963                          */
5964                         if (prov != dtrace_provider)
5965                                 continue;
5966 
5967                         /*
5968                          * Now we must further check if we are in the BEGIN
5969                          * probe.  If we are, we will only continue processing
5970                          * if we're still in WARMUP -- if one BEGIN enabling
5971                          * has invoked the exit() action, we don't want to
5972                          * evaluate subsequent BEGIN enablings.
5973                          */
5974                         if (probe->dtpr_id == dtrace_probeid_begin &&
5975                             state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
5976                                 ASSERT(state->dts_activity ==
5977                                     DTRACE_ACTIVITY_DRAINING);
5978                                 continue;
5979                         }
5980                 }
5981 
5982                 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb))
5983                         continue;
5984 
5985                 if (now - state->dts_alive > dtrace_deadman_timeout) {
5986                         /*
5987                          * We seem to be dead.  Unless we (a) have kernel
5988                          * destructive permissions (b) have explicitly enabled
5989                          * destructive actions and (c) destructive actions have
5990                          * not been disabled, we're going to transition into
5991                          * the KILLED state, from which no further processing
5992                          * on this state will be performed.
5993                          */
5994                         if (!dtrace_priv_kernel_destructive(state) ||
5995                             !state->dts_cred.dcr_destructive ||
5996                             dtrace_destructive_disallow) {
5997                                 void *activity = &state->dts_activity;
5998                                 dtrace_activity_t current;
5999 
6000                                 do {
6001                                         current = state->dts_activity;
6002                                 } while (dtrace_cas32(activity, current,
6003                                     DTRACE_ACTIVITY_KILLED) != current);
6004 
6005                                 continue;
6006                         }
6007                 }
6008 
6009                 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
6010                     ecb->dte_alignment, state, &mstate)) < 0)
6011                         continue;
6012 
6013                 tomax = buf->dtb_tomax;
6014                 ASSERT(tomax != NULL);
6015 
6016                 if (ecb->dte_size != 0) {
6017                         dtrace_rechdr_t dtrh;
6018                         if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
6019                                 mstate.dtms_timestamp = dtrace_gethrtime();
6020                                 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP;
6021                         }
6022                         ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
6023                         dtrh.dtrh_epid = ecb->dte_epid;
6024                         DTRACE_RECORD_STORE_TIMESTAMP(&dtrh,
6025                             mstate.dtms_timestamp);
6026                         *((dtrace_rechdr_t *)(tomax + offs)) = dtrh;
6027                 }
6028 
6029                 mstate.dtms_epid = ecb->dte_epid;
6030                 mstate.dtms_present |= DTRACE_MSTATE_EPID;
6031 
6032                 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
6033                         mstate.dtms_access |= DTRACE_ACCESS_KERNEL;
6034 
6035                 if (pred != NULL) {
6036                         dtrace_difo_t *dp = pred->dtp_difo;
6037                         int rval;
6038 
6039                         rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
6040 
6041                         if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
6042                                 dtrace_cacheid_t cid = probe->dtpr_predcache;
6043 
6044                                 if (cid != DTRACE_CACHEIDNONE && !onintr) {
6045                                         /*
6046                                          * Update the predicate cache...
6047                                          */
6048                                         ASSERT(cid == pred->dtp_cacheid);
6049                                         curthread->t_predcache = cid;
6050                                 }
6051 
6052                                 continue;
6053                         }
6054                 }
6055 
6056                 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
6057                     act != NULL; act = act->dta_next) {
6058                         size_t valoffs;
6059                         dtrace_difo_t *dp;
6060                         dtrace_recdesc_t *rec = &act->dta_rec;
6061 
6062                         size = rec->dtrd_size;
6063                         valoffs = offs + rec->dtrd_offset;
6064 
6065                         if (DTRACEACT_ISAGG(act->dta_kind)) {
6066                                 uint64_t v = 0xbad;
6067                                 dtrace_aggregation_t *agg;
6068 
6069                                 agg = (dtrace_aggregation_t *)act;
6070 
6071                                 if ((dp = act->dta_difo) != NULL)
6072                                         v = dtrace_dif_emulate(dp,
6073                                             &mstate, vstate, state);
6074 
6075                                 if (*flags & CPU_DTRACE_ERROR)
6076                                         continue;
6077 
6078                                 /*
6079                                  * Note that we always pass the expression
6080                                  * value from the previous iteration of the
6081                                  * action loop.  This value will only be used
6082                                  * if there is an expression argument to the
6083                                  * aggregating action, denoted by the
6084                                  * dtag_hasarg field.
6085                                  */
6086                                 dtrace_aggregate(agg, buf,
6087                                     offs, aggbuf, v, val);
6088                                 continue;
6089                         }
6090 
6091                         switch (act->dta_kind) {
6092                         case DTRACEACT_STOP:
6093                                 if (dtrace_priv_proc_destructive(state,
6094                                     &mstate))
6095                                         dtrace_action_stop();
6096                                 continue;
6097 
6098                         case DTRACEACT_BREAKPOINT:
6099                                 if (dtrace_priv_kernel_destructive(state))
6100                                         dtrace_action_breakpoint(ecb);
6101                                 continue;
6102 
6103                         case DTRACEACT_PANIC:
6104                                 if (dtrace_priv_kernel_destructive(state))
6105                                         dtrace_action_panic(ecb);
6106                                 continue;
6107 
6108                         case DTRACEACT_STACK:
6109                                 if (!dtrace_priv_kernel(state))
6110                                         continue;
6111 
6112                                 dtrace_getpcstack((pc_t *)(tomax + valoffs),
6113                                     size / sizeof (pc_t), probe->dtpr_aframes,
6114                                     DTRACE_ANCHORED(probe) ? NULL :
6115                                     (uint32_t *)arg0);
6116 
6117                                 continue;
6118 
6119                         case DTRACEACT_JSTACK:
6120                         case DTRACEACT_USTACK:
6121                                 if (!dtrace_priv_proc(state, &mstate))
6122                                         continue;
6123 
6124                                 /*
6125                                  * See comment in DIF_VAR_PID.
6126                                  */
6127                                 if (DTRACE_ANCHORED(mstate.dtms_probe) &&
6128                                     CPU_ON_INTR(CPU)) {
6129                                         int depth = DTRACE_USTACK_NFRAMES(
6130                                             rec->dtrd_arg) + 1;
6131 
6132                                         dtrace_bzero((void *)(tomax + valoffs),
6133                                             DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
6134                                             + depth * sizeof (uint64_t));
6135 
6136                                         continue;
6137                                 }
6138 
6139                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
6140                                     curproc->p_dtrace_helpers != NULL) {
6141                                         /*
6142                                          * This is the slow path -- we have
6143                                          * allocated string space, and we're
6144                                          * getting the stack of a process that
6145                                          * has helpers.  Call into a separate
6146                                          * routine to perform this processing.
6147                                          */
6148                                         dtrace_action_ustack(&mstate, state,
6149                                             (uint64_t *)(tomax + valoffs),
6150                                             rec->dtrd_arg);
6151                                         continue;
6152                                 }
6153 
6154                                 /*
6155                                  * Clear the string space, since there's no
6156                                  * helper to do it for us.
6157                                  */
6158                                 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) {
6159                                         int depth = DTRACE_USTACK_NFRAMES(
6160                                             rec->dtrd_arg);
6161                                         size_t strsize = DTRACE_USTACK_STRSIZE(
6162                                             rec->dtrd_arg);
6163                                         uint64_t *buf = (uint64_t *)(tomax +
6164                                             valoffs);
6165                                         void *strspace = &buf[depth + 1];
6166 
6167                                         dtrace_bzero(strspace,
6168                                             MIN(depth, strsize));
6169                                 }
6170 
6171                                 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6172                                 dtrace_getupcstack((uint64_t *)
6173                                     (tomax + valoffs),
6174                                     DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
6175                                 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6176                                 continue;
6177 
6178                         default:
6179                                 break;
6180                         }
6181 
6182                         dp = act->dta_difo;
6183                         ASSERT(dp != NULL);
6184 
6185                         val = dtrace_dif_emulate(dp, &mstate, vstate, state);
6186 
6187                         if (*flags & CPU_DTRACE_ERROR)
6188                                 continue;
6189 
6190                         switch (act->dta_kind) {
6191                         case DTRACEACT_SPECULATE: {
6192                                 dtrace_rechdr_t *dtrh;
6193 
6194                                 ASSERT(buf == &state->dts_buffer[cpuid]);
6195                                 buf = dtrace_speculation_buffer(state,
6196                                     cpuid, val);
6197 
6198                                 if (buf == NULL) {
6199                                         *flags |= CPU_DTRACE_DROP;
6200                                         continue;
6201                                 }
6202 
6203                                 offs = dtrace_buffer_reserve(buf,
6204                                     ecb->dte_needed, ecb->dte_alignment,
6205                                     state, NULL);
6206 
6207                                 if (offs < 0) {
6208                                         *flags |= CPU_DTRACE_DROP;
6209                                         continue;
6210                                 }
6211 
6212                                 tomax = buf->dtb_tomax;
6213                                 ASSERT(tomax != NULL);
6214 
6215                                 if (ecb->dte_size == 0)
6216                                         continue;
6217 
6218                                 ASSERT3U(ecb->dte_size, >=,
6219                                     sizeof (dtrace_rechdr_t));
6220                                 dtrh = ((void *)(tomax + offs));
6221                                 dtrh->dtrh_epid = ecb->dte_epid;
6222                                 /*
6223                                  * When the speculation is committed, all of
6224                                  * the records in the speculative buffer will
6225                                  * have their timestamps set to the commit
6226                                  * time.  Until then, it is set to a sentinel
6227                                  * value, for debugability.
6228                                  */
6229                                 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX);
6230                                 continue;
6231                         }
6232 
6233                         case DTRACEACT_CHILL:
6234                                 if (dtrace_priv_kernel_destructive(state))
6235                                         dtrace_action_chill(&mstate, val);
6236                                 continue;
6237 
6238                         case DTRACEACT_RAISE:
6239                                 if (dtrace_priv_proc_destructive(state,
6240                                     &mstate))
6241                                         dtrace_action_raise(val);
6242                                 continue;
6243 
6244                         case DTRACEACT_COMMIT:
6245                                 ASSERT(!committed);
6246 
6247                                 /*
6248                                  * We need to commit our buffer state.
6249                                  */
6250                                 if (ecb->dte_size)
6251                                         buf->dtb_offset = offs + ecb->dte_size;
6252                                 buf = &state->dts_buffer[cpuid];
6253                                 dtrace_speculation_commit(state, cpuid, val);
6254                                 committed = 1;
6255                                 continue;
6256 
6257                         case DTRACEACT_DISCARD:
6258                                 dtrace_speculation_discard(state, cpuid, val);
6259                                 continue;
6260 
6261                         case DTRACEACT_DIFEXPR:
6262                         case DTRACEACT_LIBACT:
6263                         case DTRACEACT_PRINTF:
6264                         case DTRACEACT_PRINTA:
6265                         case DTRACEACT_SYSTEM:
6266                         case DTRACEACT_FREOPEN:
6267                         case DTRACEACT_TRACEMEM:
6268                                 break;
6269 
6270                         case DTRACEACT_TRACEMEM_DYNSIZE:
6271                                 tracememsize = val;
6272                                 break;
6273 
6274                         case DTRACEACT_SYM:
6275                         case DTRACEACT_MOD:
6276                                 if (!dtrace_priv_kernel(state))
6277                                         continue;
6278                                 break;
6279 
6280                         case DTRACEACT_USYM:
6281                         case DTRACEACT_UMOD:
6282                         case DTRACEACT_UADDR: {
6283                                 struct pid *pid = curthread->t_procp->p_pidp;
6284 
6285                                 if (!dtrace_priv_proc(state, &mstate))
6286                                         continue;
6287 
6288                                 DTRACE_STORE(uint64_t, tomax,
6289                                     valoffs, (uint64_t)pid->pid_id);
6290                                 DTRACE_STORE(uint64_t, tomax,
6291                                     valoffs + sizeof (uint64_t), val);
6292 
6293                                 continue;
6294                         }
6295 
6296                         case DTRACEACT_EXIT: {
6297                                 /*
6298                                  * For the exit action, we are going to attempt
6299                                  * to atomically set our activity to be
6300                                  * draining.  If this fails (either because
6301                                  * another CPU has beat us to the exit action,
6302                                  * or because our current activity is something
6303                                  * other than ACTIVE or WARMUP), we will
6304                                  * continue.  This assures that the exit action
6305                                  * can be successfully recorded at most once
6306                                  * when we're in the ACTIVE state.  If we're
6307                                  * encountering the exit() action while in
6308                                  * COOLDOWN, however, we want to honor the new
6309                                  * status code.  (We know that we're the only
6310                                  * thread in COOLDOWN, so there is no race.)
6311                                  */
6312                                 void *activity = &state->dts_activity;
6313                                 dtrace_activity_t current = state->dts_activity;
6314 
6315                                 if (current == DTRACE_ACTIVITY_COOLDOWN)
6316                                         break;
6317 
6318                                 if (current != DTRACE_ACTIVITY_WARMUP)
6319                                         current = DTRACE_ACTIVITY_ACTIVE;
6320 
6321                                 if (dtrace_cas32(activity, current,
6322                                     DTRACE_ACTIVITY_DRAINING) != current) {
6323                                         *flags |= CPU_DTRACE_DROP;
6324                                         continue;
6325                                 }
6326 
6327                                 break;
6328                         }
6329 
6330                         default:
6331                                 ASSERT(0);
6332                         }
6333 
6334                         if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
6335                                 uintptr_t end = valoffs + size;
6336 
6337                                 if (tracememsize != 0 &&
6338                                     valoffs + tracememsize < end) {
6339                                         end = valoffs + tracememsize;
6340                                         tracememsize = 0;
6341                                 }
6342 
6343                                 if (!dtrace_vcanload((void *)(uintptr_t)val,
6344                                     &dp->dtdo_rtype, &mstate, vstate))
6345                                         continue;
6346 
6347                                 /*
6348                                  * If this is a string, we're going to only
6349                                  * load until we find the zero byte -- after
6350                                  * which we'll store zero bytes.
6351                                  */
6352                                 if (dp->dtdo_rtype.dtdt_kind ==
6353                                     DIF_TYPE_STRING) {
6354                                         char c = '\0' + 1;
6355                                         int intuple = act->dta_intuple;
6356                                         size_t s;
6357 
6358                                         for (s = 0; s < size; s++) {
6359                                                 if (c != '\0')
6360                                                         c = dtrace_load8(val++);
6361 
6362                                                 DTRACE_STORE(uint8_t, tomax,
6363                                                     valoffs++, c);
6364 
6365                                                 if (c == '\0' && intuple)
6366                                                         break;
6367                                         }
6368 
6369                                         continue;
6370                                 }
6371 
6372                                 while (valoffs < end) {
6373                                         DTRACE_STORE(uint8_t, tomax, valoffs++,
6374                                             dtrace_load8(val++));
6375                                 }
6376 
6377                                 continue;
6378                         }
6379 
6380                         switch (size) {
6381                         case 0:
6382                                 break;
6383 
6384                         case sizeof (uint8_t):
6385                                 DTRACE_STORE(uint8_t, tomax, valoffs, val);
6386                                 break;
6387                         case sizeof (uint16_t):
6388                                 DTRACE_STORE(uint16_t, tomax, valoffs, val);
6389                                 break;
6390                         case sizeof (uint32_t):
6391                                 DTRACE_STORE(uint32_t, tomax, valoffs, val);
6392                                 break;
6393                         case sizeof (uint64_t):
6394                                 DTRACE_STORE(uint64_t, tomax, valoffs, val);
6395                                 break;
6396                         default:
6397                                 /*
6398                                  * Any other size should have been returned by
6399                                  * reference, not by value.
6400                                  */
6401                                 ASSERT(0);
6402                                 break;
6403                         }
6404                 }
6405 
6406                 if (*flags & CPU_DTRACE_DROP)
6407                         continue;
6408 
6409                 if (*flags & CPU_DTRACE_FAULT) {
6410                         int ndx;
6411                         dtrace_action_t *err;
6412 
6413                         buf->dtb_errors++;
6414 
6415                         if (probe->dtpr_id == dtrace_probeid_error) {
6416                                 /*
6417                                  * There's nothing we can do -- we had an
6418                                  * error on the error probe.  We bump an
6419                                  * error counter to at least indicate that
6420                                  * this condition happened.
6421                                  */
6422                                 dtrace_error(&state->dts_dblerrors);
6423                                 continue;
6424                         }
6425 
6426                         if (vtime) {
6427                                 /*
6428                                  * Before recursing on dtrace_probe(), we
6429                                  * need to explicitly clear out our start
6430                                  * time to prevent it from being accumulated
6431                                  * into t_dtrace_vtime.
6432                                  */
6433                                 curthread->t_dtrace_start = 0;
6434                         }
6435 
6436                         /*
6437                          * Iterate over the actions to figure out which action
6438                          * we were processing when we experienced the error.
6439                          * Note that act points _past_ the faulting action; if
6440                          * act is ecb->dte_action, the fault was in the
6441                          * predicate, if it's ecb->dte_action->dta_next it's
6442                          * in action #1, and so on.
6443                          */
6444                         for (err = ecb->dte_action, ndx = 0;
6445                             err != act; err = err->dta_next, ndx++)
6446                                 continue;
6447 
6448                         dtrace_probe_error(state, ecb->dte_epid, ndx,
6449                             (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
6450                             mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
6451                             cpu_core[cpuid].cpuc_dtrace_illval);
6452 
6453                         continue;
6454                 }
6455 
6456                 if (!committed)
6457                         buf->dtb_offset = offs + ecb->dte_size;
6458         }
6459 
6460         end = dtrace_gethrtime();
6461         if (vtime)
6462                 curthread->t_dtrace_start = end;
6463 
6464         CPU->cpu_dtrace_nsec += end - now;
6465 
6466         dtrace_interrupt_enable(cookie);
6467 }
6468 
6469 /*
6470  * DTrace Probe Hashing Functions
6471  *
6472  * The functions in this section (and indeed, the functions in remaining
6473  * sections) are not _called_ from probe context.  (Any exceptions to this are
6474  * marked with a "Note:".)  Rather, they are called from elsewhere in the
6475  * DTrace framework to look-up probes in, add probes to and remove probes from
6476  * the DTrace probe hashes.  (Each probe is hashed by each element of the
6477  * probe tuple -- allowing for fast lookups, regardless of what was
6478  * specified.)
6479  */
6480 static uint_t
6481 dtrace_hash_str(char *p)
6482 {
6483         unsigned int g;
6484         uint_t hval = 0;
6485 
6486         while (*p) {
6487                 hval = (hval << 4) + *p++;
6488                 if ((g = (hval & 0xf0000000)) != 0)
6489                         hval ^= g >> 24;
6490                 hval &= ~g;
6491         }
6492         return (hval);
6493 }
6494 
6495 static dtrace_hash_t *
6496 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
6497 {
6498         dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
6499 
6500         hash->dth_stroffs = stroffs;
6501         hash->dth_nextoffs = nextoffs;
6502         hash->dth_prevoffs = prevoffs;
6503 
6504         hash->dth_size = 1;
6505         hash->dth_mask = hash->dth_size - 1;
6506 
6507         hash->dth_tab = kmem_zalloc(hash->dth_size *
6508             sizeof (dtrace_hashbucket_t *), KM_SLEEP);
6509 
6510         return (hash);
6511 }
6512 
6513 static void
6514 dtrace_hash_destroy(dtrace_hash_t *hash)
6515 {
6516 #ifdef DEBUG
6517         int i;
6518 
6519         for (i = 0; i < hash->dth_size; i++)
6520                 ASSERT(hash->dth_tab[i] == NULL);
6521 #endif
6522 
6523         kmem_free(hash->dth_tab,
6524             hash->dth_size * sizeof (dtrace_hashbucket_t *));
6525         kmem_free(hash, sizeof (dtrace_hash_t));
6526 }
6527 
6528 static void
6529 dtrace_hash_resize(dtrace_hash_t *hash)
6530 {
6531         int size = hash->dth_size, i, ndx;
6532         int new_size = hash->dth_size << 1;
6533         int new_mask = new_size - 1;
6534         dtrace_hashbucket_t **new_tab, *bucket, *next;
6535 
6536         ASSERT((new_size & new_mask) == 0);
6537 
6538         new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
6539 
6540         for (i = 0; i < size; i++) {
6541                 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
6542                         dtrace_probe_t *probe = bucket->dthb_chain;
6543 
6544                         ASSERT(probe != NULL);
6545                         ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
6546 
6547                         next = bucket->dthb_next;
6548                         bucket->dthb_next = new_tab[ndx];
6549                         new_tab[ndx] = bucket;
6550                 }
6551         }
6552 
6553         kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
6554         hash->dth_tab = new_tab;
6555         hash->dth_size = new_size;
6556         hash->dth_mask = new_mask;
6557 }
6558 
6559 static void
6560 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
6561 {
6562         int hashval = DTRACE_HASHSTR(hash, new);
6563         int ndx = hashval & hash->dth_mask;
6564         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6565         dtrace_probe_t **nextp, **prevp;
6566 
6567         for (; bucket != NULL; bucket = bucket->dthb_next) {
6568                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
6569                         goto add;
6570         }
6571 
6572         if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
6573                 dtrace_hash_resize(hash);
6574                 dtrace_hash_add(hash, new);
6575                 return;
6576         }
6577 
6578         bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
6579         bucket->dthb_next = hash->dth_tab[ndx];
6580         hash->dth_tab[ndx] = bucket;
6581         hash->dth_nbuckets++;
6582 
6583 add:
6584         nextp = DTRACE_HASHNEXT(hash, new);
6585         ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
6586         *nextp = bucket->dthb_chain;
6587 
6588         if (bucket->dthb_chain != NULL) {
6589                 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
6590                 ASSERT(*prevp == NULL);
6591                 *prevp = new;
6592         }
6593 
6594         bucket->dthb_chain = new;
6595         bucket->dthb_len++;
6596 }
6597 
6598 static dtrace_probe_t *
6599 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
6600 {
6601         int hashval = DTRACE_HASHSTR(hash, template);
6602         int ndx = hashval & hash->dth_mask;
6603         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6604 
6605         for (; bucket != NULL; bucket = bucket->dthb_next) {
6606                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
6607                         return (bucket->dthb_chain);
6608         }
6609 
6610         return (NULL);
6611 }
6612 
6613 static int
6614 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
6615 {
6616         int hashval = DTRACE_HASHSTR(hash, template);
6617         int ndx = hashval & hash->dth_mask;
6618         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6619 
6620         for (; bucket != NULL; bucket = bucket->dthb_next) {
6621                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
6622                         return (bucket->dthb_len);
6623         }
6624 
6625         return (NULL);
6626 }
6627 
6628 static void
6629 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
6630 {
6631         int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
6632         dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6633 
6634         dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
6635         dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
6636 
6637         /*
6638          * Find the bucket that we're removing this probe from.
6639          */
6640         for (; bucket != NULL; bucket = bucket->dthb_next) {
6641                 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
6642                         break;
6643         }
6644 
6645         ASSERT(bucket != NULL);
6646 
6647         if (*prevp == NULL) {
6648                 if (*nextp == NULL) {
6649                         /*
6650                          * The removed probe was the only probe on this
6651                          * bucket; we need to remove the bucket.
6652                          */
6653                         dtrace_hashbucket_t *b = hash->dth_tab[ndx];
6654 
6655                         ASSERT(bucket->dthb_chain == probe);
6656                         ASSERT(b != NULL);
6657 
6658                         if (b == bucket) {
6659                                 hash->dth_tab[ndx] = bucket->dthb_next;
6660                         } else {
6661                                 while (b->dthb_next != bucket)
6662                                         b = b->dthb_next;
6663                                 b->dthb_next = bucket->dthb_next;
6664                         }
6665 
6666                         ASSERT(hash->dth_nbuckets > 0);
6667                         hash->dth_nbuckets--;
6668                         kmem_free(bucket, sizeof (dtrace_hashbucket_t));
6669                         return;
6670                 }
6671 
6672                 bucket->dthb_chain = *nextp;
6673         } else {
6674                 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
6675         }
6676 
6677         if (*nextp != NULL)
6678                 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
6679 }
6680 
6681 /*
6682  * DTrace Utility Functions
6683  *
6684  * These are random utility functions that are _not_ called from probe context.
6685  */
6686 static int
6687 dtrace_badattr(const dtrace_attribute_t *a)
6688 {
6689         return (a->dtat_name > DTRACE_STABILITY_MAX ||
6690             a->dtat_data > DTRACE_STABILITY_MAX ||
6691             a->dtat_class > DTRACE_CLASS_MAX);
6692 }
6693 
6694 /*
6695  * Return a duplicate copy of a string.  If the specified string is NULL,
6696  * this function returns a zero-length string.
6697  */
6698 static char *
6699 dtrace_strdup(const char *str)
6700 {
6701         char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
6702 
6703         if (str != NULL)
6704                 (void) strcpy(new, str);
6705 
6706         return (new);
6707 }
6708 
6709 #define DTRACE_ISALPHA(c)       \
6710         (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
6711 
6712 static int
6713 dtrace_badname(const char *s)
6714 {
6715         char c;
6716 
6717         if (s == NULL || (c = *s++) == '\0')
6718                 return (0);
6719 
6720         if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
6721                 return (1);
6722 
6723         while ((c = *s++) != '\0') {
6724                 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
6725                     c != '-' && c != '_' && c != '.' && c != '`')
6726                         return (1);
6727         }
6728 
6729         return (0);
6730 }
6731 
6732 static void
6733 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
6734 {
6735         uint32_t priv;
6736 
6737         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
6738                 /*
6739                  * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
6740                  */
6741                 priv = DTRACE_PRIV_ALL;
6742         } else {
6743                 *uidp = crgetuid(cr);
6744                 *zoneidp = crgetzoneid(cr);
6745 
6746                 priv = 0;
6747                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
6748                         priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
6749                 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
6750                         priv |= DTRACE_PRIV_USER;
6751                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
6752                         priv |= DTRACE_PRIV_PROC;
6753                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
6754                         priv |= DTRACE_PRIV_OWNER;
6755                 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
6756                         priv |= DTRACE_PRIV_ZONEOWNER;
6757         }
6758 
6759         *privp = priv;
6760 }
6761 
6762 #ifdef DTRACE_ERRDEBUG
6763 static void
6764 dtrace_errdebug(const char *str)
6765 {
6766         int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
6767         int occupied = 0;
6768 
6769         mutex_enter(&dtrace_errlock);
6770         dtrace_errlast = str;
6771         dtrace_errthread = curthread;
6772 
6773         while (occupied++ < DTRACE_ERRHASHSZ) {
6774                 if (dtrace_errhash[hval].dter_msg == str) {
6775                         dtrace_errhash[hval].dter_count++;
6776                         goto out;
6777                 }
6778 
6779                 if (dtrace_errhash[hval].dter_msg != NULL) {
6780                         hval = (hval + 1) % DTRACE_ERRHASHSZ;
6781                         continue;
6782                 }
6783 
6784                 dtrace_errhash[hval].dter_msg = str;
6785                 dtrace_errhash[hval].dter_count = 1;
6786                 goto out;
6787         }
6788 
6789         panic("dtrace: undersized error hash");
6790 out:
6791         mutex_exit(&dtrace_errlock);
6792 }
6793 #endif
6794 
6795 /*
6796  * DTrace Matching Functions
6797  *
6798  * These functions are used to match groups of probes, given some elements of
6799  * a probe tuple, or some globbed expressions for elements of a probe tuple.
6800  */
6801 static int
6802 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
6803     zoneid_t zoneid)
6804 {
6805         if (priv != DTRACE_PRIV_ALL) {
6806                 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
6807                 uint32_t match = priv & ppriv;
6808 
6809                 /*
6810                  * No PRIV_DTRACE_* privileges...
6811                  */
6812                 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
6813                     DTRACE_PRIV_KERNEL)) == 0)
6814                         return (0);
6815 
6816                 /*
6817                  * No matching bits, but there were bits to match...
6818                  */
6819                 if (match == 0 && ppriv != 0)
6820                         return (0);
6821 
6822                 /*
6823                  * Need to have permissions to the process, but don't...
6824                  */
6825                 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
6826                     uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
6827                         return (0);
6828                 }
6829 
6830                 /*
6831                  * Need to be in the same zone unless we possess the
6832                  * privilege to examine all zones.
6833                  */
6834                 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
6835                     zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
6836                         return (0);
6837                 }
6838         }
6839 
6840         return (1);
6841 }
6842 
6843 /*
6844  * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
6845  * consists of input pattern strings and an ops-vector to evaluate them.
6846  * This function returns >0 for match, 0 for no match, and <0 for error.
6847  */
6848 static int
6849 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
6850     uint32_t priv, uid_t uid, zoneid_t zoneid)
6851 {
6852         dtrace_provider_t *pvp = prp->dtpr_provider;
6853         int rv;
6854 
6855         if (pvp->dtpv_defunct)
6856                 return (0);
6857 
6858         if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
6859                 return (rv);
6860 
6861         if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
6862                 return (rv);
6863 
6864         if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
6865                 return (rv);
6866 
6867         if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
6868                 return (rv);
6869 
6870         if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
6871                 return (0);
6872 
6873         return (rv);
6874 }
6875 
6876 /*
6877  * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
6878  * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
6879  * libc's version, the kernel version only applies to 8-bit ASCII strings.
6880  * In addition, all of the recursion cases except for '*' matching have been
6881  * unwound.  For '*', we still implement recursive evaluation, but a depth
6882  * counter is maintained and matching is aborted if we recurse too deep.
6883  * The function returns 0 if no match, >0 if match, and <0 if recursion error.
6884  */
6885 static int
6886 dtrace_match_glob(const char *s, const char *p, int depth)
6887 {
6888         const char *olds;
6889         char s1, c;
6890         int gs;
6891 
6892         if (depth > DTRACE_PROBEKEY_MAXDEPTH)
6893                 return (-1);
6894 
6895         if (s == NULL)
6896                 s = ""; /* treat NULL as empty string */
6897 
6898 top:
6899         olds = s;
6900         s1 = *s++;
6901 
6902         if (p == NULL)
6903                 return (0);
6904 
6905         if ((c = *p++) == '\0')
6906                 return (s1 == '\0');
6907 
6908         switch (c) {
6909         case '[': {
6910                 int ok = 0, notflag = 0;
6911                 char lc = '\0';
6912 
6913                 if (s1 == '\0')
6914                         return (0);
6915 
6916                 if (*p == '!') {
6917                         notflag = 1;
6918                         p++;
6919                 }
6920 
6921                 if ((c = *p++) == '\0')
6922                         return (0);
6923 
6924                 do {
6925                         if (c == '-' && lc != '\0' && *p != ']') {
6926                                 if ((c = *p++) == '\0')
6927                                         return (0);
6928                                 if (c == '\\' && (c = *p++) == '\0')
6929                                         return (0);
6930 
6931                                 if (notflag) {
6932                                         if (s1 < lc || s1 > c)
6933                                                 ok++;
6934                                         else
6935                                                 return (0);
6936                                 } else if (lc <= s1 && s1 <= c)
6937                                         ok++;
6938 
6939                         } else if (c == '\\' && (c = *p++) == '\0')
6940                                 return (0);
6941 
6942                         lc = c; /* save left-hand 'c' for next iteration */
6943 
6944                         if (notflag) {
6945                                 if (s1 != c)
6946                                         ok++;
6947                                 else
6948                                         return (0);
6949                         } else if (s1 == c)
6950                                 ok++;
6951 
6952                         if ((c = *p++) == '\0')
6953                                 return (0);
6954 
6955                 } while (c != ']');
6956 
6957                 if (ok)
6958                         goto top;
6959 
6960                 return (0);
6961         }
6962 
6963         case '\\':
6964                 if ((c = *p++) == '\0')
6965                         return (0);
6966                 /*FALLTHRU*/
6967 
6968         default:
6969                 if (c != s1)
6970                         return (0);
6971                 /*FALLTHRU*/
6972 
6973         case '?':
6974                 if (s1 != '\0')
6975                         goto top;
6976                 return (0);
6977 
6978         case '*':
6979                 while (*p == '*')
6980                         p++; /* consecutive *'s are identical to a single one */
6981 
6982                 if (*p == '\0')
6983                         return (1);
6984 
6985                 for (s = olds; *s != '\0'; s++) {
6986                         if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
6987                                 return (gs);
6988                 }
6989 
6990                 return (0);
6991         }
6992 }
6993 
6994 /*ARGSUSED*/
6995 static int
6996 dtrace_match_string(const char *s, const char *p, int depth)
6997 {
6998         return (s != NULL && strcmp(s, p) == 0);
6999 }
7000 
7001 /*ARGSUSED*/
7002 static int
7003 dtrace_match_nul(const char *s, const char *p, int depth)
7004 {
7005         return (1); /* always match the empty pattern */
7006 }
7007 
7008 /*ARGSUSED*/
7009 static int
7010 dtrace_match_nonzero(const char *s, const char *p, int depth)
7011 {
7012         return (s != NULL && s[0] != '\0');
7013 }
7014 
7015 static int
7016 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
7017     zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
7018 {
7019         dtrace_probe_t template, *probe;
7020         dtrace_hash_t *hash = NULL;
7021         int len, rc, best = INT_MAX, nmatched = 0;
7022         dtrace_id_t i;
7023 
7024         ASSERT(MUTEX_HELD(&dtrace_lock));
7025 
7026         /*
7027          * If the probe ID is specified in the key, just lookup by ID and
7028          * invoke the match callback once if a matching probe is found.
7029          */
7030         if (pkp->dtpk_id != DTRACE_IDNONE) {
7031                 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
7032                     dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
7033                         if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
7034                                 return (DTRACE_MATCH_FAIL);
7035                         nmatched++;
7036                 }
7037                 return (nmatched);
7038         }
7039 
7040         template.dtpr_mod = (char *)pkp->dtpk_mod;
7041         template.dtpr_func = (char *)pkp->dtpk_func;
7042         template.dtpr_name = (char *)pkp->dtpk_name;
7043 
7044         /*
7045          * We want to find the most distinct of the module name, function
7046          * name, and name.  So for each one that is not a glob pattern or
7047          * empty string, we perform a lookup in the corresponding hash and
7048          * use the hash table with the fewest collisions to do our search.
7049          */
7050         if (pkp->dtpk_mmatch == &dtrace_match_string &&
7051             (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
7052                 best = len;
7053                 hash = dtrace_bymod;
7054         }
7055 
7056         if (pkp->dtpk_fmatch == &dtrace_match_string &&
7057             (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
7058                 best = len;
7059                 hash = dtrace_byfunc;
7060         }
7061 
7062         if (pkp->dtpk_nmatch == &dtrace_match_string &&
7063             (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
7064                 best = len;
7065                 hash = dtrace_byname;
7066         }
7067 
7068         /*
7069          * If we did not select a hash table, iterate over every probe and
7070          * invoke our callback for each one that matches our input probe key.
7071          */
7072         if (hash == NULL) {
7073                 for (i = 0; i < dtrace_nprobes; i++) {
7074                         if ((probe = dtrace_probes[i]) == NULL ||
7075                             dtrace_match_probe(probe, pkp, priv, uid,
7076                             zoneid) <= 0)
7077                                 continue;
7078 
7079                         nmatched++;
7080 
7081                         if ((rc = (*matched)(probe, arg)) !=
7082                             DTRACE_MATCH_NEXT) {
7083                                 if (rc == DTRACE_MATCH_FAIL)
7084                                         return (DTRACE_MATCH_FAIL);
7085                                 break;
7086                         }
7087                 }
7088 
7089                 return (nmatched);
7090         }
7091 
7092         /*
7093          * If we selected a hash table, iterate over each probe of the same key
7094          * name and invoke the callback for every probe that matches the other
7095          * attributes of our input probe key.
7096          */
7097         for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
7098             probe = *(DTRACE_HASHNEXT(hash, probe))) {
7099 
7100                 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
7101                         continue;
7102 
7103                 nmatched++;
7104 
7105                 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
7106                         if (rc == DTRACE_MATCH_FAIL)
7107                                 return (DTRACE_MATCH_FAIL);
7108                         break;
7109                 }
7110         }
7111 
7112         return (nmatched);
7113 }
7114 
7115 /*
7116  * Return the function pointer dtrace_probecmp() should use to compare the
7117  * specified pattern with a string.  For NULL or empty patterns, we select
7118  * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
7119  * For non-empty non-glob strings, we use dtrace_match_string().
7120  */
7121 static dtrace_probekey_f *
7122 dtrace_probekey_func(const char *p)
7123 {
7124         char c;
7125 
7126         if (p == NULL || *p == '\0')
7127                 return (&dtrace_match_nul);
7128 
7129         while ((c = *p++) != '\0') {
7130                 if (c == '[' || c == '?' || c == '*' || c == '\\')
7131                         return (&dtrace_match_glob);
7132         }
7133 
7134         return (&dtrace_match_string);
7135 }
7136 
7137 /*
7138  * Build a probe comparison key for use with dtrace_match_probe() from the
7139  * given probe description.  By convention, a null key only matches anchored
7140  * probes: if each field is the empty string, reset dtpk_fmatch to
7141  * dtrace_match_nonzero().
7142  */
7143 static void
7144 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
7145 {
7146         pkp->dtpk_prov = pdp->dtpd_provider;
7147         pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
7148 
7149         pkp->dtpk_mod = pdp->dtpd_mod;
7150         pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
7151 
7152         pkp->dtpk_func = pdp->dtpd_func;
7153         pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
7154 
7155         pkp->dtpk_name = pdp->dtpd_name;
7156         pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
7157 
7158         pkp->dtpk_id = pdp->dtpd_id;
7159 
7160         if (pkp->dtpk_id == DTRACE_IDNONE &&
7161             pkp->dtpk_pmatch == &dtrace_match_nul &&
7162             pkp->dtpk_mmatch == &dtrace_match_nul &&
7163             pkp->dtpk_fmatch == &dtrace_match_nul &&
7164             pkp->dtpk_nmatch == &dtrace_match_nul)
7165                 pkp->dtpk_fmatch = &dtrace_match_nonzero;
7166 }
7167 
7168 /*
7169  * DTrace Provider-to-Framework API Functions
7170  *
7171  * These functions implement much of the Provider-to-Framework API, as
7172  * described in <sys/dtrace.h>.  The parts of the API not in this section are
7173  * the functions in the API for probe management (found below), and
7174  * dtrace_probe() itself (found above).
7175  */
7176 
7177 /*
7178  * Register the calling provider with the DTrace framework.  This should
7179  * generally be called by DTrace providers in their attach(9E) entry point.
7180  */
7181 int
7182 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
7183     cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
7184 {
7185         dtrace_provider_t *provider;
7186 
7187         if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
7188                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7189                     "arguments", name ? name : "<NULL>");
7190                 return (EINVAL);
7191         }
7192 
7193         if (name[0] == '\0' || dtrace_badname(name)) {
7194                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7195                     "provider name", name);
7196                 return (EINVAL);
7197         }
7198 
7199         if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
7200             pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
7201             pops->dtps_destroy == NULL ||
7202             ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
7203                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7204                     "provider ops", name);
7205                 return (EINVAL);
7206         }
7207 
7208         if (dtrace_badattr(&pap->dtpa_provider) ||
7209             dtrace_badattr(&pap->dtpa_mod) ||
7210             dtrace_badattr(&pap->dtpa_func) ||
7211             dtrace_badattr(&pap->dtpa_name) ||
7212             dtrace_badattr(&pap->dtpa_args)) {
7213                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7214                     "provider attributes", name);
7215                 return (EINVAL);
7216         }
7217 
7218         if (priv & ~DTRACE_PRIV_ALL) {
7219                 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7220                     "privilege attributes", name);
7221                 return (EINVAL);
7222         }
7223 
7224         if ((priv & DTRACE_PRIV_KERNEL) &&
7225             (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
7226             pops->dtps_mode == NULL) {
7227                 cmn_err(CE_WARN, "failed to register provider '%s': need "
7228                     "dtps_mode() op for given privilege attributes", name);
7229                 return (EINVAL);
7230         }
7231 
7232         provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
7233         provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
7234         (void) strcpy(provider->dtpv_name, name);
7235 
7236         provider->dtpv_attr = *pap;
7237         provider->dtpv_priv.dtpp_flags = priv;
7238         if (cr != NULL) {
7239                 provider->dtpv_priv.dtpp_uid = crgetuid(cr);
7240                 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
7241         }
7242         provider->dtpv_pops = *pops;
7243 
7244         if (pops->dtps_provide == NULL) {
7245                 ASSERT(pops->dtps_provide_module != NULL);
7246                 provider->dtpv_pops.dtps_provide =
7247                     (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
7248         }
7249 
7250         if (pops->dtps_provide_module == NULL) {
7251                 ASSERT(pops->dtps_provide != NULL);
7252                 provider->dtpv_pops.dtps_provide_module =
7253                     (void (*)(void *, struct modctl *))dtrace_nullop;
7254         }
7255 
7256         if (pops->dtps_suspend == NULL) {
7257                 ASSERT(pops->dtps_resume == NULL);
7258                 provider->dtpv_pops.dtps_suspend =
7259                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7260                 provider->dtpv_pops.dtps_resume =
7261                     (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7262         }
7263 
7264         provider->dtpv_arg = arg;
7265         *idp = (dtrace_provider_id_t)provider;
7266 
7267         if (pops == &dtrace_provider_ops) {
7268                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7269                 ASSERT(MUTEX_HELD(&dtrace_lock));
7270                 ASSERT(dtrace_anon.dta_enabling == NULL);
7271 
7272                 /*
7273                  * We make sure that the DTrace provider is at the head of
7274                  * the provider chain.
7275                  */
7276                 provider->dtpv_next = dtrace_provider;
7277                 dtrace_provider = provider;
7278                 return (0);
7279         }
7280 
7281         mutex_enter(&dtrace_provider_lock);
7282         mutex_enter(&dtrace_lock);
7283 
7284         /*
7285          * If there is at least one provider registered, we'll add this
7286          * provider after the first provider.
7287          */
7288         if (dtrace_provider != NULL) {
7289                 provider->dtpv_next = dtrace_provider->dtpv_next;
7290                 dtrace_provider->dtpv_next = provider;
7291         } else {
7292                 dtrace_provider = provider;
7293         }
7294 
7295         if (dtrace_retained != NULL) {
7296                 dtrace_enabling_provide(provider);
7297 
7298                 /*
7299                  * Now we need to call dtrace_enabling_matchall() -- which
7300                  * will acquire cpu_lock and dtrace_lock.  We therefore need
7301                  * to drop all of our locks before calling into it...
7302                  */
7303                 mutex_exit(&dtrace_lock);
7304                 mutex_exit(&dtrace_provider_lock);
7305                 dtrace_enabling_matchall();
7306 
7307                 return (0);
7308         }
7309 
7310         mutex_exit(&dtrace_lock);
7311         mutex_exit(&dtrace_provider_lock);
7312 
7313         return (0);
7314 }
7315 
7316 /*
7317  * Unregister the specified provider from the DTrace framework.  This should
7318  * generally be called by DTrace providers in their detach(9E) entry point.
7319  */
7320 int
7321 dtrace_unregister(dtrace_provider_id_t id)
7322 {
7323         dtrace_provider_t *old = (dtrace_provider_t *)id;
7324         dtrace_provider_t *prev = NULL;
7325         int i, self = 0, noreap = 0;
7326         dtrace_probe_t *probe, *first = NULL;
7327 
7328         if (old->dtpv_pops.dtps_enable ==
7329             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) {
7330                 /*
7331                  * If DTrace itself is the provider, we're called with locks
7332                  * already held.
7333                  */
7334                 ASSERT(old == dtrace_provider);
7335                 ASSERT(dtrace_devi != NULL);
7336                 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7337                 ASSERT(MUTEX_HELD(&dtrace_lock));
7338                 self = 1;
7339 
7340                 if (dtrace_provider->dtpv_next != NULL) {
7341                         /*
7342                          * There's another provider here; return failure.
7343                          */
7344                         return (EBUSY);
7345                 }
7346         } else {
7347                 mutex_enter(&dtrace_provider_lock);
7348                 mutex_enter(&mod_lock);
7349                 mutex_enter(&dtrace_lock);
7350         }
7351 
7352         /*
7353          * If anyone has /dev/dtrace open, or if there are anonymous enabled
7354          * probes, we refuse to let providers slither away, unless this
7355          * provider has already been explicitly invalidated.
7356          */
7357         if (!old->dtpv_defunct &&
7358             (dtrace_opens || (dtrace_anon.dta_state != NULL &&
7359             dtrace_anon.dta_state->dts_necbs > 0))) {
7360                 if (!self) {
7361                         mutex_exit(&dtrace_lock);
7362                         mutex_exit(&mod_lock);
7363                         mutex_exit(&dtrace_provider_lock);
7364                 }
7365                 return (EBUSY);
7366         }
7367 
7368         /*
7369          * Attempt to destroy the probes associated with this provider.
7370          */
7371         for (i = 0; i < dtrace_nprobes; i++) {
7372                 if ((probe = dtrace_probes[i]) == NULL)
7373                         continue;
7374 
7375                 if (probe->dtpr_provider != old)
7376                         continue;
7377 
7378                 if (probe->dtpr_ecb == NULL)
7379                         continue;
7380 
7381                 /*
7382                  * If we are trying to unregister a defunct provider, and the
7383                  * provider was made defunct within the interval dictated by
7384                  * dtrace_unregister_defunct_reap, we'll (asynchronously)
7385                  * attempt to reap our enablings.  To denote that the provider
7386                  * should reattempt to unregister itself at some point in the
7387                  * future, we will return a differentiable error code (EAGAIN
7388                  * instead of EBUSY) in this case.
7389                  */
7390                 if (dtrace_gethrtime() - old->dtpv_defunct >
7391                     dtrace_unregister_defunct_reap)
7392                         noreap = 1;
7393 
7394                 if (!self) {
7395                         mutex_exit(&dtrace_lock);
7396                         mutex_exit(&mod_lock);
7397                         mutex_exit(&dtrace_provider_lock);
7398                 }
7399 
7400                 if (noreap)
7401                         return (EBUSY);
7402 
7403                 (void) taskq_dispatch(dtrace_taskq,
7404                     (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
7405 
7406                 return (EAGAIN);
7407         }
7408 
7409         /*
7410          * All of the probes for this provider are disabled; we can safely
7411          * remove all of them from their hash chains and from the probe array.
7412          */
7413         for (i = 0; i < dtrace_nprobes; i++) {
7414                 if ((probe = dtrace_probes[i]) == NULL)
7415                         continue;
7416 
7417                 if (probe->dtpr_provider != old)
7418                         continue;
7419 
7420                 dtrace_probes[i] = NULL;
7421 
7422                 dtrace_hash_remove(dtrace_bymod, probe);
7423                 dtrace_hash_remove(dtrace_byfunc, probe);
7424                 dtrace_hash_remove(dtrace_byname, probe);
7425 
7426                 if (first == NULL) {
7427                         first = probe;
7428                         probe->dtpr_nextmod = NULL;
7429                 } else {
7430                         probe->dtpr_nextmod = first;
7431                         first = probe;
7432                 }
7433         }
7434 
7435         /*
7436          * The provider's probes have been removed from the hash chains and
7437          * from the probe array.  Now issue a dtrace_sync() to be sure that
7438          * everyone has cleared out from any probe array processing.
7439          */
7440         dtrace_sync();
7441 
7442         for (probe = first; probe != NULL; probe = first) {
7443                 first = probe->dtpr_nextmod;
7444 
7445                 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
7446                     probe->dtpr_arg);
7447                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
7448                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
7449                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
7450                 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
7451                 kmem_free(probe, sizeof (dtrace_probe_t));
7452         }
7453 
7454         if ((prev = dtrace_provider) == old) {
7455                 ASSERT(self || dtrace_devi == NULL);
7456                 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
7457                 dtrace_provider = old->dtpv_next;
7458         } else {
7459                 while (prev != NULL && prev->dtpv_next != old)
7460                         prev = prev->dtpv_next;
7461 
7462                 if (prev == NULL) {
7463                         panic("attempt to unregister non-existent "
7464                             "dtrace provider %p\n", (void *)id);
7465                 }
7466 
7467                 prev->dtpv_next = old->dtpv_next;
7468         }
7469 
7470         if (!self) {
7471                 mutex_exit(&dtrace_lock);
7472                 mutex_exit(&mod_lock);
7473                 mutex_exit(&dtrace_provider_lock);
7474         }
7475 
7476         kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
7477         kmem_free(old, sizeof (dtrace_provider_t));
7478 
7479         return (0);
7480 }
7481 
7482 /*
7483  * Invalidate the specified provider.  All subsequent probe lookups for the
7484  * specified provider will fail, but its probes will not be removed.
7485  */
7486 void
7487 dtrace_invalidate(dtrace_provider_id_t id)
7488 {
7489         dtrace_provider_t *pvp = (dtrace_provider_t *)id;
7490 
7491         ASSERT(pvp->dtpv_pops.dtps_enable !=
7492             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
7493 
7494         mutex_enter(&dtrace_provider_lock);
7495         mutex_enter(&dtrace_lock);
7496 
7497         pvp->dtpv_defunct = dtrace_gethrtime();
7498 
7499         mutex_exit(&dtrace_lock);
7500         mutex_exit(&dtrace_provider_lock);
7501 }
7502 
7503 /*
7504  * Indicate whether or not DTrace has attached.
7505  */
7506 int
7507 dtrace_attached(void)
7508 {
7509         /*
7510          * dtrace_provider will be non-NULL iff the DTrace driver has
7511          * attached.  (It's non-NULL because DTrace is always itself a
7512          * provider.)
7513          */
7514         return (dtrace_provider != NULL);
7515 }
7516 
7517 /*
7518  * Remove all the unenabled probes for the given provider.  This function is
7519  * not unlike dtrace_unregister(), except that it doesn't remove the provider
7520  * -- just as many of its associated probes as it can.
7521  */
7522 int
7523 dtrace_condense(dtrace_provider_id_t id)
7524 {
7525         dtrace_provider_t *prov = (dtrace_provider_t *)id;
7526         int i;
7527         dtrace_probe_t *probe;
7528 
7529         /*
7530          * Make sure this isn't the dtrace provider itself.
7531          */
7532         ASSERT(prov->dtpv_pops.dtps_enable !=
7533             (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
7534 
7535         mutex_enter(&dtrace_provider_lock);
7536         mutex_enter(&dtrace_lock);
7537 
7538         /*
7539          * Attempt to destroy the probes associated with this provider.
7540          */
7541         for (i = 0; i < dtrace_nprobes; i++) {
7542                 if ((probe = dtrace_probes[i]) == NULL)
7543                         continue;
7544 
7545                 if (probe->dtpr_provider != prov)
7546                         continue;
7547 
7548                 if (probe->dtpr_ecb != NULL)
7549                         continue;
7550 
7551                 dtrace_probes[i] = NULL;
7552 
7553                 dtrace_hash_remove(dtrace_bymod, probe);
7554                 dtrace_hash_remove(dtrace_byfunc, probe);
7555                 dtrace_hash_remove(dtrace_byname, probe);
7556 
7557                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
7558                     probe->dtpr_arg);
7559                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
7560                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
7561                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
7562                 kmem_free(probe, sizeof (dtrace_probe_t));
7563                 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
7564         }
7565 
7566         mutex_exit(&dtrace_lock);
7567         mutex_exit(&dtrace_provider_lock);
7568 
7569         return (0);
7570 }
7571 
7572 /*
7573  * DTrace Probe Management Functions
7574  *
7575  * The functions in this section perform the DTrace probe management,
7576  * including functions to create probes, look-up probes, and call into the
7577  * providers to request that probes be provided.  Some of these functions are
7578  * in the Provider-to-Framework API; these functions can be identified by the
7579  * fact that they are not declared "static".
7580  */
7581 
7582 /*
7583  * Create a probe with the specified module name, function name, and name.
7584  */
7585 dtrace_id_t
7586 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
7587     const char *func, const char *name, int aframes, void *arg)
7588 {
7589         dtrace_probe_t *probe, **probes;
7590         dtrace_provider_t *provider = (dtrace_provider_t *)prov;
7591         dtrace_id_t id;
7592 
7593         if (provider == dtrace_provider) {
7594                 ASSERT(MUTEX_HELD(&dtrace_lock));
7595         } else {
7596                 mutex_enter(&dtrace_lock);
7597         }
7598 
7599         id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
7600             VM_BESTFIT | VM_SLEEP);
7601         probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
7602 
7603         probe->dtpr_id = id;
7604         probe->dtpr_gen = dtrace_probegen++;
7605         probe->dtpr_mod = dtrace_strdup(mod);
7606         probe->dtpr_func = dtrace_strdup(func);
7607         probe->dtpr_name = dtrace_strdup(name);
7608         probe->dtpr_arg = arg;
7609         probe->dtpr_aframes = aframes;
7610         probe->dtpr_provider = provider;
7611 
7612         dtrace_hash_add(dtrace_bymod, probe);
7613         dtrace_hash_add(dtrace_byfunc, probe);
7614         dtrace_hash_add(dtrace_byname, probe);
7615 
7616         if (id - 1 >= dtrace_nprobes) {
7617                 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
7618                 size_t nsize = osize << 1;
7619 
7620                 if (nsize == 0) {
7621                         ASSERT(osize == 0);
7622                         ASSERT(dtrace_probes == NULL);
7623                         nsize = sizeof (dtrace_probe_t *);
7624                 }
7625 
7626                 probes = kmem_zalloc(nsize, KM_SLEEP);
7627 
7628                 if (dtrace_probes == NULL) {
7629                         ASSERT(osize == 0);
7630                         dtrace_probes = probes;
7631                         dtrace_nprobes = 1;
7632                 } else {
7633                         dtrace_probe_t **oprobes = dtrace_probes;
7634 
7635                         bcopy(oprobes, probes, osize);
7636                         dtrace_membar_producer();
7637                         dtrace_probes = probes;
7638 
7639                         dtrace_sync();
7640 
7641                         /*
7642                          * All CPUs are now seeing the new probes array; we can
7643                          * safely free the old array.
7644                          */
7645                         kmem_free(oprobes, osize);
7646                         dtrace_nprobes <<= 1;
7647                 }
7648 
7649                 ASSERT(id - 1 < dtrace_nprobes);
7650         }
7651 
7652         ASSERT(dtrace_probes[id - 1] == NULL);
7653         dtrace_probes[id - 1] = probe;
7654 
7655         if (provider != dtrace_provider)
7656                 mutex_exit(&dtrace_lock);
7657 
7658         return (id);
7659 }
7660 
7661 static dtrace_probe_t *
7662 dtrace_probe_lookup_id(dtrace_id_t id)
7663 {
7664         ASSERT(MUTEX_HELD(&dtrace_lock));
7665 
7666         if (id == 0 || id > dtrace_nprobes)
7667                 return (NULL);
7668 
7669         return (dtrace_probes[id - 1]);
7670 }
7671 
7672 static int
7673 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
7674 {
7675         *((dtrace_id_t *)arg) = probe->dtpr_id;
7676 
7677         return (DTRACE_MATCH_DONE);
7678 }
7679 
7680 /*
7681  * Look up a probe based on provider and one or more of module name, function
7682  * name and probe name.
7683  */
7684 dtrace_id_t
7685 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
7686     const char *func, const char *name)
7687 {
7688         dtrace_probekey_t pkey;
7689         dtrace_id_t id;
7690         int match;
7691 
7692         pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
7693         pkey.dtpk_pmatch = &dtrace_match_string;
7694         pkey.dtpk_mod = mod;
7695         pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
7696         pkey.dtpk_func = func;
7697         pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
7698         pkey.dtpk_name = name;
7699         pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
7700         pkey.dtpk_id = DTRACE_IDNONE;
7701 
7702         mutex_enter(&dtrace_lock);
7703         match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
7704             dtrace_probe_lookup_match, &id);
7705         mutex_exit(&dtrace_lock);
7706 
7707         ASSERT(match == 1 || match == 0);
7708         return (match ? id : 0);
7709 }
7710 
7711 /*
7712  * Returns the probe argument associated with the specified probe.
7713  */
7714 void *
7715 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
7716 {
7717         dtrace_probe_t *probe;
7718         void *rval = NULL;
7719 
7720         mutex_enter(&dtrace_lock);
7721 
7722         if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
7723             probe->dtpr_provider == (dtrace_provider_t *)id)
7724                 rval = probe->dtpr_arg;
7725 
7726         mutex_exit(&dtrace_lock);
7727 
7728         return (rval);
7729 }
7730 
7731 /*
7732  * Copy a probe into a probe description.
7733  */
7734 static void
7735 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
7736 {
7737         bzero(pdp, sizeof (dtrace_probedesc_t));
7738         pdp->dtpd_id = prp->dtpr_id;
7739 
7740         (void) strncpy(pdp->dtpd_provider,
7741             prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
7742 
7743         (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
7744         (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
7745         (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
7746 }
7747 
7748 /*
7749  * Called to indicate that a probe -- or probes -- should be provided by a
7750  * specfied provider.  If the specified description is NULL, the provider will
7751  * be told to provide all of its probes.  (This is done whenever a new
7752  * consumer comes along, or whenever a retained enabling is to be matched.) If
7753  * the specified description is non-NULL, the provider is given the
7754  * opportunity to dynamically provide the specified probe, allowing providers
7755  * to support the creation of probes on-the-fly.  (So-called _autocreated_
7756  * probes.)  If the provider is NULL, the operations will be applied to all
7757  * providers; if the provider is non-NULL the operations will only be applied
7758  * to the specified provider.  The dtrace_provider_lock must be held, and the
7759  * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
7760  * will need to grab the dtrace_lock when it reenters the framework through
7761  * dtrace_probe_lookup(), dtrace_probe_create(), etc.
7762  */
7763 static void
7764 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
7765 {
7766         struct modctl *ctl;
7767         int all = 0;
7768 
7769         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7770 
7771         if (prv == NULL) {
7772                 all = 1;
7773                 prv = dtrace_provider;
7774         }
7775 
7776         do {
7777                 /*
7778                  * First, call the blanket provide operation.
7779                  */
7780                 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
7781 
7782                 /*
7783                  * Now call the per-module provide operation.  We will grab
7784                  * mod_lock to prevent the list from being modified.  Note
7785                  * that this also prevents the mod_busy bits from changing.
7786                  * (mod_busy can only be changed with mod_lock held.)
7787                  */
7788                 mutex_enter(&mod_lock);
7789 
7790                 ctl = &modules;
7791                 do {
7792                         if (ctl->mod_busy || ctl->mod_mp == NULL)
7793                                 continue;
7794 
7795                         prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
7796 
7797                 } while ((ctl = ctl->mod_next) != &modules);
7798 
7799                 mutex_exit(&mod_lock);
7800         } while (all && (prv = prv->dtpv_next) != NULL);
7801 }
7802 
7803 /*
7804  * Iterate over each probe, and call the Framework-to-Provider API function
7805  * denoted by offs.
7806  */
7807 static void
7808 dtrace_probe_foreach(uintptr_t offs)
7809 {
7810         dtrace_provider_t *prov;
7811         void (*func)(void *, dtrace_id_t, void *);
7812         dtrace_probe_t *probe;
7813         dtrace_icookie_t cookie;
7814         int i;
7815 
7816         /*
7817          * We disable interrupts to walk through the probe array.  This is
7818          * safe -- the dtrace_sync() in dtrace_unregister() assures that we
7819          * won't see stale data.
7820          */
7821         cookie = dtrace_interrupt_disable();
7822 
7823         for (i = 0; i < dtrace_nprobes; i++) {
7824                 if ((probe = dtrace_probes[i]) == NULL)
7825                         continue;
7826 
7827                 if (probe->dtpr_ecb == NULL) {
7828                         /*
7829                          * This probe isn't enabled -- don't call the function.
7830                          */
7831                         continue;
7832                 }
7833 
7834                 prov = probe->dtpr_provider;
7835                 func = *((void(**)(void *, dtrace_id_t, void *))
7836                     ((uintptr_t)&prov->dtpv_pops + offs));
7837 
7838                 func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
7839         }
7840 
7841         dtrace_interrupt_enable(cookie);
7842 }
7843 
7844 static int
7845 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
7846 {
7847         dtrace_probekey_t pkey;
7848         uint32_t priv;
7849         uid_t uid;
7850         zoneid_t zoneid;
7851 
7852         ASSERT(MUTEX_HELD(&dtrace_lock));
7853         dtrace_ecb_create_cache = NULL;
7854 
7855         if (desc == NULL) {
7856                 /*
7857                  * If we're passed a NULL description, we're being asked to
7858                  * create an ECB with a NULL probe.
7859                  */
7860                 (void) dtrace_ecb_create_enable(NULL, enab);
7861                 return (0);
7862         }
7863 
7864         dtrace_probekey(desc, &pkey);
7865         dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
7866             &priv, &uid, &zoneid);
7867 
7868         return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
7869             enab));
7870 }
7871 
7872 /*
7873  * DTrace Helper Provider Functions
7874  */
7875 static void
7876 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
7877 {
7878         attr->dtat_name = DOF_ATTR_NAME(dofattr);
7879         attr->dtat_data = DOF_ATTR_DATA(dofattr);
7880         attr->dtat_class = DOF_ATTR_CLASS(dofattr);
7881 }
7882 
7883 static void
7884 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
7885     const dof_provider_t *dofprov, char *strtab)
7886 {
7887         hprov->dthpv_provname = strtab + dofprov->dofpv_name;
7888         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
7889             dofprov->dofpv_provattr);
7890         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
7891             dofprov->dofpv_modattr);
7892         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
7893             dofprov->dofpv_funcattr);
7894         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
7895             dofprov->dofpv_nameattr);
7896         dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
7897             dofprov->dofpv_argsattr);
7898 }
7899 
7900 static void
7901 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
7902 {
7903         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
7904         dof_hdr_t *dof = (dof_hdr_t *)daddr;
7905         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
7906         dof_provider_t *provider;
7907         dof_probe_t *probe;
7908         uint32_t *off, *enoff;
7909         uint8_t *arg;
7910         char *strtab;
7911         uint_t i, nprobes;
7912         dtrace_helper_provdesc_t dhpv;
7913         dtrace_helper_probedesc_t dhpb;
7914         dtrace_meta_t *meta = dtrace_meta_pid;
7915         dtrace_mops_t *mops = &meta->dtm_mops;
7916         void *parg;
7917 
7918         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
7919         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7920             provider->dofpv_strtab * dof->dofh_secsize);
7921         prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7922             provider->dofpv_probes * dof->dofh_secsize);
7923         arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7924             provider->dofpv_prargs * dof->dofh_secsize);
7925         off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7926             provider->dofpv_proffs * dof->dofh_secsize);
7927 
7928         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
7929         off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
7930         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
7931         enoff = NULL;
7932 
7933         /*
7934          * See dtrace_helper_provider_validate().
7935          */
7936         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
7937             provider->dofpv_prenoffs != DOF_SECT_NONE) {
7938                 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7939                     provider->dofpv_prenoffs * dof->dofh_secsize);
7940                 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
7941         }
7942 
7943         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
7944 
7945         /*
7946          * Create the provider.
7947          */
7948         dtrace_dofprov2hprov(&dhpv, provider, strtab);
7949 
7950         if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
7951                 return;
7952 
7953         meta->dtm_count++;
7954 
7955         /*
7956          * Create the probes.
7957          */
7958         for (i = 0; i < nprobes; i++) {
7959                 probe = (dof_probe_t *)(uintptr_t)(daddr +
7960                     prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
7961 
7962                 dhpb.dthpb_mod = dhp->dofhp_mod;
7963                 dhpb.dthpb_func = strtab + probe->dofpr_func;
7964                 dhpb.dthpb_name = strtab + probe->dofpr_name;
7965                 dhpb.dthpb_base = probe->dofpr_addr;
7966                 dhpb.dthpb_offs = off + probe->dofpr_offidx;
7967                 dhpb.dthpb_noffs = probe->dofpr_noffs;
7968                 if (enoff != NULL) {
7969                         dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
7970                         dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
7971                 } else {
7972                         dhpb.dthpb_enoffs = NULL;
7973                         dhpb.dthpb_nenoffs = 0;
7974                 }
7975                 dhpb.dthpb_args = arg + probe->dofpr_argidx;
7976                 dhpb.dthpb_nargc = probe->dofpr_nargc;
7977                 dhpb.dthpb_xargc = probe->dofpr_xargc;
7978                 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
7979                 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
7980 
7981                 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
7982         }
7983 }
7984 
7985 static void
7986 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
7987 {
7988         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
7989         dof_hdr_t *dof = (dof_hdr_t *)daddr;
7990         int i;
7991 
7992         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
7993 
7994         for (i = 0; i < dof->dofh_secnum; i++) {
7995                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
7996                     dof->dofh_secoff + i * dof->dofh_secsize);
7997 
7998                 if (sec->dofs_type != DOF_SECT_PROVIDER)
7999                         continue;
8000 
8001                 dtrace_helper_provide_one(dhp, sec, pid);
8002         }
8003 
8004         /*
8005          * We may have just created probes, so we must now rematch against
8006          * any retained enablings.  Note that this call will acquire both
8007          * cpu_lock and dtrace_lock; the fact that we are holding
8008          * dtrace_meta_lock now is what defines the ordering with respect to
8009          * these three locks.
8010          */
8011         dtrace_enabling_matchall();
8012 }
8013 
8014 static void
8015 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
8016 {
8017         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8018         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8019         dof_sec_t *str_sec;
8020         dof_provider_t *provider;
8021         char *strtab;
8022         dtrace_helper_provdesc_t dhpv;
8023         dtrace_meta_t *meta = dtrace_meta_pid;
8024         dtrace_mops_t *mops = &meta->dtm_mops;
8025 
8026         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
8027         str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
8028             provider->dofpv_strtab * dof->dofh_secsize);
8029 
8030         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
8031 
8032         /*
8033          * Create the provider.
8034          */
8035         dtrace_dofprov2hprov(&dhpv, provider, strtab);
8036 
8037         mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
8038 
8039         meta->dtm_count--;
8040 }
8041 
8042 static void
8043 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
8044 {
8045         uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8046         dof_hdr_t *dof = (dof_hdr_t *)daddr;
8047         int i;
8048 
8049         ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8050 
8051         for (i = 0; i < dof->dofh_secnum; i++) {
8052                 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8053                     dof->dofh_secoff + i * dof->dofh_secsize);
8054 
8055                 if (sec->dofs_type != DOF_SECT_PROVIDER)
8056                         continue;
8057 
8058                 dtrace_helper_provider_remove_one(dhp, sec, pid);
8059         }
8060 }
8061 
8062 /*
8063  * DTrace Meta Provider-to-Framework API Functions
8064  *
8065  * These functions implement the Meta Provider-to-Framework API, as described
8066  * in <sys/dtrace.h>.
8067  */
8068 int
8069 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
8070     dtrace_meta_provider_id_t *idp)
8071 {
8072         dtrace_meta_t *meta;
8073         dtrace_helpers_t *help, *next;
8074         int i;
8075 
8076         *idp = DTRACE_METAPROVNONE;
8077 
8078         /*
8079          * We strictly don't need the name, but we hold onto it for
8080          * debuggability. All hail error queues!
8081          */
8082         if (name == NULL) {
8083                 cmn_err(CE_WARN, "failed to register meta-provider: "
8084                     "invalid name");
8085                 return (EINVAL);
8086         }
8087 
8088         if (mops == NULL ||
8089             mops->dtms_create_probe == NULL ||
8090             mops->dtms_provide_pid == NULL ||
8091             mops->dtms_remove_pid == NULL) {
8092                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8093                     "invalid ops", name);
8094                 return (EINVAL);
8095         }
8096 
8097         meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
8098         meta->dtm_mops = *mops;
8099         meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
8100         (void) strcpy(meta->dtm_name, name);
8101         meta->dtm_arg = arg;
8102 
8103         mutex_enter(&dtrace_meta_lock);
8104         mutex_enter(&dtrace_lock);
8105 
8106         if (dtrace_meta_pid != NULL) {
8107                 mutex_exit(&dtrace_lock);
8108                 mutex_exit(&dtrace_meta_lock);
8109                 cmn_err(CE_WARN, "failed to register meta-register %s: "
8110                     "user-land meta-provider exists", name);
8111                 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
8112                 kmem_free(meta, sizeof (dtrace_meta_t));
8113                 return (EINVAL);
8114         }
8115 
8116         dtrace_meta_pid = meta;
8117         *idp = (dtrace_meta_provider_id_t)meta;
8118 
8119         /*
8120          * If there are providers and probes ready to go, pass them
8121          * off to the new meta provider now.
8122          */
8123 
8124         help = dtrace_deferred_pid;
8125         dtrace_deferred_pid = NULL;
8126 
8127         mutex_exit(&dtrace_lock);
8128 
8129         while (help != NULL) {
8130                 for (i = 0; i < help->dthps_nprovs; i++) {
8131                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
8132                             help->dthps_pid);
8133                 }
8134 
8135                 next = help->dthps_next;
8136                 help->dthps_next = NULL;
8137                 help->dthps_prev = NULL;
8138                 help->dthps_deferred = 0;
8139                 help = next;
8140         }
8141 
8142         mutex_exit(&dtrace_meta_lock);
8143 
8144         return (0);
8145 }
8146 
8147 int
8148 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
8149 {
8150         dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
8151 
8152         mutex_enter(&dtrace_meta_lock);
8153         mutex_enter(&dtrace_lock);
8154 
8155         if (old == dtrace_meta_pid) {
8156                 pp = &dtrace_meta_pid;
8157         } else {
8158                 panic("attempt to unregister non-existent "
8159                     "dtrace meta-provider %p\n", (void *)old);
8160         }
8161 
8162         if (old->dtm_count != 0) {
8163                 mutex_exit(&dtrace_lock);
8164                 mutex_exit(&dtrace_meta_lock);
8165                 return (EBUSY);
8166         }
8167 
8168         *pp = NULL;
8169 
8170         mutex_exit(&dtrace_lock);
8171         mutex_exit(&dtrace_meta_lock);
8172 
8173         kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
8174         kmem_free(old, sizeof (dtrace_meta_t));
8175 
8176         return (0);
8177 }
8178 
8179 
8180 /*
8181  * DTrace DIF Object Functions
8182  */
8183 static int
8184 dtrace_difo_err(uint_t pc, const char *format, ...)
8185 {
8186         if (dtrace_err_verbose) {
8187                 va_list alist;
8188 
8189                 (void) uprintf("dtrace DIF object error: [%u]: ", pc);
8190                 va_start(alist, format);
8191                 (void) vuprintf(format, alist);
8192                 va_end(alist);
8193         }
8194 
8195 #ifdef DTRACE_ERRDEBUG
8196         dtrace_errdebug(format);
8197 #endif
8198         return (1);
8199 }
8200 
8201 /*
8202  * Validate a DTrace DIF object by checking the IR instructions.  The following
8203  * rules are currently enforced by dtrace_difo_validate():
8204  *
8205  * 1. Each instruction must have a valid opcode
8206  * 2. Each register, string, variable, or subroutine reference must be valid
8207  * 3. No instruction can modify register %r0 (must be zero)
8208  * 4. All instruction reserved bits must be set to zero
8209  * 5. The last instruction must be a "ret" instruction
8210  * 6. All branch targets must reference a valid instruction _after_ the branch
8211  */
8212 static int
8213 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
8214     cred_t *cr)
8215 {
8216         int err = 0, i;
8217         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8218         int kcheckload;
8219         uint_t pc;
8220 
8221         kcheckload = cr == NULL ||
8222             (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
8223 
8224         dp->dtdo_destructive = 0;
8225 
8226         for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
8227                 dif_instr_t instr = dp->dtdo_buf[pc];
8228 
8229                 uint_t r1 = DIF_INSTR_R1(instr);
8230                 uint_t r2 = DIF_INSTR_R2(instr);
8231                 uint_t rd = DIF_INSTR_RD(instr);
8232                 uint_t rs = DIF_INSTR_RS(instr);
8233                 uint_t label = DIF_INSTR_LABEL(instr);
8234                 uint_t v = DIF_INSTR_VAR(instr);
8235                 uint_t subr = DIF_INSTR_SUBR(instr);
8236                 uint_t type = DIF_INSTR_TYPE(instr);
8237                 uint_t op = DIF_INSTR_OP(instr);
8238 
8239                 switch (op) {
8240                 case DIF_OP_OR:
8241                 case DIF_OP_XOR:
8242                 case DIF_OP_AND:
8243                 case DIF_OP_SLL:
8244                 case DIF_OP_SRL:
8245                 case DIF_OP_SRA:
8246                 case DIF_OP_SUB:
8247                 case DIF_OP_ADD:
8248                 case DIF_OP_MUL:
8249                 case DIF_OP_SDIV:
8250                 case DIF_OP_UDIV:
8251                 case DIF_OP_SREM:
8252                 case DIF_OP_UREM:
8253                 case DIF_OP_COPYS:
8254                         if (r1 >= nregs)
8255                                 err += efunc(pc, "invalid register %u\n", r1);
8256                         if (r2 >= nregs)
8257                                 err += efunc(pc, "invalid register %u\n", r2);
8258                         if (rd >= nregs)
8259                                 err += efunc(pc, "invalid register %u\n", rd);
8260                         if (rd == 0)
8261                                 err += efunc(pc, "cannot write to %r0\n");
8262                         break;
8263                 case DIF_OP_NOT:
8264                 case DIF_OP_MOV:
8265                 case DIF_OP_ALLOCS:
8266                         if (r1 >= nregs)
8267                                 err += efunc(pc, "invalid register %u\n", r1);
8268                         if (r2 != 0)
8269                                 err += efunc(pc, "non-zero reserved bits\n");
8270                         if (rd >= nregs)
8271                                 err += efunc(pc, "invalid register %u\n", rd);
8272                         if (rd == 0)
8273                                 err += efunc(pc, "cannot write to %r0\n");
8274                         break;
8275                 case DIF_OP_LDSB:
8276                 case DIF_OP_LDSH:
8277                 case DIF_OP_LDSW:
8278                 case DIF_OP_LDUB:
8279                 case DIF_OP_LDUH:
8280                 case DIF_OP_LDUW:
8281                 case DIF_OP_LDX:
8282                         if (r1 >= nregs)
8283                                 err += efunc(pc, "invalid register %u\n", r1);
8284                         if (r2 != 0)
8285                                 err += efunc(pc, "non-zero reserved bits\n");
8286                         if (rd >= nregs)
8287                                 err += efunc(pc, "invalid register %u\n", rd);
8288                         if (rd == 0)
8289                                 err += efunc(pc, "cannot write to %r0\n");
8290                         if (kcheckload)
8291                                 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
8292                                     DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
8293                         break;
8294                 case DIF_OP_RLDSB:
8295                 case DIF_OP_RLDSH:
8296                 case DIF_OP_RLDSW:
8297                 case DIF_OP_RLDUB:
8298                 case DIF_OP_RLDUH:
8299                 case DIF_OP_RLDUW:
8300                 case DIF_OP_RLDX:
8301                         if (r1 >= nregs)
8302                                 err += efunc(pc, "invalid register %u\n", r1);
8303                         if (r2 != 0)
8304                                 err += efunc(pc, "non-zero reserved bits\n");
8305                         if (rd >= nregs)
8306                                 err += efunc(pc, "invalid register %u\n", rd);
8307                         if (rd == 0)
8308                                 err += efunc(pc, "cannot write to %r0\n");
8309                         break;
8310                 case DIF_OP_ULDSB:
8311                 case DIF_OP_ULDSH:
8312                 case DIF_OP_ULDSW:
8313                 case DIF_OP_ULDUB:
8314                 case DIF_OP_ULDUH:
8315                 case DIF_OP_ULDUW:
8316                 case DIF_OP_ULDX:
8317                         if (r1 >= nregs)
8318                                 err += efunc(pc, "invalid register %u\n", r1);
8319                         if (r2 != 0)
8320                                 err += efunc(pc, "non-zero reserved bits\n");
8321                         if (rd >= nregs)
8322                                 err += efunc(pc, "invalid register %u\n", rd);
8323                         if (rd == 0)
8324                                 err += efunc(pc, "cannot write to %r0\n");
8325                         break;
8326                 case DIF_OP_STB:
8327                 case DIF_OP_STH:
8328                 case DIF_OP_STW:
8329                 case DIF_OP_STX:
8330                         if (r1 >= nregs)
8331                                 err += efunc(pc, "invalid register %u\n", r1);
8332                         if (r2 != 0)
8333                                 err += efunc(pc, "non-zero reserved bits\n");
8334                         if (rd >= nregs)
8335                                 err += efunc(pc, "invalid register %u\n", rd);
8336                         if (rd == 0)
8337                                 err += efunc(pc, "cannot write to 0 address\n");
8338                         break;
8339                 case DIF_OP_CMP:
8340                 case DIF_OP_SCMP:
8341                         if (r1 >= nregs)
8342                                 err += efunc(pc, "invalid register %u\n", r1);
8343                         if (r2 >= nregs)
8344                                 err += efunc(pc, "invalid register %u\n", r2);
8345                         if (rd != 0)
8346                                 err += efunc(pc, "non-zero reserved bits\n");
8347                         break;
8348                 case DIF_OP_TST:
8349                         if (r1 >= nregs)
8350                                 err += efunc(pc, "invalid register %u\n", r1);
8351                         if (r2 != 0 || rd != 0)
8352                                 err += efunc(pc, "non-zero reserved bits\n");
8353                         break;
8354                 case DIF_OP_BA:
8355                 case DIF_OP_BE:
8356                 case DIF_OP_BNE:
8357                 case DIF_OP_BG:
8358                 case DIF_OP_BGU:
8359                 case DIF_OP_BGE:
8360                 case DIF_OP_BGEU:
8361                 case DIF_OP_BL:
8362                 case DIF_OP_BLU:
8363                 case DIF_OP_BLE:
8364                 case DIF_OP_BLEU:
8365                         if (label >= dp->dtdo_len) {
8366                                 err += efunc(pc, "invalid branch target %u\n",
8367                                     label);
8368                         }
8369                         if (label <= pc) {
8370                                 err += efunc(pc, "backward branch to %u\n",
8371                                     label);
8372                         }
8373                         break;
8374                 case DIF_OP_RET:
8375                         if (r1 != 0 || r2 != 0)
8376                                 err += efunc(pc, "non-zero reserved bits\n");
8377                         if (rd >= nregs)
8378                                 err += efunc(pc, "invalid register %u\n", rd);
8379                         break;
8380                 case DIF_OP_NOP:
8381                 case DIF_OP_POPTS:
8382                 case DIF_OP_FLUSHTS:
8383                         if (r1 != 0 || r2 != 0 || rd != 0)
8384                                 err += efunc(pc, "non-zero reserved bits\n");
8385                         break;
8386                 case DIF_OP_SETX:
8387                         if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
8388                                 err += efunc(pc, "invalid integer ref %u\n",
8389                                     DIF_INSTR_INTEGER(instr));
8390                         }
8391                         if (rd >= nregs)
8392                                 err += efunc(pc, "invalid register %u\n", rd);
8393                         if (rd == 0)
8394                                 err += efunc(pc, "cannot write to %r0\n");
8395                         break;
8396                 case DIF_OP_SETS:
8397                         if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
8398                                 err += efunc(pc, "invalid string ref %u\n",
8399                                     DIF_INSTR_STRING(instr));
8400                         }
8401                         if (rd >= nregs)
8402                                 err += efunc(pc, "invalid register %u\n", rd);
8403                         if (rd == 0)
8404                                 err += efunc(pc, "cannot write to %r0\n");
8405                         break;
8406                 case DIF_OP_LDGA:
8407                 case DIF_OP_LDTA:
8408                         if (r1 > DIF_VAR_ARRAY_MAX)
8409                                 err += efunc(pc, "invalid array %u\n", r1);
8410                         if (r2 >= nregs)
8411                                 err += efunc(pc, "invalid register %u\n", r2);
8412                         if (rd >= nregs)
8413                                 err += efunc(pc, "invalid register %u\n", rd);
8414                         if (rd == 0)
8415                                 err += efunc(pc, "cannot write to %r0\n");
8416                         break;
8417                 case DIF_OP_LDGS:
8418                 case DIF_OP_LDTS:
8419                 case DIF_OP_LDLS:
8420                 case DIF_OP_LDGAA:
8421                 case DIF_OP_LDTAA:
8422                         if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
8423                                 err += efunc(pc, "invalid variable %u\n", v);
8424                         if (rd >= nregs)
8425                                 err += efunc(pc, "invalid register %u\n", rd);
8426                         if (rd == 0)
8427                                 err += efunc(pc, "cannot write to %r0\n");
8428                         break;
8429                 case DIF_OP_STGS:
8430                 case DIF_OP_STTS:
8431                 case DIF_OP_STLS:
8432                 case DIF_OP_STGAA:
8433                 case DIF_OP_STTAA:
8434                         if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
8435                                 err += efunc(pc, "invalid variable %u\n", v);
8436                         if (rs >= nregs)
8437                                 err += efunc(pc, "invalid register %u\n", rd);
8438                         break;
8439                 case DIF_OP_CALL:
8440                         if (subr > DIF_SUBR_MAX)
8441                                 err += efunc(pc, "invalid subr %u\n", subr);
8442                         if (rd >= nregs)
8443                                 err += efunc(pc, "invalid register %u\n", rd);
8444                         if (rd == 0)
8445                                 err += efunc(pc, "cannot write to %r0\n");
8446 
8447                         if (subr == DIF_SUBR_COPYOUT ||
8448                             subr == DIF_SUBR_COPYOUTSTR) {
8449                                 dp->dtdo_destructive = 1;
8450                         }
8451                         break;
8452                 case DIF_OP_PUSHTR:
8453                         if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
8454                                 err += efunc(pc, "invalid ref type %u\n", type);
8455                         if (r2 >= nregs)
8456                                 err += efunc(pc, "invalid register %u\n", r2);
8457                         if (rs >= nregs)
8458                                 err += efunc(pc, "invalid register %u\n", rs);
8459                         break;
8460                 case DIF_OP_PUSHTV:
8461                         if (type != DIF_TYPE_CTF)
8462                                 err += efunc(pc, "invalid val type %u\n", type);
8463                         if (r2 >= nregs)
8464                                 err += efunc(pc, "invalid register %u\n", r2);
8465                         if (rs >= nregs)
8466                                 err += efunc(pc, "invalid register %u\n", rs);
8467                         break;
8468                 default:
8469                         err += efunc(pc, "invalid opcode %u\n",
8470                             DIF_INSTR_OP(instr));
8471                 }
8472         }
8473 
8474         if (dp->dtdo_len != 0 &&
8475             DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
8476                 err += efunc(dp->dtdo_len - 1,
8477                     "expected 'ret' as last DIF instruction\n");
8478         }
8479 
8480         if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
8481                 /*
8482                  * If we're not returning by reference, the size must be either
8483                  * 0 or the size of one of the base types.
8484                  */
8485                 switch (dp->dtdo_rtype.dtdt_size) {
8486                 case 0:
8487                 case sizeof (uint8_t):
8488                 case sizeof (uint16_t):
8489                 case sizeof (uint32_t):
8490                 case sizeof (uint64_t):
8491                         break;
8492 
8493                 default:
8494                         err += efunc(dp->dtdo_len - 1, "bad return size\n");
8495                 }
8496         }
8497 
8498         for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
8499                 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
8500                 dtrace_diftype_t *vt, *et;
8501                 uint_t id, ndx;
8502 
8503                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
8504                     v->dtdv_scope != DIFV_SCOPE_THREAD &&
8505                     v->dtdv_scope != DIFV_SCOPE_LOCAL) {
8506                         err += efunc(i, "unrecognized variable scope %d\n",
8507                             v->dtdv_scope);
8508                         break;
8509                 }
8510 
8511                 if (v->dtdv_kind != DIFV_KIND_ARRAY &&
8512                     v->dtdv_kind != DIFV_KIND_SCALAR) {
8513                         err += efunc(i, "unrecognized variable type %d\n",
8514                             v->dtdv_kind);
8515                         break;
8516                 }
8517 
8518                 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
8519                         err += efunc(i, "%d exceeds variable id limit\n", id);
8520                         break;
8521                 }
8522 
8523                 if (id < DIF_VAR_OTHER_UBASE)
8524                         continue;
8525 
8526                 /*
8527                  * For user-defined variables, we need to check that this
8528                  * definition is identical to any previous definition that we
8529                  * encountered.
8530                  */
8531                 ndx = id - DIF_VAR_OTHER_UBASE;
8532 
8533                 switch (v->dtdv_scope) {
8534                 case DIFV_SCOPE_GLOBAL:
8535                         if (ndx < vstate->dtvs_nglobals) {
8536                                 dtrace_statvar_t *svar;
8537 
8538                                 if ((svar = vstate->dtvs_globals[ndx]) != NULL)
8539                                         existing = &svar->dtsv_var;
8540                         }
8541 
8542                         break;
8543 
8544                 case DIFV_SCOPE_THREAD:
8545                         if (ndx < vstate->dtvs_ntlocals)
8546                                 existing = &vstate->dtvs_tlocals[ndx];
8547                         break;
8548 
8549                 case DIFV_SCOPE_LOCAL:
8550                         if (ndx < vstate->dtvs_nlocals) {
8551                                 dtrace_statvar_t *svar;
8552 
8553                                 if ((svar = vstate->dtvs_locals[ndx]) != NULL)
8554                                         existing = &svar->dtsv_var;
8555                         }
8556 
8557                         break;
8558                 }
8559 
8560                 vt = &v->dtdv_type;
8561 
8562                 if (vt->dtdt_flags & DIF_TF_BYREF) {
8563                         if (vt->dtdt_size == 0) {
8564                                 err += efunc(i, "zero-sized variable\n");
8565                                 break;
8566                         }
8567 
8568                         if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
8569                             vt->dtdt_size > dtrace_global_maxsize) {
8570                                 err += efunc(i, "oversized by-ref global\n");
8571                                 break;
8572                         }
8573                 }
8574 
8575                 if (existing == NULL || existing->dtdv_id == 0)
8576                         continue;
8577 
8578                 ASSERT(existing->dtdv_id == v->dtdv_id);
8579                 ASSERT(existing->dtdv_scope == v->dtdv_scope);
8580 
8581                 if (existing->dtdv_kind != v->dtdv_kind)
8582                         err += efunc(i, "%d changed variable kind\n", id);
8583 
8584                 et = &existing->dtdv_type;
8585 
8586                 if (vt->dtdt_flags != et->dtdt_flags) {
8587                         err += efunc(i, "%d changed variable type flags\n", id);
8588                         break;
8589                 }
8590 
8591                 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
8592                         err += efunc(i, "%d changed variable type size\n", id);
8593                         break;
8594                 }
8595         }
8596 
8597         return (err);
8598 }
8599 
8600 /*
8601  * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
8602  * are much more constrained than normal DIFOs.  Specifically, they may
8603  * not:
8604  *
8605  * 1. Make calls to subroutines other than copyin(), copyinstr() or
8606  *    miscellaneous string routines
8607  * 2. Access DTrace variables other than the args[] array, and the
8608  *    curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
8609  * 3. Have thread-local variables.
8610  * 4. Have dynamic variables.
8611  */
8612 static int
8613 dtrace_difo_validate_helper(dtrace_difo_t *dp)
8614 {
8615         int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8616         int err = 0;
8617         uint_t pc;
8618 
8619         for (pc = 0; pc < dp->dtdo_len; pc++) {
8620                 dif_instr_t instr = dp->dtdo_buf[pc];
8621 
8622                 uint_t v = DIF_INSTR_VAR(instr);
8623                 uint_t subr = DIF_INSTR_SUBR(instr);
8624                 uint_t op = DIF_INSTR_OP(instr);
8625 
8626                 switch (op) {
8627                 case DIF_OP_OR:
8628                 case DIF_OP_XOR:
8629                 case DIF_OP_AND:
8630                 case DIF_OP_SLL:
8631                 case DIF_OP_SRL:
8632                 case DIF_OP_SRA:
8633                 case DIF_OP_SUB:
8634                 case DIF_OP_ADD:
8635                 case DIF_OP_MUL:
8636                 case DIF_OP_SDIV:
8637                 case DIF_OP_UDIV:
8638                 case DIF_OP_SREM:
8639                 case DIF_OP_UREM:
8640                 case DIF_OP_COPYS:
8641                 case DIF_OP_NOT:
8642                 case DIF_OP_MOV:
8643                 case DIF_OP_RLDSB:
8644                 case DIF_OP_RLDSH:
8645                 case DIF_OP_RLDSW:
8646                 case DIF_OP_RLDUB:
8647                 case DIF_OP_RLDUH:
8648                 case DIF_OP_RLDUW:
8649                 case DIF_OP_RLDX:
8650                 case DIF_OP_ULDSB:
8651                 case DIF_OP_ULDSH:
8652                 case DIF_OP_ULDSW:
8653                 case DIF_OP_ULDUB:
8654                 case DIF_OP_ULDUH:
8655                 case DIF_OP_ULDUW:
8656                 case DIF_OP_ULDX:
8657                 case DIF_OP_STB:
8658                 case DIF_OP_STH:
8659                 case DIF_OP_STW:
8660                 case DIF_OP_STX:
8661                 case DIF_OP_ALLOCS:
8662                 case DIF_OP_CMP:
8663                 case DIF_OP_SCMP:
8664                 case DIF_OP_TST:
8665                 case DIF_OP_BA:
8666                 case DIF_OP_BE:
8667                 case DIF_OP_BNE:
8668                 case DIF_OP_BG:
8669                 case DIF_OP_BGU:
8670                 case DIF_OP_BGE:
8671                 case DIF_OP_BGEU:
8672                 case DIF_OP_BL:
8673                 case DIF_OP_BLU:
8674                 case DIF_OP_BLE:
8675                 case DIF_OP_BLEU:
8676                 case DIF_OP_RET:
8677                 case DIF_OP_NOP:
8678                 case DIF_OP_POPTS:
8679                 case DIF_OP_FLUSHTS:
8680                 case DIF_OP_SETX:
8681                 case DIF_OP_SETS:
8682                 case DIF_OP_LDGA:
8683                 case DIF_OP_LDLS:
8684                 case DIF_OP_STGS:
8685                 case DIF_OP_STLS:
8686                 case DIF_OP_PUSHTR:
8687                 case DIF_OP_PUSHTV:
8688                         break;
8689 
8690                 case DIF_OP_LDGS:
8691                         if (v >= DIF_VAR_OTHER_UBASE)
8692                                 break;
8693 
8694                         if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
8695                                 break;
8696 
8697                         if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
8698                             v == DIF_VAR_PPID || v == DIF_VAR_TID ||
8699                             v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
8700                             v == DIF_VAR_UID || v == DIF_VAR_GID)
8701                                 break;
8702 
8703                         err += efunc(pc, "illegal variable %u\n", v);
8704                         break;
8705 
8706                 case DIF_OP_LDTA:
8707                 case DIF_OP_LDTS:
8708                 case DIF_OP_LDGAA:
8709                 case DIF_OP_LDTAA:
8710                         err += efunc(pc, "illegal dynamic variable load\n");
8711                         break;
8712 
8713                 case DIF_OP_STTS:
8714                 case DIF_OP_STGAA:
8715                 case DIF_OP_STTAA:
8716                         err += efunc(pc, "illegal dynamic variable store\n");
8717                         break;
8718 
8719                 case DIF_OP_CALL:
8720                         if (subr == DIF_SUBR_ALLOCA ||
8721                             subr == DIF_SUBR_BCOPY ||
8722                             subr == DIF_SUBR_COPYIN ||
8723                             subr == DIF_SUBR_COPYINTO ||
8724                             subr == DIF_SUBR_COPYINSTR ||
8725                             subr == DIF_SUBR_INDEX ||
8726                             subr == DIF_SUBR_INET_NTOA ||
8727                             subr == DIF_SUBR_INET_NTOA6 ||
8728                             subr == DIF_SUBR_INET_NTOP ||
8729                             subr == DIF_SUBR_LLTOSTR ||
8730                             subr == DIF_SUBR_RINDEX ||
8731                             subr == DIF_SUBR_STRCHR ||
8732                             subr == DIF_SUBR_STRJOIN ||
8733                             subr == DIF_SUBR_STRRCHR ||
8734                             subr == DIF_SUBR_STRSTR ||
8735                             subr == DIF_SUBR_HTONS ||
8736                             subr == DIF_SUBR_HTONL ||
8737                             subr == DIF_SUBR_HTONLL ||
8738                             subr == DIF_SUBR_NTOHS ||
8739                             subr == DIF_SUBR_NTOHL ||
8740                             subr == DIF_SUBR_NTOHLL)
8741                                 break;
8742 
8743                         err += efunc(pc, "invalid subr %u\n", subr);
8744                         break;
8745 
8746                 default:
8747                         err += efunc(pc, "invalid opcode %u\n",
8748                             DIF_INSTR_OP(instr));
8749                 }
8750         }
8751 
8752         return (err);
8753 }
8754 
8755 /*
8756  * Returns 1 if the expression in the DIF object can be cached on a per-thread
8757  * basis; 0 if not.
8758  */
8759 static int
8760 dtrace_difo_cacheable(dtrace_difo_t *dp)
8761 {
8762         int i;
8763 
8764         if (dp == NULL)
8765                 return (0);
8766 
8767         for (i = 0; i < dp->dtdo_varlen; i++) {
8768                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8769 
8770                 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
8771                         continue;
8772 
8773                 switch (v->dtdv_id) {
8774                 case DIF_VAR_CURTHREAD:
8775                 case DIF_VAR_PID:
8776                 case DIF_VAR_TID:
8777                 case DIF_VAR_EXECNAME:
8778                 case DIF_VAR_ZONENAME:
8779                         break;
8780 
8781                 default:
8782                         return (0);
8783                 }
8784         }
8785 
8786         /*
8787          * This DIF object may be cacheable.  Now we need to look for any
8788          * array loading instructions, any memory loading instructions, or
8789          * any stores to thread-local variables.
8790          */
8791         for (i = 0; i < dp->dtdo_len; i++) {
8792                 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
8793 
8794                 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
8795                     (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
8796                     (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
8797                     op == DIF_OP_LDGA || op == DIF_OP_STTS)
8798                         return (0);
8799         }
8800 
8801         return (1);
8802 }
8803 
8804 static void
8805 dtrace_difo_hold(dtrace_difo_t *dp)
8806 {
8807         int i;
8808 
8809         ASSERT(MUTEX_HELD(&dtrace_lock));
8810 
8811         dp->dtdo_refcnt++;
8812         ASSERT(dp->dtdo_refcnt != 0);
8813 
8814         /*
8815          * We need to check this DIF object for references to the variable
8816          * DIF_VAR_VTIMESTAMP.
8817          */
8818         for (i = 0; i < dp->dtdo_varlen; i++) {
8819                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8820 
8821                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
8822                         continue;
8823 
8824                 if (dtrace_vtime_references++ == 0)
8825                         dtrace_vtime_enable();
8826         }
8827 }
8828 
8829 /*
8830  * This routine calculates the dynamic variable chunksize for a given DIF
8831  * object.  The calculation is not fool-proof, and can probably be tricked by
8832  * malicious DIF -- but it works for all compiler-generated DIF.  Because this
8833  * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
8834  * if a dynamic variable size exceeds the chunksize.
8835  */
8836 static void
8837 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
8838 {
8839         uint64_t sval;
8840         dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
8841         const dif_instr_t *text = dp->dtdo_buf;
8842         uint_t pc, srd = 0;
8843         uint_t ttop = 0;
8844         size_t size, ksize;
8845         uint_t id, i;
8846 
8847         for (pc = 0; pc < dp->dtdo_len; pc++) {
8848                 dif_instr_t instr = text[pc];
8849                 uint_t op = DIF_INSTR_OP(instr);
8850                 uint_t rd = DIF_INSTR_RD(instr);
8851                 uint_t r1 = DIF_INSTR_R1(instr);
8852                 uint_t nkeys = 0;
8853                 uchar_t scope;
8854 
8855                 dtrace_key_t *key = tupregs;
8856 
8857                 switch (op) {
8858                 case DIF_OP_SETX:
8859                         sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
8860                         srd = rd;
8861                         continue;
8862 
8863                 case DIF_OP_STTS:
8864                         key = &tupregs[DIF_DTR_NREGS];
8865                         key[0].dttk_size = 0;
8866                         key[1].dttk_size = 0;
8867                         nkeys = 2;
8868                         scope = DIFV_SCOPE_THREAD;
8869                         break;
8870 
8871                 case DIF_OP_STGAA:
8872                 case DIF_OP_STTAA:
8873                         nkeys = ttop;
8874 
8875                         if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
8876                                 key[nkeys++].dttk_size = 0;
8877 
8878                         key[nkeys++].dttk_size = 0;
8879 
8880                         if (op == DIF_OP_STTAA) {
8881                                 scope = DIFV_SCOPE_THREAD;
8882                         } else {
8883                                 scope = DIFV_SCOPE_GLOBAL;
8884                         }
8885 
8886                         break;
8887 
8888                 case DIF_OP_PUSHTR:
8889                         if (ttop == DIF_DTR_NREGS)
8890                                 return;
8891 
8892                         if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
8893                                 /*
8894                                  * If the register for the size of the "pushtr"
8895                                  * is %r0 (or the value is 0) and the type is
8896                                  * a string, we'll use the system-wide default
8897                                  * string size.
8898                                  */
8899                                 tupregs[ttop++].dttk_size =
8900                                     dtrace_strsize_default;
8901                         } else {
8902                                 if (srd == 0)
8903                                         return;
8904 
8905                                 tupregs[ttop++].dttk_size = sval;
8906                         }
8907 
8908                         break;
8909 
8910                 case DIF_OP_PUSHTV:
8911                         if (ttop == DIF_DTR_NREGS)
8912                                 return;
8913 
8914                         tupregs[ttop++].dttk_size = 0;
8915                         break;
8916 
8917                 case DIF_OP_FLUSHTS:
8918                         ttop = 0;
8919                         break;
8920 
8921                 case DIF_OP_POPTS:
8922                         if (ttop != 0)
8923                                 ttop--;
8924                         break;
8925                 }
8926 
8927                 sval = 0;
8928                 srd = 0;
8929 
8930                 if (nkeys == 0)
8931                         continue;
8932 
8933                 /*
8934                  * We have a dynamic variable allocation; calculate its size.
8935                  */
8936                 for (ksize = 0, i = 0; i < nkeys; i++)
8937                         ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
8938 
8939                 size = sizeof (dtrace_dynvar_t);
8940                 size += sizeof (dtrace_key_t) * (nkeys - 1);
8941                 size += ksize;
8942 
8943                 /*
8944                  * Now we need to determine the size of the stored data.
8945                  */
8946                 id = DIF_INSTR_VAR(instr);
8947 
8948                 for (i = 0; i < dp->dtdo_varlen; i++) {
8949                         dtrace_difv_t *v = &dp->dtdo_vartab[i];
8950 
8951                         if (v->dtdv_id == id && v->dtdv_scope == scope) {
8952                                 size += v->dtdv_type.dtdt_size;
8953                                 break;
8954                         }
8955                 }
8956 
8957                 if (i == dp->dtdo_varlen)
8958                         return;
8959 
8960                 /*
8961                  * We have the size.  If this is larger than the chunk size
8962                  * for our dynamic variable state, reset the chunk size.
8963                  */
8964                 size = P2ROUNDUP(size, sizeof (uint64_t));
8965 
8966                 if (size > vstate->dtvs_dynvars.dtds_chunksize)
8967                         vstate->dtvs_dynvars.dtds_chunksize = size;
8968         }
8969 }
8970 
8971 static void
8972 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
8973 {
8974         int i, oldsvars, osz, nsz, otlocals, ntlocals;
8975         uint_t id;
8976 
8977         ASSERT(MUTEX_HELD(&dtrace_lock));
8978         ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
8979 
8980         for (i = 0; i < dp->dtdo_varlen; i++) {
8981                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8982                 dtrace_statvar_t *svar, ***svarp;
8983                 size_t dsize = 0;
8984                 uint8_t scope = v->dtdv_scope;
8985                 int *np;
8986 
8987                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
8988                         continue;
8989 
8990                 id -= DIF_VAR_OTHER_UBASE;
8991 
8992                 switch (scope) {
8993                 case DIFV_SCOPE_THREAD:
8994                         while (id >= (otlocals = vstate->dtvs_ntlocals)) {
8995                                 dtrace_difv_t *tlocals;
8996 
8997                                 if ((ntlocals = (otlocals << 1)) == 0)
8998                                         ntlocals = 1;
8999 
9000                                 osz = otlocals * sizeof (dtrace_difv_t);
9001                                 nsz = ntlocals * sizeof (dtrace_difv_t);
9002 
9003                                 tlocals = kmem_zalloc(nsz, KM_SLEEP);
9004 
9005                                 if (osz != 0) {
9006                                         bcopy(vstate->dtvs_tlocals,
9007                                             tlocals, osz);
9008                                         kmem_free(vstate->dtvs_tlocals, osz);
9009                                 }
9010 
9011                                 vstate->dtvs_tlocals = tlocals;
9012                                 vstate->dtvs_ntlocals = ntlocals;
9013                         }
9014 
9015                         vstate->dtvs_tlocals[id] = *v;
9016                         continue;
9017 
9018                 case DIFV_SCOPE_LOCAL:
9019                         np = &vstate->dtvs_nlocals;
9020                         svarp = &vstate->dtvs_locals;
9021 
9022                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9023                                 dsize = NCPU * (v->dtdv_type.dtdt_size +
9024                                     sizeof (uint64_t));
9025                         else
9026                                 dsize = NCPU * sizeof (uint64_t);
9027 
9028                         break;
9029 
9030                 case DIFV_SCOPE_GLOBAL:
9031                         np = &vstate->dtvs_nglobals;
9032                         svarp = &vstate->dtvs_globals;
9033 
9034                         if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
9035                                 dsize = v->dtdv_type.dtdt_size +
9036                                     sizeof (uint64_t);
9037 
9038                         break;
9039 
9040                 default:
9041                         ASSERT(0);
9042                 }
9043 
9044                 while (id >= (oldsvars = *np)) {
9045                         dtrace_statvar_t **statics;
9046                         int newsvars, oldsize, newsize;
9047 
9048                         if ((newsvars = (oldsvars << 1)) == 0)
9049                                 newsvars = 1;
9050 
9051                         oldsize = oldsvars * sizeof (dtrace_statvar_t *);
9052                         newsize = newsvars * sizeof (dtrace_statvar_t *);
9053 
9054                         statics = kmem_zalloc(newsize, KM_SLEEP);
9055 
9056                         if (oldsize != 0) {
9057                                 bcopy(*svarp, statics, oldsize);
9058                                 kmem_free(*svarp, oldsize);
9059                         }
9060 
9061                         *svarp = statics;
9062                         *np = newsvars;
9063                 }
9064 
9065                 if ((svar = (*svarp)[id]) == NULL) {
9066                         svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
9067                         svar->dtsv_var = *v;
9068 
9069                         if ((svar->dtsv_size = dsize) != 0) {
9070                                 svar->dtsv_data = (uint64_t)(uintptr_t)
9071                                     kmem_zalloc(dsize, KM_SLEEP);
9072                         }
9073 
9074                         (*svarp)[id] = svar;
9075                 }
9076 
9077                 svar->dtsv_refcnt++;
9078         }
9079 
9080         dtrace_difo_chunksize(dp, vstate);
9081         dtrace_difo_hold(dp);
9082 }
9083 
9084 static dtrace_difo_t *
9085 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9086 {
9087         dtrace_difo_t *new;
9088         size_t sz;
9089 
9090         ASSERT(dp->dtdo_buf != NULL);
9091         ASSERT(dp->dtdo_refcnt != 0);
9092 
9093         new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
9094 
9095         ASSERT(dp->dtdo_buf != NULL);
9096         sz = dp->dtdo_len * sizeof (dif_instr_t);
9097         new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
9098         bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
9099         new->dtdo_len = dp->dtdo_len;
9100 
9101         if (dp->dtdo_strtab != NULL) {
9102                 ASSERT(dp->dtdo_strlen != 0);
9103                 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
9104                 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
9105                 new->dtdo_strlen = dp->dtdo_strlen;
9106         }
9107 
9108         if (dp->dtdo_inttab != NULL) {
9109                 ASSERT(dp->dtdo_intlen != 0);
9110                 sz = dp->dtdo_intlen * sizeof (uint64_t);
9111                 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
9112                 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
9113                 new->dtdo_intlen = dp->dtdo_intlen;
9114         }
9115 
9116         if (dp->dtdo_vartab != NULL) {
9117                 ASSERT(dp->dtdo_varlen != 0);
9118                 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
9119                 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
9120                 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
9121                 new->dtdo_varlen = dp->dtdo_varlen;
9122         }
9123 
9124         dtrace_difo_init(new, vstate);
9125         return (new);
9126 }
9127 
9128 static void
9129 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9130 {
9131         int i;
9132 
9133         ASSERT(dp->dtdo_refcnt == 0);
9134 
9135         for (i = 0; i < dp->dtdo_varlen; i++) {
9136                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9137                 dtrace_statvar_t *svar, **svarp;
9138                 uint_t id;
9139                 uint8_t scope = v->dtdv_scope;
9140                 int *np;
9141 
9142                 switch (scope) {
9143                 case DIFV_SCOPE_THREAD:
9144                         continue;
9145 
9146                 case DIFV_SCOPE_LOCAL:
9147                         np = &vstate->dtvs_nlocals;
9148                         svarp = vstate->dtvs_locals;
9149                         break;
9150 
9151                 case DIFV_SCOPE_GLOBAL:
9152                         np = &vstate->dtvs_nglobals;
9153                         svarp = vstate->dtvs_globals;
9154                         break;
9155 
9156                 default:
9157                         ASSERT(0);
9158                 }
9159 
9160                 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9161                         continue;
9162 
9163                 id -= DIF_VAR_OTHER_UBASE;
9164                 ASSERT(id < *np);
9165 
9166                 svar = svarp[id];
9167                 ASSERT(svar != NULL);
9168                 ASSERT(svar->dtsv_refcnt > 0);
9169 
9170                 if (--svar->dtsv_refcnt > 0)
9171                         continue;
9172 
9173                 if (svar->dtsv_size != 0) {
9174                         ASSERT(svar->dtsv_data != NULL);
9175                         kmem_free((void *)(uintptr_t)svar->dtsv_data,
9176                             svar->dtsv_size);
9177                 }
9178 
9179                 kmem_free(svar, sizeof (dtrace_statvar_t));
9180                 svarp[id] = NULL;
9181         }
9182 
9183         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
9184         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
9185         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
9186         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
9187 
9188         kmem_free(dp, sizeof (dtrace_difo_t));
9189 }
9190 
9191 static void
9192 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9193 {
9194         int i;
9195 
9196         ASSERT(MUTEX_HELD(&dtrace_lock));
9197         ASSERT(dp->dtdo_refcnt != 0);
9198 
9199         for (i = 0; i < dp->dtdo_varlen; i++) {
9200                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9201 
9202                 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9203                         continue;
9204 
9205                 ASSERT(dtrace_vtime_references > 0);
9206                 if (--dtrace_vtime_references == 0)
9207                         dtrace_vtime_disable();
9208         }
9209 
9210         if (--dp->dtdo_refcnt == 0)
9211                 dtrace_difo_destroy(dp, vstate);
9212 }
9213 
9214 /*
9215  * DTrace Format Functions
9216  */
9217 static uint16_t
9218 dtrace_format_add(dtrace_state_t *state, char *str)
9219 {
9220         char *fmt, **new;
9221         uint16_t ndx, len = strlen(str) + 1;
9222 
9223         fmt = kmem_zalloc(len, KM_SLEEP);
9224         bcopy(str, fmt, len);
9225 
9226         for (ndx = 0; ndx < state->dts_nformats; ndx++) {
9227                 if (state->dts_formats[ndx] == NULL) {
9228                         state->dts_formats[ndx] = fmt;
9229                         return (ndx + 1);
9230                 }
9231         }
9232 
9233         if (state->dts_nformats == USHRT_MAX) {
9234                 /*
9235                  * This is only likely if a denial-of-service attack is being
9236                  * attempted.  As such, it's okay to fail silently here.
9237                  */
9238                 kmem_free(fmt, len);
9239                 return (0);
9240         }
9241 
9242         /*
9243          * For simplicity, we always resize the formats array to be exactly the
9244          * number of formats.
9245          */
9246         ndx = state->dts_nformats++;
9247         new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
9248 
9249         if (state->dts_formats != NULL) {
9250                 ASSERT(ndx != 0);
9251                 bcopy(state->dts_formats, new, ndx * sizeof (char *));
9252                 kmem_free(state->dts_formats, ndx * sizeof (char *));
9253         }
9254 
9255         state->dts_formats = new;
9256         state->dts_formats[ndx] = fmt;
9257 
9258         return (ndx + 1);
9259 }
9260 
9261 static void
9262 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
9263 {
9264         char *fmt;
9265 
9266         ASSERT(state->dts_formats != NULL);
9267         ASSERT(format <= state->dts_nformats);
9268         ASSERT(state->dts_formats[format - 1] != NULL);
9269 
9270         fmt = state->dts_formats[format - 1];
9271         kmem_free(fmt, strlen(fmt) + 1);
9272         state->dts_formats[format - 1] = NULL;
9273 }
9274 
9275 static void
9276 dtrace_format_destroy(dtrace_state_t *state)
9277 {
9278         int i;
9279 
9280         if (state->dts_nformats == 0) {
9281                 ASSERT(state->dts_formats == NULL);
9282                 return;
9283         }
9284 
9285         ASSERT(state->dts_formats != NULL);
9286 
9287         for (i = 0; i < state->dts_nformats; i++) {
9288                 char *fmt = state->dts_formats[i];
9289 
9290                 if (fmt == NULL)
9291                         continue;
9292 
9293                 kmem_free(fmt, strlen(fmt) + 1);
9294         }
9295 
9296         kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
9297         state->dts_nformats = 0;
9298         state->dts_formats = NULL;
9299 }
9300 
9301 /*
9302  * DTrace Predicate Functions
9303  */
9304 static dtrace_predicate_t *
9305 dtrace_predicate_create(dtrace_difo_t *dp)
9306 {
9307         dtrace_predicate_t *pred;
9308 
9309         ASSERT(MUTEX_HELD(&dtrace_lock));
9310         ASSERT(dp->dtdo_refcnt != 0);
9311 
9312         pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
9313         pred->dtp_difo = dp;
9314         pred->dtp_refcnt = 1;
9315 
9316         if (!dtrace_difo_cacheable(dp))
9317                 return (pred);
9318 
9319         if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
9320                 /*
9321                  * This is only theoretically possible -- we have had 2^32
9322                  * cacheable predicates on this machine.  We cannot allow any
9323                  * more predicates to become cacheable:  as unlikely as it is,
9324                  * there may be a thread caching a (now stale) predicate cache
9325                  * ID. (N.B.: the temptation is being successfully resisted to
9326                  * have this cmn_err() "Holy shit -- we executed this code!")
9327                  */
9328                 return (pred);
9329         }
9330 
9331         pred->dtp_cacheid = dtrace_predcache_id++;
9332 
9333         return (pred);
9334 }
9335 
9336 static void
9337 dtrace_predicate_hold(dtrace_predicate_t *pred)
9338 {
9339         ASSERT(MUTEX_HELD(&dtrace_lock));
9340         ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
9341         ASSERT(pred->dtp_refcnt > 0);
9342 
9343         pred->dtp_refcnt++;
9344 }
9345 
9346 static void
9347 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
9348 {
9349         dtrace_difo_t *dp = pred->dtp_difo;
9350 
9351         ASSERT(MUTEX_HELD(&dtrace_lock));
9352         ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
9353         ASSERT(pred->dtp_refcnt > 0);
9354 
9355         if (--pred->dtp_refcnt == 0) {
9356                 dtrace_difo_release(pred->dtp_difo, vstate);
9357                 kmem_free(pred, sizeof (dtrace_predicate_t));
9358         }
9359 }
9360 
9361 /*
9362  * DTrace Action Description Functions
9363  */
9364 static dtrace_actdesc_t *
9365 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
9366     uint64_t uarg, uint64_t arg)
9367 {
9368         dtrace_actdesc_t *act;
9369 
9370         ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
9371             arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
9372 
9373         act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
9374         act->dtad_kind = kind;
9375         act->dtad_ntuple = ntuple;
9376         act->dtad_uarg = uarg;
9377         act->dtad_arg = arg;
9378         act->dtad_refcnt = 1;
9379 
9380         return (act);
9381 }
9382 
9383 static void
9384 dtrace_actdesc_hold(dtrace_actdesc_t *act)
9385 {
9386         ASSERT(act->dtad_refcnt >= 1);
9387         act->dtad_refcnt++;
9388 }
9389 
9390 static void
9391 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
9392 {
9393         dtrace_actkind_t kind = act->dtad_kind;
9394         dtrace_difo_t *dp;
9395 
9396         ASSERT(act->dtad_refcnt >= 1);
9397 
9398         if (--act->dtad_refcnt != 0)
9399                 return;
9400 
9401         if ((dp = act->dtad_difo) != NULL)
9402                 dtrace_difo_release(dp, vstate);
9403 
9404         if (DTRACEACT_ISPRINTFLIKE(kind)) {
9405                 char *str = (char *)(uintptr_t)act->dtad_arg;
9406 
9407                 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
9408                     (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
9409 
9410                 if (str != NULL)
9411                         kmem_free(str, strlen(str) + 1);
9412         }
9413 
9414         kmem_free(act, sizeof (dtrace_actdesc_t));
9415 }
9416 
9417 /*
9418  * DTrace ECB Functions
9419  */
9420 static dtrace_ecb_t *
9421 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
9422 {
9423         dtrace_ecb_t *ecb;
9424         dtrace_epid_t epid;
9425 
9426         ASSERT(MUTEX_HELD(&dtrace_lock));
9427 
9428         ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
9429         ecb->dte_predicate = NULL;
9430         ecb->dte_probe = probe;
9431 
9432         /*
9433          * The default size is the size of the default action: recording
9434          * the header.
9435          */
9436         ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
9437         ecb->dte_alignment = sizeof (dtrace_epid_t);
9438 
9439         epid = state->dts_epid++;
9440 
9441         if (epid - 1 >= state->dts_necbs) {
9442                 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
9443                 int necbs = state->dts_necbs << 1;
9444 
9445                 ASSERT(epid == state->dts_necbs + 1);
9446 
9447                 if (necbs == 0) {
9448                         ASSERT(oecbs == NULL);
9449                         necbs = 1;
9450                 }
9451 
9452                 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
9453 
9454                 if (oecbs != NULL)
9455                         bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
9456 
9457                 dtrace_membar_producer();
9458                 state->dts_ecbs = ecbs;
9459 
9460                 if (oecbs != NULL) {
9461                         /*
9462                          * If this state is active, we must dtrace_sync()
9463                          * before we can free the old dts_ecbs array:  we're
9464                          * coming in hot, and there may be active ring
9465                          * buffer processing (which indexes into the dts_ecbs
9466                          * array) on another CPU.
9467                          */
9468                         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
9469                                 dtrace_sync();
9470 
9471                         kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
9472                 }
9473 
9474                 dtrace_membar_producer();
9475                 state->dts_necbs = necbs;
9476         }
9477 
9478         ecb->dte_state = state;
9479 
9480         ASSERT(state->dts_ecbs[epid - 1] == NULL);
9481         dtrace_membar_producer();
9482         state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
9483 
9484         return (ecb);
9485 }
9486 
9487 static int
9488 dtrace_ecb_enable(dtrace_ecb_t *ecb)
9489 {
9490         dtrace_probe_t *probe = ecb->dte_probe;
9491 
9492         ASSERT(MUTEX_HELD(&cpu_lock));
9493         ASSERT(MUTEX_HELD(&dtrace_lock));
9494         ASSERT(ecb->dte_next == NULL);
9495 
9496         if (probe == NULL) {
9497                 /*
9498                  * This is the NULL probe -- there's nothing to do.
9499                  */
9500                 return (0);
9501         }
9502 
9503         if (probe->dtpr_ecb == NULL) {
9504                 dtrace_provider_t *prov = probe->dtpr_provider;
9505 
9506                 /*
9507                  * We're the first ECB on this probe.
9508                  */
9509                 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
9510 
9511                 if (ecb->dte_predicate != NULL)
9512                         probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
9513 
9514                 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
9515                     probe->dtpr_id, probe->dtpr_arg));
9516         } else {
9517                 /*
9518                  * This probe is already active.  Swing the last pointer to
9519                  * point to the new ECB, and issue a dtrace_sync() to assure
9520                  * that all CPUs have seen the change.
9521                  */
9522                 ASSERT(probe->dtpr_ecb_last != NULL);
9523                 probe->dtpr_ecb_last->dte_next = ecb;
9524                 probe->dtpr_ecb_last = ecb;
9525                 probe->dtpr_predcache = 0;
9526 
9527                 dtrace_sync();
9528                 return (0);
9529         }
9530 }
9531 
9532 static void
9533 dtrace_ecb_resize(dtrace_ecb_t *ecb)
9534 {
9535         dtrace_action_t *act;
9536         uint32_t curneeded = UINT32_MAX;
9537         uint32_t aggbase = UINT32_MAX;
9538 
9539         /*
9540          * If we record anything, we always record the dtrace_rechdr_t.  (And
9541          * we always record it first.)
9542          */
9543         ecb->dte_size = sizeof (dtrace_rechdr_t);
9544         ecb->dte_alignment = sizeof (dtrace_epid_t);
9545 
9546         for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
9547                 dtrace_recdesc_t *rec = &act->dta_rec;
9548                 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1);
9549 
9550                 ecb->dte_alignment = MAX(ecb->dte_alignment,
9551                     rec->dtrd_alignment);
9552 
9553                 if (DTRACEACT_ISAGG(act->dta_kind)) {
9554                         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
9555 
9556                         ASSERT(rec->dtrd_size != 0);
9557                         ASSERT(agg->dtag_first != NULL);
9558                         ASSERT(act->dta_prev->dta_intuple);
9559                         ASSERT(aggbase != UINT32_MAX);
9560                         ASSERT(curneeded != UINT32_MAX);
9561 
9562                         agg->dtag_base = aggbase;
9563 
9564                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
9565                         rec->dtrd_offset = curneeded;
9566                         curneeded += rec->dtrd_size;
9567                         ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
9568 
9569                         aggbase = UINT32_MAX;
9570                         curneeded = UINT32_MAX;
9571                 } else if (act->dta_intuple) {
9572                         if (curneeded == UINT32_MAX) {
9573                                 /*
9574                                  * This is the first record in a tuple.  Align
9575                                  * curneeded to be at offset 4 in an 8-byte
9576                                  * aligned block.
9577                                  */
9578                                 ASSERT(act->dta_prev == NULL ||
9579                                     !act->dta_prev->dta_intuple);
9580                                 ASSERT3U(aggbase, ==, UINT32_MAX);
9581                                 curneeded = P2PHASEUP(ecb->dte_size,
9582                                     sizeof (uint64_t), sizeof (dtrace_aggid_t));
9583 
9584                                 aggbase = curneeded - sizeof (dtrace_aggid_t);
9585                                 ASSERT(IS_P2ALIGNED(aggbase,
9586                                     sizeof (uint64_t)));
9587                         }
9588                         curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment);
9589                         rec->dtrd_offset = curneeded;
9590                         curneeded += rec->dtrd_size;
9591                 } else {
9592                         /* tuples must be followed by an aggregation */
9593                         ASSERT(act->dta_prev == NULL ||
9594                             !act->dta_prev->dta_intuple);
9595 
9596                         ecb->dte_size = P2ROUNDUP(ecb->dte_size,
9597                             rec->dtrd_alignment);
9598                         rec->dtrd_offset = ecb->dte_size;
9599                         ecb->dte_size += rec->dtrd_size;
9600                         ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
9601                 }
9602         }
9603 
9604         if ((act = ecb->dte_action) != NULL &&
9605             !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
9606             ecb->dte_size == sizeof (dtrace_rechdr_t)) {
9607                 /*
9608                  * If the size is still sizeof (dtrace_rechdr_t), then all
9609                  * actions store no data; set the size to 0.
9610                  */
9611                 ecb->dte_size = 0;
9612         }
9613 
9614         ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
9615         ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
9616         ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
9617             ecb->dte_needed);
9618 }
9619 
9620 static dtrace_action_t *
9621 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
9622 {
9623         dtrace_aggregation_t *agg;
9624         size_t size = sizeof (uint64_t);
9625         int ntuple = desc->dtad_ntuple;
9626         dtrace_action_t *act;
9627         dtrace_recdesc_t *frec;
9628         dtrace_aggid_t aggid;
9629         dtrace_state_t *state = ecb->dte_state;
9630 
9631         agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
9632         agg->dtag_ecb = ecb;
9633 
9634         ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
9635 
9636         switch (desc->dtad_kind) {
9637         case DTRACEAGG_MIN:
9638                 agg->dtag_initial = INT64_MAX;
9639                 agg->dtag_aggregate = dtrace_aggregate_min;
9640                 break;
9641 
9642         case DTRACEAGG_MAX:
9643                 agg->dtag_initial = INT64_MIN;
9644                 agg->dtag_aggregate = dtrace_aggregate_max;
9645                 break;
9646 
9647         case DTRACEAGG_COUNT:
9648                 agg->dtag_aggregate = dtrace_aggregate_count;
9649                 break;
9650 
9651         case DTRACEAGG_QUANTIZE:
9652                 agg->dtag_aggregate = dtrace_aggregate_quantize;
9653                 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
9654                     sizeof (uint64_t);
9655                 break;
9656 
9657         case DTRACEAGG_LQUANTIZE: {
9658                 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
9659                 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
9660 
9661                 agg->dtag_initial = desc->dtad_arg;
9662                 agg->dtag_aggregate = dtrace_aggregate_lquantize;
9663 
9664                 if (step == 0 || levels == 0)
9665                         goto err;
9666 
9667                 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
9668                 break;
9669         }
9670 
9671         case DTRACEAGG_LLQUANTIZE: {
9672                 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
9673                 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
9674                 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
9675                 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
9676                 int64_t v;
9677 
9678                 agg->dtag_initial = desc->dtad_arg;
9679                 agg->dtag_aggregate = dtrace_aggregate_llquantize;
9680 
9681                 if (factor < 2 || low >= high || nsteps < factor)
9682                         goto err;
9683 
9684                 /*
9685                  * Now check that the number of steps evenly divides a power
9686                  * of the factor.  (This assures both integer bucket size and
9687                  * linearity within each magnitude.)
9688                  */
9689                 for (v = factor; v < nsteps; v *= factor)
9690                         continue;
9691 
9692                 if ((v % nsteps) || (nsteps % factor))
9693                         goto err;
9694 
9695                 size = (dtrace_aggregate_llquantize_bucket(factor,
9696                     low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
9697                 break;
9698         }
9699 
9700         case DTRACEAGG_AVG:
9701                 agg->dtag_aggregate = dtrace_aggregate_avg;
9702                 size = sizeof (uint64_t) * 2;
9703                 break;
9704 
9705         case DTRACEAGG_STDDEV:
9706                 agg->dtag_aggregate = dtrace_aggregate_stddev;
9707                 size = sizeof (uint64_t) * 4;
9708                 break;
9709 
9710         case DTRACEAGG_SUM:
9711                 agg->dtag_aggregate = dtrace_aggregate_sum;
9712                 break;
9713 
9714         default:
9715                 goto err;
9716         }
9717 
9718         agg->dtag_action.dta_rec.dtrd_size = size;
9719 
9720         if (ntuple == 0)
9721                 goto err;
9722 
9723         /*
9724          * We must make sure that we have enough actions for the n-tuple.
9725          */
9726         for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
9727                 if (DTRACEACT_ISAGG(act->dta_kind))
9728                         break;
9729 
9730                 if (--ntuple == 0) {
9731                         /*
9732                          * This is the action with which our n-tuple begins.
9733                          */
9734                         agg->dtag_first = act;
9735                         goto success;
9736                 }
9737         }
9738 
9739         /*
9740          * This n-tuple is short by ntuple elements.  Return failure.
9741          */
9742         ASSERT(ntuple != 0);
9743 err:
9744         kmem_free(agg, sizeof (dtrace_aggregation_t));
9745         return (NULL);
9746 
9747 success:
9748         /*
9749          * If the last action in the tuple has a size of zero, it's actually
9750          * an expression argument for the aggregating action.
9751          */
9752         ASSERT(ecb->dte_action_last != NULL);
9753         act = ecb->dte_action_last;
9754 
9755         if (act->dta_kind == DTRACEACT_DIFEXPR) {
9756                 ASSERT(act->dta_difo != NULL);
9757 
9758                 if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
9759                         agg->dtag_hasarg = 1;
9760         }
9761 
9762         /*
9763          * We need to allocate an id for this aggregation.
9764          */
9765         aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
9766             VM_BESTFIT | VM_SLEEP);
9767 
9768         if (aggid - 1 >= state->dts_naggregations) {
9769                 dtrace_aggregation_t **oaggs = state->dts_aggregations;
9770                 dtrace_aggregation_t **aggs;
9771                 int naggs = state->dts_naggregations << 1;
9772                 int onaggs = state->dts_naggregations;
9773 
9774                 ASSERT(aggid == state->dts_naggregations + 1);
9775 
9776                 if (naggs == 0) {
9777                         ASSERT(oaggs == NULL);
9778                         naggs = 1;
9779                 }
9780 
9781                 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
9782 
9783                 if (oaggs != NULL) {
9784                         bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
9785                         kmem_free(oaggs, onaggs * sizeof (*aggs));
9786                 }
9787 
9788                 state->dts_aggregations = aggs;
9789                 state->dts_naggregations = naggs;
9790         }
9791 
9792         ASSERT(state->dts_aggregations[aggid - 1] == NULL);
9793         state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
9794 
9795         frec = &agg->dtag_first->dta_rec;
9796         if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
9797                 frec->dtrd_alignment = sizeof (dtrace_aggid_t);
9798 
9799         for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
9800                 ASSERT(!act->dta_intuple);
9801                 act->dta_intuple = 1;
9802         }
9803 
9804         return (&agg->dtag_action);
9805 }
9806 
9807 static void
9808 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
9809 {
9810         dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
9811         dtrace_state_t *state = ecb->dte_state;
9812         dtrace_aggid_t aggid = agg->dtag_id;
9813 
9814         ASSERT(DTRACEACT_ISAGG(act->dta_kind));
9815         vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
9816 
9817         ASSERT(state->dts_aggregations[aggid - 1] == agg);
9818         state->dts_aggregations[aggid - 1] = NULL;
9819 
9820         kmem_free(agg, sizeof (dtrace_aggregation_t));
9821 }
9822 
9823 static int
9824 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
9825 {
9826         dtrace_action_t *action, *last;
9827         dtrace_difo_t *dp = desc->dtad_difo;
9828         uint32_t size = 0, align = sizeof (uint8_t), mask;
9829         uint16_t format = 0;
9830         dtrace_recdesc_t *rec;
9831         dtrace_state_t *state = ecb->dte_state;
9832         dtrace_optval_t *opt = state->dts_options, nframes, strsize;
9833         uint64_t arg = desc->dtad_arg;
9834 
9835         ASSERT(MUTEX_HELD(&dtrace_lock));
9836         ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
9837 
9838         if (DTRACEACT_ISAGG(desc->dtad_kind)) {
9839                 /*
9840                  * If this is an aggregating action, there must be neither
9841                  * a speculate nor a commit on the action chain.
9842                  */
9843                 dtrace_action_t *act;
9844 
9845                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
9846                         if (act->dta_kind == DTRACEACT_COMMIT)
9847                                 return (EINVAL);
9848 
9849                         if (act->dta_kind == DTRACEACT_SPECULATE)
9850                                 return (EINVAL);
9851                 }
9852 
9853                 action = dtrace_ecb_aggregation_create(ecb, desc);
9854 
9855                 if (action == NULL)
9856                         return (EINVAL);
9857         } else {
9858                 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
9859                     (desc->dtad_kind == DTRACEACT_DIFEXPR &&
9860                     dp != NULL && dp->dtdo_destructive)) {
9861                         state->dts_destructive = 1;
9862                 }
9863 
9864                 switch (desc->dtad_kind) {
9865                 case DTRACEACT_PRINTF:
9866                 case DTRACEACT_PRINTA:
9867                 case DTRACEACT_SYSTEM:
9868                 case DTRACEACT_FREOPEN:
9869                 case DTRACEACT_DIFEXPR:
9870                         /*
9871                          * We know that our arg is a string -- turn it into a
9872                          * format.
9873                          */
9874                         if (arg == NULL) {
9875                                 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
9876                                     desc->dtad_kind == DTRACEACT_DIFEXPR);
9877                                 format = 0;
9878                         } else {
9879                                 ASSERT(arg != NULL);
9880                                 ASSERT(arg > KERNELBASE);
9881                                 format = dtrace_format_add(state,
9882                                     (char *)(uintptr_t)arg);
9883                         }
9884 
9885                         /*FALLTHROUGH*/
9886                 case DTRACEACT_LIBACT:
9887                 case DTRACEACT_TRACEMEM:
9888                 case DTRACEACT_TRACEMEM_DYNSIZE:
9889                         if (dp == NULL)
9890                                 return (EINVAL);
9891 
9892                         if ((size = dp->dtdo_rtype.dtdt_size) != 0)
9893                                 break;
9894 
9895                         if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
9896                                 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9897                                         return (EINVAL);
9898 
9899                                 size = opt[DTRACEOPT_STRSIZE];
9900                         }
9901 
9902                         break;
9903 
9904                 case DTRACEACT_STACK:
9905                         if ((nframes = arg) == 0) {
9906                                 nframes = opt[DTRACEOPT_STACKFRAMES];
9907                                 ASSERT(nframes > 0);
9908                                 arg = nframes;
9909                         }
9910 
9911                         size = nframes * sizeof (pc_t);
9912                         break;
9913 
9914                 case DTRACEACT_JSTACK:
9915                         if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
9916                                 strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
9917 
9918                         if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
9919                                 nframes = opt[DTRACEOPT_JSTACKFRAMES];
9920 
9921                         arg = DTRACE_USTACK_ARG(nframes, strsize);
9922 
9923                         /*FALLTHROUGH*/
9924                 case DTRACEACT_USTACK:
9925                         if (desc->dtad_kind != DTRACEACT_JSTACK &&
9926                             (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
9927                                 strsize = DTRACE_USTACK_STRSIZE(arg);
9928                                 nframes = opt[DTRACEOPT_USTACKFRAMES];
9929                                 ASSERT(nframes > 0);
9930                                 arg = DTRACE_USTACK_ARG(nframes, strsize);
9931                         }
9932 
9933                         /*
9934                          * Save a slot for the pid.
9935                          */
9936                         size = (nframes + 1) * sizeof (uint64_t);
9937                         size += DTRACE_USTACK_STRSIZE(arg);
9938                         size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
9939 
9940                         break;
9941 
9942                 case DTRACEACT_SYM:
9943                 case DTRACEACT_MOD:
9944                         if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
9945                             sizeof (uint64_t)) ||
9946                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9947                                 return (EINVAL);
9948                         break;
9949 
9950                 case DTRACEACT_USYM:
9951                 case DTRACEACT_UMOD:
9952                 case DTRACEACT_UADDR:
9953                         if (dp == NULL ||
9954                             (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
9955                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9956                                 return (EINVAL);
9957 
9958                         /*
9959                          * We have a slot for the pid, plus a slot for the
9960                          * argument.  To keep things simple (aligned with
9961                          * bitness-neutral sizing), we store each as a 64-bit
9962                          * quantity.
9963                          */
9964                         size = 2 * sizeof (uint64_t);
9965                         break;
9966 
9967                 case DTRACEACT_STOP:
9968                 case DTRACEACT_BREAKPOINT:
9969                 case DTRACEACT_PANIC:
9970                         break;
9971 
9972                 case DTRACEACT_CHILL:
9973                 case DTRACEACT_DISCARD:
9974                 case DTRACEACT_RAISE:
9975                         if (dp == NULL)
9976                                 return (EINVAL);
9977                         break;
9978 
9979                 case DTRACEACT_EXIT:
9980                         if (dp == NULL ||
9981                             (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
9982                             (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9983                                 return (EINVAL);
9984                         break;
9985 
9986                 case DTRACEACT_SPECULATE:
9987                         if (ecb->dte_size > sizeof (dtrace_rechdr_t))
9988                                 return (EINVAL);
9989 
9990                         if (dp == NULL)
9991                                 return (EINVAL);
9992 
9993                         state->dts_speculates = 1;
9994                         break;
9995 
9996                 case DTRACEACT_COMMIT: {
9997                         dtrace_action_t *act = ecb->dte_action;
9998 
9999                         for (; act != NULL; act = act->dta_next) {
10000                                 if (act->dta_kind == DTRACEACT_COMMIT)
10001                                         return (EINVAL);
10002                         }
10003 
10004                         if (dp == NULL)
10005                                 return (EINVAL);
10006                         break;
10007                 }
10008 
10009                 default:
10010                         return (EINVAL);
10011                 }
10012 
10013                 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
10014                         /*
10015                          * If this is a data-storing action or a speculate,
10016                          * we must be sure that there isn't a commit on the
10017                          * action chain.
10018                          */
10019                         dtrace_action_t *act = ecb->dte_action;
10020 
10021                         for (; act != NULL; act = act->dta_next) {
10022                                 if (act->dta_kind == DTRACEACT_COMMIT)
10023                                         return (EINVAL);
10024                         }
10025                 }
10026 
10027                 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
10028                 action->dta_rec.dtrd_size = size;
10029         }
10030 
10031         action->dta_refcnt = 1;
10032         rec = &action->dta_rec;
10033         size = rec->dtrd_size;
10034 
10035         for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
10036                 if (!(size & mask)) {
10037                         align = mask + 1;
10038                         break;
10039                 }
10040         }
10041 
10042         action->dta_kind = desc->dtad_kind;
10043 
10044         if ((action->dta_difo = dp) != NULL)
10045                 dtrace_difo_hold(dp);
10046 
10047         rec->dtrd_action = action->dta_kind;
10048         rec->dtrd_arg = arg;
10049         rec->dtrd_uarg = desc->dtad_uarg;
10050         rec->dtrd_alignment = (uint16_t)align;
10051         rec->dtrd_format = format;
10052 
10053         if ((last = ecb->dte_action_last) != NULL) {
10054                 ASSERT(ecb->dte_action != NULL);
10055                 action->dta_prev = last;
10056                 last->dta_next = action;
10057         } else {
10058                 ASSERT(ecb->dte_action == NULL);
10059                 ecb->dte_action = action;
10060         }
10061 
10062         ecb->dte_action_last = action;
10063 
10064         return (0);
10065 }
10066 
10067 static void
10068 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
10069 {
10070         dtrace_action_t *act = ecb->dte_action, *next;
10071         dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
10072         dtrace_difo_t *dp;
10073         uint16_t format;
10074 
10075         if (act != NULL && act->dta_refcnt > 1) {
10076                 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
10077                 act->dta_refcnt--;
10078         } else {
10079                 for (; act != NULL; act = next) {
10080                         next = act->dta_next;
10081                         ASSERT(next != NULL || act == ecb->dte_action_last);
10082                         ASSERT(act->dta_refcnt == 1);
10083 
10084                         if ((format = act->dta_rec.dtrd_format) != 0)
10085                                 dtrace_format_remove(ecb->dte_state, format);
10086 
10087                         if ((dp = act->dta_difo) != NULL)
10088                                 dtrace_difo_release(dp, vstate);
10089 
10090                         if (DTRACEACT_ISAGG(act->dta_kind)) {
10091                                 dtrace_ecb_aggregation_destroy(ecb, act);
10092                         } else {
10093                                 kmem_free(act, sizeof (dtrace_action_t));
10094                         }
10095                 }
10096         }
10097 
10098         ecb->dte_action = NULL;
10099         ecb->dte_action_last = NULL;
10100         ecb->dte_size = 0;
10101 }
10102 
10103 static void
10104 dtrace_ecb_disable(dtrace_ecb_t *ecb)
10105 {
10106         /*
10107          * We disable the ECB by removing it from its probe.
10108          */
10109         dtrace_ecb_t *pecb, *prev = NULL;
10110         dtrace_probe_t *probe = ecb->dte_probe;
10111 
10112         ASSERT(MUTEX_HELD(&dtrace_lock));
10113 
10114         if (probe == NULL) {
10115                 /*
10116                  * This is the NULL probe; there is nothing to disable.
10117                  */
10118                 return;
10119         }
10120 
10121         for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
10122                 if (pecb == ecb)
10123                         break;
10124                 prev = pecb;
10125         }
10126 
10127         ASSERT(pecb != NULL);
10128 
10129         if (prev == NULL) {
10130                 probe->dtpr_ecb = ecb->dte_next;
10131         } else {
10132                 prev->dte_next = ecb->dte_next;
10133         }
10134 
10135         if (ecb == probe->dtpr_ecb_last) {
10136                 ASSERT(ecb->dte_next == NULL);
10137                 probe->dtpr_ecb_last = prev;
10138         }
10139 
10140         /*
10141          * The ECB has been disconnected from the probe; now sync to assure
10142          * that all CPUs have seen the change before returning.
10143          */
10144         dtrace_sync();
10145 
10146         if (probe->dtpr_ecb == NULL) {
10147                 /*
10148                  * That was the last ECB on the probe; clear the predicate
10149                  * cache ID for the probe, disable it and sync one more time
10150                  * to assure that we'll never hit it again.
10151                  */
10152                 dtrace_provider_t *prov = probe->dtpr_provider;
10153 
10154                 ASSERT(ecb->dte_next == NULL);
10155                 ASSERT(probe->dtpr_ecb_last == NULL);
10156                 probe->dtpr_predcache = DTRACE_CACHEIDNONE;
10157                 prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
10158                     probe->dtpr_id, probe->dtpr_arg);
10159                 dtrace_sync();
10160         } else {
10161                 /*
10162                  * There is at least one ECB remaining on the probe.  If there
10163                  * is _exactly_ one, set the probe's predicate cache ID to be
10164                  * the predicate cache ID of the remaining ECB.
10165                  */
10166                 ASSERT(probe->dtpr_ecb_last != NULL);
10167                 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
10168 
10169                 if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
10170                         dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
10171 
10172                         ASSERT(probe->dtpr_ecb->dte_next == NULL);
10173 
10174                         if (p != NULL)
10175                                 probe->dtpr_predcache = p->dtp_cacheid;
10176                 }
10177 
10178                 ecb->dte_next = NULL;
10179         }
10180 }
10181 
10182 static void
10183 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
10184 {
10185         dtrace_state_t *state = ecb->dte_state;
10186         dtrace_vstate_t *vstate = &state->dts_vstate;
10187         dtrace_predicate_t *pred;
10188         dtrace_epid_t epid = ecb->dte_epid;
10189 
10190         ASSERT(MUTEX_HELD(&dtrace_lock));
10191         ASSERT(ecb->dte_next == NULL);
10192         ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
10193 
10194         if ((pred = ecb->dte_predicate) != NULL)
10195                 dtrace_predicate_release(pred, vstate);
10196 
10197         dtrace_ecb_action_remove(ecb);
10198 
10199         ASSERT(state->dts_ecbs[epid - 1] == ecb);
10200         state->dts_ecbs[epid - 1] = NULL;
10201 
10202         kmem_free(ecb, sizeof (dtrace_ecb_t));
10203 }
10204 
10205 static dtrace_ecb_t *
10206 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
10207     dtrace_enabling_t *enab)
10208 {
10209         dtrace_ecb_t *ecb;
10210         dtrace_predicate_t *pred;
10211         dtrace_actdesc_t *act;
10212         dtrace_provider_t *prov;
10213         dtrace_ecbdesc_t *desc = enab->dten_current;
10214 
10215         ASSERT(MUTEX_HELD(&dtrace_lock));
10216         ASSERT(state != NULL);
10217 
10218         ecb = dtrace_ecb_add(state, probe);
10219         ecb->dte_uarg = desc->dted_uarg;
10220 
10221         if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
10222                 dtrace_predicate_hold(pred);
10223                 ecb->dte_predicate = pred;
10224         }
10225 
10226         if (probe != NULL) {
10227                 /*
10228                  * If the provider shows more leg than the consumer is old
10229                  * enough to see, we need to enable the appropriate implicit
10230                  * predicate bits to prevent the ecb from activating at
10231                  * revealing times.
10232                  *
10233                  * Providers specifying DTRACE_PRIV_USER at register time
10234                  * are stating that they need the /proc-style privilege
10235                  * model to be enforced, and this is what DTRACE_COND_OWNER
10236                  * and DTRACE_COND_ZONEOWNER will then do at probe time.
10237                  */
10238                 prov = probe->dtpr_provider;
10239                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
10240                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10241                         ecb->dte_cond |= DTRACE_COND_OWNER;
10242 
10243                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
10244                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10245                         ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
10246 
10247                 /*
10248                  * If the provider shows us kernel innards and the user
10249                  * is lacking sufficient privilege, enable the
10250                  * DTRACE_COND_USERMODE implicit predicate.
10251                  */
10252                 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
10253                     (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
10254                         ecb->dte_cond |= DTRACE_COND_USERMODE;
10255         }
10256 
10257         if (dtrace_ecb_create_cache != NULL) {
10258                 /*
10259                  * If we have a cached ecb, we'll use its action list instead
10260                  * of creating our own (saving both time and space).
10261                  */
10262                 dtrace_ecb_t *cached = dtrace_ecb_create_cache;
10263                 dtrace_action_t *act = cached->dte_action;
10264 
10265                 if (act != NULL) {
10266                         ASSERT(act->dta_refcnt > 0);
10267                         act->dta_refcnt++;
10268                         ecb->dte_action = act;
10269                         ecb->dte_action_last = cached->dte_action_last;
10270                         ecb->dte_needed = cached->dte_needed;
10271                         ecb->dte_size = cached->dte_size;
10272                         ecb->dte_alignment = cached->dte_alignment;
10273                 }
10274 
10275                 return (ecb);
10276         }
10277 
10278         for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
10279                 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
10280                         dtrace_ecb_destroy(ecb);
10281                         return (NULL);
10282                 }
10283         }
10284 
10285         dtrace_ecb_resize(ecb);
10286 
10287         return (dtrace_ecb_create_cache = ecb);
10288 }
10289 
10290 static int
10291 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
10292 {
10293         dtrace_ecb_t *ecb;
10294         dtrace_enabling_t *enab = arg;
10295         dtrace_state_t *state = enab->dten_vstate->dtvs_state;
10296 
10297         ASSERT(state != NULL);
10298 
10299         if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
10300                 /*
10301                  * This probe was created in a generation for which this
10302                  * enabling has previously created ECBs; we don't want to
10303                  * enable it again, so just kick out.
10304                  */
10305                 return (DTRACE_MATCH_NEXT);
10306         }
10307 
10308         if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
10309                 return (DTRACE_MATCH_DONE);
10310 
10311         if (dtrace_ecb_enable(ecb) < 0)
10312                 return (DTRACE_MATCH_FAIL);
10313 
10314         return (DTRACE_MATCH_NEXT);
10315 }
10316 
10317 static dtrace_ecb_t *
10318 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
10319 {
10320         dtrace_ecb_t *ecb;
10321 
10322         ASSERT(MUTEX_HELD(&dtrace_lock));
10323 
10324         if (id == 0 || id > state->dts_necbs)
10325                 return (NULL);
10326 
10327         ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
10328         ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
10329 
10330         return (state->dts_ecbs[id - 1]);
10331 }
10332 
10333 static dtrace_aggregation_t *
10334 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
10335 {
10336         dtrace_aggregation_t *agg;
10337 
10338         ASSERT(MUTEX_HELD(&dtrace_lock));
10339 
10340         if (id == 0 || id > state->dts_naggregations)
10341                 return (NULL);
10342 
10343         ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
10344         ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
10345             agg->dtag_id == id);
10346 
10347         return (state->dts_aggregations[id - 1]);
10348 }
10349 
10350 /*
10351  * DTrace Buffer Functions
10352  *
10353  * The following functions manipulate DTrace buffers.  Most of these functions
10354  * are called in the context of establishing or processing consumer state;
10355  * exceptions are explicitly noted.
10356  */
10357 
10358 /*
10359  * Note:  called from cross call context.  This function switches the two
10360  * buffers on a given CPU.  The atomicity of this operation is assured by
10361  * disabling interrupts while the actual switch takes place; the disabling of
10362  * interrupts serializes the execution with any execution of dtrace_probe() on
10363  * the same CPU.
10364  */
10365 static void
10366 dtrace_buffer_switch(dtrace_buffer_t *buf)
10367 {
10368         caddr_t tomax = buf->dtb_tomax;
10369         caddr_t xamot = buf->dtb_xamot;
10370         dtrace_icookie_t cookie;
10371         hrtime_t now;
10372 
10373         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
10374         ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
10375 
10376         cookie = dtrace_interrupt_disable();
10377         now = dtrace_gethrtime();
10378         buf->dtb_tomax = xamot;
10379         buf->dtb_xamot = tomax;
10380         buf->dtb_xamot_drops = buf->dtb_drops;
10381         buf->dtb_xamot_offset = buf->dtb_offset;
10382         buf->dtb_xamot_errors = buf->dtb_errors;
10383         buf->dtb_xamot_flags = buf->dtb_flags;
10384         buf->dtb_offset = 0;
10385         buf->dtb_drops = 0;
10386         buf->dtb_errors = 0;
10387         buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
10388         buf->dtb_interval = now - buf->dtb_switched;
10389         buf->dtb_switched = now;
10390         dtrace_interrupt_enable(cookie);
10391 }
10392 
10393 /*
10394  * Note:  called from cross call context.  This function activates a buffer
10395  * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
10396  * is guaranteed by the disabling of interrupts.
10397  */
10398 static void
10399 dtrace_buffer_activate(dtrace_state_t *state)
10400 {
10401         dtrace_buffer_t *buf;
10402         dtrace_icookie_t cookie = dtrace_interrupt_disable();
10403 
10404         buf = &state->dts_buffer[CPU->cpu_id];
10405 
10406         if (buf->dtb_tomax != NULL) {
10407                 /*
10408                  * We might like to assert that the buffer is marked inactive,
10409                  * but this isn't necessarily true:  the buffer for the CPU
10410                  * that processes the BEGIN probe has its buffer activated
10411                  * manually.  In this case, we take the (harmless) action
10412                  * re-clearing the bit INACTIVE bit.
10413                  */
10414                 buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
10415         }
10416 
10417         dtrace_interrupt_enable(cookie);
10418 }
10419 
10420 static int
10421 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
10422     processorid_t cpu, int *factor)
10423 {
10424         cpu_t *cp;
10425         dtrace_buffer_t *buf;
10426         int allocated = 0, desired = 0;
10427 
10428         ASSERT(MUTEX_HELD(&cpu_lock));
10429         ASSERT(MUTEX_HELD(&dtrace_lock));
10430 
10431         *factor = 1;
10432 
10433         if (size > dtrace_nonroot_maxsize &&
10434             !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
10435                 return (EFBIG);
10436 
10437         cp = cpu_list;
10438 
10439         do {
10440                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
10441                         continue;
10442 
10443                 buf = &bufs[cp->cpu_id];
10444 
10445                 /*
10446                  * If there is already a buffer allocated for this CPU, it
10447                  * is only possible that this is a DR event.  In this case,
10448                  * the buffer size must match our specified size.
10449                  */
10450                 if (buf->dtb_tomax != NULL) {
10451                         ASSERT(buf->dtb_size == size);
10452                         continue;
10453                 }
10454 
10455                 ASSERT(buf->dtb_xamot == NULL);
10456 
10457                 if ((buf->dtb_tomax = kmem_zalloc(size,
10458                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
10459                         goto err;
10460 
10461                 buf->dtb_size = size;
10462                 buf->dtb_flags = flags;
10463                 buf->dtb_offset = 0;
10464                 buf->dtb_drops = 0;
10465 
10466                 if (flags & DTRACEBUF_NOSWITCH)
10467                         continue;
10468 
10469                 if ((buf->dtb_xamot = kmem_zalloc(size,
10470                     KM_NOSLEEP | KM_NORMALPRI)) == NULL)
10471                         goto err;
10472         } while ((cp = cp->cpu_next) != cpu_list);
10473 
10474         return (0);
10475 
10476 err:
10477         cp = cpu_list;
10478 
10479         do {
10480                 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
10481                         continue;
10482 
10483                 buf = &bufs[cp->cpu_id];
10484                 desired += 2;
10485 
10486                 if (buf->dtb_xamot != NULL) {
10487                         ASSERT(buf->dtb_tomax != NULL);
10488                         ASSERT(buf->dtb_size == size);
10489                         kmem_free(buf->dtb_xamot, size);
10490                         allocated++;
10491                 }
10492 
10493                 if (buf->dtb_tomax != NULL) {
10494                         ASSERT(buf->dtb_size == size);
10495                         kmem_free(buf->dtb_tomax, size);
10496                         allocated++;
10497                 }
10498 
10499                 buf->dtb_tomax = NULL;
10500                 buf->dtb_xamot = NULL;
10501                 buf->dtb_size = 0;
10502         } while ((cp = cp->cpu_next) != cpu_list);
10503 
10504         *factor = desired / (allocated > 0 ? allocated : 1);
10505 
10506         return (ENOMEM);
10507 }
10508 
10509 /*
10510  * Note:  called from probe context.  This function just increments the drop
10511  * count on a buffer.  It has been made a function to allow for the
10512  * possibility of understanding the source of mysterious drop counts.  (A
10513  * problem for which one may be particularly disappointed that DTrace cannot
10514  * be used to understand DTrace.)
10515  */
10516 static void
10517 dtrace_buffer_drop(dtrace_buffer_t *buf)
10518 {
10519         buf->dtb_drops++;
10520 }
10521 
10522 /*
10523  * Note:  called from probe context.  This function is called to reserve space
10524  * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
10525  * mstate.  Returns the new offset in the buffer, or a negative value if an
10526  * error has occurred.
10527  */
10528 static intptr_t
10529 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
10530     dtrace_state_t *state, dtrace_mstate_t *mstate)
10531 {
10532         intptr_t offs = buf->dtb_offset, soffs;
10533         intptr_t woffs;
10534         caddr_t tomax;
10535         size_t total;
10536 
10537         if (buf->dtb_flags & DTRACEBUF_INACTIVE)
10538                 return (-1);
10539 
10540         if ((tomax = buf->dtb_tomax) == NULL) {
10541                 dtrace_buffer_drop(buf);
10542                 return (-1);
10543         }
10544 
10545         if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
10546                 while (offs & (align - 1)) {
10547                         /*
10548                          * Assert that our alignment is off by a number which
10549                          * is itself sizeof (uint32_t) aligned.
10550                          */
10551                         ASSERT(!((align - (offs & (align - 1))) &
10552                             (sizeof (uint32_t) - 1)));
10553                         DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
10554                         offs += sizeof (uint32_t);
10555                 }
10556 
10557                 if ((soffs = offs + needed) > buf->dtb_size) {
10558                         dtrace_buffer_drop(buf);
10559                         return (-1);
10560                 }
10561 
10562                 if (mstate == NULL)
10563                         return (offs);
10564 
10565                 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
10566                 mstate->dtms_scratch_size = buf->dtb_size - soffs;
10567                 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
10568 
10569                 return (offs);
10570         }
10571 
10572         if (buf->dtb_flags & DTRACEBUF_FILL) {
10573                 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
10574                     (buf->dtb_flags & DTRACEBUF_FULL))
10575                         return (-1);
10576                 goto out;
10577         }
10578 
10579         total = needed + (offs & (align - 1));
10580 
10581         /*
10582          * For a ring buffer, life is quite a bit more complicated.  Before
10583          * we can store any padding, we need to adjust our wrapping offset.
10584          * (If we've never before wrapped or we're not about to, no adjustment
10585          * is required.)
10586          */
10587         if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
10588             offs + total > buf->dtb_size) {
10589                 woffs = buf->dtb_xamot_offset;
10590 
10591                 if (offs + total > buf->dtb_size) {
10592                         /*
10593                          * We can't fit in the end of the buffer.  First, a
10594                          * sanity check that we can fit in the buffer at all.
10595                          */
10596                         if (total > buf->dtb_size) {
10597                                 dtrace_buffer_drop(buf);
10598                                 return (-1);
10599                         }
10600 
10601                         /*
10602                          * We're going to be storing at the top of the buffer,
10603                          * so now we need to deal with the wrapped offset.  We
10604                          * only reset our wrapped offset to 0 if it is
10605                          * currently greater than the current offset.  If it
10606                          * is less than the current offset, it is because a
10607                          * previous allocation induced a wrap -- but the
10608                          * allocation didn't subsequently take the space due
10609                          * to an error or false predicate evaluation.  In this
10610                          * case, we'll just leave the wrapped offset alone: if
10611                          * the wrapped offset hasn't been advanced far enough
10612                          * for this allocation, it will be adjusted in the
10613                          * lower loop.
10614                          */
10615                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
10616                                 if (woffs >= offs)
10617                                         woffs = 0;
10618                         } else {
10619                                 woffs = 0;
10620                         }
10621 
10622                         /*
10623                          * Now we know that we're going to be storing to the
10624                          * top of the buffer and that there is room for us
10625                          * there.  We need to clear the buffer from the current
10626                          * offset to the end (there may be old gunk there).
10627                          */
10628                         while (offs < buf->dtb_size)
10629                                 tomax[offs++] = 0;
10630 
10631                         /*
10632                          * We need to set our offset to zero.  And because we
10633                          * are wrapping, we need to set the bit indicating as
10634                          * much.  We can also adjust our needed space back
10635                          * down to the space required by the ECB -- we know
10636                          * that the top of the buffer is aligned.
10637                          */
10638                         offs = 0;
10639                         total = needed;
10640                         buf->dtb_flags |= DTRACEBUF_WRAPPED;
10641                 } else {
10642                         /*
10643                          * There is room for us in the buffer, so we simply
10644                          * need to check the wrapped offset.
10645                          */
10646                         if (woffs < offs) {
10647                                 /*
10648                                  * The wrapped offset is less than the offset.
10649                                  * This can happen if we allocated buffer space
10650                                  * that induced a wrap, but then we didn't
10651                                  * subsequently take the space due to an error
10652                                  * or false predicate evaluation.  This is
10653                                  * okay; we know that _this_ allocation isn't
10654                                  * going to induce a wrap.  We still can't
10655                                  * reset the wrapped offset to be zero,
10656                                  * however: the space may have been trashed in
10657                                  * the previous failed probe attempt.  But at
10658                                  * least the wrapped offset doesn't need to
10659                                  * be adjusted at all...
10660                                  */
10661                                 goto out;
10662                         }
10663                 }
10664 
10665                 while (offs + total > woffs) {
10666                         dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
10667                         size_t size;
10668 
10669                         if (epid == DTRACE_EPIDNONE) {
10670                                 size = sizeof (uint32_t);
10671                         } else {
10672                                 ASSERT3U(epid, <=, state->dts_necbs);
10673                                 ASSERT(state->dts_ecbs[epid - 1] != NULL);
10674 
10675                                 size = state->dts_ecbs[epid - 1]->dte_size;
10676                         }
10677 
10678                         ASSERT(woffs + size <= buf->dtb_size);
10679                         ASSERT(size != 0);
10680 
10681                         if (woffs + size == buf->dtb_size) {
10682                                 /*
10683                                  * We've reached the end of the buffer; we want
10684                                  * to set the wrapped offset to 0 and break
10685                                  * out.  However, if the offs is 0, then we're
10686                                  * in a strange edge-condition:  the amount of
10687                                  * space that we want to reserve plus the size
10688                                  * of the record that we're overwriting is
10689                                  * greater than the size of the buffer.  This
10690                                  * is problematic because if we reserve the
10691                                  * space but subsequently don't consume it (due
10692                                  * to a failed predicate or error) the wrapped
10693                                  * offset will be 0 -- yet the EPID at offset 0
10694                                  * will not be committed.  This situation is
10695                                  * relatively easy to deal with:  if we're in
10696                                  * this case, the buffer is indistinguishable
10697                                  * from one that hasn't wrapped; we need only
10698                                  * finish the job by clearing the wrapped bit,
10699                                  * explicitly setting the offset to be 0, and
10700                                  * zero'ing out the old data in the buffer.
10701                                  */
10702                                 if (offs == 0) {
10703                                         buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
10704                                         buf->dtb_offset = 0;
10705                                         woffs = total;
10706 
10707                                         while (woffs < buf->dtb_size)
10708                                                 tomax[woffs++] = 0;
10709                                 }
10710 
10711                                 woffs = 0;
10712                                 break;
10713                         }
10714 
10715                         woffs += size;
10716                 }
10717 
10718                 /*
10719                  * We have a wrapped offset.  It may be that the wrapped offset
10720                  * has become zero -- that's okay.
10721                  */
10722                 buf->dtb_xamot_offset = woffs;
10723         }
10724 
10725 out:
10726         /*
10727          * Now we can plow the buffer with any necessary padding.
10728          */
10729         while (offs & (align - 1)) {
10730                 /*
10731                  * Assert that our alignment is off by a number which
10732                  * is itself sizeof (uint32_t) aligned.
10733                  */
10734                 ASSERT(!((align - (offs & (align - 1))) &
10735                     (sizeof (uint32_t) - 1)));
10736                 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
10737                 offs += sizeof (uint32_t);
10738         }
10739 
10740         if (buf->dtb_flags & DTRACEBUF_FILL) {
10741                 if (offs + needed > buf->dtb_size - state->dts_reserve) {
10742                         buf->dtb_flags |= DTRACEBUF_FULL;
10743                         return (-1);
10744                 }
10745         }
10746 
10747         if (mstate == NULL)
10748                 return (offs);
10749 
10750         /*
10751          * For ring buffers and fill buffers, the scratch space is always
10752          * the inactive buffer.
10753          */
10754         mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
10755         mstate->dtms_scratch_size = buf->dtb_size;
10756         mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
10757 
10758         return (offs);
10759 }
10760 
10761 static void
10762 dtrace_buffer_polish(dtrace_buffer_t *buf)
10763 {
10764         ASSERT(buf->dtb_flags & DTRACEBUF_RING);
10765         ASSERT(MUTEX_HELD(&dtrace_lock));
10766 
10767         if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
10768                 return;
10769 
10770         /*
10771          * We need to polish the ring buffer.  There are three cases:
10772          *
10773          * - The first (and presumably most common) is that there is no gap
10774          *   between the buffer offset and the wrapped offset.  In this case,
10775          *   there is nothing in the buffer that isn't valid data; we can
10776          *   mark the buffer as polished and return.
10777          *
10778          * - The second (less common than the first but still more common
10779          *   than the third) is that there is a gap between the buffer offset
10780          *   and the wrapped offset, and the wrapped offset is larger than the
10781          *   buffer offset.  This can happen because of an alignment issue, or
10782          *   can happen because of a call to dtrace_buffer_reserve() that
10783          *   didn't subsequently consume the buffer space.  In this case,
10784          *   we need to zero the data from the buffer offset to the wrapped
10785          *   offset.
10786          *
10787          * - The third (and least common) is that there is a gap between the
10788          *   buffer offset and the wrapped offset, but the wrapped offset is
10789          *   _less_ than the buffer offset.  This can only happen because a
10790          *   call to dtrace_buffer_reserve() induced a wrap, but the space
10791          *   was not subsequently consumed.  In this case, we need to zero the
10792          *   space from the offset to the end of the buffer _and_ from the
10793          *   top of the buffer to the wrapped offset.
10794          */
10795         if (buf->dtb_offset < buf->dtb_xamot_offset) {
10796                 bzero(buf->dtb_tomax + buf->dtb_offset,
10797                     buf->dtb_xamot_offset - buf->dtb_offset);
10798         }
10799 
10800         if (buf->dtb_offset > buf->dtb_xamot_offset) {
10801                 bzero(buf->dtb_tomax + buf->dtb_offset,
10802                     buf->dtb_size - buf->dtb_offset);
10803                 bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
10804         }
10805 }
10806 
10807 /*
10808  * This routine determines if data generated at the specified time has likely
10809  * been entirely consumed at user-level.  This routine is called to determine
10810  * if an ECB on a defunct probe (but for an active enabling) can be safely
10811  * disabled and destroyed.
10812  */
10813 static int
10814 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
10815 {
10816         int i;
10817 
10818         for (i = 0; i < NCPU; i++) {
10819                 dtrace_buffer_t *buf = &bufs[i];
10820 
10821                 if (buf->dtb_size == 0)
10822                         continue;
10823 
10824                 if (buf->dtb_flags & DTRACEBUF_RING)
10825                         return (0);
10826 
10827                 if (!buf->dtb_switched && buf->dtb_offset != 0)
10828                         return (0);
10829 
10830                 if (buf->dtb_switched - buf->dtb_interval < when)
10831                         return (0);
10832         }
10833 
10834         return (1);
10835 }
10836 
10837 static void
10838 dtrace_buffer_free(dtrace_buffer_t *bufs)
10839 {
10840         int i;
10841 
10842         for (i = 0; i < NCPU; i++) {
10843                 dtrace_buffer_t *buf = &bufs[i];
10844 
10845                 if (buf->dtb_tomax == NULL) {
10846                         ASSERT(buf->dtb_xamot == NULL);
10847                         ASSERT(buf->dtb_size == 0);
10848                         continue;
10849                 }
10850 
10851                 if (buf->dtb_xamot != NULL) {
10852                         ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
10853                         kmem_free(buf->dtb_xamot, buf->dtb_size);
10854                 }
10855 
10856                 kmem_free(buf->dtb_tomax, buf->dtb_size);
10857                 buf->dtb_size = 0;
10858                 buf->dtb_tomax = NULL;
10859                 buf->dtb_xamot = NULL;
10860         }
10861 }
10862 
10863 /*
10864  * DTrace Enabling Functions
10865  */
10866 static dtrace_enabling_t *
10867 dtrace_enabling_create(dtrace_vstate_t *vstate)
10868 {
10869         dtrace_enabling_t *enab;
10870 
10871         enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
10872         enab->dten_vstate = vstate;
10873 
10874         return (enab);
10875 }
10876 
10877 static void
10878 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
10879 {
10880         dtrace_ecbdesc_t **ndesc;
10881         size_t osize, nsize;
10882 
10883         /*
10884          * We can't add to enablings after we've enabled them, or after we've
10885          * retained them.
10886          */
10887         ASSERT(enab->dten_probegen == 0);
10888         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
10889 
10890         if (enab->dten_ndesc < enab->dten_maxdesc) {
10891                 enab->dten_desc[enab->dten_ndesc++] = ecb;
10892                 return;
10893         }
10894 
10895         osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
10896 
10897         if (enab->dten_maxdesc == 0) {
10898                 enab->dten_maxdesc = 1;
10899         } else {
10900                 enab->dten_maxdesc <<= 1;
10901         }
10902 
10903         ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
10904 
10905         nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
10906         ndesc = kmem_zalloc(nsize, KM_SLEEP);
10907         bcopy(enab->dten_desc, ndesc, osize);
10908         kmem_free(enab->dten_desc, osize);
10909 
10910         enab->dten_desc = ndesc;
10911         enab->dten_desc[enab->dten_ndesc++] = ecb;
10912 }
10913 
10914 static void
10915 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
10916     dtrace_probedesc_t *pd)
10917 {
10918         dtrace_ecbdesc_t *new;
10919         dtrace_predicate_t *pred;
10920         dtrace_actdesc_t *act;
10921 
10922         /*
10923          * We're going to create a new ECB description that matches the
10924          * specified ECB in every way, but has the specified probe description.
10925          */
10926         new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
10927 
10928         if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
10929                 dtrace_predicate_hold(pred);
10930 
10931         for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
10932                 dtrace_actdesc_hold(act);
10933 
10934         new->dted_action = ecb->dted_action;
10935         new->dted_pred = ecb->dted_pred;
10936         new->dted_probe = *pd;
10937         new->dted_uarg = ecb->dted_uarg;
10938 
10939         dtrace_enabling_add(enab, new);
10940 }
10941 
10942 static void
10943 dtrace_enabling_dump(dtrace_enabling_t *enab)
10944 {
10945         int i;
10946 
10947         for (i = 0; i < enab->dten_ndesc; i++) {
10948                 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
10949 
10950                 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
10951                     desc->dtpd_provider, desc->dtpd_mod,
10952                     desc->dtpd_func, desc->dtpd_name);
10953         }
10954 }
10955 
10956 static void
10957 dtrace_enabling_destroy(dtrace_enabling_t *enab)
10958 {
10959         int i;
10960         dtrace_ecbdesc_t *ep;
10961         dtrace_vstate_t *vstate = enab->dten_vstate;
10962 
10963         ASSERT(MUTEX_HELD(&dtrace_lock));
10964 
10965         for (i = 0; i < enab->dten_ndesc; i++) {
10966                 dtrace_actdesc_t *act, *next;
10967                 dtrace_predicate_t *pred;
10968 
10969                 ep = enab->dten_desc[i];
10970 
10971                 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
10972                         dtrace_predicate_release(pred, vstate);
10973 
10974                 for (act = ep->dted_action; act != NULL; act = next) {
10975                         next = act->dtad_next;
10976                         dtrace_actdesc_release(act, vstate);
10977                 }
10978 
10979                 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
10980         }
10981 
10982         kmem_free(enab->dten_desc,
10983             enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
10984 
10985         /*
10986          * If this was a retained enabling, decrement the dts_nretained count
10987          * and take it off of the dtrace_retained list.
10988          */
10989         if (enab->dten_prev != NULL || enab->dten_next != NULL ||
10990             dtrace_retained == enab) {
10991                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
10992                 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
10993                 enab->dten_vstate->dtvs_state->dts_nretained--;
10994                 dtrace_retained_gen++;
10995         }
10996 
10997         if (enab->dten_prev == NULL) {
10998                 if (dtrace_retained == enab) {
10999                         dtrace_retained = enab->dten_next;
11000 
11001                         if (dtrace_retained != NULL)
11002                                 dtrace_retained->dten_prev = NULL;
11003                 }
11004         } else {
11005                 ASSERT(enab != dtrace_retained);
11006                 ASSERT(dtrace_retained != NULL);
11007                 enab->dten_prev->dten_next = enab->dten_next;
11008         }
11009 
11010         if (enab->dten_next != NULL) {
11011                 ASSERT(dtrace_retained != NULL);
11012                 enab->dten_next->dten_prev = enab->dten_prev;
11013         }
11014 
11015         kmem_free(enab, sizeof (dtrace_enabling_t));
11016 }
11017 
11018 static int
11019 dtrace_enabling_retain(dtrace_enabling_t *enab)
11020 {
11021         dtrace_state_t *state;
11022 
11023         ASSERT(MUTEX_HELD(&dtrace_lock));
11024         ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11025         ASSERT(enab->dten_vstate != NULL);
11026 
11027         state = enab->dten_vstate->dtvs_state;
11028         ASSERT(state != NULL);
11029 
11030         /*
11031          * We only allow each state to retain dtrace_retain_max enablings.
11032          */
11033         if (state->dts_nretained >= dtrace_retain_max)
11034                 return (ENOSPC);
11035 
11036         state->dts_nretained++;
11037         dtrace_retained_gen++;
11038 
11039         if (dtrace_retained == NULL) {
11040                 dtrace_retained = enab;
11041                 return (0);
11042         }
11043 
11044         enab->dten_next = dtrace_retained;
11045         dtrace_retained->dten_prev = enab;
11046         dtrace_retained = enab;
11047 
11048         return (0);
11049 }
11050 
11051 static int
11052 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
11053     dtrace_probedesc_t *create)
11054 {
11055         dtrace_enabling_t *new, *enab;
11056         int found = 0, err = ENOENT;
11057 
11058         ASSERT(MUTEX_HELD(&dtrace_lock));
11059         ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
11060         ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
11061         ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
11062         ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
11063 
11064         new = dtrace_enabling_create(&state->dts_vstate);
11065 
11066         /*
11067          * Iterate over all retained enablings, looking for enablings that
11068          * match the specified state.
11069          */
11070         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11071                 int i;
11072 
11073                 /*
11074                  * dtvs_state can only be NULL for helper enablings -- and
11075                  * helper enablings can't be retained.
11076                  */
11077                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11078 
11079                 if (enab->dten_vstate->dtvs_state != state)
11080                         continue;
11081 
11082                 /*
11083                  * Now iterate over each probe description; we're looking for
11084                  * an exact match to the specified probe description.
11085                  */
11086                 for (i = 0; i < enab->dten_ndesc; i++) {
11087                         dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11088                         dtrace_probedesc_t *pd = &ep->dted_probe;
11089 
11090                         if (strcmp(pd->dtpd_provider, match->dtpd_provider))
11091                                 continue;
11092 
11093                         if (strcmp(pd->dtpd_mod, match->dtpd_mod))
11094                                 continue;
11095 
11096                         if (strcmp(pd->dtpd_func, match->dtpd_func))
11097                                 continue;
11098 
11099                         if (strcmp(pd->dtpd_name, match->dtpd_name))
11100                                 continue;
11101 
11102                         /*
11103                          * We have a winning probe!  Add it to our growing
11104                          * enabling.
11105                          */
11106                         found = 1;
11107                         dtrace_enabling_addlike(new, ep, create);
11108                 }
11109         }
11110 
11111         if (!found || (err = dtrace_enabling_retain(new)) != 0) {
11112                 dtrace_enabling_destroy(new);
11113                 return (err);
11114         }
11115 
11116         return (0);
11117 }
11118 
11119 static void
11120 dtrace_enabling_retract(dtrace_state_t *state)
11121 {
11122         dtrace_enabling_t *enab, *next;
11123 
11124         ASSERT(MUTEX_HELD(&dtrace_lock));
11125 
11126         /*
11127          * Iterate over all retained enablings, destroy the enablings retained
11128          * for the specified state.
11129          */
11130         for (enab = dtrace_retained; enab != NULL; enab = next) {
11131                 next = enab->dten_next;
11132 
11133                 /*
11134                  * dtvs_state can only be NULL for helper enablings -- and
11135                  * helper enablings can't be retained.
11136                  */
11137                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11138 
11139                 if (enab->dten_vstate->dtvs_state == state) {
11140                         ASSERT(state->dts_nretained > 0);
11141                         dtrace_enabling_destroy(enab);
11142                 }
11143         }
11144 
11145         ASSERT(state->dts_nretained == 0);
11146 }
11147 
11148 static int
11149 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
11150 {
11151         int i = 0;
11152         int total_matched = 0, matched = 0;
11153 
11154         ASSERT(MUTEX_HELD(&cpu_lock));
11155         ASSERT(MUTEX_HELD(&dtrace_lock));
11156 
11157         for (i = 0; i < enab->dten_ndesc; i++) {
11158                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11159 
11160                 enab->dten_current = ep;
11161                 enab->dten_error = 0;
11162 
11163                 /*
11164                  * If a provider failed to enable a probe then get out and
11165                  * let the consumer know we failed.
11166                  */
11167                 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
11168                         return (EBUSY);
11169 
11170                 total_matched += matched;
11171 
11172                 if (enab->dten_error != 0) {
11173                         /*
11174                          * If we get an error half-way through enabling the
11175                          * probes, we kick out -- perhaps with some number of
11176                          * them enabled.  Leaving enabled probes enabled may
11177                          * be slightly confusing for user-level, but we expect
11178                          * that no one will attempt to actually drive on in
11179                          * the face of such errors.  If this is an anonymous
11180                          * enabling (indicated with a NULL nmatched pointer),
11181                          * we cmn_err() a message.  We aren't expecting to
11182                          * get such an error -- such as it can exist at all,
11183                          * it would be a result of corrupted DOF in the driver
11184                          * properties.
11185                          */
11186                         if (nmatched == NULL) {
11187                                 cmn_err(CE_WARN, "dtrace_enabling_match() "
11188                                     "error on %p: %d", (void *)ep,
11189                                     enab->dten_error);
11190                         }
11191 
11192                         return (enab->dten_error);
11193                 }
11194         }
11195 
11196         enab->dten_probegen = dtrace_probegen;
11197         if (nmatched != NULL)
11198                 *nmatched = total_matched;
11199 
11200         return (0);
11201 }
11202 
11203 static void
11204 dtrace_enabling_matchall(void)
11205 {
11206         dtrace_enabling_t *enab;
11207 
11208         mutex_enter(&cpu_lock);
11209         mutex_enter(&dtrace_lock);
11210 
11211         /*
11212          * Iterate over all retained enablings to see if any probes match
11213          * against them.  We only perform this operation on enablings for which
11214          * we have sufficient permissions by virtue of being in the global zone
11215          * or in the same zone as the DTrace client.  Because we can be called
11216          * after dtrace_detach() has been called, we cannot assert that there
11217          * are retained enablings.  We can safely load from dtrace_retained,
11218          * however:  the taskq_destroy() at the end of dtrace_detach() will
11219          * block pending our completion.
11220          */
11221         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11222                 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred;
11223                 cred_t *cr = dcr->dcr_cred;
11224                 zoneid_t zone = cr != NULL ? crgetzoneid(cr) : 0;
11225 
11226                 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL &&
11227                     (zone == GLOBAL_ZONEID || getzoneid() == zone)))
11228                         (void) dtrace_enabling_match(enab, NULL);
11229         }
11230 
11231         mutex_exit(&dtrace_lock);
11232         mutex_exit(&cpu_lock);
11233 }
11234 
11235 /*
11236  * If an enabling is to be enabled without having matched probes (that is, if
11237  * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
11238  * enabling must be _primed_ by creating an ECB for every ECB description.
11239  * This must be done to assure that we know the number of speculations, the
11240  * number of aggregations, the minimum buffer size needed, etc. before we
11241  * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
11242  * enabling any probes, we create ECBs for every ECB decription, but with a
11243  * NULL probe -- which is exactly what this function does.
11244  */
11245 static void
11246 dtrace_enabling_prime(dtrace_state_t *state)
11247 {
11248         dtrace_enabling_t *enab;
11249         int i;
11250 
11251         for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11252                 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11253 
11254                 if (enab->dten_vstate->dtvs_state != state)
11255                         continue;
11256 
11257                 /*
11258                  * We don't want to prime an enabling more than once, lest
11259                  * we allow a malicious user to induce resource exhaustion.
11260                  * (The ECBs that result from priming an enabling aren't
11261                  * leaked -- but they also aren't deallocated until the
11262                  * consumer state is destroyed.)
11263                  */
11264                 if (enab->dten_primed)
11265                         continue;
11266 
11267                 for (i = 0; i < enab->dten_ndesc; i++) {
11268                         enab->dten_current = enab->dten_desc[i];
11269                         (void) dtrace_probe_enable(NULL, enab);
11270                 }
11271 
11272                 enab->dten_primed = 1;
11273         }
11274 }
11275 
11276 /*
11277  * Called to indicate that probes should be provided due to retained
11278  * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
11279  * must take an initial lap through the enabling calling the dtps_provide()
11280  * entry point explicitly to allow for autocreated probes.
11281  */
11282 static void
11283 dtrace_enabling_provide(dtrace_provider_t *prv)
11284 {
11285         int i, all = 0;
11286         dtrace_probedesc_t desc;
11287         dtrace_genid_t gen;
11288 
11289         ASSERT(MUTEX_HELD(&dtrace_lock));
11290         ASSERT(MUTEX_HELD(&dtrace_provider_lock));
11291 
11292         if (prv == NULL) {
11293                 all = 1;
11294                 prv = dtrace_provider;
11295         }
11296 
11297         do {
11298                 dtrace_enabling_t *enab;
11299                 void *parg = prv->dtpv_arg;
11300 
11301 retry:
11302                 gen = dtrace_retained_gen;
11303                 for (enab = dtrace_retained; enab != NULL;
11304                     enab = enab->dten_next) {
11305                         for (i = 0; i < enab->dten_ndesc; i++) {
11306                                 desc = enab->dten_desc[i]->dted_probe;
11307                                 mutex_exit(&dtrace_lock);
11308                                 prv->dtpv_pops.dtps_provide(parg, &desc);
11309                                 mutex_enter(&dtrace_lock);
11310                                 /*
11311                                  * Process the retained enablings again if
11312                                  * they have changed while we weren't holding
11313                                  * dtrace_lock.
11314                                  */
11315                                 if (gen != dtrace_retained_gen)
11316                                         goto retry;
11317                         }
11318                 }
11319         } while (all && (prv = prv->dtpv_next) != NULL);
11320 
11321         mutex_exit(&dtrace_lock);
11322         dtrace_probe_provide(NULL, all ? NULL : prv);
11323         mutex_enter(&dtrace_lock);
11324 }
11325 
11326 /*
11327  * Called to reap ECBs that are attached to probes from defunct providers.
11328  */
11329 static void
11330 dtrace_enabling_reap(void)
11331 {
11332         dtrace_provider_t *prov;
11333         dtrace_probe_t *probe;
11334         dtrace_ecb_t *ecb;
11335         hrtime_t when;
11336         int i;
11337 
11338         mutex_enter(&cpu_lock);
11339         mutex_enter(&dtrace_lock);
11340 
11341         for (i = 0; i < dtrace_nprobes; i++) {
11342                 if ((probe = dtrace_probes[i]) == NULL)
11343                         continue;
11344 
11345                 if (probe->dtpr_ecb == NULL)
11346                         continue;
11347 
11348                 prov = probe->dtpr_provider;
11349 
11350                 if ((when = prov->dtpv_defunct) == 0)
11351                         continue;
11352 
11353                 /*
11354                  * We have ECBs on a defunct provider:  we want to reap these
11355                  * ECBs to allow the provider to unregister.  The destruction
11356                  * of these ECBs must be done carefully:  if we destroy the ECB
11357                  * and the consumer later wishes to consume an EPID that
11358                  * corresponds to the destroyed ECB (and if the EPID metadata
11359                  * has not been previously consumed), the consumer will abort
11360                  * processing on the unknown EPID.  To reduce (but not, sadly,
11361                  * eliminate) the possibility of this, we will only destroy an
11362                  * ECB for a defunct provider if, for the state that
11363                  * corresponds to the ECB:
11364                  *
11365                  *  (a) There is no speculative tracing (which can effectively
11366                  *      cache an EPID for an arbitrary amount of time).
11367                  *
11368                  *  (b) The principal buffers have been switched twice since the
11369                  *      provider became defunct.
11370                  *
11371                  *  (c) The aggregation buffers are of zero size or have been
11372                  *      switched twice since the provider became defunct.
11373                  *
11374                  * We use dts_speculates to determine (a) and call a function
11375                  * (dtrace_buffer_consumed()) to determine (b) and (c).  Note
11376                  * that as soon as we've been unable to destroy one of the ECBs
11377                  * associated with the probe, we quit trying -- reaping is only
11378                  * fruitful in as much as we can destroy all ECBs associated
11379                  * with the defunct provider's probes.
11380                  */
11381                 while ((ecb = probe->dtpr_ecb) != NULL) {
11382                         dtrace_state_t *state = ecb->dte_state;
11383                         dtrace_buffer_t *buf = state->dts_buffer;
11384                         dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
11385 
11386                         if (state->dts_speculates)
11387                                 break;
11388 
11389                         if (!dtrace_buffer_consumed(buf, when))
11390                                 break;
11391 
11392                         if (!dtrace_buffer_consumed(aggbuf, when))
11393                                 break;
11394 
11395                         dtrace_ecb_disable(ecb);
11396                         ASSERT(probe->dtpr_ecb != ecb);
11397                         dtrace_ecb_destroy(ecb);
11398                 }
11399         }
11400 
11401         mutex_exit(&dtrace_lock);
11402         mutex_exit(&cpu_lock);
11403 }
11404 
11405 /*
11406  * DTrace DOF Functions
11407  */
11408 /*ARGSUSED*/
11409 static void
11410 dtrace_dof_error(dof_hdr_t *dof, const char *str)
11411 {
11412         if (dtrace_err_verbose)
11413                 cmn_err(CE_WARN, "failed to process DOF: %s", str);
11414 
11415 #ifdef DTRACE_ERRDEBUG
11416         dtrace_errdebug(str);
11417 #endif
11418 }
11419 
11420 /*
11421  * Create DOF out of a currently enabled state.  Right now, we only create
11422  * DOF containing the run-time options -- but this could be expanded to create
11423  * complete DOF representing the enabled state.
11424  */
11425 static dof_hdr_t *
11426 dtrace_dof_create(dtrace_state_t *state)
11427 {
11428         dof_hdr_t *dof;
11429         dof_sec_t *sec;
11430         dof_optdesc_t *opt;
11431         int i, len = sizeof (dof_hdr_t) +
11432             roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
11433             sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
11434 
11435         ASSERT(MUTEX_HELD(&dtrace_lock));
11436 
11437         dof = kmem_zalloc(len, KM_SLEEP);
11438         dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
11439         dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
11440         dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
11441         dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
11442 
11443         dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
11444         dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
11445         dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
11446         dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
11447         dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
11448         dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
11449 
11450         dof->dofh_flags = 0;
11451         dof->dofh_hdrsize = sizeof (dof_hdr_t);
11452         dof->dofh_secsize = sizeof (dof_sec_t);
11453         dof->dofh_secnum = 1;        /* only DOF_SECT_OPTDESC */
11454         dof->dofh_secoff = sizeof (dof_hdr_t);
11455         dof->dofh_loadsz = len;
11456         dof->dofh_filesz = len;
11457         dof->dofh_pad = 0;
11458 
11459         /*
11460          * Fill in the option section header...
11461          */
11462         sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
11463         sec->dofs_type = DOF_SECT_OPTDESC;
11464         sec->dofs_align = sizeof (uint64_t);
11465         sec->dofs_flags = DOF_SECF_LOAD;
11466         sec->dofs_entsize = sizeof (dof_optdesc_t);
11467 
11468         opt = (dof_optdesc_t *)((uintptr_t)sec +
11469             roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
11470 
11471         sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
11472         sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
11473 
11474         for (i = 0; i < DTRACEOPT_MAX; i++) {
11475                 opt[i].dofo_option = i;
11476                 opt[i].dofo_strtab = DOF_SECIDX_NONE;
11477                 opt[i].dofo_value = state->dts_options[i];
11478         }
11479 
11480         return (dof);
11481 }
11482 
11483 static dof_hdr_t *
11484 dtrace_dof_copyin(uintptr_t uarg, int *errp)
11485 {
11486         dof_hdr_t hdr, *dof;
11487 
11488         ASSERT(!MUTEX_HELD(&dtrace_lock));
11489 
11490         /*
11491          * First, we're going to copyin() the sizeof (dof_hdr_t).
11492          */
11493         if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
11494                 dtrace_dof_error(NULL, "failed to copyin DOF header");
11495                 *errp = EFAULT;
11496                 return (NULL);
11497         }
11498 
11499         /*
11500          * Now we'll allocate the entire DOF and copy it in -- provided
11501          * that the length isn't outrageous.
11502          */
11503         if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
11504                 dtrace_dof_error(&hdr, "load size exceeds maximum");
11505                 *errp = E2BIG;
11506                 return (NULL);
11507         }
11508 
11509         if (hdr.dofh_loadsz < sizeof (hdr)) {
11510                 dtrace_dof_error(&hdr, "invalid load size");
11511                 *errp = EINVAL;
11512                 return (NULL);
11513         }
11514 
11515         dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
11516 
11517         if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
11518             dof->dofh_loadsz != hdr.dofh_loadsz) {
11519                 kmem_free(dof, hdr.dofh_loadsz);
11520                 *errp = EFAULT;
11521                 return (NULL);
11522         }
11523 
11524         return (dof);
11525 }
11526 
11527 static dof_hdr_t *
11528 dtrace_dof_property(const char *name)
11529 {
11530         uchar_t *buf;
11531         uint64_t loadsz;
11532         unsigned int len, i;
11533         dof_hdr_t *dof;
11534 
11535         /*
11536          * Unfortunately, array of values in .conf files are always (and
11537          * only) interpreted to be integer arrays.  We must read our DOF
11538          * as an integer array, and then squeeze it into a byte array.
11539          */
11540         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
11541             (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
11542                 return (NULL);
11543 
11544         for (i = 0; i < len; i++)
11545                 buf[i] = (uchar_t)(((int *)buf)[i]);
11546 
11547         if (len < sizeof (dof_hdr_t)) {
11548                 ddi_prop_free(buf);
11549                 dtrace_dof_error(NULL, "truncated header");
11550                 return (NULL);
11551         }
11552 
11553         if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
11554                 ddi_prop_free(buf);
11555                 dtrace_dof_error(NULL, "truncated DOF");
11556                 return (NULL);
11557         }
11558 
11559         if (loadsz >= dtrace_dof_maxsize) {
11560                 ddi_prop_free(buf);
11561                 dtrace_dof_error(NULL, "oversized DOF");
11562                 return (NULL);
11563         }
11564 
11565         dof = kmem_alloc(loadsz, KM_SLEEP);
11566         bcopy(buf, dof, loadsz);
11567         ddi_prop_free(buf);
11568 
11569         return (dof);
11570 }
11571 
11572 static void
11573 dtrace_dof_destroy(dof_hdr_t *dof)
11574 {
11575         kmem_free(dof, dof->dofh_loadsz);
11576 }
11577 
11578 /*
11579  * Return the dof_sec_t pointer corresponding to a given section index.  If the
11580  * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
11581  * a type other than DOF_SECT_NONE is specified, the header is checked against
11582  * this type and NULL is returned if the types do not match.
11583  */
11584 static dof_sec_t *
11585 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
11586 {
11587         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
11588             ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
11589 
11590         if (i >= dof->dofh_secnum) {
11591                 dtrace_dof_error(dof, "referenced section index is invalid");
11592                 return (NULL);
11593         }
11594 
11595         if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
11596                 dtrace_dof_error(dof, "referenced section is not loadable");
11597                 return (NULL);
11598         }
11599 
11600         if (type != DOF_SECT_NONE && type != sec->dofs_type) {
11601                 dtrace_dof_error(dof, "referenced section is the wrong type");
11602                 return (NULL);
11603         }
11604 
11605         return (sec);
11606 }
11607 
11608 static dtrace_probedesc_t *
11609 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
11610 {
11611         dof_probedesc_t *probe;
11612         dof_sec_t *strtab;
11613         uintptr_t daddr = (uintptr_t)dof;
11614         uintptr_t str;
11615         size_t size;
11616 
11617         if (sec->dofs_type != DOF_SECT_PROBEDESC) {
11618                 dtrace_dof_error(dof, "invalid probe section");
11619                 return (NULL);
11620         }
11621 
11622         if (sec->dofs_align != sizeof (dof_secidx_t)) {
11623                 dtrace_dof_error(dof, "bad alignment in probe description");
11624                 return (NULL);
11625         }
11626 
11627         if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
11628                 dtrace_dof_error(dof, "truncated probe description");
11629                 return (NULL);
11630         }
11631 
11632         probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
11633         strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
11634 
11635         if (strtab == NULL)
11636                 return (NULL);
11637 
11638         str = daddr + strtab->dofs_offset;
11639         size = strtab->dofs_size;
11640 
11641         if (probe->dofp_provider >= strtab->dofs_size) {
11642                 dtrace_dof_error(dof, "corrupt probe provider");
11643                 return (NULL);
11644         }
11645 
11646         (void) strncpy(desc->dtpd_provider,
11647             (char *)(str + probe->dofp_provider),
11648             MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
11649 
11650         if (probe->dofp_mod >= strtab->dofs_size) {
11651                 dtrace_dof_error(dof, "corrupt probe module");
11652                 return (NULL);
11653         }
11654 
11655         (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
11656             MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
11657 
11658         if (probe->dofp_func >= strtab->dofs_size) {
11659                 dtrace_dof_error(dof, "corrupt probe function");
11660                 return (NULL);
11661         }
11662 
11663         (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
11664             MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
11665 
11666         if (probe->dofp_name >= strtab->dofs_size) {
11667                 dtrace_dof_error(dof, "corrupt probe name");
11668                 return (NULL);
11669         }
11670 
11671         (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
11672             MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
11673 
11674         return (desc);
11675 }
11676 
11677 static dtrace_difo_t *
11678 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11679     cred_t *cr)
11680 {
11681         dtrace_difo_t *dp;
11682         size_t ttl = 0;
11683         dof_difohdr_t *dofd;
11684         uintptr_t daddr = (uintptr_t)dof;
11685         size_t max = dtrace_difo_maxsize;
11686         int i, l, n;
11687 
11688         static const struct {
11689                 int section;
11690                 int bufoffs;
11691                 int lenoffs;
11692                 int entsize;
11693                 int align;
11694                 const char *msg;
11695         } difo[] = {
11696                 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
11697                 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
11698                 sizeof (dif_instr_t), "multiple DIF sections" },
11699 
11700                 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
11701                 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
11702                 sizeof (uint64_t), "multiple integer tables" },
11703 
11704                 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
11705                 offsetof(dtrace_difo_t, dtdo_strlen), 0,
11706                 sizeof (char), "multiple string tables" },
11707 
11708                 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
11709                 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
11710                 sizeof (uint_t), "multiple variable tables" },
11711 
11712                 { DOF_SECT_NONE, 0, 0, 0, NULL }
11713         };
11714 
11715         if (sec->dofs_type != DOF_SECT_DIFOHDR) {
11716                 dtrace_dof_error(dof, "invalid DIFO header section");
11717                 return (NULL);
11718         }
11719 
11720         if (sec->dofs_align != sizeof (dof_secidx_t)) {
11721                 dtrace_dof_error(dof, "bad alignment in DIFO header");
11722                 return (NULL);
11723         }
11724 
11725         if (sec->dofs_size < sizeof (dof_difohdr_t) ||
11726             sec->dofs_size % sizeof (dof_secidx_t)) {
11727                 dtrace_dof_error(dof, "bad size in DIFO header");
11728                 return (NULL);
11729         }
11730 
11731         dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
11732         n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
11733 
11734         dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
11735         dp->dtdo_rtype = dofd->dofd_rtype;
11736 
11737         for (l = 0; l < n; l++) {
11738                 dof_sec_t *subsec;
11739                 void **bufp;
11740                 uint32_t *lenp;
11741 
11742                 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
11743                     dofd->dofd_links[l])) == NULL)
11744                         goto err; /* invalid section link */
11745 
11746                 if (ttl + subsec->dofs_size > max) {
11747                         dtrace_dof_error(dof, "exceeds maximum size");
11748                         goto err;
11749                 }
11750 
11751                 ttl += subsec->dofs_size;
11752 
11753                 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
11754                         if (subsec->dofs_type != difo[i].section)
11755                                 continue;
11756 
11757                         if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
11758                                 dtrace_dof_error(dof, "section not loaded");
11759                                 goto err;
11760                         }
11761 
11762                         if (subsec->dofs_align != difo[i].align) {
11763                                 dtrace_dof_error(dof, "bad alignment");
11764                                 goto err;
11765                         }
11766 
11767                         bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
11768                         lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
11769 
11770                         if (*bufp != NULL) {
11771                                 dtrace_dof_error(dof, difo[i].msg);
11772                                 goto err;
11773                         }
11774 
11775                         if (difo[i].entsize != subsec->dofs_entsize) {
11776                                 dtrace_dof_error(dof, "entry size mismatch");
11777                                 goto err;
11778                         }
11779 
11780                         if (subsec->dofs_entsize != 0 &&
11781                             (subsec->dofs_size % subsec->dofs_entsize) != 0) {
11782                                 dtrace_dof_error(dof, "corrupt entry size");
11783                                 goto err;
11784                         }
11785 
11786                         *lenp = subsec->dofs_size;
11787                         *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
11788                         bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
11789                             *bufp, subsec->dofs_size);
11790 
11791                         if (subsec->dofs_entsize != 0)
11792                                 *lenp /= subsec->dofs_entsize;
11793 
11794                         break;
11795                 }
11796 
11797                 /*
11798                  * If we encounter a loadable DIFO sub-section that is not
11799                  * known to us, assume this is a broken program and fail.
11800                  */
11801                 if (difo[i].section == DOF_SECT_NONE &&
11802                     (subsec->dofs_flags & DOF_SECF_LOAD)) {
11803                         dtrace_dof_error(dof, "unrecognized DIFO subsection");
11804                         goto err;
11805                 }
11806         }
11807 
11808         if (dp->dtdo_buf == NULL) {
11809                 /*
11810                  * We can't have a DIF object without DIF text.
11811                  */
11812                 dtrace_dof_error(dof, "missing DIF text");
11813                 goto err;
11814         }
11815 
11816         /*
11817          * Before we validate the DIF object, run through the variable table
11818          * looking for the strings -- if any of their size are under, we'll set
11819          * their size to be the system-wide default string size.  Note that
11820          * this should _not_ happen if the "strsize" option has been set --
11821          * in this case, the compiler should have set the size to reflect the
11822          * setting of the option.
11823          */
11824         for (i = 0; i < dp->dtdo_varlen; i++) {
11825                 dtrace_difv_t *v = &dp->dtdo_vartab[i];
11826                 dtrace_diftype_t *t = &v->dtdv_type;
11827 
11828                 if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
11829                         continue;
11830 
11831                 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
11832                         t->dtdt_size = dtrace_strsize_default;
11833         }
11834 
11835         if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
11836                 goto err;
11837 
11838         dtrace_difo_init(dp, vstate);
11839         return (dp);
11840 
11841 err:
11842         kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
11843         kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
11844         kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
11845         kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
11846 
11847         kmem_free(dp, sizeof (dtrace_difo_t));
11848         return (NULL);
11849 }
11850 
11851 static dtrace_predicate_t *
11852 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11853     cred_t *cr)
11854 {
11855         dtrace_difo_t *dp;
11856 
11857         if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
11858                 return (NULL);
11859 
11860         return (dtrace_predicate_create(dp));
11861 }
11862 
11863 static dtrace_actdesc_t *
11864 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11865     cred_t *cr)
11866 {
11867         dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
11868         dof_actdesc_t *desc;
11869         dof_sec_t *difosec;
11870         size_t offs;
11871         uintptr_t daddr = (uintptr_t)dof;
11872         uint64_t arg;
11873         dtrace_actkind_t kind;
11874 
11875         if (sec->dofs_type != DOF_SECT_ACTDESC) {
11876                 dtrace_dof_error(dof, "invalid action section");
11877                 return (NULL);
11878         }
11879 
11880         if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
11881                 dtrace_dof_error(dof, "truncated action description");
11882                 return (NULL);
11883         }
11884 
11885         if (sec->dofs_align != sizeof (uint64_t)) {
11886                 dtrace_dof_error(dof, "bad alignment in action description");
11887                 return (NULL);
11888         }
11889 
11890         if (sec->dofs_size < sec->dofs_entsize) {
11891                 dtrace_dof_error(dof, "section entry size exceeds total size");
11892                 return (NULL);
11893         }
11894 
11895         if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
11896                 dtrace_dof_error(dof, "bad entry size in action description");
11897                 return (NULL);
11898         }
11899 
11900         if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
11901                 dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
11902                 return (NULL);
11903         }
11904 
11905         for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
11906                 desc = (dof_actdesc_t *)(daddr +
11907                     (uintptr_t)sec->dofs_offset + offs);
11908                 kind = (dtrace_actkind_t)desc->dofa_kind;
11909 
11910                 if ((DTRACEACT_ISPRINTFLIKE(kind) &&
11911                     (kind != DTRACEACT_PRINTA ||
11912                     desc->dofa_strtab != DOF_SECIDX_NONE)) ||
11913                     (kind == DTRACEACT_DIFEXPR &&
11914                     desc->dofa_strtab != DOF_SECIDX_NONE)) {
11915                         dof_sec_t *strtab;
11916                         char *str, *fmt;
11917                         uint64_t i;
11918 
11919                         /*
11920                          * The argument to these actions is an index into the
11921                          * DOF string table.  For printf()-like actions, this
11922                          * is the format string.  For print(), this is the
11923                          * CTF type of the expression result.
11924                          */
11925                         if ((strtab = dtrace_dof_sect(dof,
11926                             DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
11927                                 goto err;
11928 
11929                         str = (char *)((uintptr_t)dof +
11930                             (uintptr_t)strtab->dofs_offset);
11931 
11932                         for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
11933                                 if (str[i] == '\0')
11934                                         break;
11935                         }
11936 
11937                         if (i >= strtab->dofs_size) {
11938                                 dtrace_dof_error(dof, "bogus format string");
11939                                 goto err;
11940                         }
11941 
11942                         if (i == desc->dofa_arg) {
11943                                 dtrace_dof_error(dof, "empty format string");
11944                                 goto err;
11945                         }
11946 
11947                         i -= desc->dofa_arg;
11948                         fmt = kmem_alloc(i + 1, KM_SLEEP);
11949                         bcopy(&str[desc->dofa_arg], fmt, i + 1);
11950                         arg = (uint64_t)(uintptr_t)fmt;
11951                 } else {
11952                         if (kind == DTRACEACT_PRINTA) {
11953                                 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
11954                                 arg = 0;
11955                         } else {
11956                                 arg = desc->dofa_arg;
11957                         }
11958                 }
11959 
11960                 act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
11961                     desc->dofa_uarg, arg);
11962 
11963                 if (last != NULL) {
11964                         last->dtad_next = act;
11965                 } else {
11966                         first = act;
11967                 }
11968 
11969                 last = act;
11970 
11971                 if (desc->dofa_difo == DOF_SECIDX_NONE)
11972                         continue;
11973 
11974                 if ((difosec = dtrace_dof_sect(dof,
11975                     DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
11976                         goto err;
11977 
11978                 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
11979 
11980                 if (act->dtad_difo == NULL)
11981                         goto err;
11982         }
11983 
11984         ASSERT(first != NULL);
11985         return (first);
11986 
11987 err:
11988         for (act = first; act != NULL; act = next) {
11989                 next = act->dtad_next;
11990                 dtrace_actdesc_release(act, vstate);
11991         }
11992 
11993         return (NULL);
11994 }
11995 
11996 static dtrace_ecbdesc_t *
11997 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11998     cred_t *cr)
11999 {
12000         dtrace_ecbdesc_t *ep;
12001         dof_ecbdesc_t *ecb;
12002         dtrace_probedesc_t *desc;
12003         dtrace_predicate_t *pred = NULL;
12004 
12005         if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
12006                 dtrace_dof_error(dof, "truncated ECB description");
12007                 return (NULL);
12008         }
12009 
12010         if (sec->dofs_align != sizeof (uint64_t)) {
12011                 dtrace_dof_error(dof, "bad alignment in ECB description");
12012                 return (NULL);
12013         }
12014 
12015         ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
12016         sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
12017 
12018         if (sec == NULL)
12019                 return (NULL);
12020 
12021         ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
12022         ep->dted_uarg = ecb->dofe_uarg;
12023         desc = &ep->dted_probe;
12024 
12025         if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
12026                 goto err;
12027 
12028         if (ecb->dofe_pred != DOF_SECIDX_NONE) {
12029                 if ((sec = dtrace_dof_sect(dof,
12030                     DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
12031                         goto err;
12032 
12033                 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
12034                         goto err;
12035 
12036                 ep->dted_pred.dtpdd_predicate = pred;
12037         }
12038 
12039         if (ecb->dofe_actions != DOF_SECIDX_NONE) {
12040                 if ((sec = dtrace_dof_sect(dof,
12041                     DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
12042                         goto err;
12043 
12044                 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
12045 
12046                 if (ep->dted_action == NULL)
12047                         goto err;
12048         }
12049 
12050         return (ep);
12051 
12052 err:
12053         if (pred != NULL)
12054                 dtrace_predicate_release(pred, vstate);
12055         kmem_free(ep, sizeof (dtrace_ecbdesc_t));
12056         return (NULL);
12057 }
12058 
12059 /*
12060  * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
12061  * specified DOF.  At present, this amounts to simply adding 'ubase' to the
12062  * site of any user SETX relocations to account for load object base address.
12063  * In the future, if we need other relocations, this function can be extended.
12064  */
12065 static int
12066 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
12067 {
12068         uintptr_t daddr = (uintptr_t)dof;
12069         dof_relohdr_t *dofr =
12070             (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12071         dof_sec_t *ss, *rs, *ts;
12072         dof_relodesc_t *r;
12073         uint_t i, n;
12074 
12075         if (sec->dofs_size < sizeof (dof_relohdr_t) ||
12076             sec->dofs_align != sizeof (dof_secidx_t)) {
12077                 dtrace_dof_error(dof, "invalid relocation header");
12078                 return (-1);
12079         }
12080 
12081         ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
12082         rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
12083         ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
12084 
12085         if (ss == NULL || rs == NULL || ts == NULL)
12086                 return (-1); /* dtrace_dof_error() has been called already */
12087 
12088         if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
12089             rs->dofs_align != sizeof (uint64_t)) {
12090                 dtrace_dof_error(dof, "invalid relocation section");
12091                 return (-1);
12092         }
12093 
12094         r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
12095         n = rs->dofs_size / rs->dofs_entsize;
12096 
12097         for (i = 0; i < n; i++) {
12098                 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
12099 
12100                 switch (r->dofr_type) {
12101                 case DOF_RELO_NONE:
12102                         break;
12103                 case DOF_RELO_SETX:
12104                         if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
12105                             sizeof (uint64_t) > ts->dofs_size) {
12106                                 dtrace_dof_error(dof, "bad relocation offset");
12107                                 return (-1);
12108                         }
12109 
12110                         if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
12111                                 dtrace_dof_error(dof, "misaligned setx relo");
12112                                 return (-1);
12113                         }
12114 
12115                         *(uint64_t *)taddr += ubase;
12116                         break;
12117                 default:
12118                         dtrace_dof_error(dof, "invalid relocation type");
12119                         return (-1);
12120                 }
12121 
12122                 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
12123         }
12124 
12125         return (0);
12126 }
12127 
12128 /*
12129  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
12130  * header:  it should be at the front of a memory region that is at least
12131  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
12132  * size.  It need not be validated in any other way.
12133  */
12134 static int
12135 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
12136     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
12137 {
12138         uint64_t len = dof->dofh_loadsz, seclen;
12139         uintptr_t daddr = (uintptr_t)dof;
12140         dtrace_ecbdesc_t *ep;
12141         dtrace_enabling_t *enab;
12142         uint_t i;
12143 
12144         ASSERT(MUTEX_HELD(&dtrace_lock));
12145         ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
12146 
12147         /*
12148          * Check the DOF header identification bytes.  In addition to checking
12149          * valid settings, we also verify that unused bits/bytes are zeroed so
12150          * we can use them later without fear of regressing existing binaries.
12151          */
12152         if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
12153             DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
12154                 dtrace_dof_error(dof, "DOF magic string mismatch");
12155                 return (-1);
12156         }
12157 
12158         if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
12159             dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
12160                 dtrace_dof_error(dof, "DOF has invalid data model");
12161                 return (-1);
12162         }
12163 
12164         if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
12165                 dtrace_dof_error(dof, "DOF encoding mismatch");
12166                 return (-1);
12167         }
12168 
12169         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
12170             dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
12171                 dtrace_dof_error(dof, "DOF version mismatch");
12172                 return (-1);
12173         }
12174 
12175         if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
12176                 dtrace_dof_error(dof, "DOF uses unsupported instruction set");
12177                 return (-1);
12178         }
12179 
12180         if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
12181                 dtrace_dof_error(dof, "DOF uses too many integer registers");
12182                 return (-1);
12183         }
12184 
12185         if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
12186                 dtrace_dof_error(dof, "DOF uses too many tuple registers");
12187                 return (-1);
12188         }
12189 
12190         for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
12191                 if (dof->dofh_ident[i] != 0) {
12192                         dtrace_dof_error(dof, "DOF has invalid ident byte set");
12193                         return (-1);
12194                 }
12195         }
12196 
12197         if (dof->dofh_flags & ~DOF_FL_VALID) {
12198                 dtrace_dof_error(dof, "DOF has invalid flag bits set");
12199                 return (-1);
12200         }
12201 
12202         if (dof->dofh_secsize == 0) {
12203                 dtrace_dof_error(dof, "zero section header size");
12204                 return (-1);
12205         }
12206 
12207         /*
12208          * Check that the section headers don't exceed the amount of DOF
12209          * data.  Note that we cast the section size and number of sections
12210          * to uint64_t's to prevent possible overflow in the multiplication.
12211          */
12212         seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
12213 
12214         if (dof->dofh_secoff > len || seclen > len ||
12215             dof->dofh_secoff + seclen > len) {
12216                 dtrace_dof_error(dof, "truncated section headers");
12217                 return (-1);
12218         }
12219 
12220         if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
12221                 dtrace_dof_error(dof, "misaligned section headers");
12222                 return (-1);
12223         }
12224 
12225         if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
12226                 dtrace_dof_error(dof, "misaligned section size");
12227                 return (-1);
12228         }
12229 
12230         /*
12231          * Take an initial pass through the section headers to be sure that
12232          * the headers don't have stray offsets.  If the 'noprobes' flag is
12233          * set, do not permit sections relating to providers, probes, or args.
12234          */
12235         for (i = 0; i < dof->dofh_secnum; i++) {
12236                 dof_sec_t *sec = (dof_sec_t *)(daddr +
12237                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12238 
12239                 if (noprobes) {
12240                         switch (sec->dofs_type) {
12241                         case DOF_SECT_PROVIDER:
12242                         case DOF_SECT_PROBES:
12243                         case DOF_SECT_PRARGS:
12244                         case DOF_SECT_PROFFS:
12245                                 dtrace_dof_error(dof, "illegal sections "
12246                                     "for enabling");
12247                                 return (-1);
12248                         }
12249                 }
12250 
12251                 if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
12252                     !(sec->dofs_flags & DOF_SECF_LOAD)) {
12253                         dtrace_dof_error(dof, "loadable section with load "
12254                             "flag unset");
12255                         return (-1);
12256                 }
12257 
12258                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
12259                         continue; /* just ignore non-loadable sections */
12260 
12261                 if (sec->dofs_align & (sec->dofs_align - 1)) {
12262                         dtrace_dof_error(dof, "bad section alignment");
12263                         return (-1);
12264                 }
12265 
12266                 if (sec->dofs_offset & (sec->dofs_align - 1)) {
12267                         dtrace_dof_error(dof, "misaligned section");
12268                         return (-1);
12269                 }
12270 
12271                 if (sec->dofs_offset > len || sec->dofs_size > len ||
12272                     sec->dofs_offset + sec->dofs_size > len) {
12273                         dtrace_dof_error(dof, "corrupt section header");
12274                         return (-1);
12275                 }
12276 
12277                 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
12278                     sec->dofs_offset + sec->dofs_size - 1) != '\0') {
12279                         dtrace_dof_error(dof, "non-terminating string table");
12280                         return (-1);
12281                 }
12282         }
12283 
12284         /*
12285          * Take a second pass through the sections and locate and perform any
12286          * relocations that are present.  We do this after the first pass to
12287          * be sure that all sections have had their headers validated.
12288          */
12289         for (i = 0; i < dof->dofh_secnum; i++) {
12290                 dof_sec_t *sec = (dof_sec_t *)(daddr +
12291                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12292 
12293                 if (!(sec->dofs_flags & DOF_SECF_LOAD))
12294                         continue; /* skip sections that are not loadable */
12295 
12296                 switch (sec->dofs_type) {
12297                 case DOF_SECT_URELHDR:
12298                         if (dtrace_dof_relocate(dof, sec, ubase) != 0)
12299                                 return (-1);
12300                         break;
12301                 }
12302         }
12303 
12304         if ((enab = *enabp) == NULL)
12305                 enab = *enabp = dtrace_enabling_create(vstate);
12306 
12307         for (i = 0; i < dof->dofh_secnum; i++) {
12308                 dof_sec_t *sec = (dof_sec_t *)(daddr +
12309                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12310 
12311                 if (sec->dofs_type != DOF_SECT_ECBDESC)
12312                         continue;
12313 
12314                 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
12315                         dtrace_enabling_destroy(enab);
12316                         *enabp = NULL;
12317                         return (-1);
12318                 }
12319 
12320                 dtrace_enabling_add(enab, ep);
12321         }
12322 
12323         return (0);
12324 }
12325 
12326 /*
12327  * Process DOF for any options.  This routine assumes that the DOF has been
12328  * at least processed by dtrace_dof_slurp().
12329  */
12330 static int
12331 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
12332 {
12333         int i, rval;
12334         uint32_t entsize;
12335         size_t offs;
12336         dof_optdesc_t *desc;
12337 
12338         for (i = 0; i < dof->dofh_secnum; i++) {
12339                 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
12340                     (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12341 
12342                 if (sec->dofs_type != DOF_SECT_OPTDESC)
12343                         continue;
12344 
12345                 if (sec->dofs_align != sizeof (uint64_t)) {
12346                         dtrace_dof_error(dof, "bad alignment in "
12347                             "option description");
12348                         return (EINVAL);
12349                 }
12350 
12351                 if ((entsize = sec->dofs_entsize) == 0) {
12352                         dtrace_dof_error(dof, "zeroed option entry size");
12353                         return (EINVAL);
12354                 }
12355 
12356                 if (entsize < sizeof (dof_optdesc_t)) {
12357                         dtrace_dof_error(dof, "bad option entry size");
12358                         return (EINVAL);
12359                 }
12360 
12361                 for (offs = 0; offs < sec->dofs_size; offs += entsize) {
12362                         desc = (dof_optdesc_t *)((uintptr_t)dof +
12363                             (uintptr_t)sec->dofs_offset + offs);
12364 
12365                         if (desc->dofo_strtab != DOF_SECIDX_NONE) {
12366                                 dtrace_dof_error(dof, "non-zero option string");
12367                                 return (EINVAL);
12368                         }
12369 
12370                         if (desc->dofo_value == DTRACEOPT_UNSET) {
12371                                 dtrace_dof_error(dof, "unset option");
12372                                 return (EINVAL);
12373                         }
12374 
12375                         if ((rval = dtrace_state_option(state,
12376                             desc->dofo_option, desc->dofo_value)) != 0) {
12377                                 dtrace_dof_error(dof, "rejected option");
12378                                 return (rval);
12379                         }
12380                 }
12381         }
12382 
12383         return (0);
12384 }
12385 
12386 /*
12387  * DTrace Consumer State Functions
12388  */
12389 int
12390 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
12391 {
12392         size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
12393         void *base;
12394         uintptr_t limit;
12395         dtrace_dynvar_t *dvar, *next, *start;
12396         int i;
12397 
12398         ASSERT(MUTEX_HELD(&dtrace_lock));
12399         ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
12400 
12401         bzero(dstate, sizeof (dtrace_dstate_t));
12402 
12403         if ((dstate->dtds_chunksize = chunksize) == 0)
12404                 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
12405 
12406         if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
12407                 size = min;
12408 
12409         if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
12410                 return (ENOMEM);
12411 
12412         dstate->dtds_size = size;
12413         dstate->dtds_base = base;
12414         dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
12415         bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
12416 
12417         hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
12418 
12419         if (hashsize != 1 && (hashsize & 1))
12420                 hashsize--;
12421 
12422         dstate->dtds_hashsize = hashsize;
12423         dstate->dtds_hash = dstate->dtds_base;
12424 
12425         /*
12426          * Set all of our hash buckets to point to the single sink, and (if
12427          * it hasn't already been set), set the sink's hash value to be the
12428          * sink sentinel value.  The sink is needed for dynamic variable
12429          * lookups to know that they have iterated over an entire, valid hash
12430          * chain.
12431          */
12432         for (i = 0; i < hashsize; i++)
12433                 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
12434 
12435         if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
12436                 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
12437 
12438         /*
12439          * Determine number of active CPUs.  Divide free list evenly among
12440          * active CPUs.
12441          */
12442         start = (dtrace_dynvar_t *)
12443             ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
12444         limit = (uintptr_t)base + size;
12445 
12446         maxper = (limit - (uintptr_t)start) / NCPU;
12447         maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
12448 
12449         for (i = 0; i < NCPU; i++) {
12450                 dstate->dtds_percpu[i].dtdsc_free = dvar = start;
12451 
12452                 /*
12453                  * If we don't even have enough chunks to make it once through
12454                  * NCPUs, we're just going to allocate everything to the first
12455                  * CPU.  And if we're on the last CPU, we're going to allocate
12456                  * whatever is left over.  In either case, we set the limit to
12457                  * be the limit of the dynamic variable space.
12458                  */
12459                 if (maxper == 0 || i == NCPU - 1) {
12460                         limit = (uintptr_t)base + size;
12461                         start = NULL;
12462                 } else {
12463                         limit = (uintptr_t)start + maxper;
12464                         start = (dtrace_dynvar_t *)limit;
12465                 }
12466 
12467                 ASSERT(limit <= (uintptr_t)base + size);
12468 
12469                 for (;;) {
12470                         next = (dtrace_dynvar_t *)((uintptr_t)dvar +
12471                             dstate->dtds_chunksize);
12472 
12473                         if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
12474                                 break;
12475 
12476                         dvar->dtdv_next = next;
12477                         dvar = next;
12478                 }
12479 
12480                 if (maxper == 0)
12481                         break;
12482         }
12483 
12484         return (0);
12485 }
12486 
12487 void
12488 dtrace_dstate_fini(dtrace_dstate_t *dstate)
12489 {
12490         ASSERT(MUTEX_HELD(&cpu_lock));
12491 
12492         if (dstate->dtds_base == NULL)
12493                 return;
12494 
12495         kmem_free(dstate->dtds_base, dstate->dtds_size);
12496         kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
12497 }
12498 
12499 static void
12500 dtrace_vstate_fini(dtrace_vstate_t *vstate)
12501 {
12502         /*
12503          * Logical XOR, where are you?
12504          */
12505         ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
12506 
12507         if (vstate->dtvs_nglobals > 0) {
12508                 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
12509                     sizeof (dtrace_statvar_t *));
12510         }
12511 
12512         if (vstate->dtvs_ntlocals > 0) {
12513                 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
12514                     sizeof (dtrace_difv_t));
12515         }
12516 
12517         ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
12518 
12519         if (vstate->dtvs_nlocals > 0) {
12520                 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
12521                     sizeof (dtrace_statvar_t *));
12522         }
12523 }
12524 
12525 static void
12526 dtrace_state_clean(dtrace_state_t *state)
12527 {
12528         if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
12529                 return;
12530 
12531         dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
12532         dtrace_speculation_clean(state);
12533 }
12534 
12535 static void
12536 dtrace_state_deadman(dtrace_state_t *state)
12537 {
12538         hrtime_t now;
12539 
12540         dtrace_sync();
12541 
12542         now = dtrace_gethrtime();
12543 
12544         if (state != dtrace_anon.dta_state &&
12545             now - state->dts_laststatus >= dtrace_deadman_user)
12546                 return;
12547 
12548         /*
12549          * We must be sure that dts_alive never appears to be less than the
12550          * value upon entry to dtrace_state_deadman(), and because we lack a
12551          * dtrace_cas64(), we cannot store to it atomically.  We thus instead
12552          * store INT64_MAX to it, followed by a memory barrier, followed by
12553          * the new value.  This assures that dts_alive never appears to be
12554          * less than its true value, regardless of the order in which the
12555          * stores to the underlying storage are issued.
12556          */
12557         state->dts_alive = INT64_MAX;
12558         dtrace_membar_producer();
12559         state->dts_alive = now;
12560 }
12561 
12562 dtrace_state_t *
12563 dtrace_state_create(dev_t *devp, cred_t *cr)
12564 {
12565         minor_t minor;
12566         major_t major;
12567         char c[30];
12568         dtrace_state_t *state;
12569         dtrace_optval_t *opt;
12570         int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
12571 
12572         ASSERT(MUTEX_HELD(&dtrace_lock));
12573         ASSERT(MUTEX_HELD(&cpu_lock));
12574 
12575         minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
12576             VM_BESTFIT | VM_SLEEP);
12577 
12578         if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
12579                 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
12580                 return (NULL);
12581         }
12582 
12583         state = ddi_get_soft_state(dtrace_softstate, minor);
12584         state->dts_epid = DTRACE_EPIDNONE + 1;
12585 
12586         (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
12587         state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
12588             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
12589 
12590         if (devp != NULL) {
12591                 major = getemajor(*devp);
12592         } else {
12593                 major = ddi_driver_major(dtrace_devi);
12594         }
12595 
12596         state->dts_dev = makedevice(major, minor);
12597 
12598         if (devp != NULL)
12599                 *devp = state->dts_dev;
12600 
12601         /*
12602          * We allocate NCPU buffers.  On the one hand, this can be quite
12603          * a bit of memory per instance (nearly 36K on a Starcat).  On the
12604          * other hand, it saves an additional memory reference in the probe
12605          * path.
12606          */
12607         state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
12608         state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
12609         state->dts_cleaner = CYCLIC_NONE;
12610         state->dts_deadman = CYCLIC_NONE;
12611         state->dts_vstate.dtvs_state = state;
12612 
12613         for (i = 0; i < DTRACEOPT_MAX; i++)
12614                 state->dts_options[i] = DTRACEOPT_UNSET;
12615 
12616         /*
12617          * Set the default options.
12618          */
12619         opt = state->dts_options;
12620         opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
12621         opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
12622         opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
12623         opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
12624         opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
12625         opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
12626         opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
12627         opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
12628         opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
12629         opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
12630         opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
12631         opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
12632         opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
12633         opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
12634 
12635         state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
12636 
12637         /*
12638          * Depending on the user credentials, we set flag bits which alter probe
12639          * visibility or the amount of destructiveness allowed.  In the case of
12640          * actual anonymous tracing, or the possession of all privileges, all of
12641          * the normal checks are bypassed.
12642          */
12643         if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
12644                 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
12645                 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
12646         } else {
12647                 /*
12648                  * Set up the credentials for this instantiation.  We take a
12649                  * hold on the credential to prevent it from disappearing on
12650                  * us; this in turn prevents the zone_t referenced by this
12651                  * credential from disappearing.  This means that we can
12652                  * examine the credential and the zone from probe context.
12653                  */
12654                 crhold(cr);
12655                 state->dts_cred.dcr_cred = cr;
12656 
12657                 /*
12658                  * CRA_PROC means "we have *some* privilege for dtrace" and
12659                  * unlocks the use of variables like pid, zonename, etc.
12660                  */
12661                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
12662                     PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
12663                         state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
12664                 }
12665 
12666                 /*
12667                  * dtrace_user allows use of syscall and profile providers.
12668                  * If the user also has proc_owner and/or proc_zone, we
12669                  * extend the scope to include additional visibility and
12670                  * destructive power.
12671                  */
12672                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
12673                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
12674                                 state->dts_cred.dcr_visible |=
12675                                     DTRACE_CRV_ALLPROC;
12676 
12677                                 state->dts_cred.dcr_action |=
12678                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12679                         }
12680 
12681                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
12682                                 state->dts_cred.dcr_visible |=
12683                                     DTRACE_CRV_ALLZONE;
12684 
12685                                 state->dts_cred.dcr_action |=
12686                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12687                         }
12688 
12689                         /*
12690                          * If we have all privs in whatever zone this is,
12691                          * we can do destructive things to processes which
12692                          * have altered credentials.
12693                          */
12694                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
12695                             cr->cr_zone->zone_privset)) {
12696                                 state->dts_cred.dcr_action |=
12697                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
12698                         }
12699                 }
12700 
12701                 /*
12702                  * Holding the dtrace_kernel privilege also implies that
12703                  * the user has the dtrace_user privilege from a visibility
12704                  * perspective.  But without further privileges, some
12705                  * destructive actions are not available.
12706                  */
12707                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
12708                         /*
12709                          * Make all probes in all zones visible.  However,
12710                          * this doesn't mean that all actions become available
12711                          * to all zones.
12712                          */
12713                         state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
12714                             DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
12715 
12716                         state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
12717                             DTRACE_CRA_PROC;
12718                         /*
12719                          * Holding proc_owner means that destructive actions
12720                          * for *this* zone are allowed.
12721                          */
12722                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
12723                                 state->dts_cred.dcr_action |=
12724                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12725 
12726                         /*
12727                          * Holding proc_zone means that destructive actions
12728                          * for this user/group ID in all zones is allowed.
12729                          */
12730                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
12731                                 state->dts_cred.dcr_action |=
12732                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12733 
12734                         /*
12735                          * If we have all privs in whatever zone this is,
12736                          * we can do destructive things to processes which
12737                          * have altered credentials.
12738                          */
12739                         if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
12740                             cr->cr_zone->zone_privset)) {
12741                                 state->dts_cred.dcr_action |=
12742                                     DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
12743                         }
12744                 }
12745 
12746                 /*
12747                  * Holding the dtrace_proc privilege gives control over fasttrap
12748                  * and pid providers.  We need to grant wider destructive
12749                  * privileges in the event that the user has proc_owner and/or
12750                  * proc_zone.
12751                  */
12752                 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
12753                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
12754                                 state->dts_cred.dcr_action |=
12755                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12756 
12757                         if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
12758                                 state->dts_cred.dcr_action |=
12759                                     DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12760                 }
12761         }
12762 
12763         return (state);
12764 }
12765 
12766 static int
12767 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
12768 {
12769         dtrace_optval_t *opt = state->dts_options, size;
12770         processorid_t cpu;
12771         int flags = 0, rval, factor, divisor = 1;
12772 
12773         ASSERT(MUTEX_HELD(&dtrace_lock));
12774         ASSERT(MUTEX_HELD(&cpu_lock));
12775         ASSERT(which < DTRACEOPT_MAX);
12776         ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
12777             (state == dtrace_anon.dta_state &&
12778             state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
12779 
12780         if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
12781                 return (0);
12782 
12783         if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
12784                 cpu = opt[DTRACEOPT_CPU];
12785 
12786         if (which == DTRACEOPT_SPECSIZE)
12787                 flags |= DTRACEBUF_NOSWITCH;
12788 
12789         if (which == DTRACEOPT_BUFSIZE) {
12790                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
12791                         flags |= DTRACEBUF_RING;
12792 
12793                 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
12794                         flags |= DTRACEBUF_FILL;
12795 
12796                 if (state != dtrace_anon.dta_state ||
12797                     state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
12798                         flags |= DTRACEBUF_INACTIVE;
12799         }
12800 
12801         for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
12802                 /*
12803                  * The size must be 8-byte aligned.  If the size is not 8-byte
12804                  * aligned, drop it down by the difference.
12805                  */
12806                 if (size & (sizeof (uint64_t) - 1))
12807                         size -= size & (sizeof (uint64_t) - 1);
12808 
12809                 if (size < state->dts_reserve) {
12810                         /*
12811                          * Buffers always must be large enough to accommodate
12812                          * their prereserved space.  We return E2BIG instead
12813                          * of ENOMEM in this case to allow for user-level
12814                          * software to differentiate the cases.
12815                          */
12816                         return (E2BIG);
12817                 }
12818 
12819                 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
12820 
12821                 if (rval != ENOMEM) {
12822                         opt[which] = size;
12823                         return (rval);
12824                 }
12825 
12826                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
12827                         return (rval);
12828 
12829                 for (divisor = 2; divisor < factor; divisor <<= 1)
12830                         continue;
12831         }
12832 
12833         return (ENOMEM);
12834 }
12835 
12836 static int
12837 dtrace_state_buffers(dtrace_state_t *state)
12838 {
12839         dtrace_speculation_t *spec = state->dts_speculations;
12840         int rval, i;
12841 
12842         if ((rval = dtrace_state_buffer(state, state->dts_buffer,
12843             DTRACEOPT_BUFSIZE)) != 0)
12844                 return (rval);
12845 
12846         if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
12847             DTRACEOPT_AGGSIZE)) != 0)
12848                 return (rval);
12849 
12850         for (i = 0; i < state->dts_nspeculations; i++) {
12851                 if ((rval = dtrace_state_buffer(state,
12852                     spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
12853                         return (rval);
12854         }
12855 
12856         return (0);
12857 }
12858 
12859 static void
12860 dtrace_state_prereserve(dtrace_state_t *state)
12861 {
12862         dtrace_ecb_t *ecb;
12863         dtrace_probe_t *probe;
12864 
12865         state->dts_reserve = 0;
12866 
12867         if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
12868                 return;
12869 
12870         /*
12871          * If our buffer policy is a "fill" buffer policy, we need to set the
12872          * prereserved space to be the space required by the END probes.
12873          */
12874         probe = dtrace_probes[dtrace_probeid_end - 1];
12875         ASSERT(probe != NULL);
12876 
12877         for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
12878                 if (ecb->dte_state != state)
12879                         continue;
12880 
12881                 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
12882         }
12883 }
12884 
12885 static int
12886 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
12887 {
12888         dtrace_optval_t *opt = state->dts_options, sz, nspec;
12889         dtrace_speculation_t *spec;
12890         dtrace_buffer_t *buf;
12891         cyc_handler_t hdlr;
12892         cyc_time_t when;
12893         int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
12894         dtrace_icookie_t cookie;
12895 
12896         mutex_enter(&cpu_lock);
12897         mutex_enter(&dtrace_lock);
12898 
12899         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
12900                 rval = EBUSY;
12901                 goto out;
12902         }
12903 
12904         /*
12905          * Before we can perform any checks, we must prime all of the
12906          * retained enablings that correspond to this state.
12907          */
12908         dtrace_enabling_prime(state);
12909 
12910         if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
12911                 rval = EACCES;
12912                 goto out;
12913         }
12914 
12915         dtrace_state_prereserve(state);
12916 
12917         /*
12918          * Now we want to do is try to allocate our speculations.
12919          * We do not automatically resize the number of speculations; if
12920          * this fails, we will fail the operation.
12921          */
12922         nspec = opt[DTRACEOPT_NSPEC];
12923         ASSERT(nspec != DTRACEOPT_UNSET);
12924 
12925         if (nspec > INT_MAX) {
12926                 rval = ENOMEM;
12927                 goto out;
12928         }
12929 
12930         spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
12931             KM_NOSLEEP | KM_NORMALPRI);
12932 
12933         if (spec == NULL) {
12934                 rval = ENOMEM;
12935                 goto out;
12936         }
12937 
12938         state->dts_speculations = spec;
12939         state->dts_nspeculations = (int)nspec;
12940 
12941         for (i = 0; i < nspec; i++) {
12942                 if ((buf = kmem_zalloc(bufsize,
12943                     KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
12944                         rval = ENOMEM;
12945                         goto err;
12946                 }
12947 
12948                 spec[i].dtsp_buffer = buf;
12949         }
12950 
12951         if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
12952                 if (dtrace_anon.dta_state == NULL) {
12953                         rval = ENOENT;
12954                         goto out;
12955                 }
12956 
12957                 if (state->dts_necbs != 0) {
12958                         rval = EALREADY;
12959                         goto out;
12960                 }
12961 
12962                 state->dts_anon = dtrace_anon_grab();
12963                 ASSERT(state->dts_anon != NULL);
12964                 state = state->dts_anon;
12965 
12966                 /*
12967                  * We want "grabanon" to be set in the grabbed state, so we'll
12968                  * copy that option value from the grabbing state into the
12969                  * grabbed state.
12970                  */
12971                 state->dts_options[DTRACEOPT_GRABANON] =
12972                     opt[DTRACEOPT_GRABANON];
12973 
12974                 *cpu = dtrace_anon.dta_beganon;
12975 
12976                 /*
12977                  * If the anonymous state is active (as it almost certainly
12978                  * is if the anonymous enabling ultimately matched anything),
12979                  * we don't allow any further option processing -- but we
12980                  * don't return failure.
12981                  */
12982                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
12983                         goto out;
12984         }
12985 
12986         if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
12987             opt[DTRACEOPT_AGGSIZE] != 0) {
12988                 if (state->dts_aggregations == NULL) {
12989                         /*
12990                          * We're not going to create an aggregation buffer
12991                          * because we don't have any ECBs that contain
12992                          * aggregations -- set this option to 0.
12993                          */
12994                         opt[DTRACEOPT_AGGSIZE] = 0;
12995                 } else {
12996                         /*
12997                          * If we have an aggregation buffer, we must also have
12998                          * a buffer to use as scratch.
12999                          */
13000                         if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
13001                             opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
13002                                 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
13003                         }
13004                 }
13005         }
13006 
13007         if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
13008             opt[DTRACEOPT_SPECSIZE] != 0) {
13009                 if (!state->dts_speculates) {
13010                         /*
13011                          * We're not going to create speculation buffers
13012                          * because we don't have any ECBs that actually
13013                          * speculate -- set the speculation size to 0.
13014                          */
13015                         opt[DTRACEOPT_SPECSIZE] = 0;
13016                 }
13017         }
13018 
13019         /*
13020          * The bare minimum size for any buffer that we're actually going to
13021          * do anything to is sizeof (uint64_t).
13022          */
13023         sz = sizeof (uint64_t);
13024 
13025         if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
13026             (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
13027             (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
13028                 /*
13029                  * A buffer size has been explicitly set to 0 (or to a size
13030                  * that will be adjusted to 0) and we need the space -- we
13031                  * need to return failure.  We return ENOSPC to differentiate
13032                  * it from failing to allocate a buffer due to failure to meet
13033                  * the reserve (for which we return E2BIG).
13034                  */
13035                 rval = ENOSPC;
13036                 goto out;
13037         }
13038 
13039         if ((rval = dtrace_state_buffers(state)) != 0)
13040                 goto err;
13041 
13042         if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
13043                 sz = dtrace_dstate_defsize;
13044 
13045         do {
13046                 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
13047 
13048                 if (rval == 0)
13049                         break;
13050 
13051                 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13052                         goto err;
13053         } while (sz >>= 1);
13054 
13055         opt[DTRACEOPT_DYNVARSIZE] = sz;
13056 
13057         if (rval != 0)
13058                 goto err;
13059 
13060         if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
13061                 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
13062 
13063         if (opt[DTRACEOPT_CLEANRATE] == 0)
13064                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13065 
13066         if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
13067                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
13068 
13069         if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
13070                 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13071 
13072         hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
13073         hdlr.cyh_arg = state;
13074         hdlr.cyh_level = CY_LOW_LEVEL;
13075 
13076         when.cyt_when = 0;
13077         when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
13078 
13079         state->dts_cleaner = cyclic_add(&hdlr, &when);
13080 
13081         hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
13082         hdlr.cyh_arg = state;
13083         hdlr.cyh_level = CY_LOW_LEVEL;
13084 
13085         when.cyt_when = 0;
13086         when.cyt_interval = dtrace_deadman_interval;
13087 
13088         state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
13089         state->dts_deadman = cyclic_add(&hdlr, &when);
13090 
13091         state->dts_activity = DTRACE_ACTIVITY_WARMUP;
13092 
13093         /*
13094          * Now it's time to actually fire the BEGIN probe.  We need to disable
13095          * interrupts here both to record the CPU on which we fired the BEGIN
13096          * probe (the data from this CPU will be processed first at user
13097          * level) and to manually activate the buffer for this CPU.
13098          */
13099         cookie = dtrace_interrupt_disable();
13100         *cpu = CPU->cpu_id;
13101         ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
13102         state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
13103 
13104         dtrace_probe(dtrace_probeid_begin,
13105             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13106         dtrace_interrupt_enable(cookie);
13107         /*
13108          * We may have had an exit action from a BEGIN probe; only change our
13109          * state to ACTIVE if we're still in WARMUP.
13110          */
13111         ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
13112             state->dts_activity == DTRACE_ACTIVITY_DRAINING);
13113 
13114         if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
13115                 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
13116 
13117         /*
13118          * Regardless of whether or not now we're in ACTIVE or DRAINING, we
13119          * want each CPU to transition its principal buffer out of the
13120          * INACTIVE state.  Doing this assures that no CPU will suddenly begin
13121          * processing an ECB halfway down a probe's ECB chain; all CPUs will
13122          * atomically transition from processing none of a state's ECBs to
13123          * processing all of them.
13124          */
13125         dtrace_xcall(DTRACE_CPUALL,
13126             (dtrace_xcall_t)dtrace_buffer_activate, state);
13127         goto out;
13128 
13129 err:
13130         dtrace_buffer_free(state->dts_buffer);
13131         dtrace_buffer_free(state->dts_aggbuffer);
13132 
13133         if ((nspec = state->dts_nspeculations) == 0) {
13134                 ASSERT(state->dts_speculations == NULL);
13135                 goto out;
13136         }
13137 
13138         spec = state->dts_speculations;
13139         ASSERT(spec != NULL);
13140 
13141         for (i = 0; i < state->dts_nspeculations; i++) {
13142                 if ((buf = spec[i].dtsp_buffer) == NULL)
13143                         break;
13144 
13145                 dtrace_buffer_free(buf);
13146                 kmem_free(buf, bufsize);
13147         }
13148 
13149         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13150         state->dts_nspeculations = 0;
13151         state->dts_speculations = NULL;
13152 
13153 out:
13154         mutex_exit(&dtrace_lock);
13155         mutex_exit(&cpu_lock);
13156 
13157         return (rval);
13158 }
13159 
13160 static int
13161 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
13162 {
13163         dtrace_icookie_t cookie;
13164 
13165         ASSERT(MUTEX_HELD(&dtrace_lock));
13166 
13167         if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
13168             state->dts_activity != DTRACE_ACTIVITY_DRAINING)
13169                 return (EINVAL);
13170 
13171         /*
13172          * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
13173          * to be sure that every CPU has seen it.  See below for the details
13174          * on why this is done.
13175          */
13176         state->dts_activity = DTRACE_ACTIVITY_DRAINING;
13177         dtrace_sync();
13178 
13179         /*
13180          * By this point, it is impossible for any CPU to be still processing
13181          * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
13182          * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
13183          * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
13184          * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
13185          * iff we're in the END probe.
13186          */
13187         state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
13188         dtrace_sync();
13189         ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
13190 
13191         /*
13192          * Finally, we can release the reserve and call the END probe.  We
13193          * disable interrupts across calling the END probe to allow us to
13194          * return the CPU on which we actually called the END probe.  This
13195          * allows user-land to be sure that this CPU's principal buffer is
13196          * processed last.
13197          */
13198         state->dts_reserve = 0;
13199 
13200         cookie = dtrace_interrupt_disable();
13201         *cpu = CPU->cpu_id;
13202         dtrace_probe(dtrace_probeid_end,
13203             (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13204         dtrace_interrupt_enable(cookie);
13205 
13206         state->dts_activity = DTRACE_ACTIVITY_STOPPED;
13207         dtrace_sync();
13208 
13209         return (0);
13210 }
13211 
13212 static int
13213 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
13214     dtrace_optval_t val)
13215 {
13216         ASSERT(MUTEX_HELD(&dtrace_lock));
13217 
13218         if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
13219                 return (EBUSY);
13220 
13221         if (option >= DTRACEOPT_MAX)
13222                 return (EINVAL);
13223 
13224         if (option != DTRACEOPT_CPU && val < 0)
13225                 return (EINVAL);
13226 
13227         switch (option) {
13228         case DTRACEOPT_DESTRUCTIVE:
13229                 if (dtrace_destructive_disallow)
13230                         return (EACCES);
13231 
13232                 state->dts_cred.dcr_destructive = 1;
13233                 break;
13234 
13235         case DTRACEOPT_BUFSIZE:
13236         case DTRACEOPT_DYNVARSIZE:
13237         case DTRACEOPT_AGGSIZE:
13238         case DTRACEOPT_SPECSIZE:
13239         case DTRACEOPT_STRSIZE:
13240                 if (val < 0)
13241                         return (EINVAL);
13242 
13243                 if (val >= LONG_MAX) {
13244                         /*
13245                          * If this is an otherwise negative value, set it to
13246                          * the highest multiple of 128m less than LONG_MAX.
13247                          * Technically, we're adjusting the size without
13248                          * regard to the buffer resizing policy, but in fact,
13249                          * this has no effect -- if we set the buffer size to
13250                          * ~LONG_MAX and the buffer policy is ultimately set to
13251                          * be "manual", the buffer allocation is guaranteed to
13252                          * fail, if only because the allocation requires two
13253                          * buffers.  (We set the the size to the highest
13254                          * multiple of 128m because it ensures that the size
13255                          * will remain a multiple of a megabyte when
13256                          * repeatedly halved -- all the way down to 15m.)
13257                          */
13258                         val = LONG_MAX - (1 << 27) + 1;
13259                 }
13260         }
13261 
13262         state->dts_options[option] = val;
13263 
13264         return (0);
13265 }
13266 
13267 static void
13268 dtrace_state_destroy(dtrace_state_t *state)
13269 {
13270         dtrace_ecb_t *ecb;
13271         dtrace_vstate_t *vstate = &state->dts_vstate;
13272         minor_t minor = getminor(state->dts_dev);
13273         int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
13274         dtrace_speculation_t *spec = state->dts_speculations;
13275         int nspec = state->dts_nspeculations;
13276         uint32_t match;
13277 
13278         ASSERT(MUTEX_HELD(&dtrace_lock));
13279         ASSERT(MUTEX_HELD(&cpu_lock));
13280 
13281         /*
13282          * First, retract any retained enablings for this state.
13283          */
13284         dtrace_enabling_retract(state);
13285         ASSERT(state->dts_nretained == 0);
13286 
13287         if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
13288             state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
13289                 /*
13290                  * We have managed to come into dtrace_state_destroy() on a
13291                  * hot enabling -- almost certainly because of a disorderly
13292                  * shutdown of a consumer.  (That is, a consumer that is
13293                  * exiting without having called dtrace_stop().) In this case,
13294                  * we're going to set our activity to be KILLED, and then
13295                  * issue a sync to be sure that everyone is out of probe
13296                  * context before we start blowing away ECBs.
13297                  */
13298                 state->dts_activity = DTRACE_ACTIVITY_KILLED;
13299                 dtrace_sync();
13300         }
13301 
13302         /*
13303          * Release the credential hold we took in dtrace_state_create().
13304          */
13305         if (state->dts_cred.dcr_cred != NULL)
13306                 crfree(state->dts_cred.dcr_cred);
13307 
13308         /*
13309          * Now we can safely disable and destroy any enabled probes.  Because
13310          * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
13311          * (especially if they're all enabled), we take two passes through the
13312          * ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes, and
13313          * in the second we disable whatever is left over.
13314          */
13315         for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
13316                 for (i = 0; i < state->dts_necbs; i++) {
13317                         if ((ecb = state->dts_ecbs[i]) == NULL)
13318                                 continue;
13319 
13320                         if (match && ecb->dte_probe != NULL) {
13321                                 dtrace_probe_t *probe = ecb->dte_probe;
13322                                 dtrace_provider_t *prov = probe->dtpr_provider;
13323 
13324                                 if (!(prov->dtpv_priv.dtpp_flags & match))
13325                                         continue;
13326                         }
13327 
13328                         dtrace_ecb_disable(ecb);
13329                         dtrace_ecb_destroy(ecb);
13330                 }
13331 
13332                 if (!match)
13333                         break;
13334         }
13335 
13336         /*
13337          * Before we free the buffers, perform one more sync to assure that
13338          * every CPU is out of probe context.
13339          */
13340         dtrace_sync();
13341 
13342         dtrace_buffer_free(state->dts_buffer);
13343         dtrace_buffer_free(state->dts_aggbuffer);
13344 
13345         for (i = 0; i < nspec; i++)
13346                 dtrace_buffer_free(spec[i].dtsp_buffer);
13347 
13348         if (state->dts_cleaner != CYCLIC_NONE)
13349                 cyclic_remove(state->dts_cleaner);
13350 
13351         if (state->dts_deadman != CYCLIC_NONE)
13352                 cyclic_remove(state->dts_deadman);
13353 
13354         dtrace_dstate_fini(&vstate->dtvs_dynvars);
13355         dtrace_vstate_fini(vstate);
13356         kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
13357 
13358         if (state->dts_aggregations != NULL) {
13359 #ifdef DEBUG
13360                 for (i = 0; i < state->dts_naggregations; i++)
13361                         ASSERT(state->dts_aggregations[i] == NULL);
13362 #endif
13363                 ASSERT(state->dts_naggregations > 0);
13364                 kmem_free(state->dts_aggregations,
13365                     state->dts_naggregations * sizeof (dtrace_aggregation_t *));
13366         }
13367 
13368         kmem_free(state->dts_buffer, bufsize);
13369         kmem_free(state->dts_aggbuffer, bufsize);
13370 
13371         for (i = 0; i < nspec; i++)
13372                 kmem_free(spec[i].dtsp_buffer, bufsize);
13373 
13374         kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13375 
13376         dtrace_format_destroy(state);
13377 
13378         vmem_destroy(state->dts_aggid_arena);
13379         ddi_soft_state_free(dtrace_softstate, minor);
13380         vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
13381 }
13382 
13383 /*
13384  * DTrace Anonymous Enabling Functions
13385  */
13386 static dtrace_state_t *
13387 dtrace_anon_grab(void)
13388 {
13389         dtrace_state_t *state;
13390 
13391         ASSERT(MUTEX_HELD(&dtrace_lock));
13392 
13393         if ((state = dtrace_anon.dta_state) == NULL) {
13394                 ASSERT(dtrace_anon.dta_enabling == NULL);
13395                 return (NULL);
13396         }
13397 
13398         ASSERT(dtrace_anon.dta_enabling != NULL);
13399         ASSERT(dtrace_retained != NULL);
13400 
13401         dtrace_enabling_destroy(dtrace_anon.dta_enabling);
13402         dtrace_anon.dta_enabling = NULL;
13403         dtrace_anon.dta_state = NULL;
13404 
13405         return (state);
13406 }
13407 
13408 static void
13409 dtrace_anon_property(void)
13410 {
13411         int i, rv;
13412         dtrace_state_t *state;
13413         dof_hdr_t *dof;
13414         char c[32];             /* enough for "dof-data-" + digits */
13415 
13416         ASSERT(MUTEX_HELD(&dtrace_lock));
13417         ASSERT(MUTEX_HELD(&cpu_lock));
13418 
13419         for (i = 0; ; i++) {
13420                 (void) snprintf(c, sizeof (c), "dof-data-%d", i);
13421 
13422                 dtrace_err_verbose = 1;
13423 
13424                 if ((dof = dtrace_dof_property(c)) == NULL) {
13425                         dtrace_err_verbose = 0;
13426                         break;
13427                 }
13428 
13429                 /*
13430                  * We want to create anonymous state, so we need to transition
13431                  * the kernel debugger to indicate that DTrace is active.  If
13432                  * this fails (e.g. because the debugger has modified text in
13433                  * some way), we won't continue with the processing.
13434                  */
13435                 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
13436                         cmn_err(CE_NOTE, "kernel debugger active; anonymous "
13437                             "enabling ignored.");
13438                         dtrace_dof_destroy(dof);
13439                         break;
13440                 }
13441 
13442                 /*
13443                  * If we haven't allocated an anonymous state, we'll do so now.
13444                  */
13445                 if ((state = dtrace_anon.dta_state) == NULL) {
13446                         state = dtrace_state_create(NULL, NULL);
13447                         dtrace_anon.dta_state = state;
13448 
13449                         if (state == NULL) {
13450                                 /*
13451                                  * This basically shouldn't happen:  the only
13452                                  * failure mode from dtrace_state_create() is a
13453                                  * failure of ddi_soft_state_zalloc() that
13454                                  * itself should never happen.  Still, the
13455                                  * interface allows for a failure mode, and
13456                                  * we want to fail as gracefully as possible:
13457                                  * we'll emit an error message and cease
13458                                  * processing anonymous state in this case.
13459                                  */
13460                                 cmn_err(CE_WARN, "failed to create "
13461                                     "anonymous state");
13462                                 dtrace_dof_destroy(dof);
13463                                 break;
13464                         }
13465                 }
13466 
13467                 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
13468                     &dtrace_anon.dta_enabling, 0, B_TRUE);
13469 
13470                 if (rv == 0)
13471                         rv = dtrace_dof_options(dof, state);
13472 
13473                 dtrace_err_verbose = 0;
13474                 dtrace_dof_destroy(dof);
13475 
13476                 if (rv != 0) {
13477                         /*
13478                          * This is malformed DOF; chuck any anonymous state
13479                          * that we created.
13480                          */
13481                         ASSERT(dtrace_anon.dta_enabling == NULL);
13482                         dtrace_state_destroy(state);
13483                         dtrace_anon.dta_state = NULL;
13484                         break;
13485                 }
13486 
13487                 ASSERT(dtrace_anon.dta_enabling != NULL);
13488         }
13489 
13490         if (dtrace_anon.dta_enabling != NULL) {
13491                 int rval;
13492 
13493                 /*
13494                  * dtrace_enabling_retain() can only fail because we are
13495                  * trying to retain more enablings than are allowed -- but
13496                  * we only have one anonymous enabling, and we are guaranteed
13497                  * to be allowed at least one retained enabling; we assert
13498                  * that dtrace_enabling_retain() returns success.
13499                  */
13500                 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
13501                 ASSERT(rval == 0);
13502 
13503                 dtrace_enabling_dump(dtrace_anon.dta_enabling);
13504         }
13505 }
13506 
13507 /*
13508  * DTrace Helper Functions
13509  */
13510 static void
13511 dtrace_helper_trace(dtrace_helper_action_t *helper,
13512     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
13513 {
13514         uint32_t size, next, nnext, i;
13515         dtrace_helptrace_t *ent;
13516         uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
13517 
13518         if (!dtrace_helptrace_enabled)
13519                 return;
13520 
13521         ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
13522 
13523         /*
13524          * What would a tracing framework be without its own tracing
13525          * framework?  (Well, a hell of a lot simpler, for starters...)
13526          */
13527         size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
13528             sizeof (uint64_t) - sizeof (uint64_t);
13529 
13530         /*
13531          * Iterate until we can allocate a slot in the trace buffer.
13532          */
13533         do {
13534                 next = dtrace_helptrace_next;
13535 
13536                 if (next + size < dtrace_helptrace_bufsize) {
13537                         nnext = next + size;
13538                 } else {
13539                         nnext = size;
13540                 }
13541         } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
13542 
13543         /*
13544          * We have our slot; fill it in.
13545          */
13546         if (nnext == size)
13547                 next = 0;
13548 
13549         ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
13550         ent->dtht_helper = helper;
13551         ent->dtht_where = where;
13552         ent->dtht_nlocals = vstate->dtvs_nlocals;
13553 
13554         ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
13555             mstate->dtms_fltoffs : -1;
13556         ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
13557         ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
13558 
13559         for (i = 0; i < vstate->dtvs_nlocals; i++) {
13560                 dtrace_statvar_t *svar;
13561 
13562                 if ((svar = vstate->dtvs_locals[i]) == NULL)
13563                         continue;
13564 
13565                 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
13566                 ent->dtht_locals[i] =
13567                     ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
13568         }
13569 }
13570 
13571 static uint64_t
13572 dtrace_helper(int which, dtrace_mstate_t *mstate,
13573     dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
13574 {
13575         uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
13576         uint64_t sarg0 = mstate->dtms_arg[0];
13577         uint64_t sarg1 = mstate->dtms_arg[1];
13578         uint64_t rval;
13579         dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
13580         dtrace_helper_action_t *helper;
13581         dtrace_vstate_t *vstate;
13582         dtrace_difo_t *pred;
13583         int i, trace = dtrace_helptrace_enabled;
13584 
13585         ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
13586 
13587         if (helpers == NULL)
13588                 return (0);
13589 
13590         if ((helper = helpers->dthps_actions[which]) == NULL)
13591                 return (0);
13592 
13593         vstate = &helpers->dthps_vstate;
13594         mstate->dtms_arg[0] = arg0;
13595         mstate->dtms_arg[1] = arg1;
13596 
13597         /*
13598          * Now iterate over each helper.  If its predicate evaluates to 'true',
13599          * we'll call the corresponding actions.  Note that the below calls
13600          * to dtrace_dif_emulate() may set faults in machine state.  This is
13601          * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
13602          * the stored DIF offset with its own (which is the desired behavior).
13603          * Also, note the calls to dtrace_dif_emulate() may allocate scratch
13604          * from machine state; this is okay, too.
13605          */
13606         for (; helper != NULL; helper = helper->dtha_next) {
13607                 if ((pred = helper->dtha_predicate) != NULL) {
13608                         if (trace)
13609                                 dtrace_helper_trace(helper, mstate, vstate, 0);
13610 
13611                         if (!dtrace_dif_emulate(pred, mstate, vstate, state))
13612                                 goto next;
13613 
13614                         if (*flags & CPU_DTRACE_FAULT)
13615                                 goto err;
13616                 }
13617 
13618                 for (i = 0; i < helper->dtha_nactions; i++) {
13619                         if (trace)
13620                                 dtrace_helper_trace(helper,
13621                                     mstate, vstate, i + 1);
13622 
13623                         rval = dtrace_dif_emulate(helper->dtha_actions[i],
13624                             mstate, vstate, state);
13625 
13626                         if (*flags & CPU_DTRACE_FAULT)
13627                                 goto err;
13628                 }
13629 
13630 next:
13631                 if (trace)
13632                         dtrace_helper_trace(helper, mstate, vstate,
13633                             DTRACE_HELPTRACE_NEXT);
13634         }
13635 
13636         if (trace)
13637                 dtrace_helper_trace(helper, mstate, vstate,
13638                     DTRACE_HELPTRACE_DONE);
13639 
13640         /*
13641          * Restore the arg0 that we saved upon entry.
13642          */
13643         mstate->dtms_arg[0] = sarg0;
13644         mstate->dtms_arg[1] = sarg1;
13645 
13646         return (rval);
13647 
13648 err:
13649         if (trace)
13650                 dtrace_helper_trace(helper, mstate, vstate,
13651                     DTRACE_HELPTRACE_ERR);
13652 
13653         /*
13654          * Restore the arg0 that we saved upon entry.
13655          */
13656         mstate->dtms_arg[0] = sarg0;
13657         mstate->dtms_arg[1] = sarg1;
13658 
13659         return (NULL);
13660 }
13661 
13662 static void
13663 dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
13664     dtrace_vstate_t *vstate)
13665 {
13666         int i;
13667 
13668         if (helper->dtha_predicate != NULL)
13669                 dtrace_difo_release(helper->dtha_predicate, vstate);
13670 
13671         for (i = 0; i < helper->dtha_nactions; i++) {
13672                 ASSERT(helper->dtha_actions[i] != NULL);
13673                 dtrace_difo_release(helper->dtha_actions[i], vstate);
13674         }
13675 
13676         kmem_free(helper->dtha_actions,
13677             helper->dtha_nactions * sizeof (dtrace_difo_t *));
13678         kmem_free(helper, sizeof (dtrace_helper_action_t));
13679 }
13680 
13681 static int
13682 dtrace_helper_destroygen(int gen)
13683 {
13684         proc_t *p = curproc;
13685         dtrace_helpers_t *help = p->p_dtrace_helpers;
13686         dtrace_vstate_t *vstate;
13687         int i;
13688 
13689         ASSERT(MUTEX_HELD(&dtrace_lock));
13690 
13691         if (help == NULL || gen > help->dthps_generation)
13692                 return (EINVAL);
13693 
13694         vstate = &help->dthps_vstate;
13695 
13696         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
13697                 dtrace_helper_action_t *last = NULL, *h, *next;
13698 
13699                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
13700                         next = h->dtha_next;
13701 
13702                         if (h->dtha_generation == gen) {
13703                                 if (last != NULL) {
13704                                         last->dtha_next = next;
13705                                 } else {
13706                                         help->dthps_actions[i] = next;
13707                                 }
13708 
13709                                 dtrace_helper_action_destroy(h, vstate);
13710                         } else {
13711                                 last = h;
13712                         }
13713                 }
13714         }
13715 
13716         /*
13717          * Interate until we've cleared out all helper providers with the
13718          * given generation number.
13719          */
13720         for (;;) {
13721                 dtrace_helper_provider_t *prov;
13722 
13723                 /*
13724                  * Look for a helper provider with the right generation. We
13725                  * have to start back at the beginning of the list each time
13726                  * because we drop dtrace_lock. It's unlikely that we'll make
13727                  * more than two passes.
13728                  */
13729                 for (i = 0; i < help->dthps_nprovs; i++) {
13730                         prov = help->dthps_provs[i];
13731 
13732                         if (prov->dthp_generation == gen)
13733                                 break;
13734                 }
13735 
13736                 /*
13737                  * If there were no matches, we're done.
13738                  */
13739                 if (i == help->dthps_nprovs)
13740                         break;
13741 
13742                 /*
13743                  * Move the last helper provider into this slot.
13744                  */
13745                 help->dthps_nprovs--;
13746                 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
13747                 help->dthps_provs[help->dthps_nprovs] = NULL;
13748 
13749                 mutex_exit(&dtrace_lock);
13750 
13751                 /*
13752                  * If we have a meta provider, remove this helper provider.
13753                  */
13754                 mutex_enter(&dtrace_meta_lock);
13755                 if (dtrace_meta_pid != NULL) {
13756                         ASSERT(dtrace_deferred_pid == NULL);
13757                         dtrace_helper_provider_remove(&prov->dthp_prov,
13758                             p->p_pid);
13759                 }
13760                 mutex_exit(&dtrace_meta_lock);
13761 
13762                 dtrace_helper_provider_destroy(prov);
13763 
13764                 mutex_enter(&dtrace_lock);
13765         }
13766 
13767         return (0);
13768 }
13769 
13770 static int
13771 dtrace_helper_validate(dtrace_helper_action_t *helper)
13772 {
13773         int err = 0, i;
13774         dtrace_difo_t *dp;
13775 
13776         if ((dp = helper->dtha_predicate) != NULL)
13777                 err += dtrace_difo_validate_helper(dp);
13778 
13779         for (i = 0; i < helper->dtha_nactions; i++)
13780                 err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
13781 
13782         return (err == 0);
13783 }
13784 
13785 static int
13786 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
13787 {
13788         dtrace_helpers_t *help;
13789         dtrace_helper_action_t *helper, *last;
13790         dtrace_actdesc_t *act;
13791         dtrace_vstate_t *vstate;
13792         dtrace_predicate_t *pred;
13793         int count = 0, nactions = 0, i;
13794 
13795         if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
13796                 return (EINVAL);
13797 
13798         help = curproc->p_dtrace_helpers;
13799         last = help->dthps_actions[which];
13800         vstate = &help->dthps_vstate;
13801 
13802         for (count = 0; last != NULL; last = last->dtha_next) {
13803                 count++;
13804                 if (last->dtha_next == NULL)
13805                         break;
13806         }
13807 
13808         /*
13809          * If we already have dtrace_helper_actions_max helper actions for this
13810          * helper action type, we'll refuse to add a new one.
13811          */
13812         if (count >= dtrace_helper_actions_max)
13813                 return (ENOSPC);
13814 
13815         helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
13816         helper->dtha_generation = help->dthps_generation;
13817 
13818         if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
13819                 ASSERT(pred->dtp_difo != NULL);
13820                 dtrace_difo_hold(pred->dtp_difo);
13821                 helper->dtha_predicate = pred->dtp_difo;
13822         }
13823 
13824         for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
13825                 if (act->dtad_kind != DTRACEACT_DIFEXPR)
13826                         goto err;
13827 
13828                 if (act->dtad_difo == NULL)
13829                         goto err;
13830 
13831                 nactions++;
13832         }
13833 
13834         helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
13835             (helper->dtha_nactions = nactions), KM_SLEEP);
13836 
13837         for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
13838                 dtrace_difo_hold(act->dtad_difo);
13839                 helper->dtha_actions[i++] = act->dtad_difo;
13840         }
13841 
13842         if (!dtrace_helper_validate(helper))
13843                 goto err;
13844 
13845         if (last == NULL) {
13846                 help->dthps_actions[which] = helper;
13847         } else {
13848                 last->dtha_next = helper;
13849         }
13850 
13851         if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
13852                 dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
13853                 dtrace_helptrace_next = 0;
13854         }
13855 
13856         return (0);
13857 err:
13858         dtrace_helper_action_destroy(helper, vstate);
13859         return (EINVAL);
13860 }
13861 
13862 static void
13863 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
13864     dof_helper_t *dofhp)
13865 {
13866         ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
13867 
13868         mutex_enter(&dtrace_meta_lock);
13869         mutex_enter(&dtrace_lock);
13870 
13871         if (!dtrace_attached() || dtrace_meta_pid == NULL) {
13872                 /*
13873                  * If the dtrace module is loaded but not attached, or if
13874                  * there aren't isn't a meta provider registered to deal with
13875                  * these provider descriptions, we need to postpone creating
13876                  * the actual providers until later.
13877                  */
13878 
13879                 if (help->dthps_next == NULL && help->dthps_prev == NULL &&
13880                     dtrace_deferred_pid != help) {
13881                         help->dthps_deferred = 1;
13882                         help->dthps_pid = p->p_pid;
13883                         help->dthps_next = dtrace_deferred_pid;
13884                         help->dthps_prev = NULL;
13885                         if (dtrace_deferred_pid != NULL)
13886                                 dtrace_deferred_pid->dthps_prev = help;
13887                         dtrace_deferred_pid = help;
13888                 }
13889 
13890                 mutex_exit(&dtrace_lock);
13891 
13892         } else if (dofhp != NULL) {
13893                 /*
13894                  * If the dtrace module is loaded and we have a particular
13895                  * helper provider description, pass that off to the
13896                  * meta provider.
13897                  */
13898 
13899                 mutex_exit(&dtrace_lock);
13900 
13901                 dtrace_helper_provide(dofhp, p->p_pid);
13902 
13903         } else {
13904                 /*
13905                  * Otherwise, just pass all the helper provider descriptions
13906                  * off to the meta provider.
13907                  */
13908 
13909                 int i;
13910                 mutex_exit(&dtrace_lock);
13911 
13912                 for (i = 0; i < help->dthps_nprovs; i++) {
13913                         dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
13914                             p->p_pid);
13915                 }
13916         }
13917 
13918         mutex_exit(&dtrace_meta_lock);
13919 }
13920 
13921 static int
13922 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
13923 {
13924         dtrace_helpers_t *help;
13925         dtrace_helper_provider_t *hprov, **tmp_provs;
13926         uint_t tmp_maxprovs, i;
13927 
13928         ASSERT(MUTEX_HELD(&dtrace_lock));
13929 
13930         help = curproc->p_dtrace_helpers;
13931         ASSERT(help != NULL);
13932 
13933         /*
13934          * If we already have dtrace_helper_providers_max helper providers,
13935          * we're refuse to add a new one.
13936          */
13937         if (help->dthps_nprovs >= dtrace_helper_providers_max)
13938                 return (ENOSPC);
13939 
13940         /*
13941          * Check to make sure this isn't a duplicate.
13942          */
13943         for (i = 0; i < help->dthps_nprovs; i++) {
13944                 if (dofhp->dofhp_dof ==
13945                     help->dthps_provs[i]->dthp_prov.dofhp_dof)
13946                         return (EALREADY);
13947         }
13948 
13949         hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
13950         hprov->dthp_prov = *dofhp;
13951         hprov->dthp_ref = 1;
13952         hprov->dthp_generation = gen;
13953 
13954         /*
13955          * Allocate a bigger table for helper providers if it's already full.
13956          */
13957         if (help->dthps_maxprovs == help->dthps_nprovs) {
13958                 tmp_maxprovs = help->dthps_maxprovs;
13959                 tmp_provs = help->dthps_provs;
13960 
13961                 if (help->dthps_maxprovs == 0)
13962                         help->dthps_maxprovs = 2;
13963                 else
13964                         help->dthps_maxprovs *= 2;
13965                 if (help->dthps_maxprovs > dtrace_helper_providers_max)
13966                         help->dthps_maxprovs = dtrace_helper_providers_max;
13967 
13968                 ASSERT(tmp_maxprovs < help->dthps_maxprovs);
13969 
13970                 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
13971                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
13972 
13973                 if (tmp_provs != NULL) {
13974                         bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
13975                             sizeof (dtrace_helper_provider_t *));
13976                         kmem_free(tmp_provs, tmp_maxprovs *
13977                             sizeof (dtrace_helper_provider_t *));
13978                 }
13979         }
13980 
13981         help->dthps_provs[help->dthps_nprovs] = hprov;
13982         help->dthps_nprovs++;
13983 
13984         return (0);
13985 }
13986 
13987 static void
13988 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
13989 {
13990         mutex_enter(&dtrace_lock);
13991 
13992         if (--hprov->dthp_ref == 0) {
13993                 dof_hdr_t *dof;
13994                 mutex_exit(&dtrace_lock);
13995                 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
13996                 dtrace_dof_destroy(dof);
13997                 kmem_free(hprov, sizeof (dtrace_helper_provider_t));
13998         } else {
13999                 mutex_exit(&dtrace_lock);
14000         }
14001 }
14002 
14003 static int
14004 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
14005 {
14006         uintptr_t daddr = (uintptr_t)dof;
14007         dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
14008         dof_provider_t *provider;
14009         dof_probe_t *probe;
14010         uint8_t *arg;
14011         char *strtab, *typestr;
14012         dof_stridx_t typeidx;
14013         size_t typesz;
14014         uint_t nprobes, j, k;
14015 
14016         ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
14017 
14018         if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
14019                 dtrace_dof_error(dof, "misaligned section offset");
14020                 return (-1);
14021         }
14022 
14023         /*
14024          * The section needs to be large enough to contain the DOF provider
14025          * structure appropriate for the given version.
14026          */
14027         if (sec->dofs_size <
14028             ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
14029             offsetof(dof_provider_t, dofpv_prenoffs) :
14030             sizeof (dof_provider_t))) {
14031                 dtrace_dof_error(dof, "provider section too small");
14032                 return (-1);
14033         }
14034 
14035         provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
14036         str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
14037         prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
14038         arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
14039         off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
14040 
14041         if (str_sec == NULL || prb_sec == NULL ||
14042             arg_sec == NULL || off_sec == NULL)
14043                 return (-1);
14044 
14045         enoff_sec = NULL;
14046 
14047         if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
14048             provider->dofpv_prenoffs != DOF_SECT_NONE &&
14049             (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
14050             provider->dofpv_prenoffs)) == NULL)
14051                 return (-1);
14052 
14053         strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
14054 
14055         if (provider->dofpv_name >= str_sec->dofs_size ||
14056             strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
14057                 dtrace_dof_error(dof, "invalid provider name");
14058                 return (-1);
14059         }
14060 
14061         if (prb_sec->dofs_entsize == 0 ||
14062             prb_sec->dofs_entsize > prb_sec->dofs_size) {
14063                 dtrace_dof_error(dof, "invalid entry size");
14064                 return (-1);
14065         }
14066 
14067         if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
14068                 dtrace_dof_error(dof, "misaligned entry size");
14069                 return (-1);
14070         }
14071 
14072         if (off_sec->dofs_entsize != sizeof (uint32_t)) {
14073                 dtrace_dof_error(dof, "invalid entry size");
14074                 return (-1);
14075         }
14076 
14077         if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
14078                 dtrace_dof_error(dof, "misaligned section offset");
14079                 return (-1);
14080         }
14081 
14082         if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
14083                 dtrace_dof_error(dof, "invalid entry size");
14084                 return (-1);
14085         }
14086 
14087         arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
14088 
14089         nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
14090 
14091         /*
14092          * Take a pass through the probes to check for errors.
14093          */
14094         for (j = 0; j < nprobes; j++) {
14095                 probe = (dof_probe_t *)(uintptr_t)(daddr +
14096                     prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
14097 
14098                 if (probe->dofpr_func >= str_sec->dofs_size) {
14099                         dtrace_dof_error(dof, "invalid function name");
14100                         return (-1);
14101                 }
14102 
14103                 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
14104                         dtrace_dof_error(dof, "function name too long");
14105                         return (-1);
14106                 }
14107 
14108                 if (probe->dofpr_name >= str_sec->dofs_size ||
14109                     strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
14110                         dtrace_dof_error(dof, "invalid probe name");
14111                         return (-1);
14112                 }
14113 
14114                 /*
14115                  * The offset count must not wrap the index, and the offsets
14116                  * must also not overflow the section's data.
14117                  */
14118                 if (probe->dofpr_offidx + probe->dofpr_noffs <
14119                     probe->dofpr_offidx ||
14120                     (probe->dofpr_offidx + probe->dofpr_noffs) *
14121                     off_sec->dofs_entsize > off_sec->dofs_size) {
14122                         dtrace_dof_error(dof, "invalid probe offset");
14123                         return (-1);
14124                 }
14125 
14126                 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
14127                         /*
14128                          * If there's no is-enabled offset section, make sure
14129                          * there aren't any is-enabled offsets. Otherwise
14130                          * perform the same checks as for probe offsets
14131                          * (immediately above).
14132                          */
14133                         if (enoff_sec == NULL) {
14134                                 if (probe->dofpr_enoffidx != 0 ||
14135                                     probe->dofpr_nenoffs != 0) {
14136                                         dtrace_dof_error(dof, "is-enabled "
14137                                             "offsets with null section");
14138                                         return (-1);
14139                                 }
14140                         } else if (probe->dofpr_enoffidx +
14141                             probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
14142                             (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
14143                             enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
14144                                 dtrace_dof_error(dof, "invalid is-enabled "
14145                                     "offset");
14146                                 return (-1);
14147                         }
14148 
14149                         if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
14150                                 dtrace_dof_error(dof, "zero probe and "
14151                                     "is-enabled offsets");
14152                                 return (-1);
14153                         }
14154                 } else if (probe->dofpr_noffs == 0) {
14155                         dtrace_dof_error(dof, "zero probe offsets");
14156                         return (-1);
14157                 }
14158 
14159                 if (probe->dofpr_argidx + probe->dofpr_xargc <
14160                     probe->dofpr_argidx ||
14161                     (probe->dofpr_argidx + probe->dofpr_xargc) *
14162                     arg_sec->dofs_entsize > arg_sec->dofs_size) {
14163                         dtrace_dof_error(dof, "invalid args");
14164                         return (-1);
14165                 }
14166 
14167                 typeidx = probe->dofpr_nargv;
14168                 typestr = strtab + probe->dofpr_nargv;
14169                 for (k = 0; k < probe->dofpr_nargc; k++) {
14170                         if (typeidx >= str_sec->dofs_size) {
14171                                 dtrace_dof_error(dof, "bad "
14172                                     "native argument type");
14173                                 return (-1);
14174                         }
14175 
14176                         typesz = strlen(typestr) + 1;
14177                         if (typesz > DTRACE_ARGTYPELEN) {
14178                                 dtrace_dof_error(dof, "native "
14179                                     "argument type too long");
14180                                 return (-1);
14181                         }
14182                         typeidx += typesz;
14183                         typestr += typesz;
14184                 }
14185 
14186                 typeidx = probe->dofpr_xargv;
14187                 typestr = strtab + probe->dofpr_xargv;
14188                 for (k = 0; k < probe->dofpr_xargc; k++) {
14189                         if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
14190                                 dtrace_dof_error(dof, "bad "
14191                                     "native argument index");
14192                                 return (-1);
14193                         }
14194 
14195                         if (typeidx >= str_sec->dofs_size) {
14196                                 dtrace_dof_error(dof, "bad "
14197                                     "translated argument type");
14198                                 return (-1);
14199                         }
14200 
14201                         typesz = strlen(typestr) + 1;
14202                         if (typesz > DTRACE_ARGTYPELEN) {
14203                                 dtrace_dof_error(dof, "translated argument "
14204                                     "type too long");
14205                                 return (-1);
14206                         }
14207 
14208                         typeidx += typesz;
14209                         typestr += typesz;
14210                 }
14211         }
14212 
14213         return (0);
14214 }
14215 
14216 static int
14217 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
14218 {
14219         dtrace_helpers_t *help;
14220         dtrace_vstate_t *vstate;
14221         dtrace_enabling_t *enab = NULL;
14222         int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
14223         uintptr_t daddr = (uintptr_t)dof;
14224 
14225         ASSERT(MUTEX_HELD(&dtrace_lock));
14226 
14227         if ((help = curproc->p_dtrace_helpers) == NULL)
14228                 help = dtrace_helpers_create(curproc);
14229 
14230         vstate = &help->dthps_vstate;
14231 
14232         if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
14233             dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
14234                 dtrace_dof_destroy(dof);
14235                 return (rv);
14236         }
14237 
14238         /*
14239          * Look for helper providers and validate their descriptions.
14240          */
14241         if (dhp != NULL) {
14242                 for (i = 0; i < dof->dofh_secnum; i++) {
14243                         dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
14244                             dof->dofh_secoff + i * dof->dofh_secsize);
14245 
14246                         if (sec->dofs_type != DOF_SECT_PROVIDER)
14247                                 continue;
14248 
14249                         if (dtrace_helper_provider_validate(dof, sec) != 0) {
14250                                 dtrace_enabling_destroy(enab);
14251                                 dtrace_dof_destroy(dof);
14252                                 return (-1);
14253                         }
14254 
14255                         nprovs++;
14256                 }
14257         }
14258 
14259         /*
14260          * Now we need to walk through the ECB descriptions in the enabling.
14261          */
14262         for (i = 0; i < enab->dten_ndesc; i++) {
14263                 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
14264                 dtrace_probedesc_t *desc = &ep->dted_probe;
14265 
14266                 if (strcmp(desc->dtpd_provider, "dtrace") != 0)
14267                         continue;
14268 
14269                 if (strcmp(desc->dtpd_mod, "helper") != 0)
14270                         continue;
14271 
14272                 if (strcmp(desc->dtpd_func, "ustack") != 0)
14273                         continue;
14274 
14275                 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
14276                     ep)) != 0) {
14277                         /*
14278                          * Adding this helper action failed -- we are now going
14279                          * to rip out the entire generation and return failure.
14280                          */
14281                         (void) dtrace_helper_destroygen(help->dthps_generation);
14282                         dtrace_enabling_destroy(enab);
14283                         dtrace_dof_destroy(dof);
14284                         return (-1);
14285                 }
14286 
14287                 nhelpers++;
14288         }
14289 
14290         if (nhelpers < enab->dten_ndesc)
14291                 dtrace_dof_error(dof, "unmatched helpers");
14292 
14293         gen = help->dthps_generation++;
14294         dtrace_enabling_destroy(enab);
14295 
14296         if (dhp != NULL && nprovs > 0) {
14297                 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
14298                 if (dtrace_helper_provider_add(dhp, gen) == 0) {
14299                         mutex_exit(&dtrace_lock);
14300                         dtrace_helper_provider_register(curproc, help, dhp);
14301                         mutex_enter(&dtrace_lock);
14302 
14303                         destroy = 0;
14304                 }
14305         }
14306 
14307         if (destroy)
14308                 dtrace_dof_destroy(dof);
14309 
14310         return (gen);
14311 }
14312 
14313 static dtrace_helpers_t *
14314 dtrace_helpers_create(proc_t *p)
14315 {
14316         dtrace_helpers_t *help;
14317 
14318         ASSERT(MUTEX_HELD(&dtrace_lock));
14319         ASSERT(p->p_dtrace_helpers == NULL);
14320 
14321         help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
14322         help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
14323             DTRACE_NHELPER_ACTIONS, KM_SLEEP);
14324 
14325         p->p_dtrace_helpers = help;
14326         dtrace_helpers++;
14327 
14328         return (help);
14329 }
14330 
14331 static void
14332 dtrace_helpers_destroy(void)
14333 {
14334         dtrace_helpers_t *help;
14335         dtrace_vstate_t *vstate;
14336         proc_t *p = curproc;
14337         int i;
14338 
14339         mutex_enter(&dtrace_lock);
14340 
14341         ASSERT(p->p_dtrace_helpers != NULL);
14342         ASSERT(dtrace_helpers > 0);
14343 
14344         help = p->p_dtrace_helpers;
14345         vstate = &help->dthps_vstate;
14346 
14347         /*
14348          * We're now going to lose the help from this process.
14349          */
14350         p->p_dtrace_helpers = NULL;
14351         dtrace_sync();
14352 
14353         /*
14354          * Destory the helper actions.
14355          */
14356         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14357                 dtrace_helper_action_t *h, *next;
14358 
14359                 for (h = help->dthps_actions[i]; h != NULL; h = next) {
14360                         next = h->dtha_next;
14361                         dtrace_helper_action_destroy(h, vstate);
14362                         h = next;
14363                 }
14364         }
14365 
14366         mutex_exit(&dtrace_lock);
14367 
14368         /*
14369          * Destroy the helper providers.
14370          */
14371         if (help->dthps_maxprovs > 0) {
14372                 mutex_enter(&dtrace_meta_lock);
14373                 if (dtrace_meta_pid != NULL) {
14374                         ASSERT(dtrace_deferred_pid == NULL);
14375 
14376                         for (i = 0; i < help->dthps_nprovs; i++) {
14377                                 dtrace_helper_provider_remove(
14378                                     &help->dthps_provs[i]->dthp_prov, p->p_pid);
14379                         }
14380                 } else {
14381                         mutex_enter(&dtrace_lock);
14382                         ASSERT(help->dthps_deferred == 0 ||
14383                             help->dthps_next != NULL ||
14384                             help->dthps_prev != NULL ||
14385                             help == dtrace_deferred_pid);
14386 
14387                         /*
14388                          * Remove the helper from the deferred list.
14389                          */
14390                         if (help->dthps_next != NULL)
14391                                 help->dthps_next->dthps_prev = help->dthps_prev;
14392                         if (help->dthps_prev != NULL)
14393                                 help->dthps_prev->dthps_next = help->dthps_next;
14394                         if (dtrace_deferred_pid == help) {
14395                                 dtrace_deferred_pid = help->dthps_next;
14396                                 ASSERT(help->dthps_prev == NULL);
14397                         }
14398 
14399                         mutex_exit(&dtrace_lock);
14400                 }
14401 
14402                 mutex_exit(&dtrace_meta_lock);
14403 
14404                 for (i = 0; i < help->dthps_nprovs; i++) {
14405                         dtrace_helper_provider_destroy(help->dthps_provs[i]);
14406                 }
14407 
14408                 kmem_free(help->dthps_provs, help->dthps_maxprovs *
14409                     sizeof (dtrace_helper_provider_t *));
14410         }
14411 
14412         mutex_enter(&dtrace_lock);
14413 
14414         dtrace_vstate_fini(&help->dthps_vstate);
14415         kmem_free(help->dthps_actions,
14416             sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
14417         kmem_free(help, sizeof (dtrace_helpers_t));
14418 
14419         --dtrace_helpers;
14420         mutex_exit(&dtrace_lock);
14421 }
14422 
14423 static void
14424 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
14425 {
14426         dtrace_helpers_t *help, *newhelp;
14427         dtrace_helper_action_t *helper, *new, *last;
14428         dtrace_difo_t *dp;
14429         dtrace_vstate_t *vstate;
14430         int i, j, sz, hasprovs = 0;
14431 
14432         mutex_enter(&dtrace_lock);
14433         ASSERT(from->p_dtrace_helpers != NULL);
14434         ASSERT(dtrace_helpers > 0);
14435 
14436         help = from->p_dtrace_helpers;
14437         newhelp = dtrace_helpers_create(to);
14438         ASSERT(to->p_dtrace_helpers != NULL);
14439 
14440         newhelp->dthps_generation = help->dthps_generation;
14441         vstate = &newhelp->dthps_vstate;
14442 
14443         /*
14444          * Duplicate the helper actions.
14445          */
14446         for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14447                 if ((helper = help->dthps_actions[i]) == NULL)
14448                         continue;
14449 
14450                 for (last = NULL; helper != NULL; helper = helper->dtha_next) {
14451                         new = kmem_zalloc(sizeof (dtrace_helper_action_t),
14452                             KM_SLEEP);
14453                         new->dtha_generation = helper->dtha_generation;
14454 
14455                         if ((dp = helper->dtha_predicate) != NULL) {
14456                                 dp = dtrace_difo_duplicate(dp, vstate);
14457                                 new->dtha_predicate = dp;
14458                         }
14459 
14460                         new->dtha_nactions = helper->dtha_nactions;
14461                         sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
14462                         new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
14463 
14464                         for (j = 0; j < new->dtha_nactions; j++) {
14465                                 dtrace_difo_t *dp = helper->dtha_actions[j];
14466 
14467                                 ASSERT(dp != NULL);
14468                                 dp = dtrace_difo_duplicate(dp, vstate);
14469                                 new->dtha_actions[j] = dp;
14470                         }
14471 
14472                         if (last != NULL) {
14473                                 last->dtha_next = new;
14474                         } else {
14475                                 newhelp->dthps_actions[i] = new;
14476                         }
14477 
14478                         last = new;
14479                 }
14480         }
14481 
14482         /*
14483          * Duplicate the helper providers and register them with the
14484          * DTrace framework.
14485          */
14486         if (help->dthps_nprovs > 0) {
14487                 newhelp->dthps_nprovs = help->dthps_nprovs;
14488                 newhelp->dthps_maxprovs = help->dthps_nprovs;
14489                 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
14490                     sizeof (dtrace_helper_provider_t *), KM_SLEEP);
14491                 for (i = 0; i < newhelp->dthps_nprovs; i++) {
14492                         newhelp->dthps_provs[i] = help->dthps_provs[i];
14493                         newhelp->dthps_provs[i]->dthp_ref++;
14494                 }
14495 
14496                 hasprovs = 1;
14497         }
14498 
14499         mutex_exit(&dtrace_lock);
14500 
14501         if (hasprovs)
14502                 dtrace_helper_provider_register(to, newhelp, NULL);
14503 }
14504 
14505 /*
14506  * DTrace Hook Functions
14507  */
14508 static void
14509 dtrace_module_loaded(struct modctl *ctl)
14510 {
14511         dtrace_provider_t *prv;
14512 
14513         mutex_enter(&dtrace_provider_lock);
14514         mutex_enter(&mod_lock);
14515 
14516         ASSERT(ctl->mod_busy);
14517 
14518         /*
14519          * We're going to call each providers per-module provide operation
14520          * specifying only this module.
14521          */
14522         for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
14523                 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
14524 
14525         mutex_exit(&mod_lock);
14526         mutex_exit(&dtrace_provider_lock);
14527 
14528         /*
14529          * If we have any retained enablings, we need to match against them.
14530          * Enabling probes requires that cpu_lock be held, and we cannot hold
14531          * cpu_lock here -- it is legal for cpu_lock to be held when loading a
14532          * module.  (In particular, this happens when loading scheduling
14533          * classes.)  So if we have any retained enablings, we need to dispatch
14534          * our task queue to do the match for us.
14535          */
14536         mutex_enter(&dtrace_lock);
14537 
14538         if (dtrace_retained == NULL) {
14539                 mutex_exit(&dtrace_lock);
14540                 return;
14541         }
14542 
14543         (void) taskq_dispatch(dtrace_taskq,
14544             (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
14545 
14546         mutex_exit(&dtrace_lock);
14547 
14548         /*
14549          * And now, for a little heuristic sleaze:  in general, we want to
14550          * match modules as soon as they load.  However, we cannot guarantee
14551          * this, because it would lead us to the lock ordering violation
14552          * outlined above.  The common case, of course, is that cpu_lock is
14553          * _not_ held -- so we delay here for a clock tick, hoping that that's
14554          * long enough for the task queue to do its work.  If it's not, it's
14555          * not a serious problem -- it just means that the module that we
14556          * just loaded may not be immediately instrumentable.
14557          */
14558         delay(1);
14559 }
14560 
14561 static void
14562 dtrace_module_unloaded(struct modctl *ctl)
14563 {
14564         dtrace_probe_t template, *probe, *first, *next;
14565         dtrace_provider_t *prov;
14566 
14567         template.dtpr_mod = ctl->mod_modname;
14568 
14569         mutex_enter(&dtrace_provider_lock);
14570         mutex_enter(&mod_lock);
14571         mutex_enter(&dtrace_lock);
14572 
14573         if (dtrace_bymod == NULL) {
14574                 /*
14575                  * The DTrace module is loaded (obviously) but not attached;
14576                  * we don't have any work to do.
14577                  */
14578                 mutex_exit(&dtrace_provider_lock);
14579                 mutex_exit(&mod_lock);
14580                 mutex_exit(&dtrace_lock);
14581                 return;
14582         }
14583 
14584         for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
14585             probe != NULL; probe = probe->dtpr_nextmod) {
14586                 if (probe->dtpr_ecb != NULL) {
14587                         mutex_exit(&dtrace_provider_lock);
14588                         mutex_exit(&mod_lock);
14589                         mutex_exit(&dtrace_lock);
14590 
14591                         /*
14592                          * This shouldn't _actually_ be possible -- we're
14593                          * unloading a module that has an enabled probe in it.
14594                          * (It's normally up to the provider to make sure that
14595                          * this can't happen.)  However, because dtps_enable()
14596                          * doesn't have a failure mode, there can be an
14597                          * enable/unload race.  Upshot:  we don't want to
14598                          * assert, but we're not going to disable the
14599                          * probe, either.
14600                          */
14601                         if (dtrace_err_verbose) {
14602                                 cmn_err(CE_WARN, "unloaded module '%s' had "
14603                                     "enabled probes", ctl->mod_modname);
14604                         }
14605 
14606                         return;
14607                 }
14608         }
14609 
14610         probe = first;
14611 
14612         for (first = NULL; probe != NULL; probe = next) {
14613                 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
14614 
14615                 dtrace_probes[probe->dtpr_id - 1] = NULL;
14616 
14617                 next = probe->dtpr_nextmod;
14618                 dtrace_hash_remove(dtrace_bymod, probe);
14619                 dtrace_hash_remove(dtrace_byfunc, probe);
14620                 dtrace_hash_remove(dtrace_byname, probe);
14621 
14622                 if (first == NULL) {
14623                         first = probe;
14624                         probe->dtpr_nextmod = NULL;
14625                 } else {
14626                         probe->dtpr_nextmod = first;
14627                         first = probe;
14628                 }
14629         }
14630 
14631         /*
14632          * We've removed all of the module's probes from the hash chains and
14633          * from the probe array.  Now issue a dtrace_sync() to be sure that
14634          * everyone has cleared out from any probe array processing.
14635          */
14636         dtrace_sync();
14637 
14638         for (probe = first; probe != NULL; probe = first) {
14639                 first = probe->dtpr_nextmod;
14640                 prov = probe->dtpr_provider;
14641                 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
14642                     probe->dtpr_arg);
14643                 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
14644                 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
14645                 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
14646                 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
14647                 kmem_free(probe, sizeof (dtrace_probe_t));
14648         }
14649 
14650         mutex_exit(&dtrace_lock);
14651         mutex_exit(&mod_lock);
14652         mutex_exit(&dtrace_provider_lock);
14653 }
14654 
14655 void
14656 dtrace_suspend(void)
14657 {
14658         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
14659 }
14660 
14661 void
14662 dtrace_resume(void)
14663 {
14664         dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
14665 }
14666 
14667 static int
14668 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
14669 {
14670         ASSERT(MUTEX_HELD(&cpu_lock));
14671         mutex_enter(&dtrace_lock);
14672 
14673         switch (what) {
14674         case CPU_CONFIG: {
14675                 dtrace_state_t *state;
14676                 dtrace_optval_t *opt, rs, c;
14677 
14678                 /*
14679                  * For now, we only allocate a new buffer for anonymous state.
14680                  */
14681                 if ((state = dtrace_anon.dta_state) == NULL)
14682                         break;
14683 
14684                 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
14685                         break;
14686 
14687                 opt = state->dts_options;
14688                 c = opt[DTRACEOPT_CPU];
14689 
14690                 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
14691                         break;
14692 
14693                 /*
14694                  * Regardless of what the actual policy is, we're going to
14695                  * temporarily set our resize policy to be manual.  We're
14696                  * also going to temporarily set our CPU option to denote
14697                  * the newly configured CPU.
14698                  */
14699                 rs = opt[DTRACEOPT_BUFRESIZE];
14700                 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
14701                 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
14702 
14703                 (void) dtrace_state_buffers(state);
14704 
14705                 opt[DTRACEOPT_BUFRESIZE] = rs;
14706                 opt[DTRACEOPT_CPU] = c;
14707 
14708                 break;
14709         }
14710 
14711         case CPU_UNCONFIG:
14712                 /*
14713                  * We don't free the buffer in the CPU_UNCONFIG case.  (The
14714                  * buffer will be freed when the consumer exits.)
14715                  */
14716                 break;
14717 
14718         default:
14719                 break;
14720         }
14721 
14722         mutex_exit(&dtrace_lock);
14723         return (0);
14724 }
14725 
14726 static void
14727 dtrace_cpu_setup_initial(processorid_t cpu)
14728 {
14729         (void) dtrace_cpu_setup(CPU_CONFIG, cpu);
14730 }
14731 
14732 static void
14733 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
14734 {
14735         if (dtrace_toxranges >= dtrace_toxranges_max) {
14736                 int osize, nsize;
14737                 dtrace_toxrange_t *range;
14738 
14739                 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
14740 
14741                 if (osize == 0) {
14742                         ASSERT(dtrace_toxrange == NULL);
14743                         ASSERT(dtrace_toxranges_max == 0);
14744                         dtrace_toxranges_max = 1;
14745                 } else {
14746                         dtrace_toxranges_max <<= 1;
14747                 }
14748 
14749                 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
14750                 range = kmem_zalloc(nsize, KM_SLEEP);
14751 
14752                 if (dtrace_toxrange != NULL) {
14753                         ASSERT(osize != 0);
14754                         bcopy(dtrace_toxrange, range, osize);
14755                         kmem_free(dtrace_toxrange, osize);
14756                 }
14757 
14758                 dtrace_toxrange = range;
14759         }
14760 
14761         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
14762         ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
14763 
14764         dtrace_toxrange[dtrace_toxranges].dtt_base = base;
14765         dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
14766         dtrace_toxranges++;
14767 }
14768 
14769 /*
14770  * DTrace Driver Cookbook Functions
14771  */
14772 /*ARGSUSED*/
14773 static int
14774 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
14775 {
14776         dtrace_provider_id_t id;
14777         dtrace_state_t *state = NULL;
14778         dtrace_enabling_t *enab;
14779 
14780         mutex_enter(&cpu_lock);
14781         mutex_enter(&dtrace_provider_lock);
14782         mutex_enter(&dtrace_lock);
14783 
14784         if (ddi_soft_state_init(&dtrace_softstate,
14785             sizeof (dtrace_state_t), 0) != 0) {
14786                 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
14787                 mutex_exit(&cpu_lock);
14788                 mutex_exit(&dtrace_provider_lock);
14789                 mutex_exit(&dtrace_lock);
14790                 return (DDI_FAILURE);
14791         }
14792 
14793         if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
14794             DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
14795             ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
14796             DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
14797                 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
14798                 ddi_remove_minor_node(devi, NULL);
14799                 ddi_soft_state_fini(&dtrace_softstate);
14800                 mutex_exit(&cpu_lock);
14801                 mutex_exit(&dtrace_provider_lock);
14802                 mutex_exit(&dtrace_lock);
14803                 return (DDI_FAILURE);
14804         }
14805 
14806         ddi_report_dev(devi);
14807         dtrace_devi = devi;
14808 
14809         dtrace_modload = dtrace_module_loaded;
14810         dtrace_modunload = dtrace_module_unloaded;
14811         dtrace_cpu_init = dtrace_cpu_setup_initial;
14812         dtrace_helpers_cleanup = dtrace_helpers_destroy;
14813         dtrace_helpers_fork = dtrace_helpers_duplicate;
14814         dtrace_cpustart_init = dtrace_suspend;
14815         dtrace_cpustart_fini = dtrace_resume;
14816         dtrace_debugger_init = dtrace_suspend;
14817         dtrace_debugger_fini = dtrace_resume;
14818 
14819         register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
14820 
14821         ASSERT(MUTEX_HELD(&cpu_lock));
14822 
14823         dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
14824             NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
14825         dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
14826             UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
14827             VM_SLEEP | VMC_IDENTIFIER);
14828         dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
14829             1, INT_MAX, 0);
14830 
14831         dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
14832             sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
14833             NULL, NULL, NULL, NULL, NULL, 0);
14834 
14835         ASSERT(MUTEX_HELD(&cpu_lock));
14836         dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
14837             offsetof(dtrace_probe_t, dtpr_nextmod),
14838             offsetof(dtrace_probe_t, dtpr_prevmod));
14839 
14840         dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
14841             offsetof(dtrace_probe_t, dtpr_nextfunc),
14842             offsetof(dtrace_probe_t, dtpr_prevfunc));
14843 
14844         dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
14845             offsetof(dtrace_probe_t, dtpr_nextname),
14846             offsetof(dtrace_probe_t, dtpr_prevname));
14847 
14848         if (dtrace_retain_max < 1) {
14849                 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
14850                     "setting to 1", dtrace_retain_max);
14851                 dtrace_retain_max = 1;
14852         }
14853 
14854         /*
14855          * Now discover our toxic ranges.
14856          */
14857         dtrace_toxic_ranges(dtrace_toxrange_add);
14858 
14859         /*
14860          * Before we register ourselves as a provider to our own framework,
14861          * we would like to assert that dtrace_provider is NULL -- but that's
14862          * not true if we were loaded as a dependency of a DTrace provider.
14863          * Once we've registered, we can assert that dtrace_provider is our
14864          * pseudo provider.
14865          */
14866         (void) dtrace_register("dtrace", &dtrace_provider_attr,
14867             DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
14868 
14869         ASSERT(dtrace_provider != NULL);
14870         ASSERT((dtrace_provider_id_t)dtrace_provider == id);
14871 
14872         dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
14873             dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
14874         dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
14875             dtrace_provider, NULL, NULL, "END", 0, NULL);
14876         dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
14877             dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
14878 
14879         dtrace_anon_property();
14880         mutex_exit(&cpu_lock);
14881 
14882         /*
14883          * If DTrace helper tracing is enabled, we need to allocate the
14884          * trace buffer and initialize the values.
14885          */
14886         if (dtrace_helptrace_enabled) {
14887                 ASSERT(dtrace_helptrace_buffer == NULL);
14888                 dtrace_helptrace_buffer =
14889                     kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
14890                 dtrace_helptrace_next = 0;
14891         }
14892 
14893         /*
14894          * If there are already providers, we must ask them to provide their
14895          * probes, and then match any anonymous enabling against them.  Note
14896          * that there should be no other retained enablings at this time:
14897          * the only retained enablings at this time should be the anonymous
14898          * enabling.
14899          */
14900         if (dtrace_anon.dta_enabling != NULL) {
14901                 ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
14902 
14903                 dtrace_enabling_provide(NULL);
14904                 state = dtrace_anon.dta_state;
14905 
14906                 /*
14907                  * We couldn't hold cpu_lock across the above call to
14908                  * dtrace_enabling_provide(), but we must hold it to actually
14909                  * enable the probes.  We have to drop all of our locks, pick
14910                  * up cpu_lock, and regain our locks before matching the
14911                  * retained anonymous enabling.
14912                  */
14913                 mutex_exit(&dtrace_lock);
14914                 mutex_exit(&dtrace_provider_lock);
14915 
14916                 mutex_enter(&cpu_lock);
14917                 mutex_enter(&dtrace_provider_lock);
14918                 mutex_enter(&dtrace_lock);
14919 
14920                 if ((enab = dtrace_anon.dta_enabling) != NULL)
14921                         (void) dtrace_enabling_match(enab, NULL);
14922 
14923                 mutex_exit(&cpu_lock);
14924         }
14925 
14926         mutex_exit(&dtrace_lock);
14927         mutex_exit(&dtrace_provider_lock);
14928 
14929         if (state != NULL) {
14930                 /*
14931                  * If we created any anonymous state, set it going now.
14932                  */
14933                 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
14934         }
14935 
14936         return (DDI_SUCCESS);
14937 }
14938 
14939 /*ARGSUSED*/
14940 static int
14941 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
14942 {
14943         dtrace_state_t *state;
14944         uint32_t priv;
14945         uid_t uid;
14946         zoneid_t zoneid;
14947 
14948         if (getminor(*devp) == DTRACEMNRN_HELPER)
14949                 return (0);
14950 
14951         /*
14952          * If this wasn't an open with the "helper" minor, then it must be
14953          * the "dtrace" minor.
14954          */
14955         if (getminor(*devp) != DTRACEMNRN_DTRACE)
14956                 return (ENXIO);
14957 
14958         /*
14959          * If no DTRACE_PRIV_* bits are set in the credential, then the
14960          * caller lacks sufficient permission to do anything with DTrace.
14961          */
14962         dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
14963         if (priv == DTRACE_PRIV_NONE)
14964                 return (EACCES);
14965 
14966         /*
14967          * Ask all providers to provide all their probes.
14968          */
14969         mutex_enter(&dtrace_provider_lock);
14970         dtrace_probe_provide(NULL, NULL);
14971         mutex_exit(&dtrace_provider_lock);
14972 
14973         mutex_enter(&cpu_lock);
14974         mutex_enter(&dtrace_lock);
14975         dtrace_opens++;
14976         dtrace_membar_producer();
14977 
14978         /*
14979          * If the kernel debugger is active (that is, if the kernel debugger
14980          * modified text in some way), we won't allow the open.
14981          */
14982         if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
14983                 dtrace_opens--;
14984                 mutex_exit(&cpu_lock);
14985                 mutex_exit(&dtrace_lock);
14986                 return (EBUSY);
14987         }
14988 
14989         state = dtrace_state_create(devp, cred_p);
14990         mutex_exit(&cpu_lock);
14991 
14992         if (state == NULL) {
14993                 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
14994                         (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
14995                 mutex_exit(&dtrace_lock);
14996                 return (EAGAIN);
14997         }
14998 
14999         mutex_exit(&dtrace_lock);
15000 
15001         return (0);
15002 }
15003 
15004 /*ARGSUSED*/
15005 static int
15006 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
15007 {
15008         minor_t minor = getminor(dev);
15009         dtrace_state_t *state;
15010 
15011         if (minor == DTRACEMNRN_HELPER)
15012                 return (0);
15013 
15014         state = ddi_get_soft_state(dtrace_softstate, minor);
15015 
15016         mutex_enter(&cpu_lock);
15017         mutex_enter(&dtrace_lock);
15018 
15019         if (state->dts_anon) {
15020                 /*
15021                  * There is anonymous state. Destroy that first.
15022                  */
15023                 ASSERT(dtrace_anon.dta_state == NULL);
15024                 dtrace_state_destroy(state->dts_anon);
15025         }
15026 
15027         dtrace_state_destroy(state);
15028         ASSERT(dtrace_opens > 0);
15029 
15030         /*
15031          * Only relinquish control of the kernel debugger interface when there
15032          * are no consumers and no anonymous enablings.
15033          */
15034         if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15035                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15036 
15037         mutex_exit(&dtrace_lock);
15038         mutex_exit(&cpu_lock);
15039 
15040         return (0);
15041 }
15042 
15043 /*ARGSUSED*/
15044 static int
15045 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
15046 {
15047         int rval;
15048         dof_helper_t help, *dhp = NULL;
15049 
15050         switch (cmd) {
15051         case DTRACEHIOC_ADDDOF:
15052                 if (copyin((void *)arg, &help, sizeof (help)) != 0) {
15053                         dtrace_dof_error(NULL, "failed to copyin DOF helper");
15054                         return (EFAULT);
15055                 }
15056 
15057                 dhp = &help;
15058                 arg = (intptr_t)help.dofhp_dof;
15059                 /*FALLTHROUGH*/
15060 
15061         case DTRACEHIOC_ADD: {
15062                 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
15063 
15064                 if (dof == NULL)
15065                         return (rval);
15066 
15067                 mutex_enter(&dtrace_lock);
15068 
15069                 /*
15070                  * dtrace_helper_slurp() takes responsibility for the dof --
15071                  * it may free it now or it may save it and free it later.
15072                  */
15073                 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
15074                         *rv = rval;
15075                         rval = 0;
15076                 } else {
15077                         rval = EINVAL;
15078                 }
15079 
15080                 mutex_exit(&dtrace_lock);
15081                 return (rval);
15082         }
15083 
15084         case DTRACEHIOC_REMOVE: {
15085                 mutex_enter(&dtrace_lock);
15086                 rval = dtrace_helper_destroygen(arg);
15087                 mutex_exit(&dtrace_lock);
15088 
15089                 return (rval);
15090         }
15091 
15092         default:
15093                 break;
15094         }
15095 
15096         return (ENOTTY);
15097 }
15098 
15099 /*ARGSUSED*/
15100 static int
15101 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
15102 {
15103         minor_t minor = getminor(dev);
15104         dtrace_state_t *state;
15105         int rval;
15106 
15107         if (minor == DTRACEMNRN_HELPER)
15108                 return (dtrace_ioctl_helper(cmd, arg, rv));
15109 
15110         state = ddi_get_soft_state(dtrace_softstate, minor);
15111 
15112         if (state->dts_anon) {
15113                 ASSERT(dtrace_anon.dta_state == NULL);
15114                 state = state->dts_anon;
15115         }
15116 
15117         switch (cmd) {
15118         case DTRACEIOC_PROVIDER: {
15119                 dtrace_providerdesc_t pvd;
15120                 dtrace_provider_t *pvp;
15121 
15122                 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
15123                         return (EFAULT);
15124 
15125                 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
15126                 mutex_enter(&dtrace_provider_lock);
15127 
15128                 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
15129                         if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
15130                                 break;
15131                 }
15132 
15133                 mutex_exit(&dtrace_provider_lock);
15134 
15135                 if (pvp == NULL)
15136                         return (ESRCH);
15137 
15138                 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
15139                 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
15140                 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
15141                         return (EFAULT);
15142 
15143                 return (0);
15144         }
15145 
15146         case DTRACEIOC_EPROBE: {
15147                 dtrace_eprobedesc_t epdesc;
15148                 dtrace_ecb_t *ecb;
15149                 dtrace_action_t *act;
15150                 void *buf;
15151                 size_t size;
15152                 uintptr_t dest;
15153                 int nrecs;
15154 
15155                 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
15156                         return (EFAULT);
15157 
15158                 mutex_enter(&dtrace_lock);
15159 
15160                 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
15161                         mutex_exit(&dtrace_lock);
15162                         return (EINVAL);
15163                 }
15164 
15165                 if (ecb->dte_probe == NULL) {
15166                         mutex_exit(&dtrace_lock);
15167                         return (EINVAL);
15168                 }
15169 
15170                 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
15171                 epdesc.dtepd_uarg = ecb->dte_uarg;
15172                 epdesc.dtepd_size = ecb->dte_size;
15173 
15174                 nrecs = epdesc.dtepd_nrecs;
15175                 epdesc.dtepd_nrecs = 0;
15176                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15177                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15178                                 continue;
15179 
15180                         epdesc.dtepd_nrecs++;
15181                 }
15182 
15183                 /*
15184                  * Now that we have the size, we need to allocate a temporary
15185                  * buffer in which to store the complete description.  We need
15186                  * the temporary buffer to be able to drop dtrace_lock()
15187                  * across the copyout(), below.
15188                  */
15189                 size = sizeof (dtrace_eprobedesc_t) +
15190                     (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
15191 
15192                 buf = kmem_alloc(size, KM_SLEEP);
15193                 dest = (uintptr_t)buf;
15194 
15195                 bcopy(&epdesc, (void *)dest, sizeof (epdesc));
15196                 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
15197 
15198                 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15199                         if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15200                                 continue;
15201 
15202                         if (nrecs-- == 0)
15203                                 break;
15204 
15205                         bcopy(&act->dta_rec, (void *)dest,
15206                             sizeof (dtrace_recdesc_t));
15207                         dest += sizeof (dtrace_recdesc_t);
15208                 }
15209 
15210                 mutex_exit(&dtrace_lock);
15211 
15212                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
15213                         kmem_free(buf, size);
15214                         return (EFAULT);
15215                 }
15216 
15217                 kmem_free(buf, size);
15218                 return (0);
15219         }
15220 
15221         case DTRACEIOC_AGGDESC: {
15222                 dtrace_aggdesc_t aggdesc;
15223                 dtrace_action_t *act;
15224                 dtrace_aggregation_t *agg;
15225                 int nrecs;
15226                 uint32_t offs;
15227                 dtrace_recdesc_t *lrec;
15228                 void *buf;
15229                 size_t size;
15230                 uintptr_t dest;
15231 
15232                 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
15233                         return (EFAULT);
15234 
15235                 mutex_enter(&dtrace_lock);
15236 
15237                 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
15238                         mutex_exit(&dtrace_lock);
15239                         return (EINVAL);
15240                 }
15241 
15242                 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
15243 
15244                 nrecs = aggdesc.dtagd_nrecs;
15245                 aggdesc.dtagd_nrecs = 0;
15246 
15247                 offs = agg->dtag_base;
15248                 lrec = &agg->dtag_action.dta_rec;
15249                 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
15250 
15251                 for (act = agg->dtag_first; ; act = act->dta_next) {
15252                         ASSERT(act->dta_intuple ||
15253                             DTRACEACT_ISAGG(act->dta_kind));
15254 
15255                         /*
15256                          * If this action has a record size of zero, it
15257                          * denotes an argument to the aggregating action.
15258                          * Because the presence of this record doesn't (or
15259                          * shouldn't) affect the way the data is interpreted,
15260                          * we don't copy it out to save user-level the
15261                          * confusion of dealing with a zero-length record.
15262                          */
15263                         if (act->dta_rec.dtrd_size == 0) {
15264                                 ASSERT(agg->dtag_hasarg);
15265                                 continue;
15266                         }
15267 
15268                         aggdesc.dtagd_nrecs++;
15269 
15270                         if (act == &agg->dtag_action)
15271                                 break;
15272                 }
15273 
15274                 /*
15275                  * Now that we have the size, we need to allocate a temporary
15276                  * buffer in which to store the complete description.  We need
15277                  * the temporary buffer to be able to drop dtrace_lock()
15278                  * across the copyout(), below.
15279                  */
15280                 size = sizeof (dtrace_aggdesc_t) +
15281                     (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
15282 
15283                 buf = kmem_alloc(size, KM_SLEEP);
15284                 dest = (uintptr_t)buf;
15285 
15286                 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
15287                 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
15288 
15289                 for (act = agg->dtag_first; ; act = act->dta_next) {
15290                         dtrace_recdesc_t rec = act->dta_rec;
15291 
15292                         /*
15293                          * See the comment in the above loop for why we pass
15294                          * over zero-length records.
15295                          */
15296                         if (rec.dtrd_size == 0) {
15297                                 ASSERT(agg->dtag_hasarg);
15298                                 continue;
15299                         }
15300 
15301                         if (nrecs-- == 0)
15302                                 break;
15303 
15304                         rec.dtrd_offset -= offs;
15305                         bcopy(&rec, (void *)dest, sizeof (rec));
15306                         dest += sizeof (dtrace_recdesc_t);
15307 
15308                         if (act == &agg->dtag_action)
15309                                 break;
15310                 }
15311 
15312                 mutex_exit(&dtrace_lock);
15313 
15314                 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
15315                         kmem_free(buf, size);
15316                         return (EFAULT);
15317                 }
15318 
15319                 kmem_free(buf, size);
15320                 return (0);
15321         }
15322 
15323         case DTRACEIOC_ENABLE: {
15324                 dof_hdr_t *dof;
15325                 dtrace_enabling_t *enab = NULL;
15326                 dtrace_vstate_t *vstate;
15327                 int err = 0;
15328 
15329                 *rv = 0;
15330 
15331                 /*
15332                  * If a NULL argument has been passed, we take this as our
15333                  * cue to reevaluate our enablings.
15334                  */
15335                 if (arg == NULL) {
15336                         dtrace_enabling_matchall();
15337 
15338                         return (0);
15339                 }
15340 
15341                 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
15342                         return (rval);
15343 
15344                 mutex_enter(&cpu_lock);
15345                 mutex_enter(&dtrace_lock);
15346                 vstate = &state->dts_vstate;
15347 
15348                 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
15349                         mutex_exit(&dtrace_lock);
15350                         mutex_exit(&cpu_lock);
15351                         dtrace_dof_destroy(dof);
15352                         return (EBUSY);
15353                 }
15354 
15355                 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
15356                         mutex_exit(&dtrace_lock);
15357                         mutex_exit(&cpu_lock);
15358                         dtrace_dof_destroy(dof);
15359                         return (EINVAL);
15360                 }
15361 
15362                 if ((rval = dtrace_dof_options(dof, state)) != 0) {
15363                         dtrace_enabling_destroy(enab);
15364                         mutex_exit(&dtrace_lock);
15365                         mutex_exit(&cpu_lock);
15366                         dtrace_dof_destroy(dof);
15367                         return (rval);
15368                 }
15369 
15370                 if ((err = dtrace_enabling_match(enab, rv)) == 0) {
15371                         err = dtrace_enabling_retain(enab);
15372                 } else {
15373                         dtrace_enabling_destroy(enab);
15374                 }
15375 
15376                 mutex_exit(&cpu_lock);
15377                 mutex_exit(&dtrace_lock);
15378                 dtrace_dof_destroy(dof);
15379 
15380                 return (err);
15381         }
15382 
15383         case DTRACEIOC_REPLICATE: {
15384                 dtrace_repldesc_t desc;
15385                 dtrace_probedesc_t *match = &desc.dtrpd_match;
15386                 dtrace_probedesc_t *create = &desc.dtrpd_create;
15387                 int err;
15388 
15389                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15390                         return (EFAULT);
15391 
15392                 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15393                 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15394                 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15395                 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15396 
15397                 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15398                 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15399                 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15400                 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15401 
15402                 mutex_enter(&dtrace_lock);
15403                 err = dtrace_enabling_replicate(state, match, create);
15404                 mutex_exit(&dtrace_lock);
15405 
15406                 return (err);
15407         }
15408 
15409         case DTRACEIOC_PROBEMATCH:
15410         case DTRACEIOC_PROBES: {
15411                 dtrace_probe_t *probe = NULL;
15412                 dtrace_probedesc_t desc;
15413                 dtrace_probekey_t pkey;
15414                 dtrace_id_t i;
15415                 int m = 0;
15416                 uint32_t priv;
15417                 uid_t uid;
15418                 zoneid_t zoneid;
15419 
15420                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15421                         return (EFAULT);
15422 
15423                 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15424                 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15425                 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15426                 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15427 
15428                 /*
15429                  * Before we attempt to match this probe, we want to give
15430                  * all providers the opportunity to provide it.
15431                  */
15432                 if (desc.dtpd_id == DTRACE_IDNONE) {
15433                         mutex_enter(&dtrace_provider_lock);
15434                         dtrace_probe_provide(&desc, NULL);
15435                         mutex_exit(&dtrace_provider_lock);
15436                         desc.dtpd_id++;
15437                 }
15438 
15439                 if (cmd == DTRACEIOC_PROBEMATCH)  {
15440                         dtrace_probekey(&desc, &pkey);
15441                         pkey.dtpk_id = DTRACE_IDNONE;
15442                 }
15443 
15444                 dtrace_cred2priv(cr, &priv, &uid, &zoneid);
15445 
15446                 mutex_enter(&dtrace_lock);
15447 
15448                 if (cmd == DTRACEIOC_PROBEMATCH) {
15449                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
15450                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
15451                                     (m = dtrace_match_probe(probe, &pkey,
15452                                     priv, uid, zoneid)) != 0)
15453                                         break;
15454                         }
15455 
15456                         if (m < 0) {
15457                                 mutex_exit(&dtrace_lock);
15458                                 return (EINVAL);
15459                         }
15460 
15461                 } else {
15462                         for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
15463                                 if ((probe = dtrace_probes[i - 1]) != NULL &&
15464                                     dtrace_match_priv(probe, priv, uid, zoneid))
15465                                         break;
15466                         }
15467                 }
15468 
15469                 if (probe == NULL) {
15470                         mutex_exit(&dtrace_lock);
15471                         return (ESRCH);
15472                 }
15473 
15474                 dtrace_probe_description(probe, &desc);
15475                 mutex_exit(&dtrace_lock);
15476 
15477                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15478                         return (EFAULT);
15479 
15480                 return (0);
15481         }
15482 
15483         case DTRACEIOC_PROBEARG: {
15484                 dtrace_argdesc_t desc;
15485                 dtrace_probe_t *probe;
15486                 dtrace_provider_t *prov;
15487 
15488                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15489                         return (EFAULT);
15490 
15491                 if (desc.dtargd_id == DTRACE_IDNONE)
15492                         return (EINVAL);
15493 
15494                 if (desc.dtargd_ndx == DTRACE_ARGNONE)
15495                         return (EINVAL);
15496 
15497                 mutex_enter(&dtrace_provider_lock);
15498                 mutex_enter(&mod_lock);
15499                 mutex_enter(&dtrace_lock);
15500 
15501                 if (desc.dtargd_id > dtrace_nprobes) {
15502                         mutex_exit(&dtrace_lock);
15503                         mutex_exit(&mod_lock);
15504                         mutex_exit(&dtrace_provider_lock);
15505                         return (EINVAL);
15506                 }
15507 
15508                 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
15509                         mutex_exit(&dtrace_lock);
15510                         mutex_exit(&mod_lock);
15511                         mutex_exit(&dtrace_provider_lock);
15512                         return (EINVAL);
15513                 }
15514 
15515                 mutex_exit(&dtrace_lock);
15516 
15517                 prov = probe->dtpr_provider;
15518 
15519                 if (prov->dtpv_pops.dtps_getargdesc == NULL) {
15520                         /*
15521                          * There isn't any typed information for this probe.
15522                          * Set the argument number to DTRACE_ARGNONE.
15523                          */
15524                         desc.dtargd_ndx = DTRACE_ARGNONE;
15525                 } else {
15526                         desc.dtargd_native[0] = '\0';
15527                         desc.dtargd_xlate[0] = '\0';
15528                         desc.dtargd_mapping = desc.dtargd_ndx;
15529 
15530                         prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
15531                             probe->dtpr_id, probe->dtpr_arg, &desc);
15532                 }
15533 
15534                 mutex_exit(&mod_lock);
15535                 mutex_exit(&dtrace_provider_lock);
15536 
15537                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15538                         return (EFAULT);
15539 
15540                 return (0);
15541         }
15542 
15543         case DTRACEIOC_GO: {
15544                 processorid_t cpuid;
15545                 rval = dtrace_state_go(state, &cpuid);
15546 
15547                 if (rval != 0)
15548                         return (rval);
15549 
15550                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
15551                         return (EFAULT);
15552 
15553                 return (0);
15554         }
15555 
15556         case DTRACEIOC_STOP: {
15557                 processorid_t cpuid;
15558 
15559                 mutex_enter(&dtrace_lock);
15560                 rval = dtrace_state_stop(state, &cpuid);
15561                 mutex_exit(&dtrace_lock);
15562 
15563                 if (rval != 0)
15564                         return (rval);
15565 
15566                 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
15567                         return (EFAULT);
15568 
15569                 return (0);
15570         }
15571 
15572         case DTRACEIOC_DOFGET: {
15573                 dof_hdr_t hdr, *dof;
15574                 uint64_t len;
15575 
15576                 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
15577                         return (EFAULT);
15578 
15579                 mutex_enter(&dtrace_lock);
15580                 dof = dtrace_dof_create(state);
15581                 mutex_exit(&dtrace_lock);
15582 
15583                 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
15584                 rval = copyout(dof, (void *)arg, len);
15585                 dtrace_dof_destroy(dof);
15586 
15587                 return (rval == 0 ? 0 : EFAULT);
15588         }
15589 
15590         case DTRACEIOC_AGGSNAP:
15591         case DTRACEIOC_BUFSNAP: {
15592                 dtrace_bufdesc_t desc;
15593                 caddr_t cached;
15594                 dtrace_buffer_t *buf;
15595 
15596                 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15597                         return (EFAULT);
15598 
15599                 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
15600                         return (EINVAL);
15601 
15602                 mutex_enter(&dtrace_lock);
15603 
15604                 if (cmd == DTRACEIOC_BUFSNAP) {
15605                         buf = &state->dts_buffer[desc.dtbd_cpu];
15606                 } else {
15607                         buf = &state->dts_aggbuffer[desc.dtbd_cpu];
15608                 }
15609 
15610                 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
15611                         size_t sz = buf->dtb_offset;
15612 
15613                         if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
15614                                 mutex_exit(&dtrace_lock);
15615                                 return (EBUSY);
15616                         }
15617 
15618                         /*
15619                          * If this buffer has already been consumed, we're
15620                          * going to indicate that there's nothing left here
15621                          * to consume.
15622                          */
15623                         if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
15624                                 mutex_exit(&dtrace_lock);
15625 
15626                                 desc.dtbd_size = 0;
15627                                 desc.dtbd_drops = 0;
15628                                 desc.dtbd_errors = 0;
15629                                 desc.dtbd_oldest = 0;
15630                                 sz = sizeof (desc);
15631 
15632                                 if (copyout(&desc, (void *)arg, sz) != 0)
15633                                         return (EFAULT);
15634 
15635                                 return (0);
15636                         }
15637 
15638                         /*
15639                          * If this is a ring buffer that has wrapped, we want
15640                          * to copy the whole thing out.
15641                          */
15642                         if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
15643                                 dtrace_buffer_polish(buf);
15644                                 sz = buf->dtb_size;
15645                         }
15646 
15647                         if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
15648                                 mutex_exit(&dtrace_lock);
15649                                 return (EFAULT);
15650                         }
15651 
15652                         desc.dtbd_size = sz;
15653                         desc.dtbd_drops = buf->dtb_drops;
15654                         desc.dtbd_errors = buf->dtb_errors;
15655                         desc.dtbd_oldest = buf->dtb_xamot_offset;
15656                         desc.dtbd_timestamp = dtrace_gethrtime();
15657 
15658                         mutex_exit(&dtrace_lock);
15659 
15660                         if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15661                                 return (EFAULT);
15662 
15663                         buf->dtb_flags |= DTRACEBUF_CONSUMED;
15664 
15665                         return (0);
15666                 }
15667 
15668                 if (buf->dtb_tomax == NULL) {
15669                         ASSERT(buf->dtb_xamot == NULL);
15670                         mutex_exit(&dtrace_lock);
15671                         return (ENOENT);
15672                 }
15673 
15674                 cached = buf->dtb_tomax;
15675                 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
15676 
15677                 dtrace_xcall(desc.dtbd_cpu,
15678                     (dtrace_xcall_t)dtrace_buffer_switch, buf);
15679 
15680                 state->dts_errors += buf->dtb_xamot_errors;
15681 
15682                 /*
15683                  * If the buffers did not actually switch, then the cross call
15684                  * did not take place -- presumably because the given CPU is
15685                  * not in the ready set.  If this is the case, we'll return
15686                  * ENOENT.
15687                  */
15688                 if (buf->dtb_tomax == cached) {
15689                         ASSERT(buf->dtb_xamot != cached);
15690                         mutex_exit(&dtrace_lock);
15691                         return (ENOENT);
15692                 }
15693 
15694                 ASSERT(cached == buf->dtb_xamot);
15695 
15696                 /*
15697                  * We have our snapshot; now copy it out.
15698                  */
15699                 if (copyout(buf->dtb_xamot, desc.dtbd_data,
15700                     buf->dtb_xamot_offset) != 0) {
15701                         mutex_exit(&dtrace_lock);
15702                         return (EFAULT);
15703                 }
15704 
15705                 desc.dtbd_size = buf->dtb_xamot_offset;
15706                 desc.dtbd_drops = buf->dtb_xamot_drops;
15707                 desc.dtbd_errors = buf->dtb_xamot_errors;
15708                 desc.dtbd_oldest = 0;
15709                 desc.dtbd_timestamp = buf->dtb_switched;
15710 
15711                 mutex_exit(&dtrace_lock);
15712 
15713                 /*
15714                  * Finally, copy out the buffer description.
15715                  */
15716                 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15717                         return (EFAULT);
15718 
15719                 return (0);
15720         }
15721 
15722         case DTRACEIOC_CONF: {
15723                 dtrace_conf_t conf;
15724 
15725                 bzero(&conf, sizeof (conf));
15726                 conf.dtc_difversion = DIF_VERSION;
15727                 conf.dtc_difintregs = DIF_DIR_NREGS;
15728                 conf.dtc_diftupregs = DIF_DTR_NREGS;
15729                 conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
15730 
15731                 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
15732                         return (EFAULT);
15733 
15734                 return (0);
15735         }
15736 
15737         case DTRACEIOC_STATUS: {
15738                 dtrace_status_t stat;
15739                 dtrace_dstate_t *dstate;
15740                 int i, j;
15741                 uint64_t nerrs;
15742 
15743                 /*
15744                  * See the comment in dtrace_state_deadman() for the reason
15745                  * for setting dts_laststatus to INT64_MAX before setting
15746                  * it to the correct value.
15747                  */
15748                 state->dts_laststatus = INT64_MAX;
15749                 dtrace_membar_producer();
15750                 state->dts_laststatus = dtrace_gethrtime();
15751 
15752                 bzero(&stat, sizeof (stat));
15753 
15754                 mutex_enter(&dtrace_lock);
15755 
15756                 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
15757                         mutex_exit(&dtrace_lock);
15758                         return (ENOENT);
15759                 }
15760 
15761                 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
15762                         stat.dtst_exiting = 1;
15763 
15764                 nerrs = state->dts_errors;
15765                 dstate = &state->dts_vstate.dtvs_dynvars;
15766 
15767                 for (i = 0; i < NCPU; i++) {
15768                         dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
15769 
15770                         stat.dtst_dyndrops += dcpu->dtdsc_drops;
15771                         stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
15772                         stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
15773 
15774                         if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
15775                                 stat.dtst_filled++;
15776 
15777                         nerrs += state->dts_buffer[i].dtb_errors;
15778 
15779                         for (j = 0; j < state->dts_nspeculations; j++) {
15780                                 dtrace_speculation_t *spec;
15781                                 dtrace_buffer_t *buf;
15782 
15783                                 spec = &state->dts_speculations[j];
15784                                 buf = &spec->dtsp_buffer[i];
15785                                 stat.dtst_specdrops += buf->dtb_xamot_drops;
15786                         }
15787                 }
15788 
15789                 stat.dtst_specdrops_busy = state->dts_speculations_busy;
15790                 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
15791                 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
15792                 stat.dtst_dblerrors = state->dts_dblerrors;
15793                 stat.dtst_killed =
15794                     (state->dts_activity == DTRACE_ACTIVITY_KILLED);
15795                 stat.dtst_errors = nerrs;
15796 
15797                 mutex_exit(&dtrace_lock);
15798 
15799                 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
15800                         return (EFAULT);
15801 
15802                 return (0);
15803         }
15804 
15805         case DTRACEIOC_FORMAT: {
15806                 dtrace_fmtdesc_t fmt;
15807                 char *str;
15808                 int len;
15809 
15810                 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
15811                         return (EFAULT);
15812 
15813                 mutex_enter(&dtrace_lock);
15814 
15815                 if (fmt.dtfd_format == 0 ||
15816                     fmt.dtfd_format > state->dts_nformats) {
15817                         mutex_exit(&dtrace_lock);
15818                         return (EINVAL);
15819                 }
15820 
15821                 /*
15822                  * Format strings are allocated contiguously and they are
15823                  * never freed; if a format index is less than the number
15824                  * of formats, we can assert that the format map is non-NULL
15825                  * and that the format for the specified index is non-NULL.
15826                  */
15827                 ASSERT(state->dts_formats != NULL);
15828                 str = state->dts_formats[fmt.dtfd_format - 1];
15829                 ASSERT(str != NULL);
15830 
15831                 len = strlen(str) + 1;
15832 
15833                 if (len > fmt.dtfd_length) {
15834                         fmt.dtfd_length = len;
15835 
15836                         if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
15837                                 mutex_exit(&dtrace_lock);
15838                                 return (EINVAL);
15839                         }
15840                 } else {
15841                         if (copyout(str, fmt.dtfd_string, len) != 0) {
15842                                 mutex_exit(&dtrace_lock);
15843                                 return (EINVAL);
15844                         }
15845                 }
15846 
15847                 mutex_exit(&dtrace_lock);
15848                 return (0);
15849         }
15850 
15851         default:
15852                 break;
15853         }
15854 
15855         return (ENOTTY);
15856 }
15857 
15858 /*ARGSUSED*/
15859 static int
15860 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
15861 {
15862         dtrace_state_t *state;
15863 
15864         switch (cmd) {
15865         case DDI_DETACH:
15866                 break;
15867 
15868         case DDI_SUSPEND:
15869                 return (DDI_SUCCESS);
15870 
15871         default:
15872                 return (DDI_FAILURE);
15873         }
15874 
15875         mutex_enter(&cpu_lock);
15876         mutex_enter(&dtrace_provider_lock);
15877         mutex_enter(&dtrace_lock);
15878 
15879         ASSERT(dtrace_opens == 0);
15880 
15881         if (dtrace_helpers > 0) {
15882                 mutex_exit(&dtrace_provider_lock);
15883                 mutex_exit(&dtrace_lock);
15884                 mutex_exit(&cpu_lock);
15885                 return (DDI_FAILURE);
15886         }
15887 
15888         if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
15889                 mutex_exit(&dtrace_provider_lock);
15890                 mutex_exit(&dtrace_lock);
15891                 mutex_exit(&cpu_lock);
15892                 return (DDI_FAILURE);
15893         }
15894 
15895         dtrace_provider = NULL;
15896 
15897         if ((state = dtrace_anon_grab()) != NULL) {
15898                 /*
15899                  * If there were ECBs on this state, the provider should
15900                  * have not been allowed to detach; assert that there is
15901                  * none.
15902                  */
15903                 ASSERT(state->dts_necbs == 0);
15904                 dtrace_state_destroy(state);
15905 
15906                 /*
15907                  * If we're being detached with anonymous state, we need to
15908                  * indicate to the kernel debugger that DTrace is now inactive.
15909                  */
15910                 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15911         }
15912 
15913         bzero(&dtrace_anon, sizeof (dtrace_anon_t));
15914         unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
15915         dtrace_cpu_init = NULL;
15916         dtrace_helpers_cleanup = NULL;
15917         dtrace_helpers_fork = NULL;
15918         dtrace_cpustart_init = NULL;
15919         dtrace_cpustart_fini = NULL;
15920         dtrace_debugger_init = NULL;
15921         dtrace_debugger_fini = NULL;
15922         dtrace_modload = NULL;
15923         dtrace_modunload = NULL;
15924 
15925         mutex_exit(&cpu_lock);
15926 
15927         if (dtrace_helptrace_enabled) {
15928                 kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
15929                 dtrace_helptrace_buffer = NULL;
15930         }
15931 
15932         kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
15933         dtrace_probes = NULL;
15934         dtrace_nprobes = 0;
15935 
15936         dtrace_hash_destroy(dtrace_bymod);
15937         dtrace_hash_destroy(dtrace_byfunc);
15938         dtrace_hash_destroy(dtrace_byname);
15939         dtrace_bymod = NULL;
15940         dtrace_byfunc = NULL;
15941         dtrace_byname = NULL;
15942 
15943         kmem_cache_destroy(dtrace_state_cache);
15944         vmem_destroy(dtrace_minor);
15945         vmem_destroy(dtrace_arena);
15946 
15947         if (dtrace_toxrange != NULL) {
15948                 kmem_free(dtrace_toxrange,
15949                     dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
15950                 dtrace_toxrange = NULL;
15951                 dtrace_toxranges = 0;
15952                 dtrace_toxranges_max = 0;
15953         }
15954 
15955         ddi_remove_minor_node(dtrace_devi, NULL);
15956         dtrace_devi = NULL;
15957 
15958         ddi_soft_state_fini(&dtrace_softstate);
15959 
15960         ASSERT(dtrace_vtime_references == 0);
15961         ASSERT(dtrace_opens == 0);
15962         ASSERT(dtrace_retained == NULL);
15963 
15964         mutex_exit(&dtrace_lock);
15965         mutex_exit(&dtrace_provider_lock);
15966 
15967         /*
15968          * We don't destroy the task queue until after we have dropped our
15969          * locks (taskq_destroy() may block on running tasks).  To prevent
15970          * attempting to do work after we have effectively detached but before
15971          * the task queue has been destroyed, all tasks dispatched via the
15972          * task queue must check that DTrace is still attached before
15973          * performing any operation.
15974          */
15975         taskq_destroy(dtrace_taskq);
15976         dtrace_taskq = NULL;
15977 
15978         return (DDI_SUCCESS);
15979 }
15980 
15981 /*ARGSUSED*/
15982 static int
15983 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
15984 {
15985         int error;
15986 
15987         switch (infocmd) {
15988         case DDI_INFO_DEVT2DEVINFO:
15989                 *result = (void *)dtrace_devi;
15990                 error = DDI_SUCCESS;
15991                 break;
15992         case DDI_INFO_DEVT2INSTANCE:
15993                 *result = (void *)0;
15994                 error = DDI_SUCCESS;
15995                 break;
15996         default:
15997                 error = DDI_FAILURE;
15998         }
15999         return (error);
16000 }
16001 
16002 static struct cb_ops dtrace_cb_ops = {
16003         dtrace_open,            /* open */
16004         dtrace_close,           /* close */
16005         nulldev,                /* strategy */
16006         nulldev,                /* print */
16007         nodev,                  /* dump */
16008         nodev,                  /* read */
16009         nodev,                  /* write */
16010         dtrace_ioctl,           /* ioctl */
16011         nodev,                  /* devmap */
16012         nodev,                  /* mmap */
16013         nodev,                  /* segmap */
16014         nochpoll,               /* poll */
16015         ddi_prop_op,            /* cb_prop_op */
16016         0,                      /* streamtab  */
16017         D_NEW | D_MP            /* Driver compatibility flag */
16018 };
16019 
16020 static struct dev_ops dtrace_ops = {
16021         DEVO_REV,               /* devo_rev */
16022         0,                      /* refcnt */
16023         dtrace_info,            /* get_dev_info */
16024         nulldev,                /* identify */
16025         nulldev,                /* probe */
16026         dtrace_attach,          /* attach */
16027         dtrace_detach,          /* detach */
16028         nodev,                  /* reset */
16029         &dtrace_cb_ops,             /* driver operations */
16030         NULL,                   /* bus operations */
16031         nodev,                  /* dev power */
16032         ddi_quiesce_not_needed,         /* quiesce */
16033 };
16034 
16035 static struct modldrv modldrv = {
16036         &mod_driverops,             /* module type (this is a pseudo driver) */
16037         "Dynamic Tracing",      /* name of module */
16038         &dtrace_ops,                /* driver ops */
16039 };
16040 
16041 static struct modlinkage modlinkage = {
16042         MODREV_1,
16043         (void *)&modldrv,
16044         NULL
16045 };
16046 
16047 int
16048 _init(void)
16049 {
16050         return (mod_install(&modlinkage));
16051 }
16052 
16053 int
16054 _info(struct modinfo *modinfop)
16055 {
16056         return (mod_info(&modlinkage, modinfop));
16057 }
16058 
16059 int
16060 _fini(void)
16061 {
16062         return (mod_remove(&modlinkage));
16063 }