Print this page
2916 DTrace in a zone should be able to access fds[]
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/dtrace/dtrace.c
+++ new/usr/src/uts/common/dtrace/dtrace.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
25 25 */
26 26
27 27 /*
28 28 * DTrace - Dynamic Tracing for Solaris
29 29 *
30 30 * This is the implementation of the Solaris Dynamic Tracing framework
31 31 * (DTrace). The user-visible interface to DTrace is described at length in
32 32 * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace
33 33 * library, the in-kernel DTrace framework, and the DTrace providers are
34 34 * described in the block comments in the <sys/dtrace.h> header file. The
35 35 * internal architecture of DTrace is described in the block comments in the
36 36 * <sys/dtrace_impl.h> header file. The comments contained within the DTrace
37 37 * implementation very much assume mastery of all of these sources; if one has
38 38 * an unanswered question about the implementation, one should consult them
39 39 * first.
40 40 *
41 41 * The functions here are ordered roughly as follows:
42 42 *
43 43 * - Probe context functions
44 44 * - Probe hashing functions
45 45 * - Non-probe context utility functions
46 46 * - Matching functions
47 47 * - Provider-to-Framework API functions
48 48 * - Probe management functions
49 49 * - DIF object functions
50 50 * - Format functions
51 51 * - Predicate functions
52 52 * - ECB functions
53 53 * - Buffer functions
54 54 * - Enabling functions
55 55 * - DOF functions
56 56 * - Anonymous enabling functions
57 57 * - Consumer state functions
58 58 * - Helper functions
59 59 * - Hook functions
60 60 * - Driver cookbook functions
61 61 *
62 62 * Each group of functions begins with a block comment labelled the "DTrace
63 63 * [Group] Functions", allowing one to find each block by searching forward
64 64 * on capital-f functions.
65 65 */
66 66 #include <sys/errno.h>
67 67 #include <sys/stat.h>
68 68 #include <sys/modctl.h>
69 69 #include <sys/conf.h>
70 70 #include <sys/systm.h>
71 71 #include <sys/ddi.h>
72 72 #include <sys/sunddi.h>
73 73 #include <sys/cpuvar.h>
74 74 #include <sys/kmem.h>
75 75 #include <sys/strsubr.h>
76 76 #include <sys/sysmacros.h>
77 77 #include <sys/dtrace_impl.h>
78 78 #include <sys/atomic.h>
79 79 #include <sys/cmn_err.h>
80 80 #include <sys/mutex_impl.h>
81 81 #include <sys/rwlock_impl.h>
82 82 #include <sys/ctf_api.h>
83 83 #include <sys/panic.h>
84 84 #include <sys/priv_impl.h>
85 85 #include <sys/policy.h>
86 86 #include <sys/cred_impl.h>
87 87 #include <sys/procfs_isa.h>
88 88 #include <sys/taskq.h>
89 89 #include <sys/mkdev.h>
90 90 #include <sys/kdi.h>
91 91 #include <sys/zone.h>
92 92 #include <sys/socket.h>
93 93 #include <netinet/in.h>
94 94
95 95 /*
96 96 * DTrace Tunable Variables
97 97 *
98 98 * The following variables may be tuned by adding a line to /etc/system that
99 99 * includes both the name of the DTrace module ("dtrace") and the name of the
100 100 * variable. For example:
101 101 *
102 102 * set dtrace:dtrace_destructive_disallow = 1
103 103 *
104 104 * In general, the only variables that one should be tuning this way are those
105 105 * that affect system-wide DTrace behavior, and for which the default behavior
106 106 * is undesirable. Most of these variables are tunable on a per-consumer
107 107 * basis using DTrace options, and need not be tuned on a system-wide basis.
108 108 * When tuning these variables, avoid pathological values; while some attempt
109 109 * is made to verify the integrity of these variables, they are not considered
110 110 * part of the supported interface to DTrace, and they are therefore not
111 111 * checked comprehensively. Further, these variables should not be tuned
112 112 * dynamically via "mdb -kw" or other means; they should only be tuned via
113 113 * /etc/system.
114 114 */
115 115 int dtrace_destructive_disallow = 0;
116 116 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024);
117 117 size_t dtrace_difo_maxsize = (256 * 1024);
118 118 dtrace_optval_t dtrace_dof_maxsize = (256 * 1024);
119 119 size_t dtrace_global_maxsize = (16 * 1024);
120 120 size_t dtrace_actions_max = (16 * 1024);
121 121 size_t dtrace_retain_max = 1024;
122 122 dtrace_optval_t dtrace_helper_actions_max = 1024;
123 123 dtrace_optval_t dtrace_helper_providers_max = 32;
124 124 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024);
125 125 size_t dtrace_strsize_default = 256;
126 126 dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */
127 127 dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */
128 128 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */
129 129 dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */
130 130 dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */
131 131 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */
132 132 dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */
133 133 dtrace_optval_t dtrace_nspec_default = 1;
134 134 dtrace_optval_t dtrace_specsize_default = 32 * 1024;
135 135 dtrace_optval_t dtrace_stackframes_default = 20;
136 136 dtrace_optval_t dtrace_ustackframes_default = 20;
137 137 dtrace_optval_t dtrace_jstackframes_default = 50;
138 138 dtrace_optval_t dtrace_jstackstrsize_default = 512;
139 139 int dtrace_msgdsize_max = 128;
140 140 hrtime_t dtrace_chill_max = 500 * (NANOSEC / MILLISEC); /* 500 ms */
141 141 hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */
142 142 int dtrace_devdepth_max = 32;
143 143 int dtrace_err_verbose;
144 144 hrtime_t dtrace_deadman_interval = NANOSEC;
145 145 hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
146 146 hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
147 147 hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC;
148 148
149 149 /*
150 150 * DTrace External Variables
151 151 *
152 152 * As dtrace(7D) is a kernel module, any DTrace variables are obviously
153 153 * available to DTrace consumers via the backtick (`) syntax. One of these,
154 154 * dtrace_zero, is made deliberately so: it is provided as a source of
155 155 * well-known, zero-filled memory. While this variable is not documented,
156 156 * it is used by some translators as an implementation detail.
157 157 */
158 158 const char dtrace_zero[256] = { 0 }; /* zero-filled memory */
159 159
160 160 /*
161 161 * DTrace Internal Variables
162 162 */
|
↓ open down ↓ |
162 lines elided |
↑ open up ↑ |
163 163 static dev_info_t *dtrace_devi; /* device info */
164 164 static vmem_t *dtrace_arena; /* probe ID arena */
165 165 static vmem_t *dtrace_minor; /* minor number arena */
166 166 static taskq_t *dtrace_taskq; /* task queue */
167 167 static dtrace_probe_t **dtrace_probes; /* array of all probes */
168 168 static int dtrace_nprobes; /* number of probes */
169 169 static dtrace_provider_t *dtrace_provider; /* provider list */
170 170 static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */
171 171 static int dtrace_opens; /* number of opens */
172 172 static int dtrace_helpers; /* number of helpers */
173 +static int dtrace_getf; /* number of unpriv getf()s */
173 174 static void *dtrace_softstate; /* softstate pointer */
174 175 static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */
175 176 static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */
176 177 static dtrace_hash_t *dtrace_byname; /* probes hashed by name */
177 178 static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */
178 179 static int dtrace_toxranges; /* number of toxic ranges */
179 180 static int dtrace_toxranges_max; /* size of toxic range array */
180 181 static dtrace_anon_t dtrace_anon; /* anonymous enabling */
181 182 static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */
182 183 static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */
183 184 static kthread_t *dtrace_panicked; /* panicking thread */
184 185 static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */
185 186 static dtrace_genid_t dtrace_probegen; /* current probe generation */
186 187 static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */
187 188 static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */
188 189 static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */
189 190 static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */
190 191 static int dtrace_dynvar_failclean; /* dynvars failed to clean */
191 192
192 193 /*
193 194 * DTrace Locking
194 195 * DTrace is protected by three (relatively coarse-grained) locks:
195 196 *
196 197 * (1) dtrace_lock is required to manipulate essentially any DTrace state,
197 198 * including enabling state, probes, ECBs, consumer state, helper state,
198 199 * etc. Importantly, dtrace_lock is _not_ required when in probe context;
199 200 * probe context is lock-free -- synchronization is handled via the
200 201 * dtrace_sync() cross call mechanism.
201 202 *
202 203 * (2) dtrace_provider_lock is required when manipulating provider state, or
203 204 * when provider state must be held constant.
204 205 *
205 206 * (3) dtrace_meta_lock is required when manipulating meta provider state, or
206 207 * when meta provider state must be held constant.
207 208 *
208 209 * The lock ordering between these three locks is dtrace_meta_lock before
209 210 * dtrace_provider_lock before dtrace_lock. (In particular, there are
210 211 * several places where dtrace_provider_lock is held by the framework as it
211 212 * calls into the providers -- which then call back into the framework,
212 213 * grabbing dtrace_lock.)
213 214 *
214 215 * There are two other locks in the mix: mod_lock and cpu_lock. With respect
215 216 * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
216 217 * role as a coarse-grained lock; it is acquired before both of these locks.
217 218 * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must
218 219 * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
219 220 * mod_lock is similar with respect to dtrace_provider_lock in that it must be
220 221 * acquired _between_ dtrace_provider_lock and dtrace_lock.
221 222 */
222 223 static kmutex_t dtrace_lock; /* probe state lock */
223 224 static kmutex_t dtrace_provider_lock; /* provider state lock */
224 225 static kmutex_t dtrace_meta_lock; /* meta-provider state lock */
225 226
226 227 /*
227 228 * DTrace Provider Variables
228 229 *
229 230 * These are the variables relating to DTrace as a provider (that is, the
230 231 * provider of the BEGIN, END, and ERROR probes).
231 232 */
232 233 static dtrace_pattr_t dtrace_provider_attr = {
233 234 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
234 235 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
235 236 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
236 237 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
237 238 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
238 239 };
239 240
240 241 static void
241 242 dtrace_nullop(void)
242 243 {}
243 244
244 245 static int
245 246 dtrace_enable_nullop(void)
246 247 {
247 248 return (0);
248 249 }
249 250
250 251 static dtrace_pops_t dtrace_provider_ops = {
251 252 (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
252 253 (void (*)(void *, struct modctl *))dtrace_nullop,
253 254 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
254 255 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
255 256 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
256 257 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
257 258 NULL,
258 259 NULL,
259 260 NULL,
260 261 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop
261 262 };
262 263
263 264 static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */
264 265 static dtrace_id_t dtrace_probeid_end; /* special END probe */
265 266 dtrace_id_t dtrace_probeid_error; /* special ERROR probe */
266 267
267 268 /*
268 269 * DTrace Helper Tracing Variables
269 270 */
270 271 uint32_t dtrace_helptrace_next = 0;
271 272 uint32_t dtrace_helptrace_nlocals;
272 273 char *dtrace_helptrace_buffer;
273 274 int dtrace_helptrace_bufsize = 512 * 1024;
274 275
275 276 #ifdef DEBUG
276 277 int dtrace_helptrace_enabled = 1;
277 278 #else
278 279 int dtrace_helptrace_enabled = 0;
279 280 #endif
280 281
281 282 /*
282 283 * DTrace Error Hashing
283 284 *
284 285 * On DEBUG kernels, DTrace will track the errors that has seen in a hash
285 286 * table. This is very useful for checking coverage of tests that are
286 287 * expected to induce DIF or DOF processing errors, and may be useful for
287 288 * debugging problems in the DIF code generator or in DOF generation . The
288 289 * error hash may be examined with the ::dtrace_errhash MDB dcmd.
289 290 */
290 291 #ifdef DEBUG
291 292 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ];
292 293 static const char *dtrace_errlast;
293 294 static kthread_t *dtrace_errthread;
294 295 static kmutex_t dtrace_errlock;
295 296 #endif
296 297
297 298 /*
298 299 * DTrace Macros and Constants
299 300 *
300 301 * These are various macros that are useful in various spots in the
301 302 * implementation, along with a few random constants that have no meaning
302 303 * outside of the implementation. There is no real structure to this cpp
303 304 * mishmash -- but is there ever?
304 305 */
305 306 #define DTRACE_HASHSTR(hash, probe) \
306 307 dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
307 308
308 309 #define DTRACE_HASHNEXT(hash, probe) \
309 310 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
310 311
311 312 #define DTRACE_HASHPREV(hash, probe) \
312 313 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
313 314
314 315 #define DTRACE_HASHEQ(hash, lhs, rhs) \
315 316 (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
316 317 *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
317 318
318 319 #define DTRACE_AGGHASHSIZE_SLEW 17
319 320
320 321 #define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3)
321 322
322 323 /*
323 324 * The key for a thread-local variable consists of the lower 61 bits of the
324 325 * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
325 326 * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
326 327 * equal to a variable identifier. This is necessary (but not sufficient) to
327 328 * assure that global associative arrays never collide with thread-local
328 329 * variables. To guarantee that they cannot collide, we must also define the
329 330 * order for keying dynamic variables. That order is:
330 331 *
331 332 * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
332 333 *
333 334 * Because the variable-key and the tls-key are in orthogonal spaces, there is
334 335 * no way for a global variable key signature to match a thread-local key
335 336 * signature.
336 337 */
337 338 #define DTRACE_TLS_THRKEY(where) { \
338 339 uint_t intr = 0; \
339 340 uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
340 341 for (; actv; actv >>= 1) \
341 342 intr++; \
342 343 ASSERT(intr < (1 << 3)); \
343 344 (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
344 345 (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
345 346 }
346 347
347 348 #define DT_BSWAP_8(x) ((x) & 0xff)
348 349 #define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8))
349 350 #define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16))
350 351 #define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32))
351 352
352 353 #define DT_MASK_LO 0x00000000FFFFFFFFULL
353 354
354 355 #define DTRACE_STORE(type, tomax, offset, what) \
355 356 *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
356 357
357 358 #ifndef __i386
358 359 #define DTRACE_ALIGNCHECK(addr, size, flags) \
359 360 if (addr & (size - 1)) { \
360 361 *flags |= CPU_DTRACE_BADALIGN; \
361 362 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
362 363 return (0); \
363 364 }
364 365 #else
365 366 #define DTRACE_ALIGNCHECK(addr, size, flags)
366 367 #endif
367 368
368 369 /*
369 370 * Test whether a range of memory starting at testaddr of size testsz falls
370 371 * within the range of memory described by addr, sz. We take care to avoid
371 372 * problems with overflow and underflow of the unsigned quantities, and
372 373 * disallow all negative sizes. Ranges of size 0 are allowed.
373 374 */
374 375 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \
375 376 ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \
376 377 (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \
377 378 (testaddr) + (testsz) >= (testaddr))
378 379
379 380 /*
380 381 * Test whether alloc_sz bytes will fit in the scratch region. We isolate
381 382 * alloc_sz on the righthand side of the comparison in order to avoid overflow
382 383 * or underflow in the comparison with it. This is simpler than the INRANGE
383 384 * check above, because we know that the dtms_scratch_ptr is valid in the
384 385 * range. Allocations of size zero are allowed.
385 386 */
386 387 #define DTRACE_INSCRATCH(mstate, alloc_sz) \
387 388 ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \
388 389 (mstate)->dtms_scratch_ptr >= (alloc_sz))
389 390
390 391 #define DTRACE_LOADFUNC(bits) \
391 392 /*CSTYLED*/ \
392 393 uint##bits##_t \
393 394 dtrace_load##bits(uintptr_t addr) \
394 395 { \
395 396 size_t size = bits / NBBY; \
396 397 /*CSTYLED*/ \
397 398 uint##bits##_t rval; \
398 399 int i; \
399 400 volatile uint16_t *flags = (volatile uint16_t *) \
400 401 &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; \
401 402 \
402 403 DTRACE_ALIGNCHECK(addr, size, flags); \
403 404 \
404 405 for (i = 0; i < dtrace_toxranges; i++) { \
405 406 if (addr >= dtrace_toxrange[i].dtt_limit) \
406 407 continue; \
407 408 \
408 409 if (addr + size <= dtrace_toxrange[i].dtt_base) \
409 410 continue; \
410 411 \
411 412 /* \
412 413 * This address falls within a toxic region; return 0. \
413 414 */ \
414 415 *flags |= CPU_DTRACE_BADADDR; \
415 416 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \
416 417 return (0); \
417 418 } \
418 419 \
419 420 *flags |= CPU_DTRACE_NOFAULT; \
420 421 /*CSTYLED*/ \
421 422 rval = *((volatile uint##bits##_t *)addr); \
422 423 *flags &= ~CPU_DTRACE_NOFAULT; \
423 424 \
424 425 return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \
425 426 }
426 427
427 428 #ifdef _LP64
428 429 #define dtrace_loadptr dtrace_load64
429 430 #else
430 431 #define dtrace_loadptr dtrace_load32
431 432 #endif
432 433
433 434 #define DTRACE_DYNHASH_FREE 0
434 435 #define DTRACE_DYNHASH_SINK 1
435 436 #define DTRACE_DYNHASH_VALID 2
436 437
437 438 #define DTRACE_MATCH_FAIL -1
438 439 #define DTRACE_MATCH_NEXT 0
439 440 #define DTRACE_MATCH_DONE 1
440 441 #define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0')
441 442 #define DTRACE_STATE_ALIGN 64
442 443
443 444 #define DTRACE_FLAGS2FLT(flags) \
444 445 (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \
445 446 ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \
446 447 ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \
447 448 ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \
448 449 ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \
449 450 ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \
450 451 ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \
451 452 ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \
452 453 ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \
453 454 DTRACEFLT_UNKNOWN)
454 455
455 456 #define DTRACEACT_ISSTRING(act) \
456 457 ((act)->dta_kind == DTRACEACT_DIFEXPR && \
457 458 (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
458 459
459 460 static size_t dtrace_strlen(const char *, size_t);
460 461 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
461 462 static void dtrace_enabling_provide(dtrace_provider_t *);
462 463 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
463 464 static void dtrace_enabling_matchall(void);
464 465 static void dtrace_enabling_reap(void);
465 466 static dtrace_state_t *dtrace_anon_grab(void);
466 467 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
467 468 dtrace_state_t *, uint64_t, uint64_t);
|
↓ open down ↓ |
285 lines elided |
↑ open up ↑ |
468 469 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
469 470 static void dtrace_buffer_drop(dtrace_buffer_t *);
470 471 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when);
471 472 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
472 473 dtrace_state_t *, dtrace_mstate_t *);
473 474 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
474 475 dtrace_optval_t);
475 476 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
476 477 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
477 478 static int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *);
479 +static void dtrace_getf_barrier(void);
478 480
479 481 /*
480 482 * DTrace Probe Context Functions
481 483 *
482 484 * These functions are called from probe context. Because probe context is
483 485 * any context in which C may be called, arbitrarily locks may be held,
484 486 * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
485 487 * As a result, functions called from probe context may only call other DTrace
486 488 * support functions -- they may not interact at all with the system at large.
487 489 * (Note that the ASSERT macro is made probe-context safe by redefining it in
488 490 * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
489 491 * loads are to be performed from probe context, they _must_ be in terms of
490 492 * the safe dtrace_load*() variants.
491 493 *
492 494 * Some functions in this block are not actually called from probe context;
493 495 * for these functions, there will be a comment above the function reading
494 496 * "Note: not called from probe context."
495 497 */
496 498 void
497 499 dtrace_panic(const char *format, ...)
498 500 {
499 501 va_list alist;
500 502
501 503 va_start(alist, format);
502 504 dtrace_vpanic(format, alist);
503 505 va_end(alist);
504 506 }
505 507
506 508 int
507 509 dtrace_assfail(const char *a, const char *f, int l)
508 510 {
509 511 dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
510 512
511 513 /*
512 514 * We just need something here that even the most clever compiler
513 515 * cannot optimize away.
514 516 */
515 517 return (a[(uintptr_t)f]);
516 518 }
517 519
518 520 /*
519 521 * Atomically increment a specified error counter from probe context.
520 522 */
521 523 static void
522 524 dtrace_error(uint32_t *counter)
523 525 {
524 526 /*
525 527 * Most counters stored to in probe context are per-CPU counters.
526 528 * However, there are some error conditions that are sufficiently
527 529 * arcane that they don't merit per-CPU storage. If these counters
528 530 * are incremented concurrently on different CPUs, scalability will be
529 531 * adversely affected -- but we don't expect them to be white-hot in a
530 532 * correctly constructed enabling...
531 533 */
532 534 uint32_t oval, nval;
533 535
534 536 do {
535 537 oval = *counter;
536 538
537 539 if ((nval = oval + 1) == 0) {
538 540 /*
539 541 * If the counter would wrap, set it to 1 -- assuring
540 542 * that the counter is never zero when we have seen
541 543 * errors. (The counter must be 32-bits because we
542 544 * aren't guaranteed a 64-bit compare&swap operation.)
543 545 * To save this code both the infamy of being fingered
544 546 * by a priggish news story and the indignity of being
545 547 * the target of a neo-puritan witch trial, we're
546 548 * carefully avoiding any colorful description of the
547 549 * likelihood of this condition -- but suffice it to
548 550 * say that it is only slightly more likely than the
549 551 * overflow of predicate cache IDs, as discussed in
550 552 * dtrace_predicate_create().
551 553 */
552 554 nval = 1;
553 555 }
554 556 } while (dtrace_cas32(counter, oval, nval) != oval);
555 557 }
556 558
557 559 /*
558 560 * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
559 561 * uint8_t, a uint16_t, a uint32_t and a uint64_t.
560 562 */
561 563 DTRACE_LOADFUNC(8)
562 564 DTRACE_LOADFUNC(16)
563 565 DTRACE_LOADFUNC(32)
564 566 DTRACE_LOADFUNC(64)
565 567
566 568 static int
567 569 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
568 570 {
569 571 if (dest < mstate->dtms_scratch_base)
570 572 return (0);
571 573
572 574 if (dest + size < dest)
573 575 return (0);
574 576
575 577 if (dest + size > mstate->dtms_scratch_ptr)
576 578 return (0);
577 579
578 580 return (1);
579 581 }
580 582
581 583 static int
582 584 dtrace_canstore_statvar(uint64_t addr, size_t sz,
583 585 dtrace_statvar_t **svars, int nsvars)
584 586 {
585 587 int i;
586 588
587 589 for (i = 0; i < nsvars; i++) {
588 590 dtrace_statvar_t *svar = svars[i];
589 591
590 592 if (svar == NULL || svar->dtsv_size == 0)
591 593 continue;
592 594
593 595 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
594 596 return (1);
595 597 }
596 598
597 599 return (0);
598 600 }
599 601
600 602 /*
601 603 * Check to see if the address is within a memory region to which a store may
602 604 * be issued. This includes the DTrace scratch areas, and any DTrace variable
603 605 * region. The caller of dtrace_canstore() is responsible for performing any
604 606 * alignment checks that are needed before stores are actually executed.
605 607 */
606 608 static int
607 609 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
608 610 dtrace_vstate_t *vstate)
609 611 {
610 612 /*
611 613 * First, check to see if the address is in scratch space...
612 614 */
613 615 if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base,
614 616 mstate->dtms_scratch_size))
615 617 return (1);
616 618
617 619 /*
618 620 * Now check to see if it's a dynamic variable. This check will pick
619 621 * up both thread-local variables and any global dynamically-allocated
620 622 * variables.
621 623 */
622 624 if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base,
623 625 vstate->dtvs_dynvars.dtds_size)) {
624 626 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
625 627 uintptr_t base = (uintptr_t)dstate->dtds_base +
626 628 (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t));
627 629 uintptr_t chunkoffs;
628 630
629 631 /*
630 632 * Before we assume that we can store here, we need to make
631 633 * sure that it isn't in our metadata -- storing to our
632 634 * dynamic variable metadata would corrupt our state. For
633 635 * the range to not include any dynamic variable metadata,
634 636 * it must:
635 637 *
636 638 * (1) Start above the hash table that is at the base of
637 639 * the dynamic variable space
638 640 *
639 641 * (2) Have a starting chunk offset that is beyond the
640 642 * dtrace_dynvar_t that is at the base of every chunk
641 643 *
642 644 * (3) Not span a chunk boundary
643 645 *
644 646 */
645 647 if (addr < base)
646 648 return (0);
647 649
648 650 chunkoffs = (addr - base) % dstate->dtds_chunksize;
649 651
650 652 if (chunkoffs < sizeof (dtrace_dynvar_t))
651 653 return (0);
652 654
653 655 if (chunkoffs + sz > dstate->dtds_chunksize)
654 656 return (0);
655 657
656 658 return (1);
657 659 }
658 660
659 661 /*
660 662 * Finally, check the static local and global variables. These checks
661 663 * take the longest, so we perform them last.
662 664 */
663 665 if (dtrace_canstore_statvar(addr, sz,
664 666 vstate->dtvs_locals, vstate->dtvs_nlocals))
665 667 return (1);
666 668
667 669 if (dtrace_canstore_statvar(addr, sz,
668 670 vstate->dtvs_globals, vstate->dtvs_nglobals))
669 671 return (1);
670 672
671 673 return (0);
672 674 }
673 675
674 676
675 677 /*
676 678 * Convenience routine to check to see if the address is within a memory
677 679 * region in which a load may be issued given the user's privilege level;
678 680 * if not, it sets the appropriate error flags and loads 'addr' into the
|
↓ open down ↓ |
191 lines elided |
↑ open up ↑ |
679 681 * illegal value slot.
680 682 *
681 683 * DTrace subroutines (DIF_SUBR_*) should use this helper to implement
682 684 * appropriate memory access protection.
683 685 */
684 686 static int
685 687 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
686 688 dtrace_vstate_t *vstate)
687 689 {
688 690 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
691 + file_t *fp;
689 692
690 693 /*
691 694 * If we hold the privilege to read from kernel memory, then
692 695 * everything is readable.
693 696 */
694 697 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
695 698 return (1);
696 699
697 700 /*
698 701 * You can obviously read that which you can store.
699 702 */
700 703 if (dtrace_canstore(addr, sz, mstate, vstate))
701 704 return (1);
702 705
703 706 /*
704 707 * We're allowed to read from our own string table.
705 708 */
706 709 if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab,
707 710 mstate->dtms_difo->dtdo_strlen))
708 711 return (1);
709 712
710 713 if (vstate->dtvs_state != NULL &&
711 714 dtrace_priv_proc(vstate->dtvs_state, mstate)) {
712 715 proc_t *p;
713 716
714 717 /*
715 718 * When we have privileges to the current process, there are
716 719 * several context-related kernel structures that are safe to
717 720 * read, even absent the privilege to read from kernel memory.
718 721 * These reads are safe because these structures contain only
719 722 * state that (1) we're permitted to read, (2) is harmless or
720 723 * (3) contains pointers to additional kernel state that we're
721 724 * not permitted to read (and as such, do not present an
722 725 * opportunity for privilege escalation). Finally (and
723 726 * critically), because of the nature of their relation with
724 727 * the current thread context, the memory associated with these
725 728 * structures cannot change over the duration of probe context,
726 729 * and it is therefore impossible for this memory to be
727 730 * deallocated and reallocated as something else while it's
728 731 * being operated upon.
729 732 */
730 733 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t)))
731 734 return (1);
732 735
733 736 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr,
734 737 sz, curthread->t_procp, sizeof (proc_t))) {
735 738 return (1);
736 739 }
737 740
738 741 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz,
739 742 curthread->t_cred, sizeof (cred_t))) {
740 743 return (1);
741 744 }
742 745
743 746 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz,
|
↓ open down ↓ |
45 lines elided |
↑ open up ↑ |
744 747 &(p->p_pidp->pid_id), sizeof (pid_t))) {
745 748 return (1);
746 749 }
747 750
748 751 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz,
749 752 curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) {
750 753 return (1);
751 754 }
752 755 }
753 756
757 + if ((fp = mstate->dtms_getf) != NULL) {
758 + uintptr_t psz = sizeof (void *);
759 + vnode_t *vp;
760 + vnodeops_t *op;
761 +
762 + /*
763 + * When getf() returns a file_t, the enabling is implicitly
764 + * granted the (transient) right to read the returned file_t
765 + * as well as the v_path and v_op->vnop_name of the underlying
766 + * vnode. These accesses are allowed after a successful
767 + * getf() because the members that they refer to cannot change
768 + * once set -- and the barrier logic in the kernel's closef()
769 + * path assures that the file_t and its referenced vode_t
770 + * cannot themselves be stale (that is, it impossible for
771 + * either dtms_getf itself or its f_vnode member to reference
772 + * freed memory).
773 + */
774 + if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t)))
775 + return (1);
776 +
777 + if ((vp = fp->f_vnode) != NULL) {
778 + if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz))
779 + return (1);
780 +
781 + if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz,
782 + vp->v_path, strlen(vp->v_path) + 1)) {
783 + return (1);
784 + }
785 +
786 + if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz))
787 + return (1);
788 +
789 + if ((op = vp->v_op) != NULL &&
790 + DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) {
791 + return (1);
792 + }
793 +
794 + if (op != NULL && op->vnop_name != NULL &&
795 + DTRACE_INRANGE(addr, sz, op->vnop_name,
796 + strlen(op->vnop_name) + 1)) {
797 + return (1);
798 + }
799 + }
800 + }
801 +
754 802 DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV);
755 803 *illval = addr;
756 804 return (0);
757 805 }
758 806
759 807 /*
760 808 * Convenience routine to check to see if a given string is within a memory
761 809 * region in which a load may be issued given the user's privilege level;
762 810 * this exists so that we don't need to issue unnecessary dtrace_strlen()
763 811 * calls in the event that the user has all privileges.
764 812 */
765 813 static int
766 814 dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
767 815 dtrace_vstate_t *vstate)
768 816 {
769 817 size_t strsz;
770 818
771 819 /*
772 820 * If we hold the privilege to read from kernel memory, then
773 821 * everything is readable.
774 822 */
775 823 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
776 824 return (1);
777 825
778 826 strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz);
779 827 if (dtrace_canload(addr, strsz, mstate, vstate))
780 828 return (1);
781 829
782 830 return (0);
783 831 }
784 832
785 833 /*
786 834 * Convenience routine to check to see if a given variable is within a memory
787 835 * region in which a load may be issued given the user's privilege level.
788 836 */
789 837 static int
790 838 dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate,
791 839 dtrace_vstate_t *vstate)
792 840 {
793 841 size_t sz;
794 842 ASSERT(type->dtdt_flags & DIF_TF_BYREF);
795 843
796 844 /*
797 845 * If we hold the privilege to read from kernel memory, then
798 846 * everything is readable.
799 847 */
800 848 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
801 849 return (1);
802 850
803 851 if (type->dtdt_kind == DIF_TYPE_STRING)
804 852 sz = dtrace_strlen(src,
805 853 vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1;
806 854 else
807 855 sz = type->dtdt_size;
808 856
809 857 return (dtrace_canload((uintptr_t)src, sz, mstate, vstate));
810 858 }
811 859
812 860 /*
813 861 * Compare two strings using safe loads.
814 862 */
815 863 static int
816 864 dtrace_strncmp(char *s1, char *s2, size_t limit)
817 865 {
818 866 uint8_t c1, c2;
819 867 volatile uint16_t *flags;
820 868
821 869 if (s1 == s2 || limit == 0)
822 870 return (0);
823 871
824 872 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
825 873
826 874 do {
827 875 if (s1 == NULL) {
828 876 c1 = '\0';
829 877 } else {
830 878 c1 = dtrace_load8((uintptr_t)s1++);
831 879 }
832 880
833 881 if (s2 == NULL) {
834 882 c2 = '\0';
835 883 } else {
836 884 c2 = dtrace_load8((uintptr_t)s2++);
837 885 }
838 886
839 887 if (c1 != c2)
840 888 return (c1 - c2);
841 889 } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
842 890
843 891 return (0);
844 892 }
845 893
846 894 /*
847 895 * Compute strlen(s) for a string using safe memory accesses. The additional
848 896 * len parameter is used to specify a maximum length to ensure completion.
849 897 */
850 898 static size_t
851 899 dtrace_strlen(const char *s, size_t lim)
852 900 {
853 901 uint_t len;
854 902
855 903 for (len = 0; len != lim; len++) {
856 904 if (dtrace_load8((uintptr_t)s++) == '\0')
857 905 break;
858 906 }
859 907
860 908 return (len);
861 909 }
862 910
863 911 /*
864 912 * Check if an address falls within a toxic region.
865 913 */
866 914 static int
867 915 dtrace_istoxic(uintptr_t kaddr, size_t size)
868 916 {
869 917 uintptr_t taddr, tsize;
870 918 int i;
871 919
872 920 for (i = 0; i < dtrace_toxranges; i++) {
873 921 taddr = dtrace_toxrange[i].dtt_base;
874 922 tsize = dtrace_toxrange[i].dtt_limit - taddr;
875 923
876 924 if (kaddr - taddr < tsize) {
877 925 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
878 926 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
879 927 return (1);
880 928 }
881 929
882 930 if (taddr - kaddr < size) {
883 931 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
884 932 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
885 933 return (1);
886 934 }
887 935 }
888 936
889 937 return (0);
890 938 }
891 939
892 940 /*
893 941 * Copy src to dst using safe memory accesses. The src is assumed to be unsafe
894 942 * memory specified by the DIF program. The dst is assumed to be safe memory
895 943 * that we can store to directly because it is managed by DTrace. As with
896 944 * standard bcopy, overlapping copies are handled properly.
897 945 */
898 946 static void
899 947 dtrace_bcopy(const void *src, void *dst, size_t len)
900 948 {
901 949 if (len != 0) {
902 950 uint8_t *s1 = dst;
903 951 const uint8_t *s2 = src;
904 952
905 953 if (s1 <= s2) {
906 954 do {
907 955 *s1++ = dtrace_load8((uintptr_t)s2++);
908 956 } while (--len != 0);
909 957 } else {
910 958 s2 += len;
911 959 s1 += len;
912 960
913 961 do {
914 962 *--s1 = dtrace_load8((uintptr_t)--s2);
915 963 } while (--len != 0);
916 964 }
917 965 }
918 966 }
919 967
920 968 /*
921 969 * Copy src to dst using safe memory accesses, up to either the specified
922 970 * length, or the point that a nul byte is encountered. The src is assumed to
923 971 * be unsafe memory specified by the DIF program. The dst is assumed to be
924 972 * safe memory that we can store to directly because it is managed by DTrace.
925 973 * Unlike dtrace_bcopy(), overlapping regions are not handled.
926 974 */
927 975 static void
928 976 dtrace_strcpy(const void *src, void *dst, size_t len)
929 977 {
930 978 if (len != 0) {
931 979 uint8_t *s1 = dst, c;
932 980 const uint8_t *s2 = src;
933 981
934 982 do {
935 983 *s1++ = c = dtrace_load8((uintptr_t)s2++);
936 984 } while (--len != 0 && c != '\0');
937 985 }
938 986 }
939 987
940 988 /*
941 989 * Copy src to dst, deriving the size and type from the specified (BYREF)
942 990 * variable type. The src is assumed to be unsafe memory specified by the DIF
943 991 * program. The dst is assumed to be DTrace variable memory that is of the
944 992 * specified type; we assume that we can store to directly.
945 993 */
946 994 static void
947 995 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
948 996 {
949 997 ASSERT(type->dtdt_flags & DIF_TF_BYREF);
950 998
951 999 if (type->dtdt_kind == DIF_TYPE_STRING) {
952 1000 dtrace_strcpy(src, dst, type->dtdt_size);
953 1001 } else {
954 1002 dtrace_bcopy(src, dst, type->dtdt_size);
955 1003 }
956 1004 }
957 1005
958 1006 /*
959 1007 * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be
960 1008 * unsafe memory specified by the DIF program. The s2 data is assumed to be
961 1009 * safe memory that we can access directly because it is managed by DTrace.
962 1010 */
963 1011 static int
964 1012 dtrace_bcmp(const void *s1, const void *s2, size_t len)
965 1013 {
966 1014 volatile uint16_t *flags;
967 1015
968 1016 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
969 1017
970 1018 if (s1 == s2)
971 1019 return (0);
972 1020
973 1021 if (s1 == NULL || s2 == NULL)
974 1022 return (1);
975 1023
976 1024 if (s1 != s2 && len != 0) {
977 1025 const uint8_t *ps1 = s1;
978 1026 const uint8_t *ps2 = s2;
979 1027
980 1028 do {
981 1029 if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
982 1030 return (1);
983 1031 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
984 1032 }
985 1033 return (0);
986 1034 }
987 1035
988 1036 /*
989 1037 * Zero the specified region using a simple byte-by-byte loop. Note that this
990 1038 * is for safe DTrace-managed memory only.
991 1039 */
992 1040 static void
993 1041 dtrace_bzero(void *dst, size_t len)
994 1042 {
995 1043 uchar_t *cp;
996 1044
997 1045 for (cp = dst; len != 0; len--)
998 1046 *cp++ = 0;
999 1047 }
1000 1048
1001 1049 static void
1002 1050 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
1003 1051 {
1004 1052 uint64_t result[2];
1005 1053
1006 1054 result[0] = addend1[0] + addend2[0];
1007 1055 result[1] = addend1[1] + addend2[1] +
1008 1056 (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
1009 1057
1010 1058 sum[0] = result[0];
1011 1059 sum[1] = result[1];
1012 1060 }
1013 1061
1014 1062 /*
1015 1063 * Shift the 128-bit value in a by b. If b is positive, shift left.
1016 1064 * If b is negative, shift right.
1017 1065 */
1018 1066 static void
1019 1067 dtrace_shift_128(uint64_t *a, int b)
1020 1068 {
1021 1069 uint64_t mask;
1022 1070
1023 1071 if (b == 0)
1024 1072 return;
1025 1073
1026 1074 if (b < 0) {
1027 1075 b = -b;
1028 1076 if (b >= 64) {
1029 1077 a[0] = a[1] >> (b - 64);
1030 1078 a[1] = 0;
1031 1079 } else {
1032 1080 a[0] >>= b;
1033 1081 mask = 1LL << (64 - b);
1034 1082 mask -= 1;
1035 1083 a[0] |= ((a[1] & mask) << (64 - b));
1036 1084 a[1] >>= b;
1037 1085 }
1038 1086 } else {
1039 1087 if (b >= 64) {
1040 1088 a[1] = a[0] << (b - 64);
1041 1089 a[0] = 0;
1042 1090 } else {
1043 1091 a[1] <<= b;
1044 1092 mask = a[0] >> (64 - b);
1045 1093 a[1] |= mask;
1046 1094 a[0] <<= b;
1047 1095 }
1048 1096 }
1049 1097 }
1050 1098
1051 1099 /*
1052 1100 * The basic idea is to break the 2 64-bit values into 4 32-bit values,
1053 1101 * use native multiplication on those, and then re-combine into the
1054 1102 * resulting 128-bit value.
1055 1103 *
1056 1104 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
1057 1105 * hi1 * hi2 << 64 +
1058 1106 * hi1 * lo2 << 32 +
1059 1107 * hi2 * lo1 << 32 +
1060 1108 * lo1 * lo2
1061 1109 */
1062 1110 static void
1063 1111 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
1064 1112 {
1065 1113 uint64_t hi1, hi2, lo1, lo2;
1066 1114 uint64_t tmp[2];
1067 1115
1068 1116 hi1 = factor1 >> 32;
1069 1117 hi2 = factor2 >> 32;
1070 1118
1071 1119 lo1 = factor1 & DT_MASK_LO;
1072 1120 lo2 = factor2 & DT_MASK_LO;
1073 1121
1074 1122 product[0] = lo1 * lo2;
1075 1123 product[1] = hi1 * hi2;
1076 1124
1077 1125 tmp[0] = hi1 * lo2;
1078 1126 tmp[1] = 0;
1079 1127 dtrace_shift_128(tmp, 32);
1080 1128 dtrace_add_128(product, tmp, product);
1081 1129
1082 1130 tmp[0] = hi2 * lo1;
1083 1131 tmp[1] = 0;
1084 1132 dtrace_shift_128(tmp, 32);
1085 1133 dtrace_add_128(product, tmp, product);
1086 1134 }
1087 1135
1088 1136 /*
1089 1137 * This privilege check should be used by actions and subroutines to
1090 1138 * verify that the user credentials of the process that enabled the
1091 1139 * invoking ECB match the target credentials
1092 1140 */
1093 1141 static int
1094 1142 dtrace_priv_proc_common_user(dtrace_state_t *state)
1095 1143 {
1096 1144 cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1097 1145
1098 1146 /*
1099 1147 * We should always have a non-NULL state cred here, since if cred
1100 1148 * is null (anonymous tracing), we fast-path bypass this routine.
1101 1149 */
1102 1150 ASSERT(s_cr != NULL);
1103 1151
1104 1152 if ((cr = CRED()) != NULL &&
1105 1153 s_cr->cr_uid == cr->cr_uid &&
1106 1154 s_cr->cr_uid == cr->cr_ruid &&
1107 1155 s_cr->cr_uid == cr->cr_suid &&
1108 1156 s_cr->cr_gid == cr->cr_gid &&
1109 1157 s_cr->cr_gid == cr->cr_rgid &&
1110 1158 s_cr->cr_gid == cr->cr_sgid)
1111 1159 return (1);
1112 1160
1113 1161 return (0);
1114 1162 }
1115 1163
1116 1164 /*
1117 1165 * This privilege check should be used by actions and subroutines to
1118 1166 * verify that the zone of the process that enabled the invoking ECB
1119 1167 * matches the target credentials
1120 1168 */
1121 1169 static int
|
↓ open down ↓ |
358 lines elided |
↑ open up ↑ |
1122 1170 dtrace_priv_proc_common_zone(dtrace_state_t *state)
1123 1171 {
1124 1172 cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
1125 1173
1126 1174 /*
1127 1175 * We should always have a non-NULL state cred here, since if cred
1128 1176 * is null (anonymous tracing), we fast-path bypass this routine.
1129 1177 */
1130 1178 ASSERT(s_cr != NULL);
1131 1179
1132 - if ((cr = CRED()) != NULL &&
1133 - s_cr->cr_zone == cr->cr_zone)
1180 + if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone)
1134 1181 return (1);
1135 1182
1136 1183 return (0);
1137 1184 }
1138 1185
1139 1186 /*
1140 1187 * This privilege check should be used by actions and subroutines to
1141 1188 * verify that the process has not setuid or changed credentials.
1142 1189 */
1143 1190 static int
1144 1191 dtrace_priv_proc_common_nocd()
1145 1192 {
1146 1193 proc_t *proc;
1147 1194
1148 1195 if ((proc = ttoproc(curthread)) != NULL &&
1149 1196 !(proc->p_flag & SNOCD))
1150 1197 return (1);
1151 1198
1152 1199 return (0);
1153 1200 }
1154 1201
1155 1202 static int
1156 1203 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate)
1157 1204 {
1158 1205 int action = state->dts_cred.dcr_action;
1159 1206
1160 1207 if (!(mstate->dtms_access & DTRACE_ACCESS_PROC))
1161 1208 goto bad;
1162 1209
1163 1210 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
1164 1211 dtrace_priv_proc_common_zone(state) == 0)
1165 1212 goto bad;
1166 1213
1167 1214 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
1168 1215 dtrace_priv_proc_common_user(state) == 0)
1169 1216 goto bad;
1170 1217
1171 1218 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
1172 1219 dtrace_priv_proc_common_nocd() == 0)
1173 1220 goto bad;
1174 1221
1175 1222 return (1);
1176 1223
1177 1224 bad:
1178 1225 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1179 1226
1180 1227 return (0);
1181 1228 }
1182 1229
1183 1230 static int
1184 1231 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate)
1185 1232 {
1186 1233 if (mstate->dtms_access & DTRACE_ACCESS_PROC) {
1187 1234 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
1188 1235 return (1);
1189 1236
1190 1237 if (dtrace_priv_proc_common_zone(state) &&
1191 1238 dtrace_priv_proc_common_user(state) &&
1192 1239 dtrace_priv_proc_common_nocd())
1193 1240 return (1);
1194 1241 }
1195 1242
1196 1243 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1197 1244
1198 1245 return (0);
1199 1246 }
1200 1247
1201 1248 static int
1202 1249 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate)
1203 1250 {
1204 1251 if ((mstate->dtms_access & DTRACE_ACCESS_PROC) &&
1205 1252 (state->dts_cred.dcr_action & DTRACE_CRA_PROC))
1206 1253 return (1);
1207 1254
1208 1255 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
1209 1256
1210 1257 return (0);
1211 1258 }
1212 1259
1213 1260 static int
1214 1261 dtrace_priv_kernel(dtrace_state_t *state)
1215 1262 {
1216 1263 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
1217 1264 return (1);
1218 1265
1219 1266 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1220 1267
1221 1268 return (0);
1222 1269 }
1223 1270
1224 1271 static int
1225 1272 dtrace_priv_kernel_destructive(dtrace_state_t *state)
1226 1273 {
1227 1274 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
1228 1275 return (1);
1229 1276
1230 1277 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
1231 1278
1232 1279 return (0);
1233 1280 }
1234 1281
1235 1282 /*
1236 1283 * Determine if the dte_cond of the specified ECB allows for processing of
1237 1284 * the current probe to continue. Note that this routine may allow continued
1238 1285 * processing, but with access(es) stripped from the mstate's dtms_access
1239 1286 * field.
1240 1287 */
1241 1288 static int
1242 1289 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate,
1243 1290 dtrace_ecb_t *ecb)
1244 1291 {
1245 1292 dtrace_probe_t *probe = ecb->dte_probe;
1246 1293 dtrace_provider_t *prov = probe->dtpr_provider;
1247 1294 dtrace_pops_t *pops = &prov->dtpv_pops;
1248 1295 int mode = DTRACE_MODE_NOPRIV_DROP;
1249 1296
1250 1297 ASSERT(ecb->dte_cond);
1251 1298
1252 1299 if (pops->dtps_mode != NULL) {
1253 1300 mode = pops->dtps_mode(prov->dtpv_arg,
1254 1301 probe->dtpr_id, probe->dtpr_arg);
1255 1302
1256 1303 ASSERT((mode & DTRACE_MODE_USER) ||
1257 1304 (mode & DTRACE_MODE_KERNEL));
1258 1305 ASSERT((mode & DTRACE_MODE_NOPRIV_RESTRICT) ||
1259 1306 (mode & DTRACE_MODE_NOPRIV_DROP));
1260 1307 }
1261 1308
1262 1309 /*
1263 1310 * If the dte_cond bits indicate that this consumer is only allowed to
1264 1311 * see user-mode firings of this probe, call the provider's dtps_mode()
1265 1312 * entry point to check that the probe was fired while in a user
1266 1313 * context. If that's not the case, use the policy specified by the
1267 1314 * provider to determine if we drop the probe or merely restrict
1268 1315 * operation.
1269 1316 */
1270 1317 if (ecb->dte_cond & DTRACE_COND_USERMODE) {
1271 1318 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP);
1272 1319
1273 1320 if (!(mode & DTRACE_MODE_USER)) {
1274 1321 if (mode & DTRACE_MODE_NOPRIV_DROP)
1275 1322 return (0);
1276 1323
1277 1324 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS;
1278 1325 }
1279 1326 }
1280 1327
1281 1328 /*
1282 1329 * This is more subtle than it looks. We have to be absolutely certain
1283 1330 * that CRED() isn't going to change out from under us so it's only
1284 1331 * legit to examine that structure if we're in constrained situations.
1285 1332 * Currently, the only times we'll this check is if a non-super-user
1286 1333 * has enabled the profile or syscall providers -- providers that
1287 1334 * allow visibility of all processes. For the profile case, the check
1288 1335 * above will ensure that we're examining a user context.
1289 1336 */
1290 1337 if (ecb->dte_cond & DTRACE_COND_OWNER) {
1291 1338 cred_t *cr;
1292 1339 cred_t *s_cr = state->dts_cred.dcr_cred;
1293 1340 proc_t *proc;
1294 1341
1295 1342 ASSERT(s_cr != NULL);
1296 1343
1297 1344 if ((cr = CRED()) == NULL ||
1298 1345 s_cr->cr_uid != cr->cr_uid ||
1299 1346 s_cr->cr_uid != cr->cr_ruid ||
1300 1347 s_cr->cr_uid != cr->cr_suid ||
1301 1348 s_cr->cr_gid != cr->cr_gid ||
1302 1349 s_cr->cr_gid != cr->cr_rgid ||
1303 1350 s_cr->cr_gid != cr->cr_sgid ||
1304 1351 (proc = ttoproc(curthread)) == NULL ||
1305 1352 (proc->p_flag & SNOCD)) {
1306 1353 if (mode & DTRACE_MODE_NOPRIV_DROP)
1307 1354 return (0);
1308 1355
1309 1356 mstate->dtms_access &= ~DTRACE_ACCESS_PROC;
1310 1357 }
1311 1358 }
1312 1359
1313 1360 /*
1314 1361 * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not
1315 1362 * in our zone, check to see if our mode policy is to restrict rather
1316 1363 * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC
1317 1364 * and DTRACE_ACCESS_ARGS
1318 1365 */
1319 1366 if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
1320 1367 cred_t *cr;
1321 1368 cred_t *s_cr = state->dts_cred.dcr_cred;
1322 1369
1323 1370 ASSERT(s_cr != NULL);
1324 1371
1325 1372 if ((cr = CRED()) == NULL ||
1326 1373 s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) {
1327 1374 if (mode & DTRACE_MODE_NOPRIV_DROP)
1328 1375 return (0);
1329 1376
1330 1377 mstate->dtms_access &=
1331 1378 ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS);
1332 1379 }
1333 1380 }
1334 1381
1335 1382 return (1);
1336 1383 }
1337 1384
1338 1385 /*
1339 1386 * Note: not called from probe context. This function is called
1340 1387 * asynchronously (and at a regular interval) from outside of probe context to
1341 1388 * clean the dirty dynamic variable lists on all CPUs. Dynamic variable
1342 1389 * cleaning is explained in detail in <sys/dtrace_impl.h>.
1343 1390 */
1344 1391 void
1345 1392 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
1346 1393 {
1347 1394 dtrace_dynvar_t *dirty;
1348 1395 dtrace_dstate_percpu_t *dcpu;
1349 1396 dtrace_dynvar_t **rinsep;
1350 1397 int i, j, work = 0;
1351 1398
1352 1399 for (i = 0; i < NCPU; i++) {
1353 1400 dcpu = &dstate->dtds_percpu[i];
1354 1401 rinsep = &dcpu->dtdsc_rinsing;
1355 1402
1356 1403 /*
1357 1404 * If the dirty list is NULL, there is no dirty work to do.
1358 1405 */
1359 1406 if (dcpu->dtdsc_dirty == NULL)
1360 1407 continue;
1361 1408
1362 1409 if (dcpu->dtdsc_rinsing != NULL) {
1363 1410 /*
1364 1411 * If the rinsing list is non-NULL, then it is because
1365 1412 * this CPU was selected to accept another CPU's
1366 1413 * dirty list -- and since that time, dirty buffers
1367 1414 * have accumulated. This is a highly unlikely
1368 1415 * condition, but we choose to ignore the dirty
1369 1416 * buffers -- they'll be picked up a future cleanse.
1370 1417 */
1371 1418 continue;
1372 1419 }
1373 1420
1374 1421 if (dcpu->dtdsc_clean != NULL) {
1375 1422 /*
1376 1423 * If the clean list is non-NULL, then we're in a
1377 1424 * situation where a CPU has done deallocations (we
1378 1425 * have a non-NULL dirty list) but no allocations (we
1379 1426 * also have a non-NULL clean list). We can't simply
1380 1427 * move the dirty list into the clean list on this
1381 1428 * CPU, yet we also don't want to allow this condition
1382 1429 * to persist, lest a short clean list prevent a
1383 1430 * massive dirty list from being cleaned (which in
1384 1431 * turn could lead to otherwise avoidable dynamic
1385 1432 * drops). To deal with this, we look for some CPU
1386 1433 * with a NULL clean list, NULL dirty list, and NULL
1387 1434 * rinsing list -- and then we borrow this CPU to
1388 1435 * rinse our dirty list.
1389 1436 */
1390 1437 for (j = 0; j < NCPU; j++) {
1391 1438 dtrace_dstate_percpu_t *rinser;
1392 1439
1393 1440 rinser = &dstate->dtds_percpu[j];
1394 1441
1395 1442 if (rinser->dtdsc_rinsing != NULL)
1396 1443 continue;
1397 1444
1398 1445 if (rinser->dtdsc_dirty != NULL)
1399 1446 continue;
1400 1447
1401 1448 if (rinser->dtdsc_clean != NULL)
1402 1449 continue;
1403 1450
1404 1451 rinsep = &rinser->dtdsc_rinsing;
1405 1452 break;
1406 1453 }
1407 1454
1408 1455 if (j == NCPU) {
1409 1456 /*
1410 1457 * We were unable to find another CPU that
1411 1458 * could accept this dirty list -- we are
1412 1459 * therefore unable to clean it now.
1413 1460 */
1414 1461 dtrace_dynvar_failclean++;
1415 1462 continue;
1416 1463 }
1417 1464 }
1418 1465
1419 1466 work = 1;
1420 1467
1421 1468 /*
1422 1469 * Atomically move the dirty list aside.
1423 1470 */
1424 1471 do {
1425 1472 dirty = dcpu->dtdsc_dirty;
1426 1473
1427 1474 /*
1428 1475 * Before we zap the dirty list, set the rinsing list.
1429 1476 * (This allows for a potential assertion in
1430 1477 * dtrace_dynvar(): if a free dynamic variable appears
1431 1478 * on a hash chain, either the dirty list or the
1432 1479 * rinsing list for some CPU must be non-NULL.)
1433 1480 */
1434 1481 *rinsep = dirty;
1435 1482 dtrace_membar_producer();
1436 1483 } while (dtrace_casptr(&dcpu->dtdsc_dirty,
1437 1484 dirty, NULL) != dirty);
1438 1485 }
1439 1486
1440 1487 if (!work) {
1441 1488 /*
1442 1489 * We have no work to do; we can simply return.
1443 1490 */
1444 1491 return;
1445 1492 }
1446 1493
1447 1494 dtrace_sync();
1448 1495
1449 1496 for (i = 0; i < NCPU; i++) {
1450 1497 dcpu = &dstate->dtds_percpu[i];
1451 1498
1452 1499 if (dcpu->dtdsc_rinsing == NULL)
1453 1500 continue;
1454 1501
1455 1502 /*
1456 1503 * We are now guaranteed that no hash chain contains a pointer
1457 1504 * into this dirty list; we can make it clean.
1458 1505 */
1459 1506 ASSERT(dcpu->dtdsc_clean == NULL);
1460 1507 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
1461 1508 dcpu->dtdsc_rinsing = NULL;
1462 1509 }
1463 1510
1464 1511 /*
1465 1512 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
1466 1513 * sure that all CPUs have seen all of the dtdsc_clean pointers.
1467 1514 * This prevents a race whereby a CPU incorrectly decides that
1468 1515 * the state should be something other than DTRACE_DSTATE_CLEAN
1469 1516 * after dtrace_dynvar_clean() has completed.
1470 1517 */
1471 1518 dtrace_sync();
1472 1519
1473 1520 dstate->dtds_state = DTRACE_DSTATE_CLEAN;
1474 1521 }
1475 1522
1476 1523 /*
1477 1524 * Depending on the value of the op parameter, this function looks-up,
1478 1525 * allocates or deallocates an arbitrarily-keyed dynamic variable. If an
1479 1526 * allocation is requested, this function will return a pointer to a
1480 1527 * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
1481 1528 * variable can be allocated. If NULL is returned, the appropriate counter
1482 1529 * will be incremented.
1483 1530 */
1484 1531 dtrace_dynvar_t *
1485 1532 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
1486 1533 dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op,
1487 1534 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate)
1488 1535 {
1489 1536 uint64_t hashval = DTRACE_DYNHASH_VALID;
1490 1537 dtrace_dynhash_t *hash = dstate->dtds_hash;
1491 1538 dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
1492 1539 processorid_t me = CPU->cpu_id, cpu = me;
1493 1540 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
1494 1541 size_t bucket, ksize;
1495 1542 size_t chunksize = dstate->dtds_chunksize;
1496 1543 uintptr_t kdata, lock, nstate;
1497 1544 uint_t i;
1498 1545
1499 1546 ASSERT(nkeys != 0);
1500 1547
1501 1548 /*
1502 1549 * Hash the key. As with aggregations, we use Jenkins' "One-at-a-time"
1503 1550 * algorithm. For the by-value portions, we perform the algorithm in
1504 1551 * 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a
1505 1552 * bit, and seems to have only a minute effect on distribution. For
1506 1553 * the by-reference data, we perform "One-at-a-time" iterating (safely)
1507 1554 * over each referenced byte. It's painful to do this, but it's much
1508 1555 * better than pathological hash distribution. The efficacy of the
1509 1556 * hashing algorithm (and a comparison with other algorithms) may be
1510 1557 * found by running the ::dtrace_dynstat MDB dcmd.
1511 1558 */
1512 1559 for (i = 0; i < nkeys; i++) {
1513 1560 if (key[i].dttk_size == 0) {
1514 1561 uint64_t val = key[i].dttk_value;
1515 1562
1516 1563 hashval += (val >> 48) & 0xffff;
1517 1564 hashval += (hashval << 10);
1518 1565 hashval ^= (hashval >> 6);
1519 1566
1520 1567 hashval += (val >> 32) & 0xffff;
1521 1568 hashval += (hashval << 10);
1522 1569 hashval ^= (hashval >> 6);
1523 1570
1524 1571 hashval += (val >> 16) & 0xffff;
1525 1572 hashval += (hashval << 10);
1526 1573 hashval ^= (hashval >> 6);
1527 1574
1528 1575 hashval += val & 0xffff;
1529 1576 hashval += (hashval << 10);
1530 1577 hashval ^= (hashval >> 6);
1531 1578 } else {
1532 1579 /*
1533 1580 * This is incredibly painful, but it beats the hell
1534 1581 * out of the alternative.
1535 1582 */
1536 1583 uint64_t j, size = key[i].dttk_size;
1537 1584 uintptr_t base = (uintptr_t)key[i].dttk_value;
1538 1585
1539 1586 if (!dtrace_canload(base, size, mstate, vstate))
1540 1587 break;
1541 1588
1542 1589 for (j = 0; j < size; j++) {
1543 1590 hashval += dtrace_load8(base + j);
1544 1591 hashval += (hashval << 10);
1545 1592 hashval ^= (hashval >> 6);
1546 1593 }
1547 1594 }
1548 1595 }
1549 1596
1550 1597 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
1551 1598 return (NULL);
1552 1599
1553 1600 hashval += (hashval << 3);
1554 1601 hashval ^= (hashval >> 11);
1555 1602 hashval += (hashval << 15);
1556 1603
1557 1604 /*
1558 1605 * There is a remote chance (ideally, 1 in 2^31) that our hashval
1559 1606 * comes out to be one of our two sentinel hash values. If this
1560 1607 * actually happens, we set the hashval to be a value known to be a
1561 1608 * non-sentinel value.
1562 1609 */
1563 1610 if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
1564 1611 hashval = DTRACE_DYNHASH_VALID;
1565 1612
1566 1613 /*
1567 1614 * Yes, it's painful to do a divide here. If the cycle count becomes
1568 1615 * important here, tricks can be pulled to reduce it. (However, it's
1569 1616 * critical that hash collisions be kept to an absolute minimum;
1570 1617 * they're much more painful than a divide.) It's better to have a
1571 1618 * solution that generates few collisions and still keeps things
1572 1619 * relatively simple.
1573 1620 */
1574 1621 bucket = hashval % dstate->dtds_hashsize;
1575 1622
1576 1623 if (op == DTRACE_DYNVAR_DEALLOC) {
1577 1624 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1578 1625
1579 1626 for (;;) {
1580 1627 while ((lock = *lockp) & 1)
1581 1628 continue;
1582 1629
1583 1630 if (dtrace_casptr((void *)lockp,
1584 1631 (void *)lock, (void *)(lock + 1)) == (void *)lock)
1585 1632 break;
1586 1633 }
1587 1634
1588 1635 dtrace_membar_producer();
1589 1636 }
1590 1637
1591 1638 top:
1592 1639 prev = NULL;
1593 1640 lock = hash[bucket].dtdh_lock;
1594 1641
1595 1642 dtrace_membar_consumer();
1596 1643
1597 1644 start = hash[bucket].dtdh_chain;
1598 1645 ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
1599 1646 start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
1600 1647 op != DTRACE_DYNVAR_DEALLOC));
1601 1648
1602 1649 for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1603 1650 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1604 1651 dtrace_key_t *dkey = &dtuple->dtt_key[0];
1605 1652
1606 1653 if (dvar->dtdv_hashval != hashval) {
1607 1654 if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
1608 1655 /*
1609 1656 * We've reached the sink, and therefore the
1610 1657 * end of the hash chain; we can kick out of
1611 1658 * the loop knowing that we have seen a valid
1612 1659 * snapshot of state.
1613 1660 */
1614 1661 ASSERT(dvar->dtdv_next == NULL);
1615 1662 ASSERT(dvar == &dtrace_dynhash_sink);
1616 1663 break;
1617 1664 }
1618 1665
1619 1666 if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
1620 1667 /*
1621 1668 * We've gone off the rails: somewhere along
1622 1669 * the line, one of the members of this hash
1623 1670 * chain was deleted. Note that we could also
1624 1671 * detect this by simply letting this loop run
1625 1672 * to completion, as we would eventually hit
1626 1673 * the end of the dirty list. However, we
1627 1674 * want to avoid running the length of the
1628 1675 * dirty list unnecessarily (it might be quite
1629 1676 * long), so we catch this as early as
1630 1677 * possible by detecting the hash marker. In
1631 1678 * this case, we simply set dvar to NULL and
1632 1679 * break; the conditional after the loop will
1633 1680 * send us back to top.
1634 1681 */
1635 1682 dvar = NULL;
1636 1683 break;
1637 1684 }
1638 1685
1639 1686 goto next;
1640 1687 }
1641 1688
1642 1689 if (dtuple->dtt_nkeys != nkeys)
1643 1690 goto next;
1644 1691
1645 1692 for (i = 0; i < nkeys; i++, dkey++) {
1646 1693 if (dkey->dttk_size != key[i].dttk_size)
1647 1694 goto next; /* size or type mismatch */
1648 1695
1649 1696 if (dkey->dttk_size != 0) {
1650 1697 if (dtrace_bcmp(
1651 1698 (void *)(uintptr_t)key[i].dttk_value,
1652 1699 (void *)(uintptr_t)dkey->dttk_value,
1653 1700 dkey->dttk_size))
1654 1701 goto next;
1655 1702 } else {
1656 1703 if (dkey->dttk_value != key[i].dttk_value)
1657 1704 goto next;
1658 1705 }
1659 1706 }
1660 1707
1661 1708 if (op != DTRACE_DYNVAR_DEALLOC)
1662 1709 return (dvar);
1663 1710
1664 1711 ASSERT(dvar->dtdv_next == NULL ||
1665 1712 dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
1666 1713
1667 1714 if (prev != NULL) {
1668 1715 ASSERT(hash[bucket].dtdh_chain != dvar);
1669 1716 ASSERT(start != dvar);
1670 1717 ASSERT(prev->dtdv_next == dvar);
1671 1718 prev->dtdv_next = dvar->dtdv_next;
1672 1719 } else {
1673 1720 if (dtrace_casptr(&hash[bucket].dtdh_chain,
1674 1721 start, dvar->dtdv_next) != start) {
1675 1722 /*
1676 1723 * We have failed to atomically swing the
1677 1724 * hash table head pointer, presumably because
1678 1725 * of a conflicting allocation on another CPU.
1679 1726 * We need to reread the hash chain and try
1680 1727 * again.
1681 1728 */
1682 1729 goto top;
1683 1730 }
1684 1731 }
1685 1732
1686 1733 dtrace_membar_producer();
1687 1734
1688 1735 /*
1689 1736 * Now set the hash value to indicate that it's free.
1690 1737 */
1691 1738 ASSERT(hash[bucket].dtdh_chain != dvar);
1692 1739 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1693 1740
1694 1741 dtrace_membar_producer();
1695 1742
1696 1743 /*
1697 1744 * Set the next pointer to point at the dirty list, and
1698 1745 * atomically swing the dirty pointer to the newly freed dvar.
1699 1746 */
1700 1747 do {
1701 1748 next = dcpu->dtdsc_dirty;
1702 1749 dvar->dtdv_next = next;
1703 1750 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1704 1751
1705 1752 /*
1706 1753 * Finally, unlock this hash bucket.
1707 1754 */
1708 1755 ASSERT(hash[bucket].dtdh_lock == lock);
1709 1756 ASSERT(lock & 1);
1710 1757 hash[bucket].dtdh_lock++;
1711 1758
1712 1759 return (NULL);
1713 1760 next:
1714 1761 prev = dvar;
1715 1762 continue;
1716 1763 }
1717 1764
1718 1765 if (dvar == NULL) {
1719 1766 /*
1720 1767 * If dvar is NULL, it is because we went off the rails:
1721 1768 * one of the elements that we traversed in the hash chain
1722 1769 * was deleted while we were traversing it. In this case,
1723 1770 * we assert that we aren't doing a dealloc (deallocs lock
1724 1771 * the hash bucket to prevent themselves from racing with
1725 1772 * one another), and retry the hash chain traversal.
1726 1773 */
1727 1774 ASSERT(op != DTRACE_DYNVAR_DEALLOC);
1728 1775 goto top;
1729 1776 }
1730 1777
1731 1778 if (op != DTRACE_DYNVAR_ALLOC) {
1732 1779 /*
1733 1780 * If we are not to allocate a new variable, we want to
1734 1781 * return NULL now. Before we return, check that the value
1735 1782 * of the lock word hasn't changed. If it has, we may have
1736 1783 * seen an inconsistent snapshot.
1737 1784 */
1738 1785 if (op == DTRACE_DYNVAR_NOALLOC) {
1739 1786 if (hash[bucket].dtdh_lock != lock)
1740 1787 goto top;
1741 1788 } else {
1742 1789 ASSERT(op == DTRACE_DYNVAR_DEALLOC);
1743 1790 ASSERT(hash[bucket].dtdh_lock == lock);
1744 1791 ASSERT(lock & 1);
1745 1792 hash[bucket].dtdh_lock++;
1746 1793 }
1747 1794
1748 1795 return (NULL);
1749 1796 }
1750 1797
1751 1798 /*
1752 1799 * We need to allocate a new dynamic variable. The size we need is the
1753 1800 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
1754 1801 * size of any auxiliary key data (rounded up to 8-byte alignment) plus
1755 1802 * the size of any referred-to data (dsize). We then round the final
1756 1803 * size up to the chunksize for allocation.
1757 1804 */
1758 1805 for (ksize = 0, i = 0; i < nkeys; i++)
1759 1806 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
1760 1807
1761 1808 /*
1762 1809 * This should be pretty much impossible, but could happen if, say,
1763 1810 * strange DIF specified the tuple. Ideally, this should be an
1764 1811 * assertion and not an error condition -- but that requires that the
1765 1812 * chunksize calculation in dtrace_difo_chunksize() be absolutely
1766 1813 * bullet-proof. (That is, it must not be able to be fooled by
1767 1814 * malicious DIF.) Given the lack of backwards branches in DIF,
1768 1815 * solving this would presumably not amount to solving the Halting
1769 1816 * Problem -- but it still seems awfully hard.
1770 1817 */
1771 1818 if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
1772 1819 ksize + dsize > chunksize) {
1773 1820 dcpu->dtdsc_drops++;
1774 1821 return (NULL);
1775 1822 }
1776 1823
1777 1824 nstate = DTRACE_DSTATE_EMPTY;
1778 1825
1779 1826 do {
1780 1827 retry:
1781 1828 free = dcpu->dtdsc_free;
1782 1829
1783 1830 if (free == NULL) {
1784 1831 dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
1785 1832 void *rval;
1786 1833
1787 1834 if (clean == NULL) {
1788 1835 /*
1789 1836 * We're out of dynamic variable space on
1790 1837 * this CPU. Unless we have tried all CPUs,
1791 1838 * we'll try to allocate from a different
1792 1839 * CPU.
1793 1840 */
1794 1841 switch (dstate->dtds_state) {
1795 1842 case DTRACE_DSTATE_CLEAN: {
1796 1843 void *sp = &dstate->dtds_state;
1797 1844
1798 1845 if (++cpu >= NCPU)
1799 1846 cpu = 0;
1800 1847
1801 1848 if (dcpu->dtdsc_dirty != NULL &&
1802 1849 nstate == DTRACE_DSTATE_EMPTY)
1803 1850 nstate = DTRACE_DSTATE_DIRTY;
1804 1851
1805 1852 if (dcpu->dtdsc_rinsing != NULL)
1806 1853 nstate = DTRACE_DSTATE_RINSING;
1807 1854
1808 1855 dcpu = &dstate->dtds_percpu[cpu];
1809 1856
1810 1857 if (cpu != me)
1811 1858 goto retry;
1812 1859
1813 1860 (void) dtrace_cas32(sp,
1814 1861 DTRACE_DSTATE_CLEAN, nstate);
1815 1862
1816 1863 /*
1817 1864 * To increment the correct bean
1818 1865 * counter, take another lap.
1819 1866 */
1820 1867 goto retry;
1821 1868 }
1822 1869
1823 1870 case DTRACE_DSTATE_DIRTY:
1824 1871 dcpu->dtdsc_dirty_drops++;
1825 1872 break;
1826 1873
1827 1874 case DTRACE_DSTATE_RINSING:
1828 1875 dcpu->dtdsc_rinsing_drops++;
1829 1876 break;
1830 1877
1831 1878 case DTRACE_DSTATE_EMPTY:
1832 1879 dcpu->dtdsc_drops++;
1833 1880 break;
1834 1881 }
1835 1882
1836 1883 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
1837 1884 return (NULL);
1838 1885 }
1839 1886
1840 1887 /*
1841 1888 * The clean list appears to be non-empty. We want to
1842 1889 * move the clean list to the free list; we start by
1843 1890 * moving the clean pointer aside.
1844 1891 */
1845 1892 if (dtrace_casptr(&dcpu->dtdsc_clean,
1846 1893 clean, NULL) != clean) {
1847 1894 /*
1848 1895 * We are in one of two situations:
1849 1896 *
1850 1897 * (a) The clean list was switched to the
1851 1898 * free list by another CPU.
1852 1899 *
1853 1900 * (b) The clean list was added to by the
1854 1901 * cleansing cyclic.
1855 1902 *
1856 1903 * In either of these situations, we can
1857 1904 * just reattempt the free list allocation.
1858 1905 */
1859 1906 goto retry;
1860 1907 }
1861 1908
1862 1909 ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
1863 1910
1864 1911 /*
1865 1912 * Now we'll move the clean list to our free list.
1866 1913 * It's impossible for this to fail: the only way
1867 1914 * the free list can be updated is through this
1868 1915 * code path, and only one CPU can own the clean list.
1869 1916 * Thus, it would only be possible for this to fail if
1870 1917 * this code were racing with dtrace_dynvar_clean().
1871 1918 * (That is, if dtrace_dynvar_clean() updated the clean
1872 1919 * list, and we ended up racing to update the free
1873 1920 * list.) This race is prevented by the dtrace_sync()
1874 1921 * in dtrace_dynvar_clean() -- which flushes the
1875 1922 * owners of the clean lists out before resetting
1876 1923 * the clean lists.
1877 1924 */
1878 1925 dcpu = &dstate->dtds_percpu[me];
1879 1926 rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
1880 1927 ASSERT(rval == NULL);
1881 1928 goto retry;
1882 1929 }
1883 1930
1884 1931 dvar = free;
1885 1932 new_free = dvar->dtdv_next;
1886 1933 } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
1887 1934
1888 1935 /*
1889 1936 * We have now allocated a new chunk. We copy the tuple keys into the
1890 1937 * tuple array and copy any referenced key data into the data space
1891 1938 * following the tuple array. As we do this, we relocate dttk_value
1892 1939 * in the final tuple to point to the key data address in the chunk.
1893 1940 */
1894 1941 kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
1895 1942 dvar->dtdv_data = (void *)(kdata + ksize);
1896 1943 dvar->dtdv_tuple.dtt_nkeys = nkeys;
1897 1944
1898 1945 for (i = 0; i < nkeys; i++) {
1899 1946 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
1900 1947 size_t kesize = key[i].dttk_size;
1901 1948
1902 1949 if (kesize != 0) {
1903 1950 dtrace_bcopy(
1904 1951 (const void *)(uintptr_t)key[i].dttk_value,
1905 1952 (void *)kdata, kesize);
1906 1953 dkey->dttk_value = kdata;
1907 1954 kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
1908 1955 } else {
1909 1956 dkey->dttk_value = key[i].dttk_value;
1910 1957 }
1911 1958
1912 1959 dkey->dttk_size = kesize;
1913 1960 }
1914 1961
1915 1962 ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
1916 1963 dvar->dtdv_hashval = hashval;
1917 1964 dvar->dtdv_next = start;
1918 1965
1919 1966 if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
1920 1967 return (dvar);
1921 1968
1922 1969 /*
1923 1970 * The cas has failed. Either another CPU is adding an element to
1924 1971 * this hash chain, or another CPU is deleting an element from this
1925 1972 * hash chain. The simplest way to deal with both of these cases
1926 1973 * (though not necessarily the most efficient) is to free our
1927 1974 * allocated block and tail-call ourselves. Note that the free is
1928 1975 * to the dirty list and _not_ to the free list. This is to prevent
1929 1976 * races with allocators, above.
1930 1977 */
1931 1978 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
1932 1979
1933 1980 dtrace_membar_producer();
1934 1981
1935 1982 do {
1936 1983 free = dcpu->dtdsc_dirty;
1937 1984 dvar->dtdv_next = free;
1938 1985 } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
1939 1986
1940 1987 return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate));
1941 1988 }
1942 1989
1943 1990 /*ARGSUSED*/
1944 1991 static void
1945 1992 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
1946 1993 {
1947 1994 if ((int64_t)nval < (int64_t)*oval)
1948 1995 *oval = nval;
1949 1996 }
1950 1997
1951 1998 /*ARGSUSED*/
1952 1999 static void
1953 2000 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
1954 2001 {
1955 2002 if ((int64_t)nval > (int64_t)*oval)
1956 2003 *oval = nval;
1957 2004 }
1958 2005
1959 2006 static void
1960 2007 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
1961 2008 {
1962 2009 int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
1963 2010 int64_t val = (int64_t)nval;
1964 2011
1965 2012 if (val < 0) {
1966 2013 for (i = 0; i < zero; i++) {
1967 2014 if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
1968 2015 quanta[i] += incr;
1969 2016 return;
1970 2017 }
1971 2018 }
1972 2019 } else {
1973 2020 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
1974 2021 if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
1975 2022 quanta[i - 1] += incr;
1976 2023 return;
1977 2024 }
1978 2025 }
1979 2026
1980 2027 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
1981 2028 return;
1982 2029 }
1983 2030
1984 2031 ASSERT(0);
1985 2032 }
1986 2033
1987 2034 static void
1988 2035 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
1989 2036 {
1990 2037 uint64_t arg = *lquanta++;
1991 2038 int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1992 2039 uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1993 2040 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
1994 2041 int32_t val = (int32_t)nval, level;
1995 2042
1996 2043 ASSERT(step != 0);
1997 2044 ASSERT(levels != 0);
1998 2045
1999 2046 if (val < base) {
2000 2047 /*
2001 2048 * This is an underflow.
2002 2049 */
2003 2050 lquanta[0] += incr;
2004 2051 return;
2005 2052 }
2006 2053
2007 2054 level = (val - base) / step;
2008 2055
2009 2056 if (level < levels) {
2010 2057 lquanta[level + 1] += incr;
2011 2058 return;
2012 2059 }
2013 2060
2014 2061 /*
2015 2062 * This is an overflow.
2016 2063 */
2017 2064 lquanta[levels + 1] += incr;
2018 2065 }
2019 2066
2020 2067 static int
2021 2068 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low,
2022 2069 uint16_t high, uint16_t nsteps, int64_t value)
2023 2070 {
2024 2071 int64_t this = 1, last, next;
2025 2072 int base = 1, order;
2026 2073
2027 2074 ASSERT(factor <= nsteps);
2028 2075 ASSERT(nsteps % factor == 0);
2029 2076
2030 2077 for (order = 0; order < low; order++)
2031 2078 this *= factor;
2032 2079
2033 2080 /*
2034 2081 * If our value is less than our factor taken to the power of the
2035 2082 * low order of magnitude, it goes into the zeroth bucket.
2036 2083 */
2037 2084 if (value < (last = this))
2038 2085 return (0);
2039 2086
2040 2087 for (this *= factor; order <= high; order++) {
2041 2088 int nbuckets = this > nsteps ? nsteps : this;
2042 2089
2043 2090 if ((next = this * factor) < this) {
2044 2091 /*
2045 2092 * We should not generally get log/linear quantizations
2046 2093 * with a high magnitude that allows 64-bits to
2047 2094 * overflow, but we nonetheless protect against this
2048 2095 * by explicitly checking for overflow, and clamping
2049 2096 * our value accordingly.
2050 2097 */
2051 2098 value = this - 1;
2052 2099 }
2053 2100
2054 2101 if (value < this) {
2055 2102 /*
2056 2103 * If our value lies within this order of magnitude,
2057 2104 * determine its position by taking the offset within
2058 2105 * the order of magnitude, dividing by the bucket
2059 2106 * width, and adding to our (accumulated) base.
2060 2107 */
2061 2108 return (base + (value - last) / (this / nbuckets));
2062 2109 }
2063 2110
2064 2111 base += nbuckets - (nbuckets / factor);
2065 2112 last = this;
2066 2113 this = next;
2067 2114 }
2068 2115
2069 2116 /*
2070 2117 * Our value is greater than or equal to our factor taken to the
2071 2118 * power of one plus the high magnitude -- return the top bucket.
2072 2119 */
2073 2120 return (base);
2074 2121 }
2075 2122
2076 2123 static void
2077 2124 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr)
2078 2125 {
2079 2126 uint64_t arg = *llquanta++;
2080 2127 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2081 2128 uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2082 2129 uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2083 2130 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2084 2131
2085 2132 llquanta[dtrace_aggregate_llquantize_bucket(factor,
2086 2133 low, high, nsteps, nval)] += incr;
2087 2134 }
2088 2135
2089 2136 /*ARGSUSED*/
2090 2137 static void
2091 2138 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
2092 2139 {
2093 2140 data[0]++;
2094 2141 data[1] += nval;
2095 2142 }
2096 2143
2097 2144 /*ARGSUSED*/
2098 2145 static void
2099 2146 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg)
2100 2147 {
2101 2148 int64_t snval = (int64_t)nval;
2102 2149 uint64_t tmp[2];
2103 2150
2104 2151 data[0]++;
2105 2152 data[1] += nval;
2106 2153
2107 2154 /*
2108 2155 * What we want to say here is:
2109 2156 *
2110 2157 * data[2] += nval * nval;
2111 2158 *
2112 2159 * But given that nval is 64-bit, we could easily overflow, so
2113 2160 * we do this as 128-bit arithmetic.
2114 2161 */
2115 2162 if (snval < 0)
2116 2163 snval = -snval;
2117 2164
2118 2165 dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp);
2119 2166 dtrace_add_128(data + 2, tmp, data + 2);
2120 2167 }
2121 2168
2122 2169 /*ARGSUSED*/
2123 2170 static void
2124 2171 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
2125 2172 {
2126 2173 *oval = *oval + 1;
2127 2174 }
2128 2175
2129 2176 /*ARGSUSED*/
2130 2177 static void
2131 2178 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
2132 2179 {
2133 2180 *oval += nval;
2134 2181 }
2135 2182
2136 2183 /*
2137 2184 * Aggregate given the tuple in the principal data buffer, and the aggregating
2138 2185 * action denoted by the specified dtrace_aggregation_t. The aggregation
2139 2186 * buffer is specified as the buf parameter. This routine does not return
2140 2187 * failure; if there is no space in the aggregation buffer, the data will be
2141 2188 * dropped, and a corresponding counter incremented.
2142 2189 */
2143 2190 static void
2144 2191 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
2145 2192 intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
2146 2193 {
2147 2194 dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
2148 2195 uint32_t i, ndx, size, fsize;
2149 2196 uint32_t align = sizeof (uint64_t) - 1;
2150 2197 dtrace_aggbuffer_t *agb;
2151 2198 dtrace_aggkey_t *key;
2152 2199 uint32_t hashval = 0, limit, isstr;
2153 2200 caddr_t tomax, data, kdata;
2154 2201 dtrace_actkind_t action;
2155 2202 dtrace_action_t *act;
2156 2203 uintptr_t offs;
2157 2204
2158 2205 if (buf == NULL)
2159 2206 return;
2160 2207
2161 2208 if (!agg->dtag_hasarg) {
2162 2209 /*
2163 2210 * Currently, only quantize() and lquantize() take additional
2164 2211 * arguments, and they have the same semantics: an increment
2165 2212 * value that defaults to 1 when not present. If additional
2166 2213 * aggregating actions take arguments, the setting of the
2167 2214 * default argument value will presumably have to become more
2168 2215 * sophisticated...
2169 2216 */
2170 2217 arg = 1;
2171 2218 }
2172 2219
2173 2220 action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
2174 2221 size = rec->dtrd_offset - agg->dtag_base;
2175 2222 fsize = size + rec->dtrd_size;
2176 2223
2177 2224 ASSERT(dbuf->dtb_tomax != NULL);
2178 2225 data = dbuf->dtb_tomax + offset + agg->dtag_base;
2179 2226
2180 2227 if ((tomax = buf->dtb_tomax) == NULL) {
2181 2228 dtrace_buffer_drop(buf);
2182 2229 return;
2183 2230 }
2184 2231
2185 2232 /*
2186 2233 * The metastructure is always at the bottom of the buffer.
2187 2234 */
2188 2235 agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
2189 2236 sizeof (dtrace_aggbuffer_t));
2190 2237
2191 2238 if (buf->dtb_offset == 0) {
2192 2239 /*
2193 2240 * We just kludge up approximately 1/8th of the size to be
2194 2241 * buckets. If this guess ends up being routinely
2195 2242 * off-the-mark, we may need to dynamically readjust this
2196 2243 * based on past performance.
2197 2244 */
2198 2245 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
2199 2246
2200 2247 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
2201 2248 (uintptr_t)tomax || hashsize == 0) {
2202 2249 /*
2203 2250 * We've been given a ludicrously small buffer;
2204 2251 * increment our drop count and leave.
2205 2252 */
2206 2253 dtrace_buffer_drop(buf);
2207 2254 return;
2208 2255 }
2209 2256
2210 2257 /*
2211 2258 * And now, a pathetic attempt to try to get a an odd (or
2212 2259 * perchance, a prime) hash size for better hash distribution.
2213 2260 */
2214 2261 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
2215 2262 hashsize -= DTRACE_AGGHASHSIZE_SLEW;
2216 2263
2217 2264 agb->dtagb_hashsize = hashsize;
2218 2265 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
2219 2266 agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
2220 2267 agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
2221 2268
2222 2269 for (i = 0; i < agb->dtagb_hashsize; i++)
2223 2270 agb->dtagb_hash[i] = NULL;
2224 2271 }
2225 2272
2226 2273 ASSERT(agg->dtag_first != NULL);
2227 2274 ASSERT(agg->dtag_first->dta_intuple);
2228 2275
2229 2276 /*
2230 2277 * Calculate the hash value based on the key. Note that we _don't_
2231 2278 * include the aggid in the hashing (but we will store it as part of
2232 2279 * the key). The hashing algorithm is Bob Jenkins' "One-at-a-time"
2233 2280 * algorithm: a simple, quick algorithm that has no known funnels, and
2234 2281 * gets good distribution in practice. The efficacy of the hashing
2235 2282 * algorithm (and a comparison with other algorithms) may be found by
2236 2283 * running the ::dtrace_aggstat MDB dcmd.
2237 2284 */
2238 2285 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2239 2286 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2240 2287 limit = i + act->dta_rec.dtrd_size;
2241 2288 ASSERT(limit <= size);
2242 2289 isstr = DTRACEACT_ISSTRING(act);
2243 2290
2244 2291 for (; i < limit; i++) {
2245 2292 hashval += data[i];
2246 2293 hashval += (hashval << 10);
2247 2294 hashval ^= (hashval >> 6);
2248 2295
2249 2296 if (isstr && data[i] == '\0')
2250 2297 break;
2251 2298 }
2252 2299 }
2253 2300
2254 2301 hashval += (hashval << 3);
2255 2302 hashval ^= (hashval >> 11);
2256 2303 hashval += (hashval << 15);
2257 2304
2258 2305 /*
2259 2306 * Yes, the divide here is expensive -- but it's generally the least
2260 2307 * of the performance issues given the amount of data that we iterate
2261 2308 * over to compute hash values, compare data, etc.
2262 2309 */
2263 2310 ndx = hashval % agb->dtagb_hashsize;
2264 2311
2265 2312 for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
2266 2313 ASSERT((caddr_t)key >= tomax);
2267 2314 ASSERT((caddr_t)key < tomax + buf->dtb_size);
2268 2315
2269 2316 if (hashval != key->dtak_hashval || key->dtak_size != size)
2270 2317 continue;
2271 2318
2272 2319 kdata = key->dtak_data;
2273 2320 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
2274 2321
2275 2322 for (act = agg->dtag_first; act->dta_intuple;
2276 2323 act = act->dta_next) {
2277 2324 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2278 2325 limit = i + act->dta_rec.dtrd_size;
2279 2326 ASSERT(limit <= size);
2280 2327 isstr = DTRACEACT_ISSTRING(act);
2281 2328
2282 2329 for (; i < limit; i++) {
2283 2330 if (kdata[i] != data[i])
2284 2331 goto next;
2285 2332
2286 2333 if (isstr && data[i] == '\0')
2287 2334 break;
2288 2335 }
2289 2336 }
2290 2337
2291 2338 if (action != key->dtak_action) {
2292 2339 /*
2293 2340 * We are aggregating on the same value in the same
2294 2341 * aggregation with two different aggregating actions.
2295 2342 * (This should have been picked up in the compiler,
2296 2343 * so we may be dealing with errant or devious DIF.)
2297 2344 * This is an error condition; we indicate as much,
2298 2345 * and return.
2299 2346 */
2300 2347 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2301 2348 return;
2302 2349 }
2303 2350
2304 2351 /*
2305 2352 * This is a hit: we need to apply the aggregator to
2306 2353 * the value at this key.
2307 2354 */
2308 2355 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
2309 2356 return;
2310 2357 next:
2311 2358 continue;
2312 2359 }
2313 2360
2314 2361 /*
2315 2362 * We didn't find it. We need to allocate some zero-filled space,
2316 2363 * link it into the hash table appropriately, and apply the aggregator
2317 2364 * to the (zero-filled) value.
2318 2365 */
2319 2366 offs = buf->dtb_offset;
2320 2367 while (offs & (align - 1))
2321 2368 offs += sizeof (uint32_t);
2322 2369
2323 2370 /*
2324 2371 * If we don't have enough room to both allocate a new key _and_
2325 2372 * its associated data, increment the drop count and return.
2326 2373 */
2327 2374 if ((uintptr_t)tomax + offs + fsize >
2328 2375 agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
2329 2376 dtrace_buffer_drop(buf);
2330 2377 return;
2331 2378 }
2332 2379
2333 2380 /*CONSTCOND*/
2334 2381 ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
2335 2382 key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
2336 2383 agb->dtagb_free -= sizeof (dtrace_aggkey_t);
2337 2384
2338 2385 key->dtak_data = kdata = tomax + offs;
2339 2386 buf->dtb_offset = offs + fsize;
2340 2387
2341 2388 /*
2342 2389 * Now copy the data across.
2343 2390 */
2344 2391 *((dtrace_aggid_t *)kdata) = agg->dtag_id;
2345 2392
2346 2393 for (i = sizeof (dtrace_aggid_t); i < size; i++)
2347 2394 kdata[i] = data[i];
2348 2395
2349 2396 /*
2350 2397 * Because strings are not zeroed out by default, we need to iterate
2351 2398 * looking for actions that store strings, and we need to explicitly
2352 2399 * pad these strings out with zeroes.
2353 2400 */
2354 2401 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
2355 2402 int nul;
2356 2403
2357 2404 if (!DTRACEACT_ISSTRING(act))
2358 2405 continue;
2359 2406
2360 2407 i = act->dta_rec.dtrd_offset - agg->dtag_base;
2361 2408 limit = i + act->dta_rec.dtrd_size;
2362 2409 ASSERT(limit <= size);
2363 2410
2364 2411 for (nul = 0; i < limit; i++) {
2365 2412 if (nul) {
2366 2413 kdata[i] = '\0';
2367 2414 continue;
2368 2415 }
2369 2416
2370 2417 if (data[i] != '\0')
2371 2418 continue;
2372 2419
2373 2420 nul = 1;
2374 2421 }
2375 2422 }
2376 2423
2377 2424 for (i = size; i < fsize; i++)
2378 2425 kdata[i] = 0;
2379 2426
2380 2427 key->dtak_hashval = hashval;
2381 2428 key->dtak_size = size;
2382 2429 key->dtak_action = action;
2383 2430 key->dtak_next = agb->dtagb_hash[ndx];
2384 2431 agb->dtagb_hash[ndx] = key;
2385 2432
2386 2433 /*
2387 2434 * Finally, apply the aggregator.
2388 2435 */
2389 2436 *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
2390 2437 agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
2391 2438 }
2392 2439
2393 2440 /*
2394 2441 * Given consumer state, this routine finds a speculation in the INACTIVE
2395 2442 * state and transitions it into the ACTIVE state. If there is no speculation
2396 2443 * in the INACTIVE state, 0 is returned. In this case, no error counter is
2397 2444 * incremented -- it is up to the caller to take appropriate action.
2398 2445 */
2399 2446 static int
2400 2447 dtrace_speculation(dtrace_state_t *state)
2401 2448 {
2402 2449 int i = 0;
2403 2450 dtrace_speculation_state_t current;
2404 2451 uint32_t *stat = &state->dts_speculations_unavail, count;
2405 2452
2406 2453 while (i < state->dts_nspeculations) {
2407 2454 dtrace_speculation_t *spec = &state->dts_speculations[i];
2408 2455
2409 2456 current = spec->dtsp_state;
2410 2457
2411 2458 if (current != DTRACESPEC_INACTIVE) {
2412 2459 if (current == DTRACESPEC_COMMITTINGMANY ||
2413 2460 current == DTRACESPEC_COMMITTING ||
2414 2461 current == DTRACESPEC_DISCARDING)
2415 2462 stat = &state->dts_speculations_busy;
2416 2463 i++;
2417 2464 continue;
2418 2465 }
2419 2466
2420 2467 if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2421 2468 current, DTRACESPEC_ACTIVE) == current)
2422 2469 return (i + 1);
2423 2470 }
2424 2471
2425 2472 /*
2426 2473 * We couldn't find a speculation. If we found as much as a single
2427 2474 * busy speculation buffer, we'll attribute this failure as "busy"
2428 2475 * instead of "unavail".
2429 2476 */
2430 2477 do {
2431 2478 count = *stat;
2432 2479 } while (dtrace_cas32(stat, count, count + 1) != count);
2433 2480
2434 2481 return (0);
2435 2482 }
2436 2483
2437 2484 /*
2438 2485 * This routine commits an active speculation. If the specified speculation
2439 2486 * is not in a valid state to perform a commit(), this routine will silently do
2440 2487 * nothing. The state of the specified speculation is transitioned according
2441 2488 * to the state transition diagram outlined in <sys/dtrace_impl.h>
2442 2489 */
2443 2490 static void
2444 2491 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
2445 2492 dtrace_specid_t which)
2446 2493 {
2447 2494 dtrace_speculation_t *spec;
2448 2495 dtrace_buffer_t *src, *dest;
2449 2496 uintptr_t daddr, saddr, dlimit;
2450 2497 dtrace_speculation_state_t current, new;
2451 2498 intptr_t offs;
2452 2499
2453 2500 if (which == 0)
2454 2501 return;
2455 2502
2456 2503 if (which > state->dts_nspeculations) {
2457 2504 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2458 2505 return;
2459 2506 }
2460 2507
2461 2508 spec = &state->dts_speculations[which - 1];
2462 2509 src = &spec->dtsp_buffer[cpu];
2463 2510 dest = &state->dts_buffer[cpu];
2464 2511
2465 2512 do {
2466 2513 current = spec->dtsp_state;
2467 2514
2468 2515 if (current == DTRACESPEC_COMMITTINGMANY)
2469 2516 break;
2470 2517
2471 2518 switch (current) {
2472 2519 case DTRACESPEC_INACTIVE:
2473 2520 case DTRACESPEC_DISCARDING:
2474 2521 return;
2475 2522
2476 2523 case DTRACESPEC_COMMITTING:
2477 2524 /*
2478 2525 * This is only possible if we are (a) commit()'ing
2479 2526 * without having done a prior speculate() on this CPU
2480 2527 * and (b) racing with another commit() on a different
2481 2528 * CPU. There's nothing to do -- we just assert that
2482 2529 * our offset is 0.
2483 2530 */
2484 2531 ASSERT(src->dtb_offset == 0);
2485 2532 return;
2486 2533
2487 2534 case DTRACESPEC_ACTIVE:
2488 2535 new = DTRACESPEC_COMMITTING;
2489 2536 break;
2490 2537
2491 2538 case DTRACESPEC_ACTIVEONE:
2492 2539 /*
2493 2540 * This speculation is active on one CPU. If our
2494 2541 * buffer offset is non-zero, we know that the one CPU
2495 2542 * must be us. Otherwise, we are committing on a
2496 2543 * different CPU from the speculate(), and we must
2497 2544 * rely on being asynchronously cleaned.
2498 2545 */
2499 2546 if (src->dtb_offset != 0) {
2500 2547 new = DTRACESPEC_COMMITTING;
2501 2548 break;
2502 2549 }
2503 2550 /*FALLTHROUGH*/
2504 2551
2505 2552 case DTRACESPEC_ACTIVEMANY:
2506 2553 new = DTRACESPEC_COMMITTINGMANY;
2507 2554 break;
2508 2555
2509 2556 default:
2510 2557 ASSERT(0);
2511 2558 }
2512 2559 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2513 2560 current, new) != current);
2514 2561
2515 2562 /*
2516 2563 * We have set the state to indicate that we are committing this
2517 2564 * speculation. Now reserve the necessary space in the destination
2518 2565 * buffer.
2519 2566 */
2520 2567 if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
2521 2568 sizeof (uint64_t), state, NULL)) < 0) {
2522 2569 dtrace_buffer_drop(dest);
2523 2570 goto out;
2524 2571 }
2525 2572
2526 2573 /*
2527 2574 * We have the space; copy the buffer across. (Note that this is a
2528 2575 * highly subobtimal bcopy(); in the unlikely event that this becomes
2529 2576 * a serious performance issue, a high-performance DTrace-specific
2530 2577 * bcopy() should obviously be invented.)
2531 2578 */
2532 2579 daddr = (uintptr_t)dest->dtb_tomax + offs;
2533 2580 dlimit = daddr + src->dtb_offset;
2534 2581 saddr = (uintptr_t)src->dtb_tomax;
2535 2582
2536 2583 /*
2537 2584 * First, the aligned portion.
2538 2585 */
2539 2586 while (dlimit - daddr >= sizeof (uint64_t)) {
2540 2587 *((uint64_t *)daddr) = *((uint64_t *)saddr);
2541 2588
2542 2589 daddr += sizeof (uint64_t);
2543 2590 saddr += sizeof (uint64_t);
2544 2591 }
2545 2592
2546 2593 /*
2547 2594 * Now any left-over bit...
2548 2595 */
2549 2596 while (dlimit - daddr)
2550 2597 *((uint8_t *)daddr++) = *((uint8_t *)saddr++);
2551 2598
2552 2599 /*
2553 2600 * Finally, commit the reserved space in the destination buffer.
2554 2601 */
2555 2602 dest->dtb_offset = offs + src->dtb_offset;
2556 2603
2557 2604 out:
2558 2605 /*
2559 2606 * If we're lucky enough to be the only active CPU on this speculation
2560 2607 * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
2561 2608 */
2562 2609 if (current == DTRACESPEC_ACTIVE ||
2563 2610 (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
2564 2611 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
2565 2612 DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
2566 2613
2567 2614 ASSERT(rval == DTRACESPEC_COMMITTING);
2568 2615 }
2569 2616
2570 2617 src->dtb_offset = 0;
2571 2618 src->dtb_xamot_drops += src->dtb_drops;
2572 2619 src->dtb_drops = 0;
2573 2620 }
2574 2621
2575 2622 /*
2576 2623 * This routine discards an active speculation. If the specified speculation
2577 2624 * is not in a valid state to perform a discard(), this routine will silently
2578 2625 * do nothing. The state of the specified speculation is transitioned
2579 2626 * according to the state transition diagram outlined in <sys/dtrace_impl.h>
2580 2627 */
2581 2628 static void
2582 2629 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
2583 2630 dtrace_specid_t which)
2584 2631 {
2585 2632 dtrace_speculation_t *spec;
2586 2633 dtrace_speculation_state_t current, new;
2587 2634 dtrace_buffer_t *buf;
2588 2635
2589 2636 if (which == 0)
2590 2637 return;
2591 2638
2592 2639 if (which > state->dts_nspeculations) {
2593 2640 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2594 2641 return;
2595 2642 }
2596 2643
2597 2644 spec = &state->dts_speculations[which - 1];
2598 2645 buf = &spec->dtsp_buffer[cpu];
2599 2646
2600 2647 do {
2601 2648 current = spec->dtsp_state;
2602 2649
2603 2650 switch (current) {
2604 2651 case DTRACESPEC_INACTIVE:
2605 2652 case DTRACESPEC_COMMITTINGMANY:
2606 2653 case DTRACESPEC_COMMITTING:
2607 2654 case DTRACESPEC_DISCARDING:
2608 2655 return;
2609 2656
2610 2657 case DTRACESPEC_ACTIVE:
2611 2658 case DTRACESPEC_ACTIVEMANY:
2612 2659 new = DTRACESPEC_DISCARDING;
2613 2660 break;
2614 2661
2615 2662 case DTRACESPEC_ACTIVEONE:
2616 2663 if (buf->dtb_offset != 0) {
2617 2664 new = DTRACESPEC_INACTIVE;
2618 2665 } else {
2619 2666 new = DTRACESPEC_DISCARDING;
2620 2667 }
2621 2668 break;
2622 2669
2623 2670 default:
2624 2671 ASSERT(0);
2625 2672 }
2626 2673 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2627 2674 current, new) != current);
2628 2675
2629 2676 buf->dtb_offset = 0;
2630 2677 buf->dtb_drops = 0;
2631 2678 }
2632 2679
2633 2680 /*
2634 2681 * Note: not called from probe context. This function is called
2635 2682 * asynchronously from cross call context to clean any speculations that are
2636 2683 * in the COMMITTINGMANY or DISCARDING states. These speculations may not be
2637 2684 * transitioned back to the INACTIVE state until all CPUs have cleaned the
2638 2685 * speculation.
2639 2686 */
2640 2687 static void
2641 2688 dtrace_speculation_clean_here(dtrace_state_t *state)
2642 2689 {
2643 2690 dtrace_icookie_t cookie;
2644 2691 processorid_t cpu = CPU->cpu_id;
2645 2692 dtrace_buffer_t *dest = &state->dts_buffer[cpu];
2646 2693 dtrace_specid_t i;
2647 2694
2648 2695 cookie = dtrace_interrupt_disable();
2649 2696
2650 2697 if (dest->dtb_tomax == NULL) {
2651 2698 dtrace_interrupt_enable(cookie);
2652 2699 return;
2653 2700 }
2654 2701
2655 2702 for (i = 0; i < state->dts_nspeculations; i++) {
2656 2703 dtrace_speculation_t *spec = &state->dts_speculations[i];
2657 2704 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
2658 2705
2659 2706 if (src->dtb_tomax == NULL)
2660 2707 continue;
2661 2708
2662 2709 if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
2663 2710 src->dtb_offset = 0;
2664 2711 continue;
2665 2712 }
2666 2713
2667 2714 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2668 2715 continue;
2669 2716
2670 2717 if (src->dtb_offset == 0)
2671 2718 continue;
2672 2719
2673 2720 dtrace_speculation_commit(state, cpu, i + 1);
2674 2721 }
2675 2722
2676 2723 dtrace_interrupt_enable(cookie);
2677 2724 }
2678 2725
2679 2726 /*
2680 2727 * Note: not called from probe context. This function is called
2681 2728 * asynchronously (and at a regular interval) to clean any speculations that
2682 2729 * are in the COMMITTINGMANY or DISCARDING states. If it discovers that there
2683 2730 * is work to be done, it cross calls all CPUs to perform that work;
2684 2731 * COMMITMANY and DISCARDING speculations may not be transitioned back to the
2685 2732 * INACTIVE state until they have been cleaned by all CPUs.
2686 2733 */
2687 2734 static void
2688 2735 dtrace_speculation_clean(dtrace_state_t *state)
2689 2736 {
2690 2737 int work = 0, rv;
2691 2738 dtrace_specid_t i;
2692 2739
2693 2740 for (i = 0; i < state->dts_nspeculations; i++) {
2694 2741 dtrace_speculation_t *spec = &state->dts_speculations[i];
2695 2742
2696 2743 ASSERT(!spec->dtsp_cleaning);
2697 2744
2698 2745 if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
2699 2746 spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
2700 2747 continue;
2701 2748
2702 2749 work++;
2703 2750 spec->dtsp_cleaning = 1;
2704 2751 }
2705 2752
2706 2753 if (!work)
2707 2754 return;
2708 2755
2709 2756 dtrace_xcall(DTRACE_CPUALL,
2710 2757 (dtrace_xcall_t)dtrace_speculation_clean_here, state);
2711 2758
2712 2759 /*
2713 2760 * We now know that all CPUs have committed or discarded their
2714 2761 * speculation buffers, as appropriate. We can now set the state
2715 2762 * to inactive.
2716 2763 */
2717 2764 for (i = 0; i < state->dts_nspeculations; i++) {
2718 2765 dtrace_speculation_t *spec = &state->dts_speculations[i];
2719 2766 dtrace_speculation_state_t current, new;
2720 2767
2721 2768 if (!spec->dtsp_cleaning)
2722 2769 continue;
2723 2770
2724 2771 current = spec->dtsp_state;
2725 2772 ASSERT(current == DTRACESPEC_DISCARDING ||
2726 2773 current == DTRACESPEC_COMMITTINGMANY);
2727 2774
2728 2775 new = DTRACESPEC_INACTIVE;
2729 2776
2730 2777 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
2731 2778 ASSERT(rv == current);
2732 2779 spec->dtsp_cleaning = 0;
2733 2780 }
2734 2781 }
2735 2782
2736 2783 /*
2737 2784 * Called as part of a speculate() to get the speculative buffer associated
2738 2785 * with a given speculation. Returns NULL if the specified speculation is not
2739 2786 * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and
2740 2787 * the active CPU is not the specified CPU -- the speculation will be
2741 2788 * atomically transitioned into the ACTIVEMANY state.
2742 2789 */
2743 2790 static dtrace_buffer_t *
2744 2791 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2745 2792 dtrace_specid_t which)
2746 2793 {
2747 2794 dtrace_speculation_t *spec;
2748 2795 dtrace_speculation_state_t current, new;
2749 2796 dtrace_buffer_t *buf;
2750 2797
2751 2798 if (which == 0)
2752 2799 return (NULL);
2753 2800
2754 2801 if (which > state->dts_nspeculations) {
2755 2802 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2756 2803 return (NULL);
2757 2804 }
2758 2805
2759 2806 spec = &state->dts_speculations[which - 1];
2760 2807 buf = &spec->dtsp_buffer[cpuid];
2761 2808
2762 2809 do {
2763 2810 current = spec->dtsp_state;
2764 2811
2765 2812 switch (current) {
2766 2813 case DTRACESPEC_INACTIVE:
2767 2814 case DTRACESPEC_COMMITTINGMANY:
2768 2815 case DTRACESPEC_DISCARDING:
2769 2816 return (NULL);
2770 2817
2771 2818 case DTRACESPEC_COMMITTING:
2772 2819 ASSERT(buf->dtb_offset == 0);
2773 2820 return (NULL);
2774 2821
2775 2822 case DTRACESPEC_ACTIVEONE:
2776 2823 /*
2777 2824 * This speculation is currently active on one CPU.
2778 2825 * Check the offset in the buffer; if it's non-zero,
2779 2826 * that CPU must be us (and we leave the state alone).
2780 2827 * If it's zero, assume that we're starting on a new
2781 2828 * CPU -- and change the state to indicate that the
2782 2829 * speculation is active on more than one CPU.
2783 2830 */
2784 2831 if (buf->dtb_offset != 0)
2785 2832 return (buf);
2786 2833
2787 2834 new = DTRACESPEC_ACTIVEMANY;
2788 2835 break;
2789 2836
2790 2837 case DTRACESPEC_ACTIVEMANY:
2791 2838 return (buf);
2792 2839
2793 2840 case DTRACESPEC_ACTIVE:
2794 2841 new = DTRACESPEC_ACTIVEONE;
2795 2842 break;
2796 2843
2797 2844 default:
2798 2845 ASSERT(0);
2799 2846 }
2800 2847 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2801 2848 current, new) != current);
2802 2849
2803 2850 ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
2804 2851 return (buf);
2805 2852 }
2806 2853
2807 2854 /*
2808 2855 * Return a string. In the event that the user lacks the privilege to access
2809 2856 * arbitrary kernel memory, we copy the string out to scratch memory so that we
2810 2857 * don't fail access checking.
2811 2858 *
2812 2859 * dtrace_dif_variable() uses this routine as a helper for various
2813 2860 * builtin values such as 'execname' and 'probefunc.'
2814 2861 */
2815 2862 uintptr_t
2816 2863 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state,
2817 2864 dtrace_mstate_t *mstate)
2818 2865 {
2819 2866 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2820 2867 uintptr_t ret;
2821 2868 size_t strsz;
2822 2869
2823 2870 /*
2824 2871 * The easy case: this probe is allowed to read all of memory, so
2825 2872 * we can just return this as a vanilla pointer.
2826 2873 */
2827 2874 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0)
2828 2875 return (addr);
2829 2876
2830 2877 /*
2831 2878 * This is the tougher case: we copy the string in question from
2832 2879 * kernel memory into scratch memory and return it that way: this
2833 2880 * ensures that we won't trip up when access checking tests the
2834 2881 * BYREF return value.
2835 2882 */
2836 2883 strsz = dtrace_strlen((char *)addr, size) + 1;
2837 2884
2838 2885 if (mstate->dtms_scratch_ptr + strsz >
2839 2886 mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2840 2887 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2841 2888 return (NULL);
2842 2889 }
2843 2890
2844 2891 dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr,
2845 2892 strsz);
2846 2893 ret = mstate->dtms_scratch_ptr;
2847 2894 mstate->dtms_scratch_ptr += strsz;
2848 2895 return (ret);
2849 2896 }
2850 2897
2851 2898 /*
2852 2899 * This function implements the DIF emulator's variable lookups. The emulator
2853 2900 * passes a reserved variable identifier and optional built-in array index.
2854 2901 */
2855 2902 static uint64_t
2856 2903 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
2857 2904 uint64_t ndx)
2858 2905 {
2859 2906 /*
2860 2907 * If we're accessing one of the uncached arguments, we'll turn this
2861 2908 * into a reference in the args array.
2862 2909 */
2863 2910 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
2864 2911 ndx = v - DIF_VAR_ARG0;
2865 2912 v = DIF_VAR_ARGS;
2866 2913 }
2867 2914
2868 2915 switch (v) {
2869 2916 case DIF_VAR_ARGS:
2870 2917 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) {
2871 2918 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |=
2872 2919 CPU_DTRACE_KPRIV;
2873 2920 return (0);
2874 2921 }
2875 2922
2876 2923 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
2877 2924 if (ndx >= sizeof (mstate->dtms_arg) /
2878 2925 sizeof (mstate->dtms_arg[0])) {
2879 2926 int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2880 2927 dtrace_provider_t *pv;
2881 2928 uint64_t val;
2882 2929
2883 2930 pv = mstate->dtms_probe->dtpr_provider;
2884 2931 if (pv->dtpv_pops.dtps_getargval != NULL)
2885 2932 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
2886 2933 mstate->dtms_probe->dtpr_id,
2887 2934 mstate->dtms_probe->dtpr_arg, ndx, aframes);
2888 2935 else
2889 2936 val = dtrace_getarg(ndx, aframes);
2890 2937
2891 2938 /*
2892 2939 * This is regrettably required to keep the compiler
2893 2940 * from tail-optimizing the call to dtrace_getarg().
2894 2941 * The condition always evaluates to true, but the
2895 2942 * compiler has no way of figuring that out a priori.
2896 2943 * (None of this would be necessary if the compiler
2897 2944 * could be relied upon to _always_ tail-optimize
2898 2945 * the call to dtrace_getarg() -- but it can't.)
2899 2946 */
2900 2947 if (mstate->dtms_probe != NULL)
2901 2948 return (val);
2902 2949
2903 2950 ASSERT(0);
2904 2951 }
2905 2952
2906 2953 return (mstate->dtms_arg[ndx]);
2907 2954
2908 2955 case DIF_VAR_UREGS: {
2909 2956 klwp_t *lwp;
2910 2957
2911 2958 if (!dtrace_priv_proc(state, mstate))
2912 2959 return (0);
2913 2960
2914 2961 if ((lwp = curthread->t_lwp) == NULL) {
2915 2962 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
2916 2963 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
2917 2964 return (0);
2918 2965 }
2919 2966
2920 2967 return (dtrace_getreg(lwp->lwp_regs, ndx));
2921 2968 }
2922 2969
2923 2970 case DIF_VAR_VMREGS: {
2924 2971 uint64_t rval;
2925 2972
2926 2973 if (!dtrace_priv_kernel(state))
2927 2974 return (0);
2928 2975
2929 2976 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2930 2977
2931 2978 rval = dtrace_getvmreg(ndx,
2932 2979 &cpu_core[CPU->cpu_id].cpuc_dtrace_flags);
2933 2980
2934 2981 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2935 2982
2936 2983 return (rval);
2937 2984 }
2938 2985
2939 2986 case DIF_VAR_CURTHREAD:
2940 2987 if (!dtrace_priv_proc(state, mstate))
2941 2988 return (0);
2942 2989 return ((uint64_t)(uintptr_t)curthread);
2943 2990
2944 2991 case DIF_VAR_TIMESTAMP:
2945 2992 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
2946 2993 mstate->dtms_timestamp = dtrace_gethrtime();
2947 2994 mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
2948 2995 }
2949 2996 return (mstate->dtms_timestamp);
2950 2997
2951 2998 case DIF_VAR_VTIMESTAMP:
2952 2999 ASSERT(dtrace_vtime_references != 0);
2953 3000 return (curthread->t_dtrace_vtime);
2954 3001
2955 3002 case DIF_VAR_WALLTIMESTAMP:
2956 3003 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
2957 3004 mstate->dtms_walltimestamp = dtrace_gethrestime();
2958 3005 mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
2959 3006 }
2960 3007 return (mstate->dtms_walltimestamp);
2961 3008
2962 3009 case DIF_VAR_IPL:
2963 3010 if (!dtrace_priv_kernel(state))
2964 3011 return (0);
2965 3012 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
2966 3013 mstate->dtms_ipl = dtrace_getipl();
2967 3014 mstate->dtms_present |= DTRACE_MSTATE_IPL;
2968 3015 }
2969 3016 return (mstate->dtms_ipl);
2970 3017
2971 3018 case DIF_VAR_EPID:
2972 3019 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
2973 3020 return (mstate->dtms_epid);
2974 3021
2975 3022 case DIF_VAR_ID:
2976 3023 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2977 3024 return (mstate->dtms_probe->dtpr_id);
2978 3025
2979 3026 case DIF_VAR_STACKDEPTH:
2980 3027 if (!dtrace_priv_kernel(state))
2981 3028 return (0);
2982 3029 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
2983 3030 int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2984 3031
2985 3032 mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
2986 3033 mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
2987 3034 }
2988 3035 return (mstate->dtms_stackdepth);
2989 3036
2990 3037 case DIF_VAR_USTACKDEPTH:
2991 3038 if (!dtrace_priv_proc(state, mstate))
2992 3039 return (0);
2993 3040 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
2994 3041 /*
2995 3042 * See comment in DIF_VAR_PID.
2996 3043 */
2997 3044 if (DTRACE_ANCHORED(mstate->dtms_probe) &&
2998 3045 CPU_ON_INTR(CPU)) {
2999 3046 mstate->dtms_ustackdepth = 0;
3000 3047 } else {
3001 3048 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3002 3049 mstate->dtms_ustackdepth =
3003 3050 dtrace_getustackdepth();
3004 3051 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3005 3052 }
3006 3053 mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
3007 3054 }
3008 3055 return (mstate->dtms_ustackdepth);
3009 3056
3010 3057 case DIF_VAR_CALLER:
3011 3058 if (!dtrace_priv_kernel(state))
3012 3059 return (0);
3013 3060 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
3014 3061 int aframes = mstate->dtms_probe->dtpr_aframes + 2;
3015 3062
3016 3063 if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
3017 3064 /*
3018 3065 * If this is an unanchored probe, we are
3019 3066 * required to go through the slow path:
3020 3067 * dtrace_caller() only guarantees correct
3021 3068 * results for anchored probes.
3022 3069 */
3023 3070 pc_t caller[2];
3024 3071
3025 3072 dtrace_getpcstack(caller, 2, aframes,
3026 3073 (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
3027 3074 mstate->dtms_caller = caller[1];
3028 3075 } else if ((mstate->dtms_caller =
3029 3076 dtrace_caller(aframes)) == -1) {
3030 3077 /*
3031 3078 * We have failed to do this the quick way;
3032 3079 * we must resort to the slower approach of
3033 3080 * calling dtrace_getpcstack().
3034 3081 */
3035 3082 pc_t caller;
3036 3083
3037 3084 dtrace_getpcstack(&caller, 1, aframes, NULL);
3038 3085 mstate->dtms_caller = caller;
3039 3086 }
3040 3087
3041 3088 mstate->dtms_present |= DTRACE_MSTATE_CALLER;
3042 3089 }
3043 3090 return (mstate->dtms_caller);
3044 3091
3045 3092 case DIF_VAR_UCALLER:
3046 3093 if (!dtrace_priv_proc(state, mstate))
3047 3094 return (0);
3048 3095
3049 3096 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
3050 3097 uint64_t ustack[3];
3051 3098
3052 3099 /*
3053 3100 * dtrace_getupcstack() fills in the first uint64_t
3054 3101 * with the current PID. The second uint64_t will
3055 3102 * be the program counter at user-level. The third
3056 3103 * uint64_t will contain the caller, which is what
3057 3104 * we're after.
3058 3105 */
3059 3106 ustack[2] = NULL;
3060 3107 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3061 3108 dtrace_getupcstack(ustack, 3);
3062 3109 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3063 3110 mstate->dtms_ucaller = ustack[2];
3064 3111 mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
3065 3112 }
3066 3113
3067 3114 return (mstate->dtms_ucaller);
3068 3115
3069 3116 case DIF_VAR_PROBEPROV:
3070 3117 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3071 3118 return (dtrace_dif_varstr(
3072 3119 (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name,
3073 3120 state, mstate));
3074 3121
3075 3122 case DIF_VAR_PROBEMOD:
3076 3123 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3077 3124 return (dtrace_dif_varstr(
3078 3125 (uintptr_t)mstate->dtms_probe->dtpr_mod,
3079 3126 state, mstate));
3080 3127
3081 3128 case DIF_VAR_PROBEFUNC:
3082 3129 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3083 3130 return (dtrace_dif_varstr(
3084 3131 (uintptr_t)mstate->dtms_probe->dtpr_func,
3085 3132 state, mstate));
3086 3133
3087 3134 case DIF_VAR_PROBENAME:
3088 3135 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
3089 3136 return (dtrace_dif_varstr(
3090 3137 (uintptr_t)mstate->dtms_probe->dtpr_name,
3091 3138 state, mstate));
3092 3139
3093 3140 case DIF_VAR_PID:
3094 3141 if (!dtrace_priv_proc(state, mstate))
3095 3142 return (0);
3096 3143
3097 3144 /*
3098 3145 * Note that we are assuming that an unanchored probe is
3099 3146 * always due to a high-level interrupt. (And we're assuming
3100 3147 * that there is only a single high level interrupt.)
3101 3148 */
3102 3149 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3103 3150 return (pid0.pid_id);
3104 3151
3105 3152 /*
3106 3153 * It is always safe to dereference one's own t_procp pointer:
3107 3154 * it always points to a valid, allocated proc structure.
3108 3155 * Further, it is always safe to dereference the p_pidp member
3109 3156 * of one's own proc structure. (These are truisms becuase
3110 3157 * threads and processes don't clean up their own state --
3111 3158 * they leave that task to whomever reaps them.)
3112 3159 */
3113 3160 return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
3114 3161
3115 3162 case DIF_VAR_PPID:
3116 3163 if (!dtrace_priv_proc(state, mstate))
3117 3164 return (0);
3118 3165
3119 3166 /*
3120 3167 * See comment in DIF_VAR_PID.
3121 3168 */
3122 3169 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3123 3170 return (pid0.pid_id);
3124 3171
3125 3172 /*
3126 3173 * It is always safe to dereference one's own t_procp pointer:
3127 3174 * it always points to a valid, allocated proc structure.
3128 3175 * (This is true because threads don't clean up their own
3129 3176 * state -- they leave that task to whomever reaps them.)
3130 3177 */
3131 3178 return ((uint64_t)curthread->t_procp->p_ppid);
3132 3179
3133 3180 case DIF_VAR_TID:
3134 3181 /*
3135 3182 * See comment in DIF_VAR_PID.
3136 3183 */
3137 3184 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3138 3185 return (0);
3139 3186
3140 3187 return ((uint64_t)curthread->t_tid);
3141 3188
3142 3189 case DIF_VAR_EXECNAME:
3143 3190 if (!dtrace_priv_proc(state, mstate))
3144 3191 return (0);
3145 3192
3146 3193 /*
3147 3194 * See comment in DIF_VAR_PID.
3148 3195 */
3149 3196 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3150 3197 return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
3151 3198
3152 3199 /*
3153 3200 * It is always safe to dereference one's own t_procp pointer:
3154 3201 * it always points to a valid, allocated proc structure.
3155 3202 * (This is true because threads don't clean up their own
3156 3203 * state -- they leave that task to whomever reaps them.)
3157 3204 */
3158 3205 return (dtrace_dif_varstr(
3159 3206 (uintptr_t)curthread->t_procp->p_user.u_comm,
3160 3207 state, mstate));
3161 3208
3162 3209 case DIF_VAR_ZONENAME:
3163 3210 if (!dtrace_priv_proc(state, mstate))
3164 3211 return (0);
3165 3212
3166 3213 /*
3167 3214 * See comment in DIF_VAR_PID.
3168 3215 */
3169 3216 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3170 3217 return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
3171 3218
3172 3219 /*
3173 3220 * It is always safe to dereference one's own t_procp pointer:
3174 3221 * it always points to a valid, allocated proc structure.
3175 3222 * (This is true because threads don't clean up their own
3176 3223 * state -- they leave that task to whomever reaps them.)
3177 3224 */
3178 3225 return (dtrace_dif_varstr(
3179 3226 (uintptr_t)curthread->t_procp->p_zone->zone_name,
3180 3227 state, mstate));
3181 3228
3182 3229 case DIF_VAR_UID:
3183 3230 if (!dtrace_priv_proc(state, mstate))
3184 3231 return (0);
3185 3232
3186 3233 /*
3187 3234 * See comment in DIF_VAR_PID.
3188 3235 */
3189 3236 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3190 3237 return ((uint64_t)p0.p_cred->cr_uid);
3191 3238
3192 3239 /*
3193 3240 * It is always safe to dereference one's own t_procp pointer:
3194 3241 * it always points to a valid, allocated proc structure.
3195 3242 * (This is true because threads don't clean up their own
3196 3243 * state -- they leave that task to whomever reaps them.)
3197 3244 *
3198 3245 * Additionally, it is safe to dereference one's own process
3199 3246 * credential, since this is never NULL after process birth.
3200 3247 */
3201 3248 return ((uint64_t)curthread->t_procp->p_cred->cr_uid);
3202 3249
3203 3250 case DIF_VAR_GID:
3204 3251 if (!dtrace_priv_proc(state, mstate))
3205 3252 return (0);
3206 3253
3207 3254 /*
3208 3255 * See comment in DIF_VAR_PID.
3209 3256 */
3210 3257 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3211 3258 return ((uint64_t)p0.p_cred->cr_gid);
3212 3259
3213 3260 /*
3214 3261 * It is always safe to dereference one's own t_procp pointer:
3215 3262 * it always points to a valid, allocated proc structure.
3216 3263 * (This is true because threads don't clean up their own
3217 3264 * state -- they leave that task to whomever reaps them.)
3218 3265 *
3219 3266 * Additionally, it is safe to dereference one's own process
3220 3267 * credential, since this is never NULL after process birth.
3221 3268 */
3222 3269 return ((uint64_t)curthread->t_procp->p_cred->cr_gid);
3223 3270
3224 3271 case DIF_VAR_ERRNO: {
3225 3272 klwp_t *lwp;
3226 3273 if (!dtrace_priv_proc(state, mstate))
3227 3274 return (0);
3228 3275
3229 3276 /*
3230 3277 * See comment in DIF_VAR_PID.
3231 3278 */
3232 3279 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
3233 3280 return (0);
3234 3281
3235 3282 /*
3236 3283 * It is always safe to dereference one's own t_lwp pointer in
3237 3284 * the event that this pointer is non-NULL. (This is true
3238 3285 * because threads and lwps don't clean up their own state --
3239 3286 * they leave that task to whomever reaps them.)
3240 3287 */
3241 3288 if ((lwp = curthread->t_lwp) == NULL)
3242 3289 return (0);
3243 3290
3244 3291 return ((uint64_t)lwp->lwp_errno);
3245 3292 }
3246 3293 default:
3247 3294 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
3248 3295 return (0);
3249 3296 }
3250 3297 }
3251 3298
3252 3299 /*
3253 3300 * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
3254 3301 * Notice that we don't bother validating the proper number of arguments or
3255 3302 * their types in the tuple stack. This isn't needed because all argument
3256 3303 * interpretation is safe because of our load safety -- the worst that can
3257 3304 * happen is that a bogus program can obtain bogus results.
3258 3305 */
3259 3306 static void
3260 3307 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
3261 3308 dtrace_key_t *tupregs, int nargs,
3262 3309 dtrace_mstate_t *mstate, dtrace_state_t *state)
3263 3310 {
3264 3311 volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
3265 3312 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
3266 3313 dtrace_vstate_t *vstate = &state->dts_vstate;
3267 3314
3268 3315 union {
3269 3316 mutex_impl_t mi;
3270 3317 uint64_t mx;
3271 3318 } m;
3272 3319
3273 3320 union {
3274 3321 krwlock_t ri;
3275 3322 uintptr_t rw;
3276 3323 } r;
3277 3324
3278 3325 switch (subr) {
3279 3326 case DIF_SUBR_RAND:
3280 3327 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
3281 3328 break;
3282 3329
3283 3330 case DIF_SUBR_MUTEX_OWNED:
3284 3331 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3285 3332 mstate, vstate)) {
3286 3333 regs[rd] = NULL;
3287 3334 break;
3288 3335 }
3289 3336
3290 3337 m.mx = dtrace_load64(tupregs[0].dttk_value);
3291 3338 if (MUTEX_TYPE_ADAPTIVE(&m.mi))
3292 3339 regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
3293 3340 else
3294 3341 regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
3295 3342 break;
3296 3343
3297 3344 case DIF_SUBR_MUTEX_OWNER:
3298 3345 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3299 3346 mstate, vstate)) {
3300 3347 regs[rd] = NULL;
3301 3348 break;
3302 3349 }
3303 3350
3304 3351 m.mx = dtrace_load64(tupregs[0].dttk_value);
3305 3352 if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
3306 3353 MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
3307 3354 regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
3308 3355 else
3309 3356 regs[rd] = 0;
3310 3357 break;
3311 3358
3312 3359 case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
3313 3360 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3314 3361 mstate, vstate)) {
3315 3362 regs[rd] = NULL;
3316 3363 break;
3317 3364 }
3318 3365
3319 3366 m.mx = dtrace_load64(tupregs[0].dttk_value);
3320 3367 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
3321 3368 break;
3322 3369
3323 3370 case DIF_SUBR_MUTEX_TYPE_SPIN:
3324 3371 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t),
3325 3372 mstate, vstate)) {
3326 3373 regs[rd] = NULL;
3327 3374 break;
3328 3375 }
3329 3376
3330 3377 m.mx = dtrace_load64(tupregs[0].dttk_value);
3331 3378 regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
3332 3379 break;
3333 3380
3334 3381 case DIF_SUBR_RW_READ_HELD: {
3335 3382 uintptr_t tmp;
3336 3383
3337 3384 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t),
3338 3385 mstate, vstate)) {
3339 3386 regs[rd] = NULL;
3340 3387 break;
3341 3388 }
3342 3389
3343 3390 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3344 3391 regs[rd] = _RW_READ_HELD(&r.ri, tmp);
3345 3392 break;
3346 3393 }
3347 3394
3348 3395 case DIF_SUBR_RW_WRITE_HELD:
3349 3396 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3350 3397 mstate, vstate)) {
3351 3398 regs[rd] = NULL;
3352 3399 break;
3353 3400 }
3354 3401
3355 3402 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3356 3403 regs[rd] = _RW_WRITE_HELD(&r.ri);
3357 3404 break;
3358 3405
3359 3406 case DIF_SUBR_RW_ISWRITER:
3360 3407 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t),
3361 3408 mstate, vstate)) {
3362 3409 regs[rd] = NULL;
3363 3410 break;
3364 3411 }
3365 3412
3366 3413 r.rw = dtrace_loadptr(tupregs[0].dttk_value);
3367 3414 regs[rd] = _RW_ISWRITER(&r.ri);
3368 3415 break;
3369 3416
3370 3417 case DIF_SUBR_BCOPY: {
3371 3418 /*
3372 3419 * We need to be sure that the destination is in the scratch
3373 3420 * region -- no other region is allowed.
3374 3421 */
3375 3422 uintptr_t src = tupregs[0].dttk_value;
3376 3423 uintptr_t dest = tupregs[1].dttk_value;
3377 3424 size_t size = tupregs[2].dttk_value;
3378 3425
3379 3426 if (!dtrace_inscratch(dest, size, mstate)) {
3380 3427 *flags |= CPU_DTRACE_BADADDR;
3381 3428 *illval = regs[rd];
3382 3429 break;
3383 3430 }
3384 3431
3385 3432 if (!dtrace_canload(src, size, mstate, vstate)) {
3386 3433 regs[rd] = NULL;
3387 3434 break;
3388 3435 }
3389 3436
3390 3437 dtrace_bcopy((void *)src, (void *)dest, size);
3391 3438 break;
3392 3439 }
3393 3440
3394 3441 case DIF_SUBR_ALLOCA:
3395 3442 case DIF_SUBR_COPYIN: {
3396 3443 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
3397 3444 uint64_t size =
3398 3445 tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
3399 3446 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
3400 3447
3401 3448 /*
3402 3449 * This action doesn't require any credential checks since
3403 3450 * probes will not activate in user contexts to which the
3404 3451 * enabling user does not have permissions.
3405 3452 */
3406 3453
3407 3454 /*
3408 3455 * Rounding up the user allocation size could have overflowed
3409 3456 * a large, bogus allocation (like -1ULL) to 0.
3410 3457 */
3411 3458 if (scratch_size < size ||
3412 3459 !DTRACE_INSCRATCH(mstate, scratch_size)) {
3413 3460 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3414 3461 regs[rd] = NULL;
3415 3462 break;
3416 3463 }
3417 3464
3418 3465 if (subr == DIF_SUBR_COPYIN) {
3419 3466 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3420 3467 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
3421 3468 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3422 3469 }
3423 3470
3424 3471 mstate->dtms_scratch_ptr += scratch_size;
3425 3472 regs[rd] = dest;
3426 3473 break;
3427 3474 }
3428 3475
3429 3476 case DIF_SUBR_COPYINTO: {
3430 3477 uint64_t size = tupregs[1].dttk_value;
3431 3478 uintptr_t dest = tupregs[2].dttk_value;
3432 3479
3433 3480 /*
3434 3481 * This action doesn't require any credential checks since
3435 3482 * probes will not activate in user contexts to which the
3436 3483 * enabling user does not have permissions.
3437 3484 */
3438 3485 if (!dtrace_inscratch(dest, size, mstate)) {
3439 3486 *flags |= CPU_DTRACE_BADADDR;
3440 3487 *illval = regs[rd];
3441 3488 break;
3442 3489 }
3443 3490
3444 3491 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3445 3492 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags);
3446 3493 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3447 3494 break;
3448 3495 }
3449 3496
3450 3497 case DIF_SUBR_COPYINSTR: {
3451 3498 uintptr_t dest = mstate->dtms_scratch_ptr;
3452 3499 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3453 3500
3454 3501 if (nargs > 1 && tupregs[1].dttk_value < size)
3455 3502 size = tupregs[1].dttk_value + 1;
3456 3503
3457 3504 /*
3458 3505 * This action doesn't require any credential checks since
3459 3506 * probes will not activate in user contexts to which the
3460 3507 * enabling user does not have permissions.
3461 3508 */
3462 3509 if (!DTRACE_INSCRATCH(mstate, size)) {
3463 3510 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3464 3511 regs[rd] = NULL;
3465 3512 break;
3466 3513 }
3467 3514
3468 3515 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3469 3516 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags);
3470 3517 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3471 3518
3472 3519 ((char *)dest)[size - 1] = '\0';
3473 3520 mstate->dtms_scratch_ptr += size;
3474 3521 regs[rd] = dest;
3475 3522 break;
3476 3523 }
3477 3524
3478 3525 case DIF_SUBR_MSGSIZE:
3479 3526 case DIF_SUBR_MSGDSIZE: {
3480 3527 uintptr_t baddr = tupregs[0].dttk_value, daddr;
3481 3528 uintptr_t wptr, rptr;
3482 3529 size_t count = 0;
3483 3530 int cont = 0;
3484 3531
3485 3532 while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
3486 3533
3487 3534 if (!dtrace_canload(baddr, sizeof (mblk_t), mstate,
3488 3535 vstate)) {
3489 3536 regs[rd] = NULL;
3490 3537 break;
3491 3538 }
3492 3539
3493 3540 wptr = dtrace_loadptr(baddr +
3494 3541 offsetof(mblk_t, b_wptr));
3495 3542
3496 3543 rptr = dtrace_loadptr(baddr +
3497 3544 offsetof(mblk_t, b_rptr));
3498 3545
3499 3546 if (wptr < rptr) {
3500 3547 *flags |= CPU_DTRACE_BADADDR;
3501 3548 *illval = tupregs[0].dttk_value;
3502 3549 break;
3503 3550 }
3504 3551
3505 3552 daddr = dtrace_loadptr(baddr +
3506 3553 offsetof(mblk_t, b_datap));
3507 3554
3508 3555 baddr = dtrace_loadptr(baddr +
3509 3556 offsetof(mblk_t, b_cont));
3510 3557
3511 3558 /*
3512 3559 * We want to prevent against denial-of-service here,
3513 3560 * so we're only going to search the list for
3514 3561 * dtrace_msgdsize_max mblks.
3515 3562 */
3516 3563 if (cont++ > dtrace_msgdsize_max) {
3517 3564 *flags |= CPU_DTRACE_ILLOP;
3518 3565 break;
3519 3566 }
3520 3567
3521 3568 if (subr == DIF_SUBR_MSGDSIZE) {
3522 3569 if (dtrace_load8(daddr +
3523 3570 offsetof(dblk_t, db_type)) != M_DATA)
3524 3571 continue;
3525 3572 }
3526 3573
3527 3574 count += wptr - rptr;
3528 3575 }
3529 3576
3530 3577 if (!(*flags & CPU_DTRACE_FAULT))
3531 3578 regs[rd] = count;
3532 3579
3533 3580 break;
3534 3581 }
3535 3582
3536 3583 case DIF_SUBR_PROGENYOF: {
3537 3584 pid_t pid = tupregs[0].dttk_value;
3538 3585 proc_t *p;
3539 3586 int rval = 0;
3540 3587
3541 3588 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3542 3589
3543 3590 for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
3544 3591 if (p->p_pidp->pid_id == pid) {
3545 3592 rval = 1;
3546 3593 break;
3547 3594 }
3548 3595 }
3549 3596
3550 3597 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3551 3598
3552 3599 regs[rd] = rval;
3553 3600 break;
3554 3601 }
3555 3602
3556 3603 case DIF_SUBR_SPECULATION:
3557 3604 regs[rd] = dtrace_speculation(state);
3558 3605 break;
3559 3606
3560 3607 case DIF_SUBR_COPYOUT: {
3561 3608 uintptr_t kaddr = tupregs[0].dttk_value;
3562 3609 uintptr_t uaddr = tupregs[1].dttk_value;
3563 3610 uint64_t size = tupregs[2].dttk_value;
3564 3611
3565 3612 if (!dtrace_destructive_disallow &&
3566 3613 dtrace_priv_proc_control(state, mstate) &&
3567 3614 !dtrace_istoxic(kaddr, size)) {
3568 3615 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3569 3616 dtrace_copyout(kaddr, uaddr, size, flags);
3570 3617 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3571 3618 }
3572 3619 break;
3573 3620 }
3574 3621
3575 3622 case DIF_SUBR_COPYOUTSTR: {
3576 3623 uintptr_t kaddr = tupregs[0].dttk_value;
3577 3624 uintptr_t uaddr = tupregs[1].dttk_value;
3578 3625 uint64_t size = tupregs[2].dttk_value;
3579 3626
3580 3627 if (!dtrace_destructive_disallow &&
3581 3628 dtrace_priv_proc_control(state, mstate) &&
3582 3629 !dtrace_istoxic(kaddr, size)) {
3583 3630 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
3584 3631 dtrace_copyoutstr(kaddr, uaddr, size, flags);
3585 3632 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
3586 3633 }
3587 3634 break;
3588 3635 }
3589 3636
3590 3637 case DIF_SUBR_STRLEN: {
3591 3638 size_t sz;
3592 3639 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value;
3593 3640 sz = dtrace_strlen((char *)addr,
3594 3641 state->dts_options[DTRACEOPT_STRSIZE]);
3595 3642
3596 3643 if (!dtrace_canload(addr, sz + 1, mstate, vstate)) {
3597 3644 regs[rd] = NULL;
3598 3645 break;
3599 3646 }
3600 3647
3601 3648 regs[rd] = sz;
3602 3649
3603 3650 break;
3604 3651 }
3605 3652
3606 3653 case DIF_SUBR_STRCHR:
3607 3654 case DIF_SUBR_STRRCHR: {
3608 3655 /*
3609 3656 * We're going to iterate over the string looking for the
3610 3657 * specified character. We will iterate until we have reached
3611 3658 * the string length or we have found the character. If this
3612 3659 * is DIF_SUBR_STRRCHR, we will look for the last occurrence
3613 3660 * of the specified character instead of the first.
3614 3661 */
3615 3662 uintptr_t saddr = tupregs[0].dttk_value;
3616 3663 uintptr_t addr = tupregs[0].dttk_value;
3617 3664 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
3618 3665 char c, target = (char)tupregs[1].dttk_value;
3619 3666
3620 3667 for (regs[rd] = NULL; addr < limit; addr++) {
3621 3668 if ((c = dtrace_load8(addr)) == target) {
3622 3669 regs[rd] = addr;
3623 3670
3624 3671 if (subr == DIF_SUBR_STRCHR)
3625 3672 break;
3626 3673 }
3627 3674
3628 3675 if (c == '\0')
3629 3676 break;
3630 3677 }
3631 3678
3632 3679 if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) {
3633 3680 regs[rd] = NULL;
3634 3681 break;
3635 3682 }
3636 3683
3637 3684 break;
3638 3685 }
3639 3686
3640 3687 case DIF_SUBR_STRSTR:
3641 3688 case DIF_SUBR_INDEX:
3642 3689 case DIF_SUBR_RINDEX: {
3643 3690 /*
3644 3691 * We're going to iterate over the string looking for the
3645 3692 * specified string. We will iterate until we have reached
3646 3693 * the string length or we have found the string. (Yes, this
3647 3694 * is done in the most naive way possible -- but considering
3648 3695 * that the string we're searching for is likely to be
3649 3696 * relatively short, the complexity of Rabin-Karp or similar
3650 3697 * hardly seems merited.)
3651 3698 */
3652 3699 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
3653 3700 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
3654 3701 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3655 3702 size_t len = dtrace_strlen(addr, size);
3656 3703 size_t sublen = dtrace_strlen(substr, size);
3657 3704 char *limit = addr + len, *orig = addr;
3658 3705 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
3659 3706 int inc = 1;
3660 3707
3661 3708 regs[rd] = notfound;
3662 3709
3663 3710 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) {
3664 3711 regs[rd] = NULL;
3665 3712 break;
3666 3713 }
3667 3714
3668 3715 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate,
3669 3716 vstate)) {
3670 3717 regs[rd] = NULL;
3671 3718 break;
3672 3719 }
3673 3720
3674 3721 /*
3675 3722 * strstr() and index()/rindex() have similar semantics if
3676 3723 * both strings are the empty string: strstr() returns a
3677 3724 * pointer to the (empty) string, and index() and rindex()
3678 3725 * both return index 0 (regardless of any position argument).
3679 3726 */
3680 3727 if (sublen == 0 && len == 0) {
3681 3728 if (subr == DIF_SUBR_STRSTR)
3682 3729 regs[rd] = (uintptr_t)addr;
3683 3730 else
3684 3731 regs[rd] = 0;
3685 3732 break;
3686 3733 }
3687 3734
3688 3735 if (subr != DIF_SUBR_STRSTR) {
3689 3736 if (subr == DIF_SUBR_RINDEX) {
3690 3737 limit = orig - 1;
3691 3738 addr += len;
3692 3739 inc = -1;
3693 3740 }
3694 3741
3695 3742 /*
3696 3743 * Both index() and rindex() take an optional position
3697 3744 * argument that denotes the starting position.
3698 3745 */
3699 3746 if (nargs == 3) {
3700 3747 int64_t pos = (int64_t)tupregs[2].dttk_value;
3701 3748
3702 3749 /*
3703 3750 * If the position argument to index() is
3704 3751 * negative, Perl implicitly clamps it at
3705 3752 * zero. This semantic is a little surprising
3706 3753 * given the special meaning of negative
3707 3754 * positions to similar Perl functions like
3708 3755 * substr(), but it appears to reflect a
3709 3756 * notion that index() can start from a
3710 3757 * negative index and increment its way up to
3711 3758 * the string. Given this notion, Perl's
3712 3759 * rindex() is at least self-consistent in
3713 3760 * that it implicitly clamps positions greater
3714 3761 * than the string length to be the string
3715 3762 * length. Where Perl completely loses
3716 3763 * coherence, however, is when the specified
3717 3764 * substring is the empty string (""). In
3718 3765 * this case, even if the position is
3719 3766 * negative, rindex() returns 0 -- and even if
3720 3767 * the position is greater than the length,
3721 3768 * index() returns the string length. These
3722 3769 * semantics violate the notion that index()
3723 3770 * should never return a value less than the
3724 3771 * specified position and that rindex() should
3725 3772 * never return a value greater than the
3726 3773 * specified position. (One assumes that
3727 3774 * these semantics are artifacts of Perl's
3728 3775 * implementation and not the results of
3729 3776 * deliberate design -- it beggars belief that
3730 3777 * even Larry Wall could desire such oddness.)
3731 3778 * While in the abstract one would wish for
3732 3779 * consistent position semantics across
3733 3780 * substr(), index() and rindex() -- or at the
3734 3781 * very least self-consistent position
3735 3782 * semantics for index() and rindex() -- we
3736 3783 * instead opt to keep with the extant Perl
3737 3784 * semantics, in all their broken glory. (Do
3738 3785 * we have more desire to maintain Perl's
3739 3786 * semantics than Perl does? Probably.)
3740 3787 */
3741 3788 if (subr == DIF_SUBR_RINDEX) {
3742 3789 if (pos < 0) {
3743 3790 if (sublen == 0)
3744 3791 regs[rd] = 0;
3745 3792 break;
3746 3793 }
3747 3794
3748 3795 if (pos > len)
3749 3796 pos = len;
3750 3797 } else {
3751 3798 if (pos < 0)
3752 3799 pos = 0;
3753 3800
3754 3801 if (pos >= len) {
3755 3802 if (sublen == 0)
3756 3803 regs[rd] = len;
3757 3804 break;
3758 3805 }
3759 3806 }
3760 3807
3761 3808 addr = orig + pos;
3762 3809 }
3763 3810 }
3764 3811
3765 3812 for (regs[rd] = notfound; addr != limit; addr += inc) {
3766 3813 if (dtrace_strncmp(addr, substr, sublen) == 0) {
3767 3814 if (subr != DIF_SUBR_STRSTR) {
3768 3815 /*
3769 3816 * As D index() and rindex() are
3770 3817 * modeled on Perl (and not on awk),
3771 3818 * we return a zero-based (and not a
3772 3819 * one-based) index. (For you Perl
3773 3820 * weenies: no, we're not going to add
3774 3821 * $[ -- and shouldn't you be at a con
3775 3822 * or something?)
3776 3823 */
3777 3824 regs[rd] = (uintptr_t)(addr - orig);
3778 3825 break;
3779 3826 }
3780 3827
3781 3828 ASSERT(subr == DIF_SUBR_STRSTR);
3782 3829 regs[rd] = (uintptr_t)addr;
3783 3830 break;
3784 3831 }
3785 3832 }
3786 3833
3787 3834 break;
3788 3835 }
3789 3836
3790 3837 case DIF_SUBR_STRTOK: {
3791 3838 uintptr_t addr = tupregs[0].dttk_value;
3792 3839 uintptr_t tokaddr = tupregs[1].dttk_value;
3793 3840 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3794 3841 uintptr_t limit, toklimit = tokaddr + size;
3795 3842 uint8_t c, tokmap[32]; /* 256 / 8 */
3796 3843 char *dest = (char *)mstate->dtms_scratch_ptr;
3797 3844 int i;
3798 3845
3799 3846 /*
3800 3847 * Check both the token buffer and (later) the input buffer,
3801 3848 * since both could be non-scratch addresses.
3802 3849 */
3803 3850 if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) {
3804 3851 regs[rd] = NULL;
3805 3852 break;
3806 3853 }
3807 3854
3808 3855 if (!DTRACE_INSCRATCH(mstate, size)) {
3809 3856 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3810 3857 regs[rd] = NULL;
3811 3858 break;
3812 3859 }
3813 3860
3814 3861 if (addr == NULL) {
3815 3862 /*
3816 3863 * If the address specified is NULL, we use our saved
3817 3864 * strtok pointer from the mstate. Note that this
3818 3865 * means that the saved strtok pointer is _only_
3819 3866 * valid within multiple enablings of the same probe --
3820 3867 * it behaves like an implicit clause-local variable.
3821 3868 */
3822 3869 addr = mstate->dtms_strtok;
3823 3870 } else {
3824 3871 /*
3825 3872 * If the user-specified address is non-NULL we must
3826 3873 * access check it. This is the only time we have
3827 3874 * a chance to do so, since this address may reside
3828 3875 * in the string table of this clause-- future calls
3829 3876 * (when we fetch addr from mstate->dtms_strtok)
3830 3877 * would fail this access check.
3831 3878 */
3832 3879 if (!dtrace_strcanload(addr, size, mstate, vstate)) {
3833 3880 regs[rd] = NULL;
3834 3881 break;
3835 3882 }
3836 3883 }
3837 3884
3838 3885 /*
3839 3886 * First, zero the token map, and then process the token
3840 3887 * string -- setting a bit in the map for every character
3841 3888 * found in the token string.
3842 3889 */
3843 3890 for (i = 0; i < sizeof (tokmap); i++)
3844 3891 tokmap[i] = 0;
3845 3892
3846 3893 for (; tokaddr < toklimit; tokaddr++) {
3847 3894 if ((c = dtrace_load8(tokaddr)) == '\0')
3848 3895 break;
3849 3896
3850 3897 ASSERT((c >> 3) < sizeof (tokmap));
3851 3898 tokmap[c >> 3] |= (1 << (c & 0x7));
3852 3899 }
3853 3900
3854 3901 for (limit = addr + size; addr < limit; addr++) {
3855 3902 /*
3856 3903 * We're looking for a character that is _not_ contained
3857 3904 * in the token string.
3858 3905 */
3859 3906 if ((c = dtrace_load8(addr)) == '\0')
3860 3907 break;
3861 3908
3862 3909 if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
3863 3910 break;
3864 3911 }
3865 3912
3866 3913 if (c == '\0') {
3867 3914 /*
3868 3915 * We reached the end of the string without finding
3869 3916 * any character that was not in the token string.
3870 3917 * We return NULL in this case, and we set the saved
3871 3918 * address to NULL as well.
3872 3919 */
3873 3920 regs[rd] = NULL;
3874 3921 mstate->dtms_strtok = NULL;
3875 3922 break;
3876 3923 }
3877 3924
3878 3925 /*
3879 3926 * From here on, we're copying into the destination string.
3880 3927 */
3881 3928 for (i = 0; addr < limit && i < size - 1; addr++) {
3882 3929 if ((c = dtrace_load8(addr)) == '\0')
3883 3930 break;
3884 3931
3885 3932 if (tokmap[c >> 3] & (1 << (c & 0x7)))
3886 3933 break;
3887 3934
3888 3935 ASSERT(i < size);
3889 3936 dest[i++] = c;
3890 3937 }
3891 3938
3892 3939 ASSERT(i < size);
3893 3940 dest[i] = '\0';
3894 3941 regs[rd] = (uintptr_t)dest;
3895 3942 mstate->dtms_scratch_ptr += size;
3896 3943 mstate->dtms_strtok = addr;
3897 3944 break;
3898 3945 }
3899 3946
3900 3947 case DIF_SUBR_SUBSTR: {
3901 3948 uintptr_t s = tupregs[0].dttk_value;
3902 3949 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3903 3950 char *d = (char *)mstate->dtms_scratch_ptr;
3904 3951 int64_t index = (int64_t)tupregs[1].dttk_value;
3905 3952 int64_t remaining = (int64_t)tupregs[2].dttk_value;
3906 3953 size_t len = dtrace_strlen((char *)s, size);
3907 3954 int64_t i;
3908 3955
3909 3956 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
3910 3957 regs[rd] = NULL;
3911 3958 break;
3912 3959 }
3913 3960
3914 3961 if (!DTRACE_INSCRATCH(mstate, size)) {
3915 3962 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3916 3963 regs[rd] = NULL;
3917 3964 break;
3918 3965 }
3919 3966
3920 3967 if (nargs <= 2)
3921 3968 remaining = (int64_t)size;
3922 3969
3923 3970 if (index < 0) {
3924 3971 index += len;
3925 3972
3926 3973 if (index < 0 && index + remaining > 0) {
3927 3974 remaining += index;
3928 3975 index = 0;
3929 3976 }
3930 3977 }
3931 3978
3932 3979 if (index >= len || index < 0) {
3933 3980 remaining = 0;
3934 3981 } else if (remaining < 0) {
3935 3982 remaining += len - index;
3936 3983 } else if (index + remaining > size) {
3937 3984 remaining = size - index;
3938 3985 }
3939 3986
3940 3987 for (i = 0; i < remaining; i++) {
3941 3988 if ((d[i] = dtrace_load8(s + index + i)) == '\0')
3942 3989 break;
3943 3990 }
3944 3991
3945 3992 d[i] = '\0';
3946 3993
3947 3994 mstate->dtms_scratch_ptr += size;
3948 3995 regs[rd] = (uintptr_t)d;
3949 3996 break;
3950 3997 }
3951 3998
3952 3999 case DIF_SUBR_TOUPPER:
3953 4000 case DIF_SUBR_TOLOWER: {
3954 4001 uintptr_t s = tupregs[0].dttk_value;
3955 4002 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3956 4003 char *dest = (char *)mstate->dtms_scratch_ptr, c;
3957 4004 size_t len = dtrace_strlen((char *)s, size);
3958 4005 char lower, upper, convert;
3959 4006 int64_t i;
3960 4007
3961 4008 if (subr == DIF_SUBR_TOUPPER) {
3962 4009 lower = 'a';
3963 4010 upper = 'z';
3964 4011 convert = 'A';
3965 4012 } else {
3966 4013 lower = 'A';
3967 4014 upper = 'Z';
3968 4015 convert = 'a';
3969 4016 }
3970 4017
3971 4018 if (!dtrace_canload(s, len + 1, mstate, vstate)) {
3972 4019 regs[rd] = NULL;
3973 4020 break;
3974 4021 }
3975 4022
3976 4023 if (!DTRACE_INSCRATCH(mstate, size)) {
3977 4024 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3978 4025 regs[rd] = NULL;
3979 4026 break;
3980 4027 }
3981 4028
3982 4029 for (i = 0; i < size - 1; i++) {
3983 4030 if ((c = dtrace_load8(s + i)) == '\0')
3984 4031 break;
3985 4032
3986 4033 if (c >= lower && c <= upper)
3987 4034 c = convert + (c - lower);
3988 4035
3989 4036 dest[i] = c;
3990 4037 }
3991 4038
3992 4039 ASSERT(i < size);
3993 4040 dest[i] = '\0';
3994 4041 regs[rd] = (uintptr_t)dest;
3995 4042 mstate->dtms_scratch_ptr += size;
3996 4043 break;
3997 4044 }
3998 4045
3999 4046 case DIF_SUBR_GETMAJOR:
4000 4047 #ifdef _LP64
4001 4048 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
4002 4049 #else
4003 4050 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
4004 4051 #endif
4005 4052 break;
4006 4053
4007 4054 case DIF_SUBR_GETMINOR:
4008 4055 #ifdef _LP64
4009 4056 regs[rd] = tupregs[0].dttk_value & MAXMIN64;
4010 4057 #else
4011 4058 regs[rd] = tupregs[0].dttk_value & MAXMIN;
4012 4059 #endif
4013 4060 break;
4014 4061
4015 4062 case DIF_SUBR_DDI_PATHNAME: {
4016 4063 /*
4017 4064 * This one is a galactic mess. We are going to roughly
4018 4065 * emulate ddi_pathname(), but it's made more complicated
4019 4066 * by the fact that we (a) want to include the minor name and
4020 4067 * (b) must proceed iteratively instead of recursively.
4021 4068 */
4022 4069 uintptr_t dest = mstate->dtms_scratch_ptr;
4023 4070 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4024 4071 char *start = (char *)dest, *end = start + size - 1;
4025 4072 uintptr_t daddr = tupregs[0].dttk_value;
4026 4073 int64_t minor = (int64_t)tupregs[1].dttk_value;
4027 4074 char *s;
4028 4075 int i, len, depth = 0;
4029 4076
4030 4077 /*
4031 4078 * Due to all the pointer jumping we do and context we must
4032 4079 * rely upon, we just mandate that the user must have kernel
4033 4080 * read privileges to use this routine.
4034 4081 */
4035 4082 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) {
4036 4083 *flags |= CPU_DTRACE_KPRIV;
4037 4084 *illval = daddr;
4038 4085 regs[rd] = NULL;
4039 4086 }
4040 4087
4041 4088 if (!DTRACE_INSCRATCH(mstate, size)) {
4042 4089 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4043 4090 regs[rd] = NULL;
4044 4091 break;
4045 4092 }
4046 4093
4047 4094 *end = '\0';
4048 4095
4049 4096 /*
4050 4097 * We want to have a name for the minor. In order to do this,
4051 4098 * we need to walk the minor list from the devinfo. We want
4052 4099 * to be sure that we don't infinitely walk a circular list,
4053 4100 * so we check for circularity by sending a scout pointer
4054 4101 * ahead two elements for every element that we iterate over;
4055 4102 * if the list is circular, these will ultimately point to the
4056 4103 * same element. You may recognize this little trick as the
4057 4104 * answer to a stupid interview question -- one that always
4058 4105 * seems to be asked by those who had to have it laboriously
4059 4106 * explained to them, and who can't even concisely describe
4060 4107 * the conditions under which one would be forced to resort to
4061 4108 * this technique. Needless to say, those conditions are
4062 4109 * found here -- and probably only here. Is this the only use
4063 4110 * of this infamous trick in shipping, production code? If it
4064 4111 * isn't, it probably should be...
4065 4112 */
4066 4113 if (minor != -1) {
4067 4114 uintptr_t maddr = dtrace_loadptr(daddr +
4068 4115 offsetof(struct dev_info, devi_minor));
4069 4116
4070 4117 uintptr_t next = offsetof(struct ddi_minor_data, next);
4071 4118 uintptr_t name = offsetof(struct ddi_minor_data,
4072 4119 d_minor) + offsetof(struct ddi_minor, name);
4073 4120 uintptr_t dev = offsetof(struct ddi_minor_data,
4074 4121 d_minor) + offsetof(struct ddi_minor, dev);
4075 4122 uintptr_t scout;
4076 4123
4077 4124 if (maddr != NULL)
4078 4125 scout = dtrace_loadptr(maddr + next);
4079 4126
4080 4127 while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4081 4128 uint64_t m;
4082 4129 #ifdef _LP64
4083 4130 m = dtrace_load64(maddr + dev) & MAXMIN64;
4084 4131 #else
4085 4132 m = dtrace_load32(maddr + dev) & MAXMIN;
4086 4133 #endif
4087 4134 if (m != minor) {
4088 4135 maddr = dtrace_loadptr(maddr + next);
4089 4136
4090 4137 if (scout == NULL)
4091 4138 continue;
4092 4139
4093 4140 scout = dtrace_loadptr(scout + next);
4094 4141
4095 4142 if (scout == NULL)
4096 4143 continue;
4097 4144
4098 4145 scout = dtrace_loadptr(scout + next);
4099 4146
4100 4147 if (scout == NULL)
4101 4148 continue;
4102 4149
4103 4150 if (scout == maddr) {
4104 4151 *flags |= CPU_DTRACE_ILLOP;
4105 4152 break;
4106 4153 }
4107 4154
4108 4155 continue;
4109 4156 }
4110 4157
4111 4158 /*
4112 4159 * We have the minor data. Now we need to
4113 4160 * copy the minor's name into the end of the
4114 4161 * pathname.
4115 4162 */
4116 4163 s = (char *)dtrace_loadptr(maddr + name);
4117 4164 len = dtrace_strlen(s, size);
4118 4165
4119 4166 if (*flags & CPU_DTRACE_FAULT)
4120 4167 break;
4121 4168
4122 4169 if (len != 0) {
4123 4170 if ((end -= (len + 1)) < start)
4124 4171 break;
4125 4172
4126 4173 *end = ':';
4127 4174 }
4128 4175
4129 4176 for (i = 1; i <= len; i++)
4130 4177 end[i] = dtrace_load8((uintptr_t)s++);
4131 4178 break;
4132 4179 }
4133 4180 }
4134 4181
4135 4182 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
4136 4183 ddi_node_state_t devi_state;
4137 4184
4138 4185 devi_state = dtrace_load32(daddr +
4139 4186 offsetof(struct dev_info, devi_node_state));
4140 4187
4141 4188 if (*flags & CPU_DTRACE_FAULT)
4142 4189 break;
4143 4190
4144 4191 if (devi_state >= DS_INITIALIZED) {
4145 4192 s = (char *)dtrace_loadptr(daddr +
4146 4193 offsetof(struct dev_info, devi_addr));
4147 4194 len = dtrace_strlen(s, size);
4148 4195
4149 4196 if (*flags & CPU_DTRACE_FAULT)
4150 4197 break;
4151 4198
4152 4199 if (len != 0) {
4153 4200 if ((end -= (len + 1)) < start)
4154 4201 break;
4155 4202
4156 4203 *end = '@';
4157 4204 }
4158 4205
4159 4206 for (i = 1; i <= len; i++)
4160 4207 end[i] = dtrace_load8((uintptr_t)s++);
4161 4208 }
4162 4209
4163 4210 /*
4164 4211 * Now for the node name...
4165 4212 */
4166 4213 s = (char *)dtrace_loadptr(daddr +
4167 4214 offsetof(struct dev_info, devi_node_name));
4168 4215
4169 4216 daddr = dtrace_loadptr(daddr +
4170 4217 offsetof(struct dev_info, devi_parent));
4171 4218
4172 4219 /*
4173 4220 * If our parent is NULL (that is, if we're the root
4174 4221 * node), we're going to use the special path
4175 4222 * "devices".
4176 4223 */
4177 4224 if (daddr == NULL)
4178 4225 s = "devices";
4179 4226
4180 4227 len = dtrace_strlen(s, size);
4181 4228 if (*flags & CPU_DTRACE_FAULT)
4182 4229 break;
4183 4230
4184 4231 if ((end -= (len + 1)) < start)
4185 4232 break;
4186 4233
4187 4234 for (i = 1; i <= len; i++)
4188 4235 end[i] = dtrace_load8((uintptr_t)s++);
4189 4236 *end = '/';
4190 4237
4191 4238 if (depth++ > dtrace_devdepth_max) {
4192 4239 *flags |= CPU_DTRACE_ILLOP;
4193 4240 break;
4194 4241 }
4195 4242 }
4196 4243
4197 4244 if (end < start)
4198 4245 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4199 4246
4200 4247 if (daddr == NULL) {
4201 4248 regs[rd] = (uintptr_t)end;
4202 4249 mstate->dtms_scratch_ptr += size;
4203 4250 }
4204 4251
4205 4252 break;
4206 4253 }
4207 4254
4208 4255 case DIF_SUBR_STRJOIN: {
4209 4256 char *d = (char *)mstate->dtms_scratch_ptr;
4210 4257 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4211 4258 uintptr_t s1 = tupregs[0].dttk_value;
4212 4259 uintptr_t s2 = tupregs[1].dttk_value;
4213 4260 int i = 0;
4214 4261
4215 4262 if (!dtrace_strcanload(s1, size, mstate, vstate) ||
4216 4263 !dtrace_strcanload(s2, size, mstate, vstate)) {
4217 4264 regs[rd] = NULL;
4218 4265 break;
4219 4266 }
4220 4267
4221 4268 if (!DTRACE_INSCRATCH(mstate, size)) {
4222 4269 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4223 4270 regs[rd] = NULL;
4224 4271 break;
4225 4272 }
4226 4273
4227 4274 for (;;) {
4228 4275 if (i >= size) {
4229 4276 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4230 4277 regs[rd] = NULL;
4231 4278 break;
4232 4279 }
4233 4280
4234 4281 if ((d[i++] = dtrace_load8(s1++)) == '\0') {
4235 4282 i--;
4236 4283 break;
4237 4284 }
4238 4285 }
4239 4286
4240 4287 for (;;) {
4241 4288 if (i >= size) {
4242 4289 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4243 4290 regs[rd] = NULL;
4244 4291 break;
4245 4292 }
4246 4293
4247 4294 if ((d[i++] = dtrace_load8(s2++)) == '\0')
4248 4295 break;
4249 4296 }
4250 4297
4251 4298 if (i < size) {
4252 4299 mstate->dtms_scratch_ptr += i;
4253 4300 regs[rd] = (uintptr_t)d;
4254 4301 }
4255 4302
4256 4303 break;
4257 4304 }
4258 4305
4259 4306 case DIF_SUBR_LLTOSTR: {
4260 4307 int64_t i = (int64_t)tupregs[0].dttk_value;
4261 4308 uint64_t val, digit;
4262 4309 uint64_t size = 65; /* enough room for 2^64 in binary */
4263 4310 char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
4264 4311 int base = 10;
4265 4312
4266 4313 if (nargs > 1) {
4267 4314 if ((base = tupregs[1].dttk_value) <= 1 ||
4268 4315 base > ('z' - 'a' + 1) + ('9' - '0' + 1)) {
4269 4316 *flags |= CPU_DTRACE_ILLOP;
4270 4317 break;
4271 4318 }
4272 4319 }
4273 4320
4274 4321 val = (base == 10 && i < 0) ? i * -1 : i;
4275 4322
4276 4323 if (!DTRACE_INSCRATCH(mstate, size)) {
4277 4324 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4278 4325 regs[rd] = NULL;
4279 4326 break;
4280 4327 }
4281 4328
4282 4329 for (*end-- = '\0'; val; val /= base) {
4283 4330 if ((digit = val % base) <= '9' - '0') {
4284 4331 *end-- = '0' + digit;
4285 4332 } else {
4286 4333 *end-- = 'a' + (digit - ('9' - '0') - 1);
4287 4334 }
4288 4335 }
4289 4336
4290 4337 if (i == 0 && base == 16)
4291 4338 *end-- = '0';
4292 4339
4293 4340 if (base == 16)
4294 4341 *end-- = 'x';
4295 4342
4296 4343 if (i == 0 || base == 8 || base == 16)
4297 4344 *end-- = '0';
4298 4345
4299 4346 if (i < 0 && base == 10)
4300 4347 *end-- = '-';
4301 4348
4302 4349 regs[rd] = (uintptr_t)end + 1;
4303 4350 mstate->dtms_scratch_ptr += size;
4304 4351 break;
4305 4352 }
4306 4353
4307 4354 case DIF_SUBR_HTONS:
4308 4355 case DIF_SUBR_NTOHS:
4309 4356 #ifdef _BIG_ENDIAN
4310 4357 regs[rd] = (uint16_t)tupregs[0].dttk_value;
4311 4358 #else
4312 4359 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value);
4313 4360 #endif
4314 4361 break;
4315 4362
4316 4363
4317 4364 case DIF_SUBR_HTONL:
4318 4365 case DIF_SUBR_NTOHL:
4319 4366 #ifdef _BIG_ENDIAN
4320 4367 regs[rd] = (uint32_t)tupregs[0].dttk_value;
4321 4368 #else
4322 4369 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value);
4323 4370 #endif
4324 4371 break;
4325 4372
4326 4373
4327 4374 case DIF_SUBR_HTONLL:
4328 4375 case DIF_SUBR_NTOHLL:
4329 4376 #ifdef _BIG_ENDIAN
4330 4377 regs[rd] = (uint64_t)tupregs[0].dttk_value;
4331 4378 #else
4332 4379 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value);
4333 4380 #endif
4334 4381 break;
4335 4382
4336 4383
4337 4384 case DIF_SUBR_DIRNAME:
4338 4385 case DIF_SUBR_BASENAME: {
4339 4386 char *dest = (char *)mstate->dtms_scratch_ptr;
4340 4387 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4341 4388 uintptr_t src = tupregs[0].dttk_value;
4342 4389 int i, j, len = dtrace_strlen((char *)src, size);
4343 4390 int lastbase = -1, firstbase = -1, lastdir = -1;
4344 4391 int start, end;
4345 4392
4346 4393 if (!dtrace_canload(src, len + 1, mstate, vstate)) {
4347 4394 regs[rd] = NULL;
4348 4395 break;
4349 4396 }
4350 4397
4351 4398 if (!DTRACE_INSCRATCH(mstate, size)) {
4352 4399 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4353 4400 regs[rd] = NULL;
4354 4401 break;
4355 4402 }
4356 4403
4357 4404 /*
4358 4405 * The basename and dirname for a zero-length string is
4359 4406 * defined to be "."
4360 4407 */
4361 4408 if (len == 0) {
4362 4409 len = 1;
4363 4410 src = (uintptr_t)".";
4364 4411 }
4365 4412
4366 4413 /*
4367 4414 * Start from the back of the string, moving back toward the
4368 4415 * front until we see a character that isn't a slash. That
4369 4416 * character is the last character in the basename.
4370 4417 */
4371 4418 for (i = len - 1; i >= 0; i--) {
4372 4419 if (dtrace_load8(src + i) != '/')
4373 4420 break;
4374 4421 }
4375 4422
4376 4423 if (i >= 0)
4377 4424 lastbase = i;
4378 4425
4379 4426 /*
4380 4427 * Starting from the last character in the basename, move
4381 4428 * towards the front until we find a slash. The character
4382 4429 * that we processed immediately before that is the first
4383 4430 * character in the basename.
4384 4431 */
4385 4432 for (; i >= 0; i--) {
4386 4433 if (dtrace_load8(src + i) == '/')
4387 4434 break;
4388 4435 }
4389 4436
4390 4437 if (i >= 0)
4391 4438 firstbase = i + 1;
4392 4439
4393 4440 /*
4394 4441 * Now keep going until we find a non-slash character. That
4395 4442 * character is the last character in the dirname.
4396 4443 */
4397 4444 for (; i >= 0; i--) {
4398 4445 if (dtrace_load8(src + i) != '/')
4399 4446 break;
4400 4447 }
4401 4448
4402 4449 if (i >= 0)
4403 4450 lastdir = i;
4404 4451
4405 4452 ASSERT(!(lastbase == -1 && firstbase != -1));
4406 4453 ASSERT(!(firstbase == -1 && lastdir != -1));
4407 4454
4408 4455 if (lastbase == -1) {
4409 4456 /*
4410 4457 * We didn't find a non-slash character. We know that
4411 4458 * the length is non-zero, so the whole string must be
4412 4459 * slashes. In either the dirname or the basename
4413 4460 * case, we return '/'.
4414 4461 */
4415 4462 ASSERT(firstbase == -1);
4416 4463 firstbase = lastbase = lastdir = 0;
4417 4464 }
4418 4465
4419 4466 if (firstbase == -1) {
4420 4467 /*
4421 4468 * The entire string consists only of a basename
4422 4469 * component. If we're looking for dirname, we need
4423 4470 * to change our string to be just "."; if we're
4424 4471 * looking for a basename, we'll just set the first
4425 4472 * character of the basename to be 0.
4426 4473 */
4427 4474 if (subr == DIF_SUBR_DIRNAME) {
4428 4475 ASSERT(lastdir == -1);
4429 4476 src = (uintptr_t)".";
4430 4477 lastdir = 0;
4431 4478 } else {
4432 4479 firstbase = 0;
4433 4480 }
4434 4481 }
4435 4482
4436 4483 if (subr == DIF_SUBR_DIRNAME) {
4437 4484 if (lastdir == -1) {
4438 4485 /*
4439 4486 * We know that we have a slash in the name --
4440 4487 * or lastdir would be set to 0, above. And
4441 4488 * because lastdir is -1, we know that this
4442 4489 * slash must be the first character. (That
4443 4490 * is, the full string must be of the form
4444 4491 * "/basename".) In this case, the last
4445 4492 * character of the directory name is 0.
4446 4493 */
4447 4494 lastdir = 0;
4448 4495 }
4449 4496
4450 4497 start = 0;
4451 4498 end = lastdir;
4452 4499 } else {
4453 4500 ASSERT(subr == DIF_SUBR_BASENAME);
4454 4501 ASSERT(firstbase != -1 && lastbase != -1);
4455 4502 start = firstbase;
4456 4503 end = lastbase;
4457 4504 }
|
↓ open down ↓ |
3314 lines elided |
↑ open up ↑ |
4458 4505
4459 4506 for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
4460 4507 dest[j] = dtrace_load8(src + i);
4461 4508
4462 4509 dest[j] = '\0';
4463 4510 regs[rd] = (uintptr_t)dest;
4464 4511 mstate->dtms_scratch_ptr += size;
4465 4512 break;
4466 4513 }
4467 4514
4515 + case DIF_SUBR_GETF: {
4516 + uintptr_t fd = tupregs[0].dttk_value;
4517 + uf_info_t *finfo = &curthread->t_procp->p_user.u_finfo;
4518 + file_t *fp;
4519 +
4520 + if (!dtrace_priv_proc(state, mstate)) {
4521 + regs[rd] = NULL;
4522 + break;
4523 + }
4524 +
4525 + /*
4526 + * This is safe because fi_nfiles only increases, and the
4527 + * fi_list array is not freed when the array size doubles.
4528 + * (See the comment in flist_grow() for details on the
4529 + * management of the u_finfo structure.)
4530 + */
4531 + fp = fd < finfo->fi_nfiles ? finfo->fi_list[fd].uf_file : NULL;
4532 +
4533 + mstate->dtms_getf = fp;
4534 + regs[rd] = (uintptr_t)fp;
4535 + break;
4536 + }
4537 +
4468 4538 case DIF_SUBR_CLEANPATH: {
4469 4539 char *dest = (char *)mstate->dtms_scratch_ptr, c;
4470 4540 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
4471 4541 uintptr_t src = tupregs[0].dttk_value;
4472 4542 int i = 0, j = 0;
4543 + zone_t *z;
4473 4544
4474 4545 if (!dtrace_strcanload(src, size, mstate, vstate)) {
4475 4546 regs[rd] = NULL;
4476 4547 break;
4477 4548 }
4478 4549
4479 4550 if (!DTRACE_INSCRATCH(mstate, size)) {
4480 4551 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4481 4552 regs[rd] = NULL;
4482 4553 break;
4483 4554 }
4484 4555
4485 4556 /*
4486 4557 * Move forward, loading each character.
4487 4558 */
4488 4559 do {
4489 4560 c = dtrace_load8(src + i++);
4490 4561 next:
4491 4562 if (j + 5 >= size) /* 5 = strlen("/..c\0") */
4492 4563 break;
4493 4564
4494 4565 if (c != '/') {
4495 4566 dest[j++] = c;
4496 4567 continue;
4497 4568 }
4498 4569
4499 4570 c = dtrace_load8(src + i++);
4500 4571
4501 4572 if (c == '/') {
4502 4573 /*
4503 4574 * We have two slashes -- we can just advance
4504 4575 * to the next character.
4505 4576 */
4506 4577 goto next;
4507 4578 }
4508 4579
4509 4580 if (c != '.') {
4510 4581 /*
4511 4582 * This is not "." and it's not ".." -- we can
4512 4583 * just store the "/" and this character and
4513 4584 * drive on.
4514 4585 */
4515 4586 dest[j++] = '/';
4516 4587 dest[j++] = c;
4517 4588 continue;
4518 4589 }
4519 4590
4520 4591 c = dtrace_load8(src + i++);
4521 4592
4522 4593 if (c == '/') {
4523 4594 /*
4524 4595 * This is a "/./" component. We're not going
4525 4596 * to store anything in the destination buffer;
4526 4597 * we're just going to go to the next component.
4527 4598 */
4528 4599 goto next;
4529 4600 }
4530 4601
4531 4602 if (c != '.') {
4532 4603 /*
4533 4604 * This is not ".." -- we can just store the
4534 4605 * "/." and this character and continue
4535 4606 * processing.
4536 4607 */
4537 4608 dest[j++] = '/';
4538 4609 dest[j++] = '.';
4539 4610 dest[j++] = c;
4540 4611 continue;
4541 4612 }
4542 4613
4543 4614 c = dtrace_load8(src + i++);
4544 4615
4545 4616 if (c != '/' && c != '\0') {
4546 4617 /*
4547 4618 * This is not ".." -- it's "..[mumble]".
4548 4619 * We'll store the "/.." and this character
4549 4620 * and continue processing.
4550 4621 */
4551 4622 dest[j++] = '/';
4552 4623 dest[j++] = '.';
4553 4624 dest[j++] = '.';
4554 4625 dest[j++] = c;
4555 4626 continue;
4556 4627 }
4557 4628
4558 4629 /*
4559 4630 * This is "/../" or "/..\0". We need to back up
4560 4631 * our destination pointer until we find a "/".
|
↓ open down ↓ |
78 lines elided |
↑ open up ↑ |
4561 4632 */
4562 4633 i--;
4563 4634 while (j != 0 && dest[--j] != '/')
4564 4635 continue;
4565 4636
4566 4637 if (c == '\0')
4567 4638 dest[++j] = '/';
4568 4639 } while (c != '\0');
4569 4640
4570 4641 dest[j] = '\0';
4642 +
4643 + if (mstate->dtms_getf != NULL &&
4644 + !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) &&
4645 + (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) {
4646 + /*
4647 + * If we've done a getf() as a part of this ECB and we
4648 + * don't have kernel access (and we're not in the global
4649 + * zone), check if the path we cleaned up begins with
4650 + * the zone's root path, and trim it off if so. Note
4651 + * that this is an output cleanliness issue, not a
4652 + * security issue: knowing one's zone root path does
4653 + * not enable privilege escalation.
4654 + */
4655 + if (strstr(dest, z->zone_rootpath) == dest)
4656 + dest += strlen(z->zone_rootpath) - 1;
4657 + }
4658 +
4571 4659 regs[rd] = (uintptr_t)dest;
4572 4660 mstate->dtms_scratch_ptr += size;
4573 4661 break;
4574 4662 }
4575 4663
4576 4664 case DIF_SUBR_INET_NTOA:
4577 4665 case DIF_SUBR_INET_NTOA6:
4578 4666 case DIF_SUBR_INET_NTOP: {
4579 4667 size_t size;
4580 4668 int af, argi, i;
4581 4669 char *base, *end;
4582 4670
4583 4671 if (subr == DIF_SUBR_INET_NTOP) {
4584 4672 af = (int)tupregs[0].dttk_value;
4585 4673 argi = 1;
4586 4674 } else {
4587 4675 af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6;
4588 4676 argi = 0;
4589 4677 }
4590 4678
4591 4679 if (af == AF_INET) {
4592 4680 ipaddr_t ip4;
4593 4681 uint8_t *ptr8, val;
4594 4682
4595 4683 /*
4596 4684 * Safely load the IPv4 address.
4597 4685 */
4598 4686 ip4 = dtrace_load32(tupregs[argi].dttk_value);
4599 4687
4600 4688 /*
4601 4689 * Check an IPv4 string will fit in scratch.
4602 4690 */
4603 4691 size = INET_ADDRSTRLEN;
4604 4692 if (!DTRACE_INSCRATCH(mstate, size)) {
4605 4693 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4606 4694 regs[rd] = NULL;
4607 4695 break;
4608 4696 }
4609 4697 base = (char *)mstate->dtms_scratch_ptr;
4610 4698 end = (char *)mstate->dtms_scratch_ptr + size - 1;
4611 4699
4612 4700 /*
4613 4701 * Stringify as a dotted decimal quad.
4614 4702 */
4615 4703 *end-- = '\0';
4616 4704 ptr8 = (uint8_t *)&ip4;
4617 4705 for (i = 3; i >= 0; i--) {
4618 4706 val = ptr8[i];
4619 4707
4620 4708 if (val == 0) {
4621 4709 *end-- = '0';
4622 4710 } else {
4623 4711 for (; val; val /= 10) {
4624 4712 *end-- = '0' + (val % 10);
4625 4713 }
4626 4714 }
4627 4715
4628 4716 if (i > 0)
4629 4717 *end-- = '.';
4630 4718 }
4631 4719 ASSERT(end + 1 >= base);
4632 4720
4633 4721 } else if (af == AF_INET6) {
4634 4722 struct in6_addr ip6;
4635 4723 int firstzero, tryzero, numzero, v6end;
4636 4724 uint16_t val;
4637 4725 const char digits[] = "0123456789abcdef";
4638 4726
4639 4727 /*
4640 4728 * Stringify using RFC 1884 convention 2 - 16 bit
4641 4729 * hexadecimal values with a zero-run compression.
4642 4730 * Lower case hexadecimal digits are used.
4643 4731 * eg, fe80::214:4fff:fe0b:76c8.
4644 4732 * The IPv4 embedded form is returned for inet_ntop,
4645 4733 * just the IPv4 string is returned for inet_ntoa6.
4646 4734 */
4647 4735
4648 4736 /*
4649 4737 * Safely load the IPv6 address.
4650 4738 */
4651 4739 dtrace_bcopy(
4652 4740 (void *)(uintptr_t)tupregs[argi].dttk_value,
4653 4741 (void *)(uintptr_t)&ip6, sizeof (struct in6_addr));
4654 4742
4655 4743 /*
4656 4744 * Check an IPv6 string will fit in scratch.
4657 4745 */
4658 4746 size = INET6_ADDRSTRLEN;
4659 4747 if (!DTRACE_INSCRATCH(mstate, size)) {
4660 4748 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4661 4749 regs[rd] = NULL;
4662 4750 break;
4663 4751 }
4664 4752 base = (char *)mstate->dtms_scratch_ptr;
4665 4753 end = (char *)mstate->dtms_scratch_ptr + size - 1;
4666 4754 *end-- = '\0';
4667 4755
4668 4756 /*
4669 4757 * Find the longest run of 16 bit zero values
4670 4758 * for the single allowed zero compression - "::".
4671 4759 */
4672 4760 firstzero = -1;
4673 4761 tryzero = -1;
4674 4762 numzero = 1;
4675 4763 for (i = 0; i < sizeof (struct in6_addr); i++) {
4676 4764 if (ip6._S6_un._S6_u8[i] == 0 &&
4677 4765 tryzero == -1 && i % 2 == 0) {
4678 4766 tryzero = i;
4679 4767 continue;
4680 4768 }
4681 4769
4682 4770 if (tryzero != -1 &&
4683 4771 (ip6._S6_un._S6_u8[i] != 0 ||
4684 4772 i == sizeof (struct in6_addr) - 1)) {
4685 4773
4686 4774 if (i - tryzero <= numzero) {
4687 4775 tryzero = -1;
4688 4776 continue;
4689 4777 }
4690 4778
4691 4779 firstzero = tryzero;
4692 4780 numzero = i - i % 2 - tryzero;
4693 4781 tryzero = -1;
4694 4782
4695 4783 if (ip6._S6_un._S6_u8[i] == 0 &&
4696 4784 i == sizeof (struct in6_addr) - 1)
4697 4785 numzero += 2;
4698 4786 }
4699 4787 }
4700 4788 ASSERT(firstzero + numzero <= sizeof (struct in6_addr));
4701 4789
4702 4790 /*
4703 4791 * Check for an IPv4 embedded address.
4704 4792 */
4705 4793 v6end = sizeof (struct in6_addr) - 2;
4706 4794 if (IN6_IS_ADDR_V4MAPPED(&ip6) ||
4707 4795 IN6_IS_ADDR_V4COMPAT(&ip6)) {
4708 4796 for (i = sizeof (struct in6_addr) - 1;
4709 4797 i >= DTRACE_V4MAPPED_OFFSET; i--) {
4710 4798 ASSERT(end >= base);
4711 4799
4712 4800 val = ip6._S6_un._S6_u8[i];
4713 4801
4714 4802 if (val == 0) {
4715 4803 *end-- = '0';
4716 4804 } else {
4717 4805 for (; val; val /= 10) {
4718 4806 *end-- = '0' + val % 10;
4719 4807 }
4720 4808 }
4721 4809
4722 4810 if (i > DTRACE_V4MAPPED_OFFSET)
4723 4811 *end-- = '.';
4724 4812 }
4725 4813
4726 4814 if (subr == DIF_SUBR_INET_NTOA6)
4727 4815 goto inetout;
4728 4816
4729 4817 /*
4730 4818 * Set v6end to skip the IPv4 address that
4731 4819 * we have already stringified.
4732 4820 */
4733 4821 v6end = 10;
4734 4822 }
4735 4823
4736 4824 /*
4737 4825 * Build the IPv6 string by working through the
4738 4826 * address in reverse.
4739 4827 */
4740 4828 for (i = v6end; i >= 0; i -= 2) {
4741 4829 ASSERT(end >= base);
4742 4830
4743 4831 if (i == firstzero + numzero - 2) {
4744 4832 *end-- = ':';
4745 4833 *end-- = ':';
4746 4834 i -= numzero - 2;
4747 4835 continue;
4748 4836 }
4749 4837
4750 4838 if (i < 14 && i != firstzero - 2)
4751 4839 *end-- = ':';
4752 4840
4753 4841 val = (ip6._S6_un._S6_u8[i] << 8) +
4754 4842 ip6._S6_un._S6_u8[i + 1];
4755 4843
4756 4844 if (val == 0) {
4757 4845 *end-- = '0';
4758 4846 } else {
4759 4847 for (; val; val /= 16) {
4760 4848 *end-- = digits[val % 16];
4761 4849 }
4762 4850 }
4763 4851 }
4764 4852 ASSERT(end + 1 >= base);
4765 4853
4766 4854 } else {
4767 4855 /*
4768 4856 * The user didn't use AH_INET or AH_INET6.
4769 4857 */
4770 4858 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
4771 4859 regs[rd] = NULL;
4772 4860 break;
4773 4861 }
4774 4862
4775 4863 inetout: regs[rd] = (uintptr_t)end + 1;
4776 4864 mstate->dtms_scratch_ptr += size;
4777 4865 break;
4778 4866 }
4779 4867
4780 4868 }
4781 4869 }
4782 4870
4783 4871 /*
4784 4872 * Emulate the execution of DTrace IR instructions specified by the given
4785 4873 * DIF object. This function is deliberately void of assertions as all of
4786 4874 * the necessary checks are handled by a call to dtrace_difo_validate().
4787 4875 */
4788 4876 static uint64_t
4789 4877 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
4790 4878 dtrace_vstate_t *vstate, dtrace_state_t *state)
4791 4879 {
4792 4880 const dif_instr_t *text = difo->dtdo_buf;
4793 4881 const uint_t textlen = difo->dtdo_len;
4794 4882 const char *strtab = difo->dtdo_strtab;
4795 4883 const uint64_t *inttab = difo->dtdo_inttab;
4796 4884
4797 4885 uint64_t rval = 0;
4798 4886 dtrace_statvar_t *svar;
4799 4887 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
4800 4888 dtrace_difv_t *v;
4801 4889 volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
4802 4890 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
4803 4891
4804 4892 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
4805 4893 uint64_t regs[DIF_DIR_NREGS];
4806 4894 uint64_t *tmp;
4807 4895
4808 4896 uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
4809 4897 int64_t cc_r;
4810 4898 uint_t pc = 0, id, opc;
4811 4899 uint8_t ttop = 0;
4812 4900 dif_instr_t instr;
4813 4901 uint_t r1, r2, rd;
4814 4902
4815 4903 /*
4816 4904 * We stash the current DIF object into the machine state: we need it
4817 4905 * for subsequent access checking.
4818 4906 */
4819 4907 mstate->dtms_difo = difo;
4820 4908
4821 4909 regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */
4822 4910
4823 4911 while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
4824 4912 opc = pc;
4825 4913
4826 4914 instr = text[pc++];
4827 4915 r1 = DIF_INSTR_R1(instr);
4828 4916 r2 = DIF_INSTR_R2(instr);
4829 4917 rd = DIF_INSTR_RD(instr);
4830 4918
4831 4919 switch (DIF_INSTR_OP(instr)) {
4832 4920 case DIF_OP_OR:
4833 4921 regs[rd] = regs[r1] | regs[r2];
4834 4922 break;
4835 4923 case DIF_OP_XOR:
4836 4924 regs[rd] = regs[r1] ^ regs[r2];
4837 4925 break;
4838 4926 case DIF_OP_AND:
4839 4927 regs[rd] = regs[r1] & regs[r2];
4840 4928 break;
4841 4929 case DIF_OP_SLL:
4842 4930 regs[rd] = regs[r1] << regs[r2];
4843 4931 break;
4844 4932 case DIF_OP_SRL:
4845 4933 regs[rd] = regs[r1] >> regs[r2];
4846 4934 break;
4847 4935 case DIF_OP_SUB:
4848 4936 regs[rd] = regs[r1] - regs[r2];
4849 4937 break;
4850 4938 case DIF_OP_ADD:
4851 4939 regs[rd] = regs[r1] + regs[r2];
4852 4940 break;
4853 4941 case DIF_OP_MUL:
4854 4942 regs[rd] = regs[r1] * regs[r2];
4855 4943 break;
4856 4944 case DIF_OP_SDIV:
4857 4945 if (regs[r2] == 0) {
4858 4946 regs[rd] = 0;
4859 4947 *flags |= CPU_DTRACE_DIVZERO;
4860 4948 } else {
4861 4949 regs[rd] = (int64_t)regs[r1] /
4862 4950 (int64_t)regs[r2];
4863 4951 }
4864 4952 break;
4865 4953
4866 4954 case DIF_OP_UDIV:
4867 4955 if (regs[r2] == 0) {
4868 4956 regs[rd] = 0;
4869 4957 *flags |= CPU_DTRACE_DIVZERO;
4870 4958 } else {
4871 4959 regs[rd] = regs[r1] / regs[r2];
4872 4960 }
4873 4961 break;
4874 4962
4875 4963 case DIF_OP_SREM:
4876 4964 if (regs[r2] == 0) {
4877 4965 regs[rd] = 0;
4878 4966 *flags |= CPU_DTRACE_DIVZERO;
4879 4967 } else {
4880 4968 regs[rd] = (int64_t)regs[r1] %
4881 4969 (int64_t)regs[r2];
4882 4970 }
4883 4971 break;
4884 4972
4885 4973 case DIF_OP_UREM:
4886 4974 if (regs[r2] == 0) {
4887 4975 regs[rd] = 0;
4888 4976 *flags |= CPU_DTRACE_DIVZERO;
4889 4977 } else {
4890 4978 regs[rd] = regs[r1] % regs[r2];
4891 4979 }
4892 4980 break;
4893 4981
4894 4982 case DIF_OP_NOT:
4895 4983 regs[rd] = ~regs[r1];
4896 4984 break;
4897 4985 case DIF_OP_MOV:
4898 4986 regs[rd] = regs[r1];
4899 4987 break;
4900 4988 case DIF_OP_CMP:
4901 4989 cc_r = regs[r1] - regs[r2];
4902 4990 cc_n = cc_r < 0;
4903 4991 cc_z = cc_r == 0;
4904 4992 cc_v = 0;
4905 4993 cc_c = regs[r1] < regs[r2];
4906 4994 break;
4907 4995 case DIF_OP_TST:
4908 4996 cc_n = cc_v = cc_c = 0;
4909 4997 cc_z = regs[r1] == 0;
4910 4998 break;
4911 4999 case DIF_OP_BA:
4912 5000 pc = DIF_INSTR_LABEL(instr);
4913 5001 break;
4914 5002 case DIF_OP_BE:
4915 5003 if (cc_z)
4916 5004 pc = DIF_INSTR_LABEL(instr);
4917 5005 break;
4918 5006 case DIF_OP_BNE:
4919 5007 if (cc_z == 0)
4920 5008 pc = DIF_INSTR_LABEL(instr);
4921 5009 break;
4922 5010 case DIF_OP_BG:
4923 5011 if ((cc_z | (cc_n ^ cc_v)) == 0)
4924 5012 pc = DIF_INSTR_LABEL(instr);
4925 5013 break;
4926 5014 case DIF_OP_BGU:
4927 5015 if ((cc_c | cc_z) == 0)
4928 5016 pc = DIF_INSTR_LABEL(instr);
4929 5017 break;
4930 5018 case DIF_OP_BGE:
4931 5019 if ((cc_n ^ cc_v) == 0)
4932 5020 pc = DIF_INSTR_LABEL(instr);
4933 5021 break;
4934 5022 case DIF_OP_BGEU:
4935 5023 if (cc_c == 0)
4936 5024 pc = DIF_INSTR_LABEL(instr);
4937 5025 break;
4938 5026 case DIF_OP_BL:
4939 5027 if (cc_n ^ cc_v)
4940 5028 pc = DIF_INSTR_LABEL(instr);
4941 5029 break;
4942 5030 case DIF_OP_BLU:
4943 5031 if (cc_c)
4944 5032 pc = DIF_INSTR_LABEL(instr);
4945 5033 break;
4946 5034 case DIF_OP_BLE:
4947 5035 if (cc_z | (cc_n ^ cc_v))
4948 5036 pc = DIF_INSTR_LABEL(instr);
4949 5037 break;
4950 5038 case DIF_OP_BLEU:
4951 5039 if (cc_c | cc_z)
4952 5040 pc = DIF_INSTR_LABEL(instr);
4953 5041 break;
4954 5042 case DIF_OP_RLDSB:
4955 5043 if (!dtrace_canload(regs[r1], 1, mstate, vstate))
4956 5044 break;
4957 5045 /*FALLTHROUGH*/
4958 5046 case DIF_OP_LDSB:
4959 5047 regs[rd] = (int8_t)dtrace_load8(regs[r1]);
4960 5048 break;
4961 5049 case DIF_OP_RLDSH:
4962 5050 if (!dtrace_canload(regs[r1], 2, mstate, vstate))
4963 5051 break;
4964 5052 /*FALLTHROUGH*/
4965 5053 case DIF_OP_LDSH:
4966 5054 regs[rd] = (int16_t)dtrace_load16(regs[r1]);
4967 5055 break;
4968 5056 case DIF_OP_RLDSW:
4969 5057 if (!dtrace_canload(regs[r1], 4, mstate, vstate))
4970 5058 break;
4971 5059 /*FALLTHROUGH*/
4972 5060 case DIF_OP_LDSW:
4973 5061 regs[rd] = (int32_t)dtrace_load32(regs[r1]);
4974 5062 break;
4975 5063 case DIF_OP_RLDUB:
4976 5064 if (!dtrace_canload(regs[r1], 1, mstate, vstate))
4977 5065 break;
4978 5066 /*FALLTHROUGH*/
4979 5067 case DIF_OP_LDUB:
4980 5068 regs[rd] = dtrace_load8(regs[r1]);
4981 5069 break;
4982 5070 case DIF_OP_RLDUH:
4983 5071 if (!dtrace_canload(regs[r1], 2, mstate, vstate))
4984 5072 break;
4985 5073 /*FALLTHROUGH*/
4986 5074 case DIF_OP_LDUH:
4987 5075 regs[rd] = dtrace_load16(regs[r1]);
4988 5076 break;
4989 5077 case DIF_OP_RLDUW:
4990 5078 if (!dtrace_canload(regs[r1], 4, mstate, vstate))
4991 5079 break;
4992 5080 /*FALLTHROUGH*/
4993 5081 case DIF_OP_LDUW:
4994 5082 regs[rd] = dtrace_load32(regs[r1]);
4995 5083 break;
4996 5084 case DIF_OP_RLDX:
4997 5085 if (!dtrace_canload(regs[r1], 8, mstate, vstate))
4998 5086 break;
4999 5087 /*FALLTHROUGH*/
5000 5088 case DIF_OP_LDX:
5001 5089 regs[rd] = dtrace_load64(regs[r1]);
5002 5090 break;
5003 5091 case DIF_OP_ULDSB:
5004 5092 regs[rd] = (int8_t)
5005 5093 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5006 5094 break;
5007 5095 case DIF_OP_ULDSH:
5008 5096 regs[rd] = (int16_t)
5009 5097 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5010 5098 break;
5011 5099 case DIF_OP_ULDSW:
5012 5100 regs[rd] = (int32_t)
5013 5101 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5014 5102 break;
5015 5103 case DIF_OP_ULDUB:
5016 5104 regs[rd] =
5017 5105 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5018 5106 break;
5019 5107 case DIF_OP_ULDUH:
5020 5108 regs[rd] =
5021 5109 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5022 5110 break;
5023 5111 case DIF_OP_ULDUW:
5024 5112 regs[rd] =
5025 5113 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5026 5114 break;
5027 5115 case DIF_OP_ULDX:
5028 5116 regs[rd] =
5029 5117 dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5030 5118 break;
5031 5119 case DIF_OP_RET:
5032 5120 rval = regs[rd];
5033 5121 pc = textlen;
5034 5122 break;
5035 5123 case DIF_OP_NOP:
5036 5124 break;
5037 5125 case DIF_OP_SETX:
5038 5126 regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5039 5127 break;
5040 5128 case DIF_OP_SETS:
5041 5129 regs[rd] = (uint64_t)(uintptr_t)
5042 5130 (strtab + DIF_INSTR_STRING(instr));
5043 5131 break;
5044 5132 case DIF_OP_SCMP: {
5045 5133 size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5046 5134 uintptr_t s1 = regs[r1];
5047 5135 uintptr_t s2 = regs[r2];
5048 5136
5049 5137 if (s1 != NULL &&
5050 5138 !dtrace_strcanload(s1, sz, mstate, vstate))
5051 5139 break;
5052 5140 if (s2 != NULL &&
5053 5141 !dtrace_strcanload(s2, sz, mstate, vstate))
5054 5142 break;
5055 5143
5056 5144 cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz);
5057 5145
5058 5146 cc_n = cc_r < 0;
5059 5147 cc_z = cc_r == 0;
5060 5148 cc_v = cc_c = 0;
5061 5149 break;
5062 5150 }
5063 5151 case DIF_OP_LDGA:
5064 5152 regs[rd] = dtrace_dif_variable(mstate, state,
5065 5153 r1, regs[r2]);
5066 5154 break;
5067 5155 case DIF_OP_LDGS:
5068 5156 id = DIF_INSTR_VAR(instr);
5069 5157
5070 5158 if (id >= DIF_VAR_OTHER_UBASE) {
5071 5159 uintptr_t a;
5072 5160
5073 5161 id -= DIF_VAR_OTHER_UBASE;
5074 5162 svar = vstate->dtvs_globals[id];
5075 5163 ASSERT(svar != NULL);
5076 5164 v = &svar->dtsv_var;
5077 5165
5078 5166 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
5079 5167 regs[rd] = svar->dtsv_data;
5080 5168 break;
5081 5169 }
5082 5170
5083 5171 a = (uintptr_t)svar->dtsv_data;
5084 5172
5085 5173 if (*(uint8_t *)a == UINT8_MAX) {
5086 5174 /*
5087 5175 * If the 0th byte is set to UINT8_MAX
5088 5176 * then this is to be treated as a
5089 5177 * reference to a NULL variable.
5090 5178 */
5091 5179 regs[rd] = NULL;
5092 5180 } else {
5093 5181 regs[rd] = a + sizeof (uint64_t);
5094 5182 }
5095 5183
5096 5184 break;
5097 5185 }
5098 5186
5099 5187 regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
5100 5188 break;
5101 5189
5102 5190 case DIF_OP_STGS:
5103 5191 id = DIF_INSTR_VAR(instr);
5104 5192
5105 5193 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5106 5194 id -= DIF_VAR_OTHER_UBASE;
5107 5195
5108 5196 svar = vstate->dtvs_globals[id];
5109 5197 ASSERT(svar != NULL);
5110 5198 v = &svar->dtsv_var;
5111 5199
5112 5200 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5113 5201 uintptr_t a = (uintptr_t)svar->dtsv_data;
5114 5202
5115 5203 ASSERT(a != NULL);
5116 5204 ASSERT(svar->dtsv_size != 0);
5117 5205
5118 5206 if (regs[rd] == NULL) {
5119 5207 *(uint8_t *)a = UINT8_MAX;
5120 5208 break;
5121 5209 } else {
5122 5210 *(uint8_t *)a = 0;
5123 5211 a += sizeof (uint64_t);
5124 5212 }
5125 5213 if (!dtrace_vcanload(
5126 5214 (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5127 5215 mstate, vstate))
5128 5216 break;
5129 5217
5130 5218 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5131 5219 (void *)a, &v->dtdv_type);
5132 5220 break;
5133 5221 }
5134 5222
5135 5223 svar->dtsv_data = regs[rd];
5136 5224 break;
5137 5225
5138 5226 case DIF_OP_LDTA:
5139 5227 /*
5140 5228 * There are no DTrace built-in thread-local arrays at
5141 5229 * present. This opcode is saved for future work.
5142 5230 */
5143 5231 *flags |= CPU_DTRACE_ILLOP;
5144 5232 regs[rd] = 0;
5145 5233 break;
5146 5234
5147 5235 case DIF_OP_LDLS:
5148 5236 id = DIF_INSTR_VAR(instr);
5149 5237
5150 5238 if (id < DIF_VAR_OTHER_UBASE) {
5151 5239 /*
5152 5240 * For now, this has no meaning.
5153 5241 */
5154 5242 regs[rd] = 0;
5155 5243 break;
5156 5244 }
5157 5245
5158 5246 id -= DIF_VAR_OTHER_UBASE;
5159 5247
5160 5248 ASSERT(id < vstate->dtvs_nlocals);
5161 5249 ASSERT(vstate->dtvs_locals != NULL);
5162 5250
5163 5251 svar = vstate->dtvs_locals[id];
5164 5252 ASSERT(svar != NULL);
5165 5253 v = &svar->dtsv_var;
5166 5254
5167 5255 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5168 5256 uintptr_t a = (uintptr_t)svar->dtsv_data;
5169 5257 size_t sz = v->dtdv_type.dtdt_size;
5170 5258
5171 5259 sz += sizeof (uint64_t);
5172 5260 ASSERT(svar->dtsv_size == NCPU * sz);
5173 5261 a += CPU->cpu_id * sz;
5174 5262
5175 5263 if (*(uint8_t *)a == UINT8_MAX) {
5176 5264 /*
5177 5265 * If the 0th byte is set to UINT8_MAX
5178 5266 * then this is to be treated as a
5179 5267 * reference to a NULL variable.
5180 5268 */
5181 5269 regs[rd] = NULL;
5182 5270 } else {
5183 5271 regs[rd] = a + sizeof (uint64_t);
5184 5272 }
5185 5273
5186 5274 break;
5187 5275 }
5188 5276
5189 5277 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5190 5278 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5191 5279 regs[rd] = tmp[CPU->cpu_id];
5192 5280 break;
5193 5281
5194 5282 case DIF_OP_STLS:
5195 5283 id = DIF_INSTR_VAR(instr);
5196 5284
5197 5285 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5198 5286 id -= DIF_VAR_OTHER_UBASE;
5199 5287 ASSERT(id < vstate->dtvs_nlocals);
5200 5288
5201 5289 ASSERT(vstate->dtvs_locals != NULL);
5202 5290 svar = vstate->dtvs_locals[id];
5203 5291 ASSERT(svar != NULL);
5204 5292 v = &svar->dtsv_var;
5205 5293
5206 5294 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5207 5295 uintptr_t a = (uintptr_t)svar->dtsv_data;
5208 5296 size_t sz = v->dtdv_type.dtdt_size;
5209 5297
5210 5298 sz += sizeof (uint64_t);
5211 5299 ASSERT(svar->dtsv_size == NCPU * sz);
5212 5300 a += CPU->cpu_id * sz;
5213 5301
5214 5302 if (regs[rd] == NULL) {
5215 5303 *(uint8_t *)a = UINT8_MAX;
5216 5304 break;
5217 5305 } else {
5218 5306 *(uint8_t *)a = 0;
5219 5307 a += sizeof (uint64_t);
5220 5308 }
5221 5309
5222 5310 if (!dtrace_vcanload(
5223 5311 (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5224 5312 mstate, vstate))
5225 5313 break;
5226 5314
5227 5315 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5228 5316 (void *)a, &v->dtdv_type);
5229 5317 break;
5230 5318 }
5231 5319
5232 5320 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
5233 5321 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
5234 5322 tmp[CPU->cpu_id] = regs[rd];
5235 5323 break;
5236 5324
5237 5325 case DIF_OP_LDTS: {
5238 5326 dtrace_dynvar_t *dvar;
5239 5327 dtrace_key_t *key;
5240 5328
5241 5329 id = DIF_INSTR_VAR(instr);
5242 5330 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5243 5331 id -= DIF_VAR_OTHER_UBASE;
5244 5332 v = &vstate->dtvs_tlocals[id];
5245 5333
5246 5334 key = &tupregs[DIF_DTR_NREGS];
5247 5335 key[0].dttk_value = (uint64_t)id;
5248 5336 key[0].dttk_size = 0;
5249 5337 DTRACE_TLS_THRKEY(key[1].dttk_value);
5250 5338 key[1].dttk_size = 0;
5251 5339
5252 5340 dvar = dtrace_dynvar(dstate, 2, key,
5253 5341 sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC,
5254 5342 mstate, vstate);
5255 5343
5256 5344 if (dvar == NULL) {
5257 5345 regs[rd] = 0;
5258 5346 break;
5259 5347 }
5260 5348
5261 5349 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5262 5350 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5263 5351 } else {
5264 5352 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5265 5353 }
5266 5354
5267 5355 break;
5268 5356 }
5269 5357
5270 5358 case DIF_OP_STTS: {
5271 5359 dtrace_dynvar_t *dvar;
5272 5360 dtrace_key_t *key;
5273 5361
5274 5362 id = DIF_INSTR_VAR(instr);
5275 5363 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5276 5364 id -= DIF_VAR_OTHER_UBASE;
5277 5365
5278 5366 key = &tupregs[DIF_DTR_NREGS];
5279 5367 key[0].dttk_value = (uint64_t)id;
5280 5368 key[0].dttk_size = 0;
5281 5369 DTRACE_TLS_THRKEY(key[1].dttk_value);
5282 5370 key[1].dttk_size = 0;
5283 5371 v = &vstate->dtvs_tlocals[id];
5284 5372
5285 5373 dvar = dtrace_dynvar(dstate, 2, key,
5286 5374 v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5287 5375 v->dtdv_type.dtdt_size : sizeof (uint64_t),
5288 5376 regs[rd] ? DTRACE_DYNVAR_ALLOC :
5289 5377 DTRACE_DYNVAR_DEALLOC, mstate, vstate);
5290 5378
5291 5379 /*
5292 5380 * Given that we're storing to thread-local data,
5293 5381 * we need to flush our predicate cache.
5294 5382 */
5295 5383 curthread->t_predcache = NULL;
5296 5384
5297 5385 if (dvar == NULL)
5298 5386 break;
5299 5387
5300 5388 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5301 5389 if (!dtrace_vcanload(
5302 5390 (void *)(uintptr_t)regs[rd],
5303 5391 &v->dtdv_type, mstate, vstate))
5304 5392 break;
5305 5393
5306 5394 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5307 5395 dvar->dtdv_data, &v->dtdv_type);
5308 5396 } else {
5309 5397 *((uint64_t *)dvar->dtdv_data) = regs[rd];
5310 5398 }
5311 5399
5312 5400 break;
5313 5401 }
5314 5402
5315 5403 case DIF_OP_SRA:
5316 5404 regs[rd] = (int64_t)regs[r1] >> regs[r2];
5317 5405 break;
5318 5406
5319 5407 case DIF_OP_CALL:
5320 5408 dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
5321 5409 regs, tupregs, ttop, mstate, state);
5322 5410 break;
5323 5411
5324 5412 case DIF_OP_PUSHTR:
5325 5413 if (ttop == DIF_DTR_NREGS) {
5326 5414 *flags |= CPU_DTRACE_TUPOFLOW;
5327 5415 break;
5328 5416 }
5329 5417
5330 5418 if (r1 == DIF_TYPE_STRING) {
5331 5419 /*
5332 5420 * If this is a string type and the size is 0,
5333 5421 * we'll use the system-wide default string
5334 5422 * size. Note that we are _not_ looking at
5335 5423 * the value of the DTRACEOPT_STRSIZE option;
5336 5424 * had this been set, we would expect to have
5337 5425 * a non-zero size value in the "pushtr".
5338 5426 */
5339 5427 tupregs[ttop].dttk_size =
5340 5428 dtrace_strlen((char *)(uintptr_t)regs[rd],
5341 5429 regs[r2] ? regs[r2] :
5342 5430 dtrace_strsize_default) + 1;
5343 5431 } else {
5344 5432 tupregs[ttop].dttk_size = regs[r2];
5345 5433 }
5346 5434
5347 5435 tupregs[ttop++].dttk_value = regs[rd];
5348 5436 break;
5349 5437
5350 5438 case DIF_OP_PUSHTV:
5351 5439 if (ttop == DIF_DTR_NREGS) {
5352 5440 *flags |= CPU_DTRACE_TUPOFLOW;
5353 5441 break;
5354 5442 }
5355 5443
5356 5444 tupregs[ttop].dttk_value = regs[rd];
5357 5445 tupregs[ttop++].dttk_size = 0;
5358 5446 break;
5359 5447
5360 5448 case DIF_OP_POPTS:
5361 5449 if (ttop != 0)
5362 5450 ttop--;
5363 5451 break;
5364 5452
5365 5453 case DIF_OP_FLUSHTS:
5366 5454 ttop = 0;
5367 5455 break;
5368 5456
5369 5457 case DIF_OP_LDGAA:
5370 5458 case DIF_OP_LDTAA: {
5371 5459 dtrace_dynvar_t *dvar;
5372 5460 dtrace_key_t *key = tupregs;
5373 5461 uint_t nkeys = ttop;
5374 5462
5375 5463 id = DIF_INSTR_VAR(instr);
5376 5464 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5377 5465 id -= DIF_VAR_OTHER_UBASE;
5378 5466
5379 5467 key[nkeys].dttk_value = (uint64_t)id;
5380 5468 key[nkeys++].dttk_size = 0;
5381 5469
5382 5470 if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
5383 5471 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
5384 5472 key[nkeys++].dttk_size = 0;
5385 5473 v = &vstate->dtvs_tlocals[id];
5386 5474 } else {
5387 5475 v = &vstate->dtvs_globals[id]->dtsv_var;
5388 5476 }
5389 5477
5390 5478 dvar = dtrace_dynvar(dstate, nkeys, key,
5391 5479 v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5392 5480 v->dtdv_type.dtdt_size : sizeof (uint64_t),
5393 5481 DTRACE_DYNVAR_NOALLOC, mstate, vstate);
5394 5482
5395 5483 if (dvar == NULL) {
5396 5484 regs[rd] = 0;
5397 5485 break;
5398 5486 }
5399 5487
5400 5488 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5401 5489 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
5402 5490 } else {
5403 5491 regs[rd] = *((uint64_t *)dvar->dtdv_data);
5404 5492 }
5405 5493
5406 5494 break;
5407 5495 }
5408 5496
5409 5497 case DIF_OP_STGAA:
5410 5498 case DIF_OP_STTAA: {
5411 5499 dtrace_dynvar_t *dvar;
5412 5500 dtrace_key_t *key = tupregs;
5413 5501 uint_t nkeys = ttop;
5414 5502
5415 5503 id = DIF_INSTR_VAR(instr);
5416 5504 ASSERT(id >= DIF_VAR_OTHER_UBASE);
5417 5505 id -= DIF_VAR_OTHER_UBASE;
5418 5506
5419 5507 key[nkeys].dttk_value = (uint64_t)id;
5420 5508 key[nkeys++].dttk_size = 0;
5421 5509
5422 5510 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
5423 5511 DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
5424 5512 key[nkeys++].dttk_size = 0;
5425 5513 v = &vstate->dtvs_tlocals[id];
5426 5514 } else {
5427 5515 v = &vstate->dtvs_globals[id]->dtsv_var;
5428 5516 }
5429 5517
5430 5518 dvar = dtrace_dynvar(dstate, nkeys, key,
5431 5519 v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
5432 5520 v->dtdv_type.dtdt_size : sizeof (uint64_t),
5433 5521 regs[rd] ? DTRACE_DYNVAR_ALLOC :
5434 5522 DTRACE_DYNVAR_DEALLOC, mstate, vstate);
5435 5523
5436 5524 if (dvar == NULL)
5437 5525 break;
5438 5526
5439 5527 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
5440 5528 if (!dtrace_vcanload(
5441 5529 (void *)(uintptr_t)regs[rd], &v->dtdv_type,
5442 5530 mstate, vstate))
5443 5531 break;
5444 5532
5445 5533 dtrace_vcopy((void *)(uintptr_t)regs[rd],
5446 5534 dvar->dtdv_data, &v->dtdv_type);
5447 5535 } else {
5448 5536 *((uint64_t *)dvar->dtdv_data) = regs[rd];
5449 5537 }
5450 5538
5451 5539 break;
5452 5540 }
5453 5541
5454 5542 case DIF_OP_ALLOCS: {
5455 5543 uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
5456 5544 size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
5457 5545
5458 5546 /*
5459 5547 * Rounding up the user allocation size could have
5460 5548 * overflowed large, bogus allocations (like -1ULL) to
5461 5549 * 0.
5462 5550 */
5463 5551 if (size < regs[r1] ||
5464 5552 !DTRACE_INSCRATCH(mstate, size)) {
5465 5553 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5466 5554 regs[rd] = NULL;
5467 5555 break;
5468 5556 }
5469 5557
5470 5558 dtrace_bzero((void *) mstate->dtms_scratch_ptr, size);
5471 5559 mstate->dtms_scratch_ptr += size;
5472 5560 regs[rd] = ptr;
5473 5561 break;
5474 5562 }
5475 5563
5476 5564 case DIF_OP_COPYS:
5477 5565 if (!dtrace_canstore(regs[rd], regs[r2],
5478 5566 mstate, vstate)) {
5479 5567 *flags |= CPU_DTRACE_BADADDR;
5480 5568 *illval = regs[rd];
5481 5569 break;
5482 5570 }
5483 5571
5484 5572 if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate))
5485 5573 break;
5486 5574
5487 5575 dtrace_bcopy((void *)(uintptr_t)regs[r1],
5488 5576 (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
5489 5577 break;
5490 5578
5491 5579 case DIF_OP_STB:
5492 5580 if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
5493 5581 *flags |= CPU_DTRACE_BADADDR;
5494 5582 *illval = regs[rd];
5495 5583 break;
5496 5584 }
5497 5585 *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
5498 5586 break;
5499 5587
5500 5588 case DIF_OP_STH:
5501 5589 if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
5502 5590 *flags |= CPU_DTRACE_BADADDR;
5503 5591 *illval = regs[rd];
5504 5592 break;
5505 5593 }
5506 5594 if (regs[rd] & 1) {
5507 5595 *flags |= CPU_DTRACE_BADALIGN;
5508 5596 *illval = regs[rd];
5509 5597 break;
5510 5598 }
5511 5599 *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
5512 5600 break;
5513 5601
5514 5602 case DIF_OP_STW:
5515 5603 if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
5516 5604 *flags |= CPU_DTRACE_BADADDR;
5517 5605 *illval = regs[rd];
5518 5606 break;
5519 5607 }
5520 5608 if (regs[rd] & 3) {
5521 5609 *flags |= CPU_DTRACE_BADALIGN;
5522 5610 *illval = regs[rd];
5523 5611 break;
5524 5612 }
5525 5613 *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
5526 5614 break;
5527 5615
5528 5616 case DIF_OP_STX:
5529 5617 if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
5530 5618 *flags |= CPU_DTRACE_BADADDR;
5531 5619 *illval = regs[rd];
5532 5620 break;
5533 5621 }
5534 5622 if (regs[rd] & 7) {
5535 5623 *flags |= CPU_DTRACE_BADALIGN;
5536 5624 *illval = regs[rd];
5537 5625 break;
5538 5626 }
5539 5627 *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
5540 5628 break;
5541 5629 }
5542 5630 }
5543 5631
5544 5632 if (!(*flags & CPU_DTRACE_FAULT))
5545 5633 return (rval);
5546 5634
5547 5635 mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
5548 5636 mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
5549 5637
5550 5638 return (0);
5551 5639 }
5552 5640
5553 5641 static void
5554 5642 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
5555 5643 {
5556 5644 dtrace_probe_t *probe = ecb->dte_probe;
5557 5645 dtrace_provider_t *prov = probe->dtpr_provider;
5558 5646 char c[DTRACE_FULLNAMELEN + 80], *str;
5559 5647 char *msg = "dtrace: breakpoint action at probe ";
5560 5648 char *ecbmsg = " (ecb ";
5561 5649 uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
5562 5650 uintptr_t val = (uintptr_t)ecb;
5563 5651 int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
5564 5652
5565 5653 if (dtrace_destructive_disallow)
5566 5654 return;
5567 5655
5568 5656 /*
5569 5657 * It's impossible to be taking action on the NULL probe.
5570 5658 */
5571 5659 ASSERT(probe != NULL);
5572 5660
5573 5661 /*
5574 5662 * This is a poor man's (destitute man's?) sprintf(): we want to
5575 5663 * print the provider name, module name, function name and name of
5576 5664 * the probe, along with the hex address of the ECB with the breakpoint
5577 5665 * action -- all of which we must place in the character buffer by
5578 5666 * hand.
5579 5667 */
5580 5668 while (*msg != '\0')
5581 5669 c[i++] = *msg++;
5582 5670
5583 5671 for (str = prov->dtpv_name; *str != '\0'; str++)
5584 5672 c[i++] = *str;
5585 5673 c[i++] = ':';
5586 5674
5587 5675 for (str = probe->dtpr_mod; *str != '\0'; str++)
5588 5676 c[i++] = *str;
5589 5677 c[i++] = ':';
5590 5678
5591 5679 for (str = probe->dtpr_func; *str != '\0'; str++)
5592 5680 c[i++] = *str;
5593 5681 c[i++] = ':';
5594 5682
5595 5683 for (str = probe->dtpr_name; *str != '\0'; str++)
5596 5684 c[i++] = *str;
5597 5685
5598 5686 while (*ecbmsg != '\0')
5599 5687 c[i++] = *ecbmsg++;
5600 5688
5601 5689 while (shift >= 0) {
5602 5690 mask = (uintptr_t)0xf << shift;
5603 5691
5604 5692 if (val >= ((uintptr_t)1 << shift))
5605 5693 c[i++] = "0123456789abcdef"[(val & mask) >> shift];
5606 5694 shift -= 4;
5607 5695 }
5608 5696
5609 5697 c[i++] = ')';
5610 5698 c[i] = '\0';
5611 5699
5612 5700 debug_enter(c);
5613 5701 }
5614 5702
5615 5703 static void
5616 5704 dtrace_action_panic(dtrace_ecb_t *ecb)
5617 5705 {
5618 5706 dtrace_probe_t *probe = ecb->dte_probe;
5619 5707
5620 5708 /*
5621 5709 * It's impossible to be taking action on the NULL probe.
5622 5710 */
5623 5711 ASSERT(probe != NULL);
5624 5712
5625 5713 if (dtrace_destructive_disallow)
5626 5714 return;
5627 5715
5628 5716 if (dtrace_panicked != NULL)
5629 5717 return;
5630 5718
5631 5719 if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
5632 5720 return;
5633 5721
5634 5722 /*
5635 5723 * We won the right to panic. (We want to be sure that only one
5636 5724 * thread calls panic() from dtrace_probe(), and that panic() is
5637 5725 * called exactly once.)
5638 5726 */
5639 5727 dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
5640 5728 probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
5641 5729 probe->dtpr_func, probe->dtpr_name, (void *)ecb);
5642 5730 }
5643 5731
5644 5732 static void
5645 5733 dtrace_action_raise(uint64_t sig)
5646 5734 {
5647 5735 if (dtrace_destructive_disallow)
5648 5736 return;
5649 5737
5650 5738 if (sig >= NSIG) {
5651 5739 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
5652 5740 return;
5653 5741 }
5654 5742
5655 5743 /*
5656 5744 * raise() has a queue depth of 1 -- we ignore all subsequent
5657 5745 * invocations of the raise() action.
5658 5746 */
5659 5747 if (curthread->t_dtrace_sig == 0)
5660 5748 curthread->t_dtrace_sig = (uint8_t)sig;
5661 5749
5662 5750 curthread->t_sig_check = 1;
5663 5751 aston(curthread);
5664 5752 }
5665 5753
5666 5754 static void
5667 5755 dtrace_action_stop(void)
5668 5756 {
5669 5757 if (dtrace_destructive_disallow)
5670 5758 return;
5671 5759
5672 5760 if (!curthread->t_dtrace_stop) {
5673 5761 curthread->t_dtrace_stop = 1;
5674 5762 curthread->t_sig_check = 1;
5675 5763 aston(curthread);
5676 5764 }
5677 5765 }
5678 5766
5679 5767 static void
5680 5768 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
5681 5769 {
5682 5770 hrtime_t now;
5683 5771 volatile uint16_t *flags;
5684 5772 cpu_t *cpu = CPU;
5685 5773
5686 5774 if (dtrace_destructive_disallow)
5687 5775 return;
5688 5776
5689 5777 flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
5690 5778
5691 5779 now = dtrace_gethrtime();
5692 5780
5693 5781 if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
5694 5782 /*
5695 5783 * We need to advance the mark to the current time.
5696 5784 */
5697 5785 cpu->cpu_dtrace_chillmark = now;
5698 5786 cpu->cpu_dtrace_chilled = 0;
5699 5787 }
5700 5788
5701 5789 /*
5702 5790 * Now check to see if the requested chill time would take us over
5703 5791 * the maximum amount of time allowed in the chill interval. (Or
5704 5792 * worse, if the calculation itself induces overflow.)
5705 5793 */
5706 5794 if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
5707 5795 cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
5708 5796 *flags |= CPU_DTRACE_ILLOP;
5709 5797 return;
5710 5798 }
5711 5799
5712 5800 while (dtrace_gethrtime() - now < val)
5713 5801 continue;
5714 5802
5715 5803 /*
5716 5804 * Normally, we assure that the value of the variable "timestamp" does
5717 5805 * not change within an ECB. The presence of chill() represents an
5718 5806 * exception to this rule, however.
5719 5807 */
5720 5808 mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
5721 5809 cpu->cpu_dtrace_chilled += val;
5722 5810 }
5723 5811
5724 5812 static void
5725 5813 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
5726 5814 uint64_t *buf, uint64_t arg)
5727 5815 {
5728 5816 int nframes = DTRACE_USTACK_NFRAMES(arg);
5729 5817 int strsize = DTRACE_USTACK_STRSIZE(arg);
5730 5818 uint64_t *pcs = &buf[1], *fps;
5731 5819 char *str = (char *)&pcs[nframes];
5732 5820 int size, offs = 0, i, j;
5733 5821 uintptr_t old = mstate->dtms_scratch_ptr, saved;
5734 5822 uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
5735 5823 char *sym;
5736 5824
5737 5825 /*
5738 5826 * Should be taking a faster path if string space has not been
5739 5827 * allocated.
5740 5828 */
5741 5829 ASSERT(strsize != 0);
5742 5830
5743 5831 /*
5744 5832 * We will first allocate some temporary space for the frame pointers.
5745 5833 */
5746 5834 fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
5747 5835 size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
5748 5836 (nframes * sizeof (uint64_t));
5749 5837
5750 5838 if (!DTRACE_INSCRATCH(mstate, size)) {
5751 5839 /*
5752 5840 * Not enough room for our frame pointers -- need to indicate
5753 5841 * that we ran out of scratch space.
5754 5842 */
5755 5843 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
5756 5844 return;
5757 5845 }
5758 5846
5759 5847 mstate->dtms_scratch_ptr += size;
5760 5848 saved = mstate->dtms_scratch_ptr;
5761 5849
5762 5850 /*
5763 5851 * Now get a stack with both program counters and frame pointers.
5764 5852 */
5765 5853 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5766 5854 dtrace_getufpstack(buf, fps, nframes + 1);
5767 5855 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5768 5856
5769 5857 /*
5770 5858 * If that faulted, we're cooked.
5771 5859 */
5772 5860 if (*flags & CPU_DTRACE_FAULT)
5773 5861 goto out;
5774 5862
5775 5863 /*
5776 5864 * Now we want to walk up the stack, calling the USTACK helper. For
5777 5865 * each iteration, we restore the scratch pointer.
5778 5866 */
5779 5867 for (i = 0; i < nframes; i++) {
5780 5868 mstate->dtms_scratch_ptr = saved;
5781 5869
5782 5870 if (offs >= strsize)
5783 5871 break;
5784 5872
5785 5873 sym = (char *)(uintptr_t)dtrace_helper(
5786 5874 DTRACE_HELPER_ACTION_USTACK,
5787 5875 mstate, state, pcs[i], fps[i]);
5788 5876
5789 5877 /*
5790 5878 * If we faulted while running the helper, we're going to
5791 5879 * clear the fault and null out the corresponding string.
5792 5880 */
5793 5881 if (*flags & CPU_DTRACE_FAULT) {
5794 5882 *flags &= ~CPU_DTRACE_FAULT;
5795 5883 str[offs++] = '\0';
5796 5884 continue;
5797 5885 }
5798 5886
5799 5887 if (sym == NULL) {
5800 5888 str[offs++] = '\0';
5801 5889 continue;
5802 5890 }
5803 5891
5804 5892 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5805 5893
5806 5894 /*
5807 5895 * Now copy in the string that the helper returned to us.
5808 5896 */
5809 5897 for (j = 0; offs + j < strsize; j++) {
5810 5898 if ((str[offs + j] = sym[j]) == '\0')
5811 5899 break;
5812 5900 }
5813 5901
5814 5902 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5815 5903
5816 5904 offs += j + 1;
5817 5905 }
5818 5906
5819 5907 if (offs >= strsize) {
5820 5908 /*
5821 5909 * If we didn't have room for all of the strings, we don't
5822 5910 * abort processing -- this needn't be a fatal error -- but we
5823 5911 * still want to increment a counter (dts_stkstroverflows) to
5824 5912 * allow this condition to be warned about. (If this is from
5825 5913 * a jstack() action, it is easily tuned via jstackstrsize.)
5826 5914 */
5827 5915 dtrace_error(&state->dts_stkstroverflows);
5828 5916 }
5829 5917
5830 5918 while (offs < strsize)
5831 5919 str[offs++] = '\0';
5832 5920
5833 5921 out:
5834 5922 mstate->dtms_scratch_ptr = old;
5835 5923 }
5836 5924
5837 5925 /*
5838 5926 * If you're looking for the epicenter of DTrace, you just found it. This
5839 5927 * is the function called by the provider to fire a probe -- from which all
5840 5928 * subsequent probe-context DTrace activity emanates.
5841 5929 */
5842 5930 void
5843 5931 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
5844 5932 uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
5845 5933 {
5846 5934 processorid_t cpuid;
5847 5935 dtrace_icookie_t cookie;
5848 5936 dtrace_probe_t *probe;
5849 5937 dtrace_mstate_t mstate;
5850 5938 dtrace_ecb_t *ecb;
5851 5939 dtrace_action_t *act;
5852 5940 intptr_t offs;
5853 5941 size_t size;
5854 5942 int vtime, onintr;
5855 5943 volatile uint16_t *flags;
5856 5944 hrtime_t now;
5857 5945
5858 5946 /*
5859 5947 * Kick out immediately if this CPU is still being born (in which case
5860 5948 * curthread will be set to -1) or the current thread can't allow
5861 5949 * probes in its current context.
5862 5950 */
5863 5951 if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))
5864 5952 return;
5865 5953
5866 5954 cookie = dtrace_interrupt_disable();
5867 5955 probe = dtrace_probes[id - 1];
5868 5956 cpuid = CPU->cpu_id;
5869 5957 onintr = CPU_ON_INTR(CPU);
5870 5958
5871 5959 if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
5872 5960 probe->dtpr_predcache == curthread->t_predcache) {
5873 5961 /*
5874 5962 * We have hit in the predicate cache; we know that
5875 5963 * this predicate would evaluate to be false.
5876 5964 */
5877 5965 dtrace_interrupt_enable(cookie);
5878 5966 return;
5879 5967 }
5880 5968
5881 5969 if (panic_quiesce) {
5882 5970 /*
5883 5971 * We don't trace anything if we're panicking.
5884 5972 */
5885 5973 dtrace_interrupt_enable(cookie);
5886 5974 return;
5887 5975 }
5888 5976
5889 5977 now = dtrace_gethrtime();
5890 5978 vtime = dtrace_vtime_references != 0;
5891 5979
5892 5980 if (vtime && curthread->t_dtrace_start)
5893 5981 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
5894 5982
5895 5983 mstate.dtms_difo = NULL;
5896 5984 mstate.dtms_probe = probe;
5897 5985 mstate.dtms_strtok = NULL;
5898 5986 mstate.dtms_arg[0] = arg0;
5899 5987 mstate.dtms_arg[1] = arg1;
5900 5988 mstate.dtms_arg[2] = arg2;
5901 5989 mstate.dtms_arg[3] = arg3;
5902 5990 mstate.dtms_arg[4] = arg4;
5903 5991
5904 5992 flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
5905 5993
5906 5994 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
5907 5995 dtrace_predicate_t *pred = ecb->dte_predicate;
5908 5996 dtrace_state_t *state = ecb->dte_state;
5909 5997 dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
5910 5998 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
5911 5999 dtrace_vstate_t *vstate = &state->dts_vstate;
5912 6000 dtrace_provider_t *prov = probe->dtpr_provider;
5913 6001 uint64_t tracememsize = 0;
5914 6002 int committed = 0;
5915 6003 caddr_t tomax;
5916 6004
5917 6005 /*
5918 6006 * A little subtlety with the following (seemingly innocuous)
5919 6007 * declaration of the automatic 'val': by looking at the
5920 6008 * code, you might think that it could be declared in the
5921 6009 * action processing loop, below. (That is, it's only used in
5922 6010 * the action processing loop.) However, it must be declared
5923 6011 * out of that scope because in the case of DIF expression
5924 6012 * arguments to aggregating actions, one iteration of the
|
↓ open down ↓ |
1344 lines elided |
↑ open up ↑ |
5925 6013 * action loop will use the last iteration's value.
5926 6014 */
5927 6015 #ifdef lint
5928 6016 uint64_t val = 0;
5929 6017 #else
5930 6018 uint64_t val;
5931 6019 #endif
5932 6020
5933 6021 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
5934 6022 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC;
6023 + mstate.dtms_getf = NULL;
6024 +
5935 6025 *flags &= ~CPU_DTRACE_ERROR;
5936 6026
5937 6027 if (prov == dtrace_provider) {
5938 6028 /*
5939 6029 * If dtrace itself is the provider of this probe,
5940 6030 * we're only going to continue processing the ECB if
5941 6031 * arg0 (the dtrace_state_t) is equal to the ECB's
5942 6032 * creating state. (This prevents disjoint consumers
5943 6033 * from seeing one another's metaprobes.)
5944 6034 */
5945 6035 if (arg0 != (uint64_t)(uintptr_t)state)
5946 6036 continue;
5947 6037 }
5948 6038
5949 6039 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
5950 6040 /*
5951 6041 * We're not currently active. If our provider isn't
5952 6042 * the dtrace pseudo provider, we're not interested.
5953 6043 */
5954 6044 if (prov != dtrace_provider)
5955 6045 continue;
5956 6046
5957 6047 /*
5958 6048 * Now we must further check if we are in the BEGIN
5959 6049 * probe. If we are, we will only continue processing
5960 6050 * if we're still in WARMUP -- if one BEGIN enabling
5961 6051 * has invoked the exit() action, we don't want to
5962 6052 * evaluate subsequent BEGIN enablings.
5963 6053 */
5964 6054 if (probe->dtpr_id == dtrace_probeid_begin &&
5965 6055 state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
5966 6056 ASSERT(state->dts_activity ==
5967 6057 DTRACE_ACTIVITY_DRAINING);
5968 6058 continue;
5969 6059 }
5970 6060 }
5971 6061
5972 6062 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb))
5973 6063 continue;
5974 6064
5975 6065 if (now - state->dts_alive > dtrace_deadman_timeout) {
5976 6066 /*
5977 6067 * We seem to be dead. Unless we (a) have kernel
5978 6068 * destructive permissions (b) have expicitly enabled
5979 6069 * destructive actions and (c) destructive actions have
5980 6070 * not been disabled, we're going to transition into
5981 6071 * the KILLED state, from which no further processing
5982 6072 * on this state will be performed.
5983 6073 */
5984 6074 if (!dtrace_priv_kernel_destructive(state) ||
5985 6075 !state->dts_cred.dcr_destructive ||
5986 6076 dtrace_destructive_disallow) {
5987 6077 void *activity = &state->dts_activity;
5988 6078 dtrace_activity_t current;
5989 6079
5990 6080 do {
5991 6081 current = state->dts_activity;
5992 6082 } while (dtrace_cas32(activity, current,
5993 6083 DTRACE_ACTIVITY_KILLED) != current);
5994 6084
5995 6085 continue;
5996 6086 }
5997 6087 }
5998 6088
5999 6089 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
6000 6090 ecb->dte_alignment, state, &mstate)) < 0)
6001 6091 continue;
6002 6092
6003 6093 tomax = buf->dtb_tomax;
6004 6094 ASSERT(tomax != NULL);
6005 6095
6006 6096 if (ecb->dte_size != 0)
6007 6097 DTRACE_STORE(uint32_t, tomax, offs, ecb->dte_epid);
6008 6098
6009 6099 mstate.dtms_epid = ecb->dte_epid;
6010 6100 mstate.dtms_present |= DTRACE_MSTATE_EPID;
6011 6101
6012 6102 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)
6013 6103 mstate.dtms_access |= DTRACE_ACCESS_KERNEL;
6014 6104
6015 6105 if (pred != NULL) {
6016 6106 dtrace_difo_t *dp = pred->dtp_difo;
6017 6107 int rval;
6018 6108
6019 6109 rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
6020 6110
6021 6111 if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
6022 6112 dtrace_cacheid_t cid = probe->dtpr_predcache;
6023 6113
6024 6114 if (cid != DTRACE_CACHEIDNONE && !onintr) {
6025 6115 /*
6026 6116 * Update the predicate cache...
6027 6117 */
6028 6118 ASSERT(cid == pred->dtp_cacheid);
6029 6119 curthread->t_predcache = cid;
6030 6120 }
6031 6121
6032 6122 continue;
6033 6123 }
6034 6124 }
6035 6125
6036 6126 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
6037 6127 act != NULL; act = act->dta_next) {
6038 6128 size_t valoffs;
6039 6129 dtrace_difo_t *dp;
6040 6130 dtrace_recdesc_t *rec = &act->dta_rec;
6041 6131
6042 6132 size = rec->dtrd_size;
6043 6133 valoffs = offs + rec->dtrd_offset;
6044 6134
6045 6135 if (DTRACEACT_ISAGG(act->dta_kind)) {
6046 6136 uint64_t v = 0xbad;
6047 6137 dtrace_aggregation_t *agg;
6048 6138
6049 6139 agg = (dtrace_aggregation_t *)act;
6050 6140
6051 6141 if ((dp = act->dta_difo) != NULL)
6052 6142 v = dtrace_dif_emulate(dp,
6053 6143 &mstate, vstate, state);
6054 6144
6055 6145 if (*flags & CPU_DTRACE_ERROR)
6056 6146 continue;
6057 6147
6058 6148 /*
6059 6149 * Note that we always pass the expression
6060 6150 * value from the previous iteration of the
6061 6151 * action loop. This value will only be used
6062 6152 * if there is an expression argument to the
6063 6153 * aggregating action, denoted by the
6064 6154 * dtag_hasarg field.
6065 6155 */
6066 6156 dtrace_aggregate(agg, buf,
6067 6157 offs, aggbuf, v, val);
6068 6158 continue;
6069 6159 }
6070 6160
6071 6161 switch (act->dta_kind) {
6072 6162 case DTRACEACT_STOP:
6073 6163 if (dtrace_priv_proc_destructive(state,
6074 6164 &mstate))
6075 6165 dtrace_action_stop();
6076 6166 continue;
6077 6167
6078 6168 case DTRACEACT_BREAKPOINT:
6079 6169 if (dtrace_priv_kernel_destructive(state))
6080 6170 dtrace_action_breakpoint(ecb);
6081 6171 continue;
6082 6172
6083 6173 case DTRACEACT_PANIC:
6084 6174 if (dtrace_priv_kernel_destructive(state))
6085 6175 dtrace_action_panic(ecb);
6086 6176 continue;
6087 6177
6088 6178 case DTRACEACT_STACK:
6089 6179 if (!dtrace_priv_kernel(state))
6090 6180 continue;
6091 6181
6092 6182 dtrace_getpcstack((pc_t *)(tomax + valoffs),
6093 6183 size / sizeof (pc_t), probe->dtpr_aframes,
6094 6184 DTRACE_ANCHORED(probe) ? NULL :
6095 6185 (uint32_t *)arg0);
6096 6186
6097 6187 continue;
6098 6188
6099 6189 case DTRACEACT_JSTACK:
6100 6190 case DTRACEACT_USTACK:
6101 6191 if (!dtrace_priv_proc(state, &mstate))
6102 6192 continue;
6103 6193
6104 6194 /*
6105 6195 * See comment in DIF_VAR_PID.
6106 6196 */
6107 6197 if (DTRACE_ANCHORED(mstate.dtms_probe) &&
6108 6198 CPU_ON_INTR(CPU)) {
6109 6199 int depth = DTRACE_USTACK_NFRAMES(
6110 6200 rec->dtrd_arg) + 1;
6111 6201
6112 6202 dtrace_bzero((void *)(tomax + valoffs),
6113 6203 DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
6114 6204 + depth * sizeof (uint64_t));
6115 6205
6116 6206 continue;
6117 6207 }
6118 6208
6119 6209 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
6120 6210 curproc->p_dtrace_helpers != NULL) {
6121 6211 /*
6122 6212 * This is the slow path -- we have
6123 6213 * allocated string space, and we're
6124 6214 * getting the stack of a process that
6125 6215 * has helpers. Call into a separate
6126 6216 * routine to perform this processing.
6127 6217 */
6128 6218 dtrace_action_ustack(&mstate, state,
6129 6219 (uint64_t *)(tomax + valoffs),
6130 6220 rec->dtrd_arg);
6131 6221 continue;
6132 6222 }
6133 6223
6134 6224 /*
6135 6225 * Clear the string space, since there's no
6136 6226 * helper to do it for us.
6137 6227 */
6138 6228 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) {
6139 6229 int depth = DTRACE_USTACK_NFRAMES(
6140 6230 rec->dtrd_arg);
6141 6231 size_t strsize = DTRACE_USTACK_STRSIZE(
6142 6232 rec->dtrd_arg);
6143 6233 uint64_t *buf = (uint64_t *)(tomax +
6144 6234 valoffs);
6145 6235 void *strspace = &buf[depth + 1];
6146 6236
6147 6237 dtrace_bzero(strspace,
6148 6238 MIN(depth, strsize));
6149 6239 }
6150 6240
6151 6241 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6152 6242 dtrace_getupcstack((uint64_t *)
6153 6243 (tomax + valoffs),
6154 6244 DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
6155 6245 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6156 6246 continue;
6157 6247
6158 6248 default:
6159 6249 break;
6160 6250 }
6161 6251
6162 6252 dp = act->dta_difo;
6163 6253 ASSERT(dp != NULL);
6164 6254
6165 6255 val = dtrace_dif_emulate(dp, &mstate, vstate, state);
6166 6256
6167 6257 if (*flags & CPU_DTRACE_ERROR)
6168 6258 continue;
6169 6259
6170 6260 switch (act->dta_kind) {
6171 6261 case DTRACEACT_SPECULATE:
6172 6262 ASSERT(buf == &state->dts_buffer[cpuid]);
6173 6263 buf = dtrace_speculation_buffer(state,
6174 6264 cpuid, val);
6175 6265
6176 6266 if (buf == NULL) {
6177 6267 *flags |= CPU_DTRACE_DROP;
6178 6268 continue;
6179 6269 }
6180 6270
6181 6271 offs = dtrace_buffer_reserve(buf,
6182 6272 ecb->dte_needed, ecb->dte_alignment,
6183 6273 state, NULL);
6184 6274
6185 6275 if (offs < 0) {
6186 6276 *flags |= CPU_DTRACE_DROP;
6187 6277 continue;
6188 6278 }
6189 6279
6190 6280 tomax = buf->dtb_tomax;
6191 6281 ASSERT(tomax != NULL);
6192 6282
6193 6283 if (ecb->dte_size != 0)
6194 6284 DTRACE_STORE(uint32_t, tomax, offs,
6195 6285 ecb->dte_epid);
6196 6286 continue;
6197 6287
6198 6288 case DTRACEACT_CHILL:
6199 6289 if (dtrace_priv_kernel_destructive(state))
6200 6290 dtrace_action_chill(&mstate, val);
6201 6291 continue;
6202 6292
6203 6293 case DTRACEACT_RAISE:
6204 6294 if (dtrace_priv_proc_destructive(state,
6205 6295 &mstate))
6206 6296 dtrace_action_raise(val);
6207 6297 continue;
6208 6298
6209 6299 case DTRACEACT_COMMIT:
6210 6300 ASSERT(!committed);
6211 6301
6212 6302 /*
6213 6303 * We need to commit our buffer state.
6214 6304 */
6215 6305 if (ecb->dte_size)
6216 6306 buf->dtb_offset = offs + ecb->dte_size;
6217 6307 buf = &state->dts_buffer[cpuid];
6218 6308 dtrace_speculation_commit(state, cpuid, val);
6219 6309 committed = 1;
6220 6310 continue;
6221 6311
6222 6312 case DTRACEACT_DISCARD:
6223 6313 dtrace_speculation_discard(state, cpuid, val);
6224 6314 continue;
6225 6315
6226 6316 case DTRACEACT_DIFEXPR:
6227 6317 case DTRACEACT_LIBACT:
6228 6318 case DTRACEACT_PRINTF:
6229 6319 case DTRACEACT_PRINTA:
6230 6320 case DTRACEACT_SYSTEM:
6231 6321 case DTRACEACT_FREOPEN:
6232 6322 case DTRACEACT_TRACEMEM:
6233 6323 break;
6234 6324
6235 6325 case DTRACEACT_TRACEMEM_DYNSIZE:
6236 6326 tracememsize = val;
6237 6327 break;
6238 6328
6239 6329 case DTRACEACT_SYM:
6240 6330 case DTRACEACT_MOD:
6241 6331 if (!dtrace_priv_kernel(state))
6242 6332 continue;
6243 6333 break;
6244 6334
6245 6335 case DTRACEACT_USYM:
6246 6336 case DTRACEACT_UMOD:
6247 6337 case DTRACEACT_UADDR: {
6248 6338 struct pid *pid = curthread->t_procp->p_pidp;
6249 6339
6250 6340 if (!dtrace_priv_proc(state, &mstate))
6251 6341 continue;
6252 6342
6253 6343 DTRACE_STORE(uint64_t, tomax,
6254 6344 valoffs, (uint64_t)pid->pid_id);
6255 6345 DTRACE_STORE(uint64_t, tomax,
6256 6346 valoffs + sizeof (uint64_t), val);
6257 6347
6258 6348 continue;
6259 6349 }
6260 6350
6261 6351 case DTRACEACT_EXIT: {
6262 6352 /*
6263 6353 * For the exit action, we are going to attempt
6264 6354 * to atomically set our activity to be
6265 6355 * draining. If this fails (either because
6266 6356 * another CPU has beat us to the exit action,
6267 6357 * or because our current activity is something
6268 6358 * other than ACTIVE or WARMUP), we will
6269 6359 * continue. This assures that the exit action
6270 6360 * can be successfully recorded at most once
6271 6361 * when we're in the ACTIVE state. If we're
6272 6362 * encountering the exit() action while in
6273 6363 * COOLDOWN, however, we want to honor the new
6274 6364 * status code. (We know that we're the only
6275 6365 * thread in COOLDOWN, so there is no race.)
6276 6366 */
6277 6367 void *activity = &state->dts_activity;
6278 6368 dtrace_activity_t current = state->dts_activity;
6279 6369
6280 6370 if (current == DTRACE_ACTIVITY_COOLDOWN)
6281 6371 break;
6282 6372
6283 6373 if (current != DTRACE_ACTIVITY_WARMUP)
6284 6374 current = DTRACE_ACTIVITY_ACTIVE;
6285 6375
6286 6376 if (dtrace_cas32(activity, current,
6287 6377 DTRACE_ACTIVITY_DRAINING) != current) {
6288 6378 *flags |= CPU_DTRACE_DROP;
6289 6379 continue;
6290 6380 }
6291 6381
6292 6382 break;
6293 6383 }
6294 6384
6295 6385 default:
6296 6386 ASSERT(0);
6297 6387 }
6298 6388
6299 6389 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
6300 6390 uintptr_t end = valoffs + size;
6301 6391
6302 6392 if (tracememsize != 0 &&
6303 6393 valoffs + tracememsize < end) {
6304 6394 end = valoffs + tracememsize;
6305 6395 tracememsize = 0;
6306 6396 }
6307 6397
6308 6398 if (!dtrace_vcanload((void *)(uintptr_t)val,
6309 6399 &dp->dtdo_rtype, &mstate, vstate))
6310 6400 continue;
6311 6401
6312 6402 /*
6313 6403 * If this is a string, we're going to only
6314 6404 * load until we find the zero byte -- after
6315 6405 * which we'll store zero bytes.
6316 6406 */
6317 6407 if (dp->dtdo_rtype.dtdt_kind ==
6318 6408 DIF_TYPE_STRING) {
6319 6409 char c = '\0' + 1;
6320 6410 int intuple = act->dta_intuple;
6321 6411 size_t s;
6322 6412
6323 6413 for (s = 0; s < size; s++) {
6324 6414 if (c != '\0')
6325 6415 c = dtrace_load8(val++);
6326 6416
6327 6417 DTRACE_STORE(uint8_t, tomax,
6328 6418 valoffs++, c);
6329 6419
6330 6420 if (c == '\0' && intuple)
6331 6421 break;
6332 6422 }
6333 6423
6334 6424 continue;
6335 6425 }
6336 6426
6337 6427 while (valoffs < end) {
6338 6428 DTRACE_STORE(uint8_t, tomax, valoffs++,
6339 6429 dtrace_load8(val++));
6340 6430 }
6341 6431
6342 6432 continue;
6343 6433 }
6344 6434
6345 6435 switch (size) {
6346 6436 case 0:
6347 6437 break;
6348 6438
6349 6439 case sizeof (uint8_t):
6350 6440 DTRACE_STORE(uint8_t, tomax, valoffs, val);
6351 6441 break;
6352 6442 case sizeof (uint16_t):
6353 6443 DTRACE_STORE(uint16_t, tomax, valoffs, val);
6354 6444 break;
6355 6445 case sizeof (uint32_t):
6356 6446 DTRACE_STORE(uint32_t, tomax, valoffs, val);
6357 6447 break;
6358 6448 case sizeof (uint64_t):
6359 6449 DTRACE_STORE(uint64_t, tomax, valoffs, val);
6360 6450 break;
6361 6451 default:
6362 6452 /*
6363 6453 * Any other size should have been returned by
6364 6454 * reference, not by value.
6365 6455 */
6366 6456 ASSERT(0);
6367 6457 break;
6368 6458 }
6369 6459 }
6370 6460
6371 6461 if (*flags & CPU_DTRACE_DROP)
6372 6462 continue;
6373 6463
6374 6464 if (*flags & CPU_DTRACE_FAULT) {
6375 6465 int ndx;
6376 6466 dtrace_action_t *err;
6377 6467
6378 6468 buf->dtb_errors++;
6379 6469
6380 6470 if (probe->dtpr_id == dtrace_probeid_error) {
6381 6471 /*
6382 6472 * There's nothing we can do -- we had an
6383 6473 * error on the error probe. We bump an
6384 6474 * error counter to at least indicate that
6385 6475 * this condition happened.
6386 6476 */
6387 6477 dtrace_error(&state->dts_dblerrors);
6388 6478 continue;
6389 6479 }
6390 6480
6391 6481 if (vtime) {
6392 6482 /*
6393 6483 * Before recursing on dtrace_probe(), we
6394 6484 * need to explicitly clear out our start
6395 6485 * time to prevent it from being accumulated
6396 6486 * into t_dtrace_vtime.
6397 6487 */
6398 6488 curthread->t_dtrace_start = 0;
6399 6489 }
6400 6490
6401 6491 /*
6402 6492 * Iterate over the actions to figure out which action
6403 6493 * we were processing when we experienced the error.
6404 6494 * Note that act points _past_ the faulting action; if
6405 6495 * act is ecb->dte_action, the fault was in the
6406 6496 * predicate, if it's ecb->dte_action->dta_next it's
6407 6497 * in action #1, and so on.
6408 6498 */
6409 6499 for (err = ecb->dte_action, ndx = 0;
6410 6500 err != act; err = err->dta_next, ndx++)
6411 6501 continue;
6412 6502
6413 6503 dtrace_probe_error(state, ecb->dte_epid, ndx,
6414 6504 (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
6415 6505 mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
6416 6506 cpu_core[cpuid].cpuc_dtrace_illval);
6417 6507
6418 6508 continue;
6419 6509 }
6420 6510
6421 6511 if (!committed)
6422 6512 buf->dtb_offset = offs + ecb->dte_size;
6423 6513 }
6424 6514
6425 6515 if (vtime)
6426 6516 curthread->t_dtrace_start = dtrace_gethrtime();
6427 6517
6428 6518 dtrace_interrupt_enable(cookie);
6429 6519 }
6430 6520
6431 6521 /*
6432 6522 * DTrace Probe Hashing Functions
6433 6523 *
6434 6524 * The functions in this section (and indeed, the functions in remaining
6435 6525 * sections) are not _called_ from probe context. (Any exceptions to this are
6436 6526 * marked with a "Note:".) Rather, they are called from elsewhere in the
6437 6527 * DTrace framework to look-up probes in, add probes to and remove probes from
6438 6528 * the DTrace probe hashes. (Each probe is hashed by each element of the
6439 6529 * probe tuple -- allowing for fast lookups, regardless of what was
6440 6530 * specified.)
6441 6531 */
6442 6532 static uint_t
6443 6533 dtrace_hash_str(char *p)
6444 6534 {
6445 6535 unsigned int g;
6446 6536 uint_t hval = 0;
6447 6537
6448 6538 while (*p) {
6449 6539 hval = (hval << 4) + *p++;
6450 6540 if ((g = (hval & 0xf0000000)) != 0)
6451 6541 hval ^= g >> 24;
6452 6542 hval &= ~g;
6453 6543 }
6454 6544 return (hval);
6455 6545 }
6456 6546
6457 6547 static dtrace_hash_t *
6458 6548 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
6459 6549 {
6460 6550 dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
6461 6551
6462 6552 hash->dth_stroffs = stroffs;
6463 6553 hash->dth_nextoffs = nextoffs;
6464 6554 hash->dth_prevoffs = prevoffs;
6465 6555
6466 6556 hash->dth_size = 1;
6467 6557 hash->dth_mask = hash->dth_size - 1;
6468 6558
6469 6559 hash->dth_tab = kmem_zalloc(hash->dth_size *
6470 6560 sizeof (dtrace_hashbucket_t *), KM_SLEEP);
6471 6561
6472 6562 return (hash);
6473 6563 }
6474 6564
6475 6565 static void
6476 6566 dtrace_hash_destroy(dtrace_hash_t *hash)
6477 6567 {
6478 6568 #ifdef DEBUG
6479 6569 int i;
6480 6570
6481 6571 for (i = 0; i < hash->dth_size; i++)
6482 6572 ASSERT(hash->dth_tab[i] == NULL);
6483 6573 #endif
6484 6574
6485 6575 kmem_free(hash->dth_tab,
6486 6576 hash->dth_size * sizeof (dtrace_hashbucket_t *));
6487 6577 kmem_free(hash, sizeof (dtrace_hash_t));
6488 6578 }
6489 6579
6490 6580 static void
6491 6581 dtrace_hash_resize(dtrace_hash_t *hash)
6492 6582 {
6493 6583 int size = hash->dth_size, i, ndx;
6494 6584 int new_size = hash->dth_size << 1;
6495 6585 int new_mask = new_size - 1;
6496 6586 dtrace_hashbucket_t **new_tab, *bucket, *next;
6497 6587
6498 6588 ASSERT((new_size & new_mask) == 0);
6499 6589
6500 6590 new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
6501 6591
6502 6592 for (i = 0; i < size; i++) {
6503 6593 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
6504 6594 dtrace_probe_t *probe = bucket->dthb_chain;
6505 6595
6506 6596 ASSERT(probe != NULL);
6507 6597 ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
6508 6598
6509 6599 next = bucket->dthb_next;
6510 6600 bucket->dthb_next = new_tab[ndx];
6511 6601 new_tab[ndx] = bucket;
6512 6602 }
6513 6603 }
6514 6604
6515 6605 kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
6516 6606 hash->dth_tab = new_tab;
6517 6607 hash->dth_size = new_size;
6518 6608 hash->dth_mask = new_mask;
6519 6609 }
6520 6610
6521 6611 static void
6522 6612 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
6523 6613 {
6524 6614 int hashval = DTRACE_HASHSTR(hash, new);
6525 6615 int ndx = hashval & hash->dth_mask;
6526 6616 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6527 6617 dtrace_probe_t **nextp, **prevp;
6528 6618
6529 6619 for (; bucket != NULL; bucket = bucket->dthb_next) {
6530 6620 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
6531 6621 goto add;
6532 6622 }
6533 6623
6534 6624 if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
6535 6625 dtrace_hash_resize(hash);
6536 6626 dtrace_hash_add(hash, new);
6537 6627 return;
6538 6628 }
6539 6629
6540 6630 bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
6541 6631 bucket->dthb_next = hash->dth_tab[ndx];
6542 6632 hash->dth_tab[ndx] = bucket;
6543 6633 hash->dth_nbuckets++;
6544 6634
6545 6635 add:
6546 6636 nextp = DTRACE_HASHNEXT(hash, new);
6547 6637 ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
6548 6638 *nextp = bucket->dthb_chain;
6549 6639
6550 6640 if (bucket->dthb_chain != NULL) {
6551 6641 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
6552 6642 ASSERT(*prevp == NULL);
6553 6643 *prevp = new;
6554 6644 }
6555 6645
6556 6646 bucket->dthb_chain = new;
6557 6647 bucket->dthb_len++;
6558 6648 }
6559 6649
6560 6650 static dtrace_probe_t *
6561 6651 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
6562 6652 {
6563 6653 int hashval = DTRACE_HASHSTR(hash, template);
6564 6654 int ndx = hashval & hash->dth_mask;
6565 6655 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6566 6656
6567 6657 for (; bucket != NULL; bucket = bucket->dthb_next) {
6568 6658 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
6569 6659 return (bucket->dthb_chain);
6570 6660 }
6571 6661
6572 6662 return (NULL);
6573 6663 }
6574 6664
6575 6665 static int
6576 6666 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
6577 6667 {
6578 6668 int hashval = DTRACE_HASHSTR(hash, template);
6579 6669 int ndx = hashval & hash->dth_mask;
6580 6670 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6581 6671
6582 6672 for (; bucket != NULL; bucket = bucket->dthb_next) {
6583 6673 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
6584 6674 return (bucket->dthb_len);
6585 6675 }
6586 6676
6587 6677 return (NULL);
6588 6678 }
6589 6679
6590 6680 static void
6591 6681 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
6592 6682 {
6593 6683 int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
6594 6684 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
6595 6685
6596 6686 dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
6597 6687 dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
6598 6688
6599 6689 /*
6600 6690 * Find the bucket that we're removing this probe from.
6601 6691 */
6602 6692 for (; bucket != NULL; bucket = bucket->dthb_next) {
6603 6693 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
6604 6694 break;
6605 6695 }
6606 6696
6607 6697 ASSERT(bucket != NULL);
6608 6698
6609 6699 if (*prevp == NULL) {
6610 6700 if (*nextp == NULL) {
6611 6701 /*
6612 6702 * The removed probe was the only probe on this
6613 6703 * bucket; we need to remove the bucket.
6614 6704 */
6615 6705 dtrace_hashbucket_t *b = hash->dth_tab[ndx];
6616 6706
6617 6707 ASSERT(bucket->dthb_chain == probe);
6618 6708 ASSERT(b != NULL);
6619 6709
6620 6710 if (b == bucket) {
6621 6711 hash->dth_tab[ndx] = bucket->dthb_next;
6622 6712 } else {
6623 6713 while (b->dthb_next != bucket)
6624 6714 b = b->dthb_next;
6625 6715 b->dthb_next = bucket->dthb_next;
6626 6716 }
6627 6717
6628 6718 ASSERT(hash->dth_nbuckets > 0);
6629 6719 hash->dth_nbuckets--;
6630 6720 kmem_free(bucket, sizeof (dtrace_hashbucket_t));
6631 6721 return;
6632 6722 }
6633 6723
6634 6724 bucket->dthb_chain = *nextp;
6635 6725 } else {
6636 6726 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
6637 6727 }
6638 6728
6639 6729 if (*nextp != NULL)
6640 6730 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
6641 6731 }
6642 6732
6643 6733 /*
6644 6734 * DTrace Utility Functions
6645 6735 *
6646 6736 * These are random utility functions that are _not_ called from probe context.
6647 6737 */
6648 6738 static int
6649 6739 dtrace_badattr(const dtrace_attribute_t *a)
6650 6740 {
6651 6741 return (a->dtat_name > DTRACE_STABILITY_MAX ||
6652 6742 a->dtat_data > DTRACE_STABILITY_MAX ||
6653 6743 a->dtat_class > DTRACE_CLASS_MAX);
6654 6744 }
6655 6745
6656 6746 /*
6657 6747 * Return a duplicate copy of a string. If the specified string is NULL,
6658 6748 * this function returns a zero-length string.
6659 6749 */
6660 6750 static char *
6661 6751 dtrace_strdup(const char *str)
6662 6752 {
6663 6753 char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
6664 6754
6665 6755 if (str != NULL)
6666 6756 (void) strcpy(new, str);
6667 6757
6668 6758 return (new);
6669 6759 }
6670 6760
6671 6761 #define DTRACE_ISALPHA(c) \
6672 6762 (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
6673 6763
6674 6764 static int
6675 6765 dtrace_badname(const char *s)
6676 6766 {
6677 6767 char c;
6678 6768
6679 6769 if (s == NULL || (c = *s++) == '\0')
6680 6770 return (0);
6681 6771
6682 6772 if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
6683 6773 return (1);
6684 6774
6685 6775 while ((c = *s++) != '\0') {
6686 6776 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
6687 6777 c != '-' && c != '_' && c != '.' && c != '`')
6688 6778 return (1);
6689 6779 }
6690 6780
6691 6781 return (0);
6692 6782 }
6693 6783
6694 6784 static void
6695 6785 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
6696 6786 {
6697 6787 uint32_t priv;
6698 6788
6699 6789 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
6700 6790 /*
6701 6791 * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
6702 6792 */
6703 6793 priv = DTRACE_PRIV_ALL;
6704 6794 } else {
6705 6795 *uidp = crgetuid(cr);
6706 6796 *zoneidp = crgetzoneid(cr);
6707 6797
6708 6798 priv = 0;
6709 6799 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
6710 6800 priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
6711 6801 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
6712 6802 priv |= DTRACE_PRIV_USER;
6713 6803 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
6714 6804 priv |= DTRACE_PRIV_PROC;
6715 6805 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
6716 6806 priv |= DTRACE_PRIV_OWNER;
6717 6807 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
6718 6808 priv |= DTRACE_PRIV_ZONEOWNER;
6719 6809 }
6720 6810
6721 6811 *privp = priv;
6722 6812 }
6723 6813
6724 6814 #ifdef DTRACE_ERRDEBUG
6725 6815 static void
6726 6816 dtrace_errdebug(const char *str)
6727 6817 {
6728 6818 int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
6729 6819 int occupied = 0;
6730 6820
6731 6821 mutex_enter(&dtrace_errlock);
6732 6822 dtrace_errlast = str;
6733 6823 dtrace_errthread = curthread;
6734 6824
6735 6825 while (occupied++ < DTRACE_ERRHASHSZ) {
6736 6826 if (dtrace_errhash[hval].dter_msg == str) {
6737 6827 dtrace_errhash[hval].dter_count++;
6738 6828 goto out;
6739 6829 }
6740 6830
6741 6831 if (dtrace_errhash[hval].dter_msg != NULL) {
6742 6832 hval = (hval + 1) % DTRACE_ERRHASHSZ;
6743 6833 continue;
6744 6834 }
6745 6835
6746 6836 dtrace_errhash[hval].dter_msg = str;
6747 6837 dtrace_errhash[hval].dter_count = 1;
6748 6838 goto out;
6749 6839 }
6750 6840
6751 6841 panic("dtrace: undersized error hash");
6752 6842 out:
6753 6843 mutex_exit(&dtrace_errlock);
6754 6844 }
6755 6845 #endif
6756 6846
6757 6847 /*
6758 6848 * DTrace Matching Functions
6759 6849 *
6760 6850 * These functions are used to match groups of probes, given some elements of
6761 6851 * a probe tuple, or some globbed expressions for elements of a probe tuple.
6762 6852 */
6763 6853 static int
6764 6854 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
6765 6855 zoneid_t zoneid)
6766 6856 {
6767 6857 if (priv != DTRACE_PRIV_ALL) {
6768 6858 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
6769 6859 uint32_t match = priv & ppriv;
6770 6860
6771 6861 /*
6772 6862 * No PRIV_DTRACE_* privileges...
6773 6863 */
6774 6864 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
6775 6865 DTRACE_PRIV_KERNEL)) == 0)
6776 6866 return (0);
6777 6867
6778 6868 /*
6779 6869 * No matching bits, but there were bits to match...
6780 6870 */
6781 6871 if (match == 0 && ppriv != 0)
6782 6872 return (0);
6783 6873
6784 6874 /*
6785 6875 * Need to have permissions to the process, but don't...
6786 6876 */
6787 6877 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
6788 6878 uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
6789 6879 return (0);
6790 6880 }
6791 6881
6792 6882 /*
6793 6883 * Need to be in the same zone unless we possess the
6794 6884 * privilege to examine all zones.
6795 6885 */
6796 6886 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
6797 6887 zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
6798 6888 return (0);
6799 6889 }
6800 6890 }
6801 6891
6802 6892 return (1);
6803 6893 }
6804 6894
6805 6895 /*
6806 6896 * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
6807 6897 * consists of input pattern strings and an ops-vector to evaluate them.
6808 6898 * This function returns >0 for match, 0 for no match, and <0 for error.
6809 6899 */
6810 6900 static int
6811 6901 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
6812 6902 uint32_t priv, uid_t uid, zoneid_t zoneid)
6813 6903 {
6814 6904 dtrace_provider_t *pvp = prp->dtpr_provider;
6815 6905 int rv;
6816 6906
6817 6907 if (pvp->dtpv_defunct)
6818 6908 return (0);
6819 6909
6820 6910 if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
6821 6911 return (rv);
6822 6912
6823 6913 if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
6824 6914 return (rv);
6825 6915
6826 6916 if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
6827 6917 return (rv);
6828 6918
6829 6919 if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
6830 6920 return (rv);
6831 6921
6832 6922 if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
6833 6923 return (0);
6834 6924
6835 6925 return (rv);
6836 6926 }
6837 6927
6838 6928 /*
6839 6929 * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
6840 6930 * interface for matching a glob pattern 'p' to an input string 's'. Unlike
6841 6931 * libc's version, the kernel version only applies to 8-bit ASCII strings.
6842 6932 * In addition, all of the recursion cases except for '*' matching have been
6843 6933 * unwound. For '*', we still implement recursive evaluation, but a depth
6844 6934 * counter is maintained and matching is aborted if we recurse too deep.
6845 6935 * The function returns 0 if no match, >0 if match, and <0 if recursion error.
6846 6936 */
6847 6937 static int
6848 6938 dtrace_match_glob(const char *s, const char *p, int depth)
6849 6939 {
6850 6940 const char *olds;
6851 6941 char s1, c;
6852 6942 int gs;
6853 6943
6854 6944 if (depth > DTRACE_PROBEKEY_MAXDEPTH)
6855 6945 return (-1);
6856 6946
6857 6947 if (s == NULL)
6858 6948 s = ""; /* treat NULL as empty string */
6859 6949
6860 6950 top:
6861 6951 olds = s;
6862 6952 s1 = *s++;
6863 6953
6864 6954 if (p == NULL)
6865 6955 return (0);
6866 6956
6867 6957 if ((c = *p++) == '\0')
6868 6958 return (s1 == '\0');
6869 6959
6870 6960 switch (c) {
6871 6961 case '[': {
6872 6962 int ok = 0, notflag = 0;
6873 6963 char lc = '\0';
6874 6964
6875 6965 if (s1 == '\0')
6876 6966 return (0);
6877 6967
6878 6968 if (*p == '!') {
6879 6969 notflag = 1;
6880 6970 p++;
6881 6971 }
6882 6972
6883 6973 if ((c = *p++) == '\0')
6884 6974 return (0);
6885 6975
6886 6976 do {
6887 6977 if (c == '-' && lc != '\0' && *p != ']') {
6888 6978 if ((c = *p++) == '\0')
6889 6979 return (0);
6890 6980 if (c == '\\' && (c = *p++) == '\0')
6891 6981 return (0);
6892 6982
6893 6983 if (notflag) {
6894 6984 if (s1 < lc || s1 > c)
6895 6985 ok++;
6896 6986 else
6897 6987 return (0);
6898 6988 } else if (lc <= s1 && s1 <= c)
6899 6989 ok++;
6900 6990
6901 6991 } else if (c == '\\' && (c = *p++) == '\0')
6902 6992 return (0);
6903 6993
6904 6994 lc = c; /* save left-hand 'c' for next iteration */
6905 6995
6906 6996 if (notflag) {
6907 6997 if (s1 != c)
6908 6998 ok++;
6909 6999 else
6910 7000 return (0);
6911 7001 } else if (s1 == c)
6912 7002 ok++;
6913 7003
6914 7004 if ((c = *p++) == '\0')
6915 7005 return (0);
6916 7006
6917 7007 } while (c != ']');
6918 7008
6919 7009 if (ok)
6920 7010 goto top;
6921 7011
6922 7012 return (0);
6923 7013 }
6924 7014
6925 7015 case '\\':
6926 7016 if ((c = *p++) == '\0')
6927 7017 return (0);
6928 7018 /*FALLTHRU*/
6929 7019
6930 7020 default:
6931 7021 if (c != s1)
6932 7022 return (0);
6933 7023 /*FALLTHRU*/
6934 7024
6935 7025 case '?':
6936 7026 if (s1 != '\0')
6937 7027 goto top;
6938 7028 return (0);
6939 7029
6940 7030 case '*':
6941 7031 while (*p == '*')
6942 7032 p++; /* consecutive *'s are identical to a single one */
6943 7033
6944 7034 if (*p == '\0')
6945 7035 return (1);
6946 7036
6947 7037 for (s = olds; *s != '\0'; s++) {
6948 7038 if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
6949 7039 return (gs);
6950 7040 }
6951 7041
6952 7042 return (0);
6953 7043 }
6954 7044 }
6955 7045
6956 7046 /*ARGSUSED*/
6957 7047 static int
6958 7048 dtrace_match_string(const char *s, const char *p, int depth)
6959 7049 {
6960 7050 return (s != NULL && strcmp(s, p) == 0);
6961 7051 }
6962 7052
6963 7053 /*ARGSUSED*/
6964 7054 static int
6965 7055 dtrace_match_nul(const char *s, const char *p, int depth)
6966 7056 {
6967 7057 return (1); /* always match the empty pattern */
6968 7058 }
6969 7059
6970 7060 /*ARGSUSED*/
6971 7061 static int
6972 7062 dtrace_match_nonzero(const char *s, const char *p, int depth)
6973 7063 {
6974 7064 return (s != NULL && s[0] != '\0');
6975 7065 }
6976 7066
6977 7067 static int
6978 7068 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
6979 7069 zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
6980 7070 {
6981 7071 dtrace_probe_t template, *probe;
6982 7072 dtrace_hash_t *hash = NULL;
6983 7073 int len, rc, best = INT_MAX, nmatched = 0;
6984 7074 dtrace_id_t i;
6985 7075
6986 7076 ASSERT(MUTEX_HELD(&dtrace_lock));
6987 7077
6988 7078 /*
6989 7079 * If the probe ID is specified in the key, just lookup by ID and
6990 7080 * invoke the match callback once if a matching probe is found.
6991 7081 */
6992 7082 if (pkp->dtpk_id != DTRACE_IDNONE) {
6993 7083 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
6994 7084 dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
6995 7085 if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
6996 7086 return (DTRACE_MATCH_FAIL);
6997 7087 nmatched++;
6998 7088 }
6999 7089 return (nmatched);
7000 7090 }
7001 7091
7002 7092 template.dtpr_mod = (char *)pkp->dtpk_mod;
7003 7093 template.dtpr_func = (char *)pkp->dtpk_func;
7004 7094 template.dtpr_name = (char *)pkp->dtpk_name;
7005 7095
7006 7096 /*
7007 7097 * We want to find the most distinct of the module name, function
7008 7098 * name, and name. So for each one that is not a glob pattern or
7009 7099 * empty string, we perform a lookup in the corresponding hash and
7010 7100 * use the hash table with the fewest collisions to do our search.
7011 7101 */
7012 7102 if (pkp->dtpk_mmatch == &dtrace_match_string &&
7013 7103 (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
7014 7104 best = len;
7015 7105 hash = dtrace_bymod;
7016 7106 }
7017 7107
7018 7108 if (pkp->dtpk_fmatch == &dtrace_match_string &&
7019 7109 (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
7020 7110 best = len;
7021 7111 hash = dtrace_byfunc;
7022 7112 }
7023 7113
7024 7114 if (pkp->dtpk_nmatch == &dtrace_match_string &&
7025 7115 (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
7026 7116 best = len;
7027 7117 hash = dtrace_byname;
7028 7118 }
7029 7119
7030 7120 /*
7031 7121 * If we did not select a hash table, iterate over every probe and
7032 7122 * invoke our callback for each one that matches our input probe key.
7033 7123 */
7034 7124 if (hash == NULL) {
7035 7125 for (i = 0; i < dtrace_nprobes; i++) {
7036 7126 if ((probe = dtrace_probes[i]) == NULL ||
7037 7127 dtrace_match_probe(probe, pkp, priv, uid,
7038 7128 zoneid) <= 0)
7039 7129 continue;
7040 7130
7041 7131 nmatched++;
7042 7132
7043 7133 if ((rc = (*matched)(probe, arg)) !=
7044 7134 DTRACE_MATCH_NEXT) {
7045 7135 if (rc == DTRACE_MATCH_FAIL)
7046 7136 return (DTRACE_MATCH_FAIL);
7047 7137 break;
7048 7138 }
7049 7139 }
7050 7140
7051 7141 return (nmatched);
7052 7142 }
7053 7143
7054 7144 /*
7055 7145 * If we selected a hash table, iterate over each probe of the same key
7056 7146 * name and invoke the callback for every probe that matches the other
7057 7147 * attributes of our input probe key.
7058 7148 */
7059 7149 for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
7060 7150 probe = *(DTRACE_HASHNEXT(hash, probe))) {
7061 7151
7062 7152 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
7063 7153 continue;
7064 7154
7065 7155 nmatched++;
7066 7156
7067 7157 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
7068 7158 if (rc == DTRACE_MATCH_FAIL)
7069 7159 return (DTRACE_MATCH_FAIL);
7070 7160 break;
7071 7161 }
7072 7162 }
7073 7163
7074 7164 return (nmatched);
7075 7165 }
7076 7166
7077 7167 /*
7078 7168 * Return the function pointer dtrace_probecmp() should use to compare the
7079 7169 * specified pattern with a string. For NULL or empty patterns, we select
7080 7170 * dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob().
7081 7171 * For non-empty non-glob strings, we use dtrace_match_string().
7082 7172 */
7083 7173 static dtrace_probekey_f *
7084 7174 dtrace_probekey_func(const char *p)
7085 7175 {
7086 7176 char c;
7087 7177
7088 7178 if (p == NULL || *p == '\0')
7089 7179 return (&dtrace_match_nul);
7090 7180
7091 7181 while ((c = *p++) != '\0') {
7092 7182 if (c == '[' || c == '?' || c == '*' || c == '\\')
7093 7183 return (&dtrace_match_glob);
7094 7184 }
7095 7185
7096 7186 return (&dtrace_match_string);
7097 7187 }
7098 7188
7099 7189 /*
7100 7190 * Build a probe comparison key for use with dtrace_match_probe() from the
7101 7191 * given probe description. By convention, a null key only matches anchored
7102 7192 * probes: if each field is the empty string, reset dtpk_fmatch to
7103 7193 * dtrace_match_nonzero().
7104 7194 */
7105 7195 static void
7106 7196 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
7107 7197 {
7108 7198 pkp->dtpk_prov = pdp->dtpd_provider;
7109 7199 pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
7110 7200
7111 7201 pkp->dtpk_mod = pdp->dtpd_mod;
7112 7202 pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
7113 7203
7114 7204 pkp->dtpk_func = pdp->dtpd_func;
7115 7205 pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
7116 7206
7117 7207 pkp->dtpk_name = pdp->dtpd_name;
7118 7208 pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
7119 7209
7120 7210 pkp->dtpk_id = pdp->dtpd_id;
7121 7211
7122 7212 if (pkp->dtpk_id == DTRACE_IDNONE &&
7123 7213 pkp->dtpk_pmatch == &dtrace_match_nul &&
7124 7214 pkp->dtpk_mmatch == &dtrace_match_nul &&
7125 7215 pkp->dtpk_fmatch == &dtrace_match_nul &&
7126 7216 pkp->dtpk_nmatch == &dtrace_match_nul)
7127 7217 pkp->dtpk_fmatch = &dtrace_match_nonzero;
7128 7218 }
7129 7219
7130 7220 /*
7131 7221 * DTrace Provider-to-Framework API Functions
7132 7222 *
7133 7223 * These functions implement much of the Provider-to-Framework API, as
7134 7224 * described in <sys/dtrace.h>. The parts of the API not in this section are
7135 7225 * the functions in the API for probe management (found below), and
7136 7226 * dtrace_probe() itself (found above).
7137 7227 */
7138 7228
7139 7229 /*
7140 7230 * Register the calling provider with the DTrace framework. This should
7141 7231 * generally be called by DTrace providers in their attach(9E) entry point.
7142 7232 */
7143 7233 int
7144 7234 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
7145 7235 cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
7146 7236 {
7147 7237 dtrace_provider_t *provider;
7148 7238
7149 7239 if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
7150 7240 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7151 7241 "arguments", name ? name : "<NULL>");
7152 7242 return (EINVAL);
7153 7243 }
7154 7244
7155 7245 if (name[0] == '\0' || dtrace_badname(name)) {
7156 7246 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7157 7247 "provider name", name);
7158 7248 return (EINVAL);
7159 7249 }
7160 7250
7161 7251 if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
7162 7252 pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
7163 7253 pops->dtps_destroy == NULL ||
7164 7254 ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
7165 7255 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7166 7256 "provider ops", name);
7167 7257 return (EINVAL);
7168 7258 }
7169 7259
7170 7260 if (dtrace_badattr(&pap->dtpa_provider) ||
7171 7261 dtrace_badattr(&pap->dtpa_mod) ||
7172 7262 dtrace_badattr(&pap->dtpa_func) ||
7173 7263 dtrace_badattr(&pap->dtpa_name) ||
7174 7264 dtrace_badattr(&pap->dtpa_args)) {
7175 7265 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7176 7266 "provider attributes", name);
7177 7267 return (EINVAL);
7178 7268 }
7179 7269
7180 7270 if (priv & ~DTRACE_PRIV_ALL) {
7181 7271 cmn_err(CE_WARN, "failed to register provider '%s': invalid "
7182 7272 "privilege attributes", name);
7183 7273 return (EINVAL);
7184 7274 }
7185 7275
7186 7276 if ((priv & DTRACE_PRIV_KERNEL) &&
7187 7277 (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
7188 7278 pops->dtps_mode == NULL) {
7189 7279 cmn_err(CE_WARN, "failed to register provider '%s': need "
7190 7280 "dtps_mode() op for given privilege attributes", name);
7191 7281 return (EINVAL);
7192 7282 }
7193 7283
7194 7284 provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
7195 7285 provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
7196 7286 (void) strcpy(provider->dtpv_name, name);
7197 7287
7198 7288 provider->dtpv_attr = *pap;
7199 7289 provider->dtpv_priv.dtpp_flags = priv;
7200 7290 if (cr != NULL) {
7201 7291 provider->dtpv_priv.dtpp_uid = crgetuid(cr);
7202 7292 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
7203 7293 }
7204 7294 provider->dtpv_pops = *pops;
7205 7295
7206 7296 if (pops->dtps_provide == NULL) {
7207 7297 ASSERT(pops->dtps_provide_module != NULL);
7208 7298 provider->dtpv_pops.dtps_provide =
7209 7299 (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
7210 7300 }
7211 7301
7212 7302 if (pops->dtps_provide_module == NULL) {
7213 7303 ASSERT(pops->dtps_provide != NULL);
7214 7304 provider->dtpv_pops.dtps_provide_module =
7215 7305 (void (*)(void *, struct modctl *))dtrace_nullop;
7216 7306 }
7217 7307
7218 7308 if (pops->dtps_suspend == NULL) {
7219 7309 ASSERT(pops->dtps_resume == NULL);
7220 7310 provider->dtpv_pops.dtps_suspend =
7221 7311 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7222 7312 provider->dtpv_pops.dtps_resume =
7223 7313 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
7224 7314 }
7225 7315
7226 7316 provider->dtpv_arg = arg;
7227 7317 *idp = (dtrace_provider_id_t)provider;
7228 7318
7229 7319 if (pops == &dtrace_provider_ops) {
7230 7320 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7231 7321 ASSERT(MUTEX_HELD(&dtrace_lock));
7232 7322 ASSERT(dtrace_anon.dta_enabling == NULL);
7233 7323
7234 7324 /*
7235 7325 * We make sure that the DTrace provider is at the head of
7236 7326 * the provider chain.
7237 7327 */
7238 7328 provider->dtpv_next = dtrace_provider;
7239 7329 dtrace_provider = provider;
7240 7330 return (0);
7241 7331 }
7242 7332
7243 7333 mutex_enter(&dtrace_provider_lock);
7244 7334 mutex_enter(&dtrace_lock);
7245 7335
7246 7336 /*
7247 7337 * If there is at least one provider registered, we'll add this
7248 7338 * provider after the first provider.
7249 7339 */
7250 7340 if (dtrace_provider != NULL) {
7251 7341 provider->dtpv_next = dtrace_provider->dtpv_next;
7252 7342 dtrace_provider->dtpv_next = provider;
7253 7343 } else {
7254 7344 dtrace_provider = provider;
7255 7345 }
7256 7346
7257 7347 if (dtrace_retained != NULL) {
7258 7348 dtrace_enabling_provide(provider);
7259 7349
7260 7350 /*
7261 7351 * Now we need to call dtrace_enabling_matchall() -- which
7262 7352 * will acquire cpu_lock and dtrace_lock. We therefore need
7263 7353 * to drop all of our locks before calling into it...
7264 7354 */
7265 7355 mutex_exit(&dtrace_lock);
7266 7356 mutex_exit(&dtrace_provider_lock);
7267 7357 dtrace_enabling_matchall();
7268 7358
7269 7359 return (0);
7270 7360 }
7271 7361
7272 7362 mutex_exit(&dtrace_lock);
7273 7363 mutex_exit(&dtrace_provider_lock);
7274 7364
7275 7365 return (0);
7276 7366 }
7277 7367
7278 7368 /*
7279 7369 * Unregister the specified provider from the DTrace framework. This should
7280 7370 * generally be called by DTrace providers in their detach(9E) entry point.
7281 7371 */
7282 7372 int
7283 7373 dtrace_unregister(dtrace_provider_id_t id)
7284 7374 {
7285 7375 dtrace_provider_t *old = (dtrace_provider_t *)id;
7286 7376 dtrace_provider_t *prev = NULL;
7287 7377 int i, self = 0, noreap = 0;
7288 7378 dtrace_probe_t *probe, *first = NULL;
7289 7379
7290 7380 if (old->dtpv_pops.dtps_enable ==
7291 7381 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) {
7292 7382 /*
7293 7383 * If DTrace itself is the provider, we're called with locks
7294 7384 * already held.
7295 7385 */
7296 7386 ASSERT(old == dtrace_provider);
7297 7387 ASSERT(dtrace_devi != NULL);
7298 7388 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7299 7389 ASSERT(MUTEX_HELD(&dtrace_lock));
7300 7390 self = 1;
7301 7391
7302 7392 if (dtrace_provider->dtpv_next != NULL) {
7303 7393 /*
7304 7394 * There's another provider here; return failure.
7305 7395 */
7306 7396 return (EBUSY);
7307 7397 }
7308 7398 } else {
7309 7399 mutex_enter(&dtrace_provider_lock);
7310 7400 mutex_enter(&mod_lock);
7311 7401 mutex_enter(&dtrace_lock);
7312 7402 }
7313 7403
7314 7404 /*
7315 7405 * If anyone has /dev/dtrace open, or if there are anonymous enabled
7316 7406 * probes, we refuse to let providers slither away, unless this
7317 7407 * provider has already been explicitly invalidated.
7318 7408 */
7319 7409 if (!old->dtpv_defunct &&
7320 7410 (dtrace_opens || (dtrace_anon.dta_state != NULL &&
7321 7411 dtrace_anon.dta_state->dts_necbs > 0))) {
7322 7412 if (!self) {
7323 7413 mutex_exit(&dtrace_lock);
7324 7414 mutex_exit(&mod_lock);
7325 7415 mutex_exit(&dtrace_provider_lock);
7326 7416 }
7327 7417 return (EBUSY);
7328 7418 }
7329 7419
7330 7420 /*
7331 7421 * Attempt to destroy the probes associated with this provider.
7332 7422 */
7333 7423 for (i = 0; i < dtrace_nprobes; i++) {
7334 7424 if ((probe = dtrace_probes[i]) == NULL)
7335 7425 continue;
7336 7426
7337 7427 if (probe->dtpr_provider != old)
7338 7428 continue;
7339 7429
7340 7430 if (probe->dtpr_ecb == NULL)
7341 7431 continue;
7342 7432
7343 7433 /*
7344 7434 * If we are trying to unregister a defunct provider, and the
7345 7435 * provider was made defunct within the interval dictated by
7346 7436 * dtrace_unregister_defunct_reap, we'll (asynchronously)
7347 7437 * attempt to reap our enablings. To denote that the provider
7348 7438 * should reattempt to unregister itself at some point in the
7349 7439 * future, we will return a differentiable error code (EAGAIN
7350 7440 * instead of EBUSY) in this case.
7351 7441 */
7352 7442 if (dtrace_gethrtime() - old->dtpv_defunct >
7353 7443 dtrace_unregister_defunct_reap)
7354 7444 noreap = 1;
7355 7445
7356 7446 if (!self) {
7357 7447 mutex_exit(&dtrace_lock);
7358 7448 mutex_exit(&mod_lock);
7359 7449 mutex_exit(&dtrace_provider_lock);
7360 7450 }
7361 7451
7362 7452 if (noreap)
7363 7453 return (EBUSY);
7364 7454
7365 7455 (void) taskq_dispatch(dtrace_taskq,
7366 7456 (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP);
7367 7457
7368 7458 return (EAGAIN);
7369 7459 }
7370 7460
7371 7461 /*
7372 7462 * All of the probes for this provider are disabled; we can safely
7373 7463 * remove all of them from their hash chains and from the probe array.
7374 7464 */
7375 7465 for (i = 0; i < dtrace_nprobes; i++) {
7376 7466 if ((probe = dtrace_probes[i]) == NULL)
7377 7467 continue;
7378 7468
7379 7469 if (probe->dtpr_provider != old)
7380 7470 continue;
7381 7471
7382 7472 dtrace_probes[i] = NULL;
7383 7473
7384 7474 dtrace_hash_remove(dtrace_bymod, probe);
7385 7475 dtrace_hash_remove(dtrace_byfunc, probe);
7386 7476 dtrace_hash_remove(dtrace_byname, probe);
7387 7477
7388 7478 if (first == NULL) {
7389 7479 first = probe;
7390 7480 probe->dtpr_nextmod = NULL;
7391 7481 } else {
7392 7482 probe->dtpr_nextmod = first;
7393 7483 first = probe;
7394 7484 }
7395 7485 }
7396 7486
7397 7487 /*
7398 7488 * The provider's probes have been removed from the hash chains and
7399 7489 * from the probe array. Now issue a dtrace_sync() to be sure that
7400 7490 * everyone has cleared out from any probe array processing.
7401 7491 */
7402 7492 dtrace_sync();
7403 7493
7404 7494 for (probe = first; probe != NULL; probe = first) {
7405 7495 first = probe->dtpr_nextmod;
7406 7496
7407 7497 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
7408 7498 probe->dtpr_arg);
7409 7499 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
7410 7500 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
7411 7501 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
7412 7502 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
7413 7503 kmem_free(probe, sizeof (dtrace_probe_t));
7414 7504 }
7415 7505
7416 7506 if ((prev = dtrace_provider) == old) {
7417 7507 ASSERT(self || dtrace_devi == NULL);
7418 7508 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
7419 7509 dtrace_provider = old->dtpv_next;
7420 7510 } else {
7421 7511 while (prev != NULL && prev->dtpv_next != old)
7422 7512 prev = prev->dtpv_next;
7423 7513
7424 7514 if (prev == NULL) {
7425 7515 panic("attempt to unregister non-existent "
7426 7516 "dtrace provider %p\n", (void *)id);
7427 7517 }
7428 7518
7429 7519 prev->dtpv_next = old->dtpv_next;
7430 7520 }
7431 7521
7432 7522 if (!self) {
7433 7523 mutex_exit(&dtrace_lock);
7434 7524 mutex_exit(&mod_lock);
7435 7525 mutex_exit(&dtrace_provider_lock);
7436 7526 }
7437 7527
7438 7528 kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
7439 7529 kmem_free(old, sizeof (dtrace_provider_t));
7440 7530
7441 7531 return (0);
7442 7532 }
7443 7533
7444 7534 /*
7445 7535 * Invalidate the specified provider. All subsequent probe lookups for the
7446 7536 * specified provider will fail, but its probes will not be removed.
7447 7537 */
7448 7538 void
7449 7539 dtrace_invalidate(dtrace_provider_id_t id)
7450 7540 {
7451 7541 dtrace_provider_t *pvp = (dtrace_provider_t *)id;
7452 7542
7453 7543 ASSERT(pvp->dtpv_pops.dtps_enable !=
7454 7544 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
7455 7545
7456 7546 mutex_enter(&dtrace_provider_lock);
7457 7547 mutex_enter(&dtrace_lock);
7458 7548
7459 7549 pvp->dtpv_defunct = dtrace_gethrtime();
7460 7550
7461 7551 mutex_exit(&dtrace_lock);
7462 7552 mutex_exit(&dtrace_provider_lock);
7463 7553 }
7464 7554
7465 7555 /*
7466 7556 * Indicate whether or not DTrace has attached.
7467 7557 */
7468 7558 int
7469 7559 dtrace_attached(void)
7470 7560 {
7471 7561 /*
7472 7562 * dtrace_provider will be non-NULL iff the DTrace driver has
7473 7563 * attached. (It's non-NULL because DTrace is always itself a
7474 7564 * provider.)
7475 7565 */
7476 7566 return (dtrace_provider != NULL);
7477 7567 }
7478 7568
7479 7569 /*
7480 7570 * Remove all the unenabled probes for the given provider. This function is
7481 7571 * not unlike dtrace_unregister(), except that it doesn't remove the provider
7482 7572 * -- just as many of its associated probes as it can.
7483 7573 */
7484 7574 int
7485 7575 dtrace_condense(dtrace_provider_id_t id)
7486 7576 {
7487 7577 dtrace_provider_t *prov = (dtrace_provider_t *)id;
7488 7578 int i;
7489 7579 dtrace_probe_t *probe;
7490 7580
7491 7581 /*
7492 7582 * Make sure this isn't the dtrace provider itself.
7493 7583 */
7494 7584 ASSERT(prov->dtpv_pops.dtps_enable !=
7495 7585 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
7496 7586
7497 7587 mutex_enter(&dtrace_provider_lock);
7498 7588 mutex_enter(&dtrace_lock);
7499 7589
7500 7590 /*
7501 7591 * Attempt to destroy the probes associated with this provider.
7502 7592 */
7503 7593 for (i = 0; i < dtrace_nprobes; i++) {
7504 7594 if ((probe = dtrace_probes[i]) == NULL)
7505 7595 continue;
7506 7596
7507 7597 if (probe->dtpr_provider != prov)
7508 7598 continue;
7509 7599
7510 7600 if (probe->dtpr_ecb != NULL)
7511 7601 continue;
7512 7602
7513 7603 dtrace_probes[i] = NULL;
7514 7604
7515 7605 dtrace_hash_remove(dtrace_bymod, probe);
7516 7606 dtrace_hash_remove(dtrace_byfunc, probe);
7517 7607 dtrace_hash_remove(dtrace_byname, probe);
7518 7608
7519 7609 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
7520 7610 probe->dtpr_arg);
7521 7611 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
7522 7612 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
7523 7613 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
7524 7614 kmem_free(probe, sizeof (dtrace_probe_t));
7525 7615 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
7526 7616 }
7527 7617
7528 7618 mutex_exit(&dtrace_lock);
7529 7619 mutex_exit(&dtrace_provider_lock);
7530 7620
7531 7621 return (0);
7532 7622 }
7533 7623
7534 7624 /*
7535 7625 * DTrace Probe Management Functions
7536 7626 *
7537 7627 * The functions in this section perform the DTrace probe management,
7538 7628 * including functions to create probes, look-up probes, and call into the
7539 7629 * providers to request that probes be provided. Some of these functions are
7540 7630 * in the Provider-to-Framework API; these functions can be identified by the
7541 7631 * fact that they are not declared "static".
7542 7632 */
7543 7633
7544 7634 /*
7545 7635 * Create a probe with the specified module name, function name, and name.
7546 7636 */
7547 7637 dtrace_id_t
7548 7638 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
7549 7639 const char *func, const char *name, int aframes, void *arg)
7550 7640 {
7551 7641 dtrace_probe_t *probe, **probes;
7552 7642 dtrace_provider_t *provider = (dtrace_provider_t *)prov;
7553 7643 dtrace_id_t id;
7554 7644
7555 7645 if (provider == dtrace_provider) {
7556 7646 ASSERT(MUTEX_HELD(&dtrace_lock));
7557 7647 } else {
7558 7648 mutex_enter(&dtrace_lock);
7559 7649 }
7560 7650
7561 7651 id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
7562 7652 VM_BESTFIT | VM_SLEEP);
7563 7653 probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
7564 7654
7565 7655 probe->dtpr_id = id;
7566 7656 probe->dtpr_gen = dtrace_probegen++;
7567 7657 probe->dtpr_mod = dtrace_strdup(mod);
7568 7658 probe->dtpr_func = dtrace_strdup(func);
7569 7659 probe->dtpr_name = dtrace_strdup(name);
7570 7660 probe->dtpr_arg = arg;
7571 7661 probe->dtpr_aframes = aframes;
7572 7662 probe->dtpr_provider = provider;
7573 7663
7574 7664 dtrace_hash_add(dtrace_bymod, probe);
7575 7665 dtrace_hash_add(dtrace_byfunc, probe);
7576 7666 dtrace_hash_add(dtrace_byname, probe);
7577 7667
7578 7668 if (id - 1 >= dtrace_nprobes) {
7579 7669 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
7580 7670 size_t nsize = osize << 1;
7581 7671
7582 7672 if (nsize == 0) {
7583 7673 ASSERT(osize == 0);
7584 7674 ASSERT(dtrace_probes == NULL);
7585 7675 nsize = sizeof (dtrace_probe_t *);
7586 7676 }
7587 7677
7588 7678 probes = kmem_zalloc(nsize, KM_SLEEP);
7589 7679
7590 7680 if (dtrace_probes == NULL) {
7591 7681 ASSERT(osize == 0);
7592 7682 dtrace_probes = probes;
7593 7683 dtrace_nprobes = 1;
7594 7684 } else {
7595 7685 dtrace_probe_t **oprobes = dtrace_probes;
7596 7686
7597 7687 bcopy(oprobes, probes, osize);
7598 7688 dtrace_membar_producer();
7599 7689 dtrace_probes = probes;
7600 7690
7601 7691 dtrace_sync();
7602 7692
7603 7693 /*
7604 7694 * All CPUs are now seeing the new probes array; we can
7605 7695 * safely free the old array.
7606 7696 */
7607 7697 kmem_free(oprobes, osize);
7608 7698 dtrace_nprobes <<= 1;
7609 7699 }
7610 7700
7611 7701 ASSERT(id - 1 < dtrace_nprobes);
7612 7702 }
7613 7703
7614 7704 ASSERT(dtrace_probes[id - 1] == NULL);
7615 7705 dtrace_probes[id - 1] = probe;
7616 7706
7617 7707 if (provider != dtrace_provider)
7618 7708 mutex_exit(&dtrace_lock);
7619 7709
7620 7710 return (id);
7621 7711 }
7622 7712
7623 7713 static dtrace_probe_t *
7624 7714 dtrace_probe_lookup_id(dtrace_id_t id)
7625 7715 {
7626 7716 ASSERT(MUTEX_HELD(&dtrace_lock));
7627 7717
7628 7718 if (id == 0 || id > dtrace_nprobes)
7629 7719 return (NULL);
7630 7720
7631 7721 return (dtrace_probes[id - 1]);
7632 7722 }
7633 7723
7634 7724 static int
7635 7725 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
7636 7726 {
7637 7727 *((dtrace_id_t *)arg) = probe->dtpr_id;
7638 7728
7639 7729 return (DTRACE_MATCH_DONE);
7640 7730 }
7641 7731
7642 7732 /*
7643 7733 * Look up a probe based on provider and one or more of module name, function
7644 7734 * name and probe name.
7645 7735 */
7646 7736 dtrace_id_t
7647 7737 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
7648 7738 const char *func, const char *name)
7649 7739 {
7650 7740 dtrace_probekey_t pkey;
7651 7741 dtrace_id_t id;
7652 7742 int match;
7653 7743
7654 7744 pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
7655 7745 pkey.dtpk_pmatch = &dtrace_match_string;
7656 7746 pkey.dtpk_mod = mod;
7657 7747 pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
7658 7748 pkey.dtpk_func = func;
7659 7749 pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
7660 7750 pkey.dtpk_name = name;
7661 7751 pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
7662 7752 pkey.dtpk_id = DTRACE_IDNONE;
7663 7753
7664 7754 mutex_enter(&dtrace_lock);
7665 7755 match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
7666 7756 dtrace_probe_lookup_match, &id);
7667 7757 mutex_exit(&dtrace_lock);
7668 7758
7669 7759 ASSERT(match == 1 || match == 0);
7670 7760 return (match ? id : 0);
7671 7761 }
7672 7762
7673 7763 /*
7674 7764 * Returns the probe argument associated with the specified probe.
7675 7765 */
7676 7766 void *
7677 7767 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
7678 7768 {
7679 7769 dtrace_probe_t *probe;
7680 7770 void *rval = NULL;
7681 7771
7682 7772 mutex_enter(&dtrace_lock);
7683 7773
7684 7774 if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
7685 7775 probe->dtpr_provider == (dtrace_provider_t *)id)
7686 7776 rval = probe->dtpr_arg;
7687 7777
7688 7778 mutex_exit(&dtrace_lock);
7689 7779
7690 7780 return (rval);
7691 7781 }
7692 7782
7693 7783 /*
7694 7784 * Copy a probe into a probe description.
7695 7785 */
7696 7786 static void
7697 7787 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
7698 7788 {
7699 7789 bzero(pdp, sizeof (dtrace_probedesc_t));
7700 7790 pdp->dtpd_id = prp->dtpr_id;
7701 7791
7702 7792 (void) strncpy(pdp->dtpd_provider,
7703 7793 prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
7704 7794
7705 7795 (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
7706 7796 (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
7707 7797 (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
7708 7798 }
7709 7799
7710 7800 /*
7711 7801 * Called to indicate that a probe -- or probes -- should be provided by a
7712 7802 * specfied provider. If the specified description is NULL, the provider will
7713 7803 * be told to provide all of its probes. (This is done whenever a new
7714 7804 * consumer comes along, or whenever a retained enabling is to be matched.) If
7715 7805 * the specified description is non-NULL, the provider is given the
7716 7806 * opportunity to dynamically provide the specified probe, allowing providers
7717 7807 * to support the creation of probes on-the-fly. (So-called _autocreated_
7718 7808 * probes.) If the provider is NULL, the operations will be applied to all
7719 7809 * providers; if the provider is non-NULL the operations will only be applied
7720 7810 * to the specified provider. The dtrace_provider_lock must be held, and the
7721 7811 * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
7722 7812 * will need to grab the dtrace_lock when it reenters the framework through
7723 7813 * dtrace_probe_lookup(), dtrace_probe_create(), etc.
7724 7814 */
7725 7815 static void
7726 7816 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
7727 7817 {
7728 7818 struct modctl *ctl;
7729 7819 int all = 0;
7730 7820
7731 7821 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
7732 7822
7733 7823 if (prv == NULL) {
7734 7824 all = 1;
7735 7825 prv = dtrace_provider;
7736 7826 }
7737 7827
7738 7828 do {
7739 7829 /*
7740 7830 * First, call the blanket provide operation.
7741 7831 */
7742 7832 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
7743 7833
7744 7834 /*
7745 7835 * Now call the per-module provide operation. We will grab
7746 7836 * mod_lock to prevent the list from being modified. Note
7747 7837 * that this also prevents the mod_busy bits from changing.
7748 7838 * (mod_busy can only be changed with mod_lock held.)
7749 7839 */
7750 7840 mutex_enter(&mod_lock);
7751 7841
7752 7842 ctl = &modules;
7753 7843 do {
7754 7844 if (ctl->mod_busy || ctl->mod_mp == NULL)
7755 7845 continue;
7756 7846
7757 7847 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
7758 7848
7759 7849 } while ((ctl = ctl->mod_next) != &modules);
7760 7850
7761 7851 mutex_exit(&mod_lock);
7762 7852 } while (all && (prv = prv->dtpv_next) != NULL);
7763 7853 }
7764 7854
7765 7855 /*
7766 7856 * Iterate over each probe, and call the Framework-to-Provider API function
7767 7857 * denoted by offs.
7768 7858 */
7769 7859 static void
7770 7860 dtrace_probe_foreach(uintptr_t offs)
7771 7861 {
7772 7862 dtrace_provider_t *prov;
7773 7863 void (*func)(void *, dtrace_id_t, void *);
7774 7864 dtrace_probe_t *probe;
7775 7865 dtrace_icookie_t cookie;
7776 7866 int i;
7777 7867
7778 7868 /*
7779 7869 * We disable interrupts to walk through the probe array. This is
7780 7870 * safe -- the dtrace_sync() in dtrace_unregister() assures that we
7781 7871 * won't see stale data.
7782 7872 */
7783 7873 cookie = dtrace_interrupt_disable();
7784 7874
7785 7875 for (i = 0; i < dtrace_nprobes; i++) {
7786 7876 if ((probe = dtrace_probes[i]) == NULL)
7787 7877 continue;
7788 7878
7789 7879 if (probe->dtpr_ecb == NULL) {
7790 7880 /*
7791 7881 * This probe isn't enabled -- don't call the function.
7792 7882 */
7793 7883 continue;
7794 7884 }
7795 7885
7796 7886 prov = probe->dtpr_provider;
7797 7887 func = *((void(**)(void *, dtrace_id_t, void *))
7798 7888 ((uintptr_t)&prov->dtpv_pops + offs));
7799 7889
7800 7890 func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
7801 7891 }
7802 7892
7803 7893 dtrace_interrupt_enable(cookie);
7804 7894 }
7805 7895
7806 7896 static int
7807 7897 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
7808 7898 {
7809 7899 dtrace_probekey_t pkey;
7810 7900 uint32_t priv;
7811 7901 uid_t uid;
7812 7902 zoneid_t zoneid;
7813 7903
7814 7904 ASSERT(MUTEX_HELD(&dtrace_lock));
7815 7905 dtrace_ecb_create_cache = NULL;
7816 7906
7817 7907 if (desc == NULL) {
7818 7908 /*
7819 7909 * If we're passed a NULL description, we're being asked to
7820 7910 * create an ECB with a NULL probe.
7821 7911 */
7822 7912 (void) dtrace_ecb_create_enable(NULL, enab);
7823 7913 return (0);
7824 7914 }
7825 7915
7826 7916 dtrace_probekey(desc, &pkey);
7827 7917 dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
7828 7918 &priv, &uid, &zoneid);
7829 7919
7830 7920 return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
7831 7921 enab));
7832 7922 }
7833 7923
7834 7924 /*
7835 7925 * DTrace Helper Provider Functions
7836 7926 */
7837 7927 static void
7838 7928 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
7839 7929 {
7840 7930 attr->dtat_name = DOF_ATTR_NAME(dofattr);
7841 7931 attr->dtat_data = DOF_ATTR_DATA(dofattr);
7842 7932 attr->dtat_class = DOF_ATTR_CLASS(dofattr);
7843 7933 }
7844 7934
7845 7935 static void
7846 7936 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
7847 7937 const dof_provider_t *dofprov, char *strtab)
7848 7938 {
7849 7939 hprov->dthpv_provname = strtab + dofprov->dofpv_name;
7850 7940 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
7851 7941 dofprov->dofpv_provattr);
7852 7942 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
7853 7943 dofprov->dofpv_modattr);
7854 7944 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
7855 7945 dofprov->dofpv_funcattr);
7856 7946 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
7857 7947 dofprov->dofpv_nameattr);
7858 7948 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
7859 7949 dofprov->dofpv_argsattr);
7860 7950 }
7861 7951
7862 7952 static void
7863 7953 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
7864 7954 {
7865 7955 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
7866 7956 dof_hdr_t *dof = (dof_hdr_t *)daddr;
7867 7957 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
7868 7958 dof_provider_t *provider;
7869 7959 dof_probe_t *probe;
7870 7960 uint32_t *off, *enoff;
7871 7961 uint8_t *arg;
7872 7962 char *strtab;
7873 7963 uint_t i, nprobes;
7874 7964 dtrace_helper_provdesc_t dhpv;
7875 7965 dtrace_helper_probedesc_t dhpb;
7876 7966 dtrace_meta_t *meta = dtrace_meta_pid;
7877 7967 dtrace_mops_t *mops = &meta->dtm_mops;
7878 7968 void *parg;
7879 7969
7880 7970 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
7881 7971 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7882 7972 provider->dofpv_strtab * dof->dofh_secsize);
7883 7973 prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7884 7974 provider->dofpv_probes * dof->dofh_secsize);
7885 7975 arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7886 7976 provider->dofpv_prargs * dof->dofh_secsize);
7887 7977 off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7888 7978 provider->dofpv_proffs * dof->dofh_secsize);
7889 7979
7890 7980 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
7891 7981 off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
7892 7982 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
7893 7983 enoff = NULL;
7894 7984
7895 7985 /*
7896 7986 * See dtrace_helper_provider_validate().
7897 7987 */
7898 7988 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
7899 7989 provider->dofpv_prenoffs != DOF_SECT_NONE) {
7900 7990 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7901 7991 provider->dofpv_prenoffs * dof->dofh_secsize);
7902 7992 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
7903 7993 }
7904 7994
7905 7995 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
7906 7996
7907 7997 /*
7908 7998 * Create the provider.
7909 7999 */
7910 8000 dtrace_dofprov2hprov(&dhpv, provider, strtab);
7911 8001
7912 8002 if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
7913 8003 return;
7914 8004
7915 8005 meta->dtm_count++;
7916 8006
7917 8007 /*
7918 8008 * Create the probes.
7919 8009 */
7920 8010 for (i = 0; i < nprobes; i++) {
7921 8011 probe = (dof_probe_t *)(uintptr_t)(daddr +
7922 8012 prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
7923 8013
7924 8014 dhpb.dthpb_mod = dhp->dofhp_mod;
7925 8015 dhpb.dthpb_func = strtab + probe->dofpr_func;
7926 8016 dhpb.dthpb_name = strtab + probe->dofpr_name;
7927 8017 dhpb.dthpb_base = probe->dofpr_addr;
7928 8018 dhpb.dthpb_offs = off + probe->dofpr_offidx;
7929 8019 dhpb.dthpb_noffs = probe->dofpr_noffs;
7930 8020 if (enoff != NULL) {
7931 8021 dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
7932 8022 dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
7933 8023 } else {
7934 8024 dhpb.dthpb_enoffs = NULL;
7935 8025 dhpb.dthpb_nenoffs = 0;
7936 8026 }
7937 8027 dhpb.dthpb_args = arg + probe->dofpr_argidx;
7938 8028 dhpb.dthpb_nargc = probe->dofpr_nargc;
7939 8029 dhpb.dthpb_xargc = probe->dofpr_xargc;
7940 8030 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
7941 8031 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
7942 8032
7943 8033 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
7944 8034 }
7945 8035 }
7946 8036
7947 8037 static void
7948 8038 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
7949 8039 {
7950 8040 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
7951 8041 dof_hdr_t *dof = (dof_hdr_t *)daddr;
7952 8042 int i;
7953 8043
7954 8044 ASSERT(MUTEX_HELD(&dtrace_meta_lock));
7955 8045
7956 8046 for (i = 0; i < dof->dofh_secnum; i++) {
7957 8047 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
7958 8048 dof->dofh_secoff + i * dof->dofh_secsize);
7959 8049
7960 8050 if (sec->dofs_type != DOF_SECT_PROVIDER)
7961 8051 continue;
7962 8052
7963 8053 dtrace_helper_provide_one(dhp, sec, pid);
7964 8054 }
7965 8055
7966 8056 /*
7967 8057 * We may have just created probes, so we must now rematch against
7968 8058 * any retained enablings. Note that this call will acquire both
7969 8059 * cpu_lock and dtrace_lock; the fact that we are holding
7970 8060 * dtrace_meta_lock now is what defines the ordering with respect to
7971 8061 * these three locks.
7972 8062 */
7973 8063 dtrace_enabling_matchall();
7974 8064 }
7975 8065
7976 8066 static void
7977 8067 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
7978 8068 {
7979 8069 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
7980 8070 dof_hdr_t *dof = (dof_hdr_t *)daddr;
7981 8071 dof_sec_t *str_sec;
7982 8072 dof_provider_t *provider;
7983 8073 char *strtab;
7984 8074 dtrace_helper_provdesc_t dhpv;
7985 8075 dtrace_meta_t *meta = dtrace_meta_pid;
7986 8076 dtrace_mops_t *mops = &meta->dtm_mops;
7987 8077
7988 8078 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
7989 8079 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
7990 8080 provider->dofpv_strtab * dof->dofh_secsize);
7991 8081
7992 8082 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
7993 8083
7994 8084 /*
7995 8085 * Create the provider.
7996 8086 */
7997 8087 dtrace_dofprov2hprov(&dhpv, provider, strtab);
7998 8088
7999 8089 mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
8000 8090
8001 8091 meta->dtm_count--;
8002 8092 }
8003 8093
8004 8094 static void
8005 8095 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
8006 8096 {
8007 8097 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
8008 8098 dof_hdr_t *dof = (dof_hdr_t *)daddr;
8009 8099 int i;
8010 8100
8011 8101 ASSERT(MUTEX_HELD(&dtrace_meta_lock));
8012 8102
8013 8103 for (i = 0; i < dof->dofh_secnum; i++) {
8014 8104 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
8015 8105 dof->dofh_secoff + i * dof->dofh_secsize);
8016 8106
8017 8107 if (sec->dofs_type != DOF_SECT_PROVIDER)
8018 8108 continue;
8019 8109
8020 8110 dtrace_helper_provider_remove_one(dhp, sec, pid);
8021 8111 }
8022 8112 }
8023 8113
8024 8114 /*
8025 8115 * DTrace Meta Provider-to-Framework API Functions
8026 8116 *
8027 8117 * These functions implement the Meta Provider-to-Framework API, as described
8028 8118 * in <sys/dtrace.h>.
8029 8119 */
8030 8120 int
8031 8121 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
8032 8122 dtrace_meta_provider_id_t *idp)
8033 8123 {
8034 8124 dtrace_meta_t *meta;
8035 8125 dtrace_helpers_t *help, *next;
8036 8126 int i;
8037 8127
8038 8128 *idp = DTRACE_METAPROVNONE;
8039 8129
8040 8130 /*
8041 8131 * We strictly don't need the name, but we hold onto it for
8042 8132 * debuggability. All hail error queues!
8043 8133 */
8044 8134 if (name == NULL) {
8045 8135 cmn_err(CE_WARN, "failed to register meta-provider: "
8046 8136 "invalid name");
8047 8137 return (EINVAL);
8048 8138 }
8049 8139
8050 8140 if (mops == NULL ||
8051 8141 mops->dtms_create_probe == NULL ||
8052 8142 mops->dtms_provide_pid == NULL ||
8053 8143 mops->dtms_remove_pid == NULL) {
8054 8144 cmn_err(CE_WARN, "failed to register meta-register %s: "
8055 8145 "invalid ops", name);
8056 8146 return (EINVAL);
8057 8147 }
8058 8148
8059 8149 meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
8060 8150 meta->dtm_mops = *mops;
8061 8151 meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
8062 8152 (void) strcpy(meta->dtm_name, name);
8063 8153 meta->dtm_arg = arg;
8064 8154
8065 8155 mutex_enter(&dtrace_meta_lock);
8066 8156 mutex_enter(&dtrace_lock);
8067 8157
8068 8158 if (dtrace_meta_pid != NULL) {
8069 8159 mutex_exit(&dtrace_lock);
8070 8160 mutex_exit(&dtrace_meta_lock);
8071 8161 cmn_err(CE_WARN, "failed to register meta-register %s: "
8072 8162 "user-land meta-provider exists", name);
8073 8163 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
8074 8164 kmem_free(meta, sizeof (dtrace_meta_t));
8075 8165 return (EINVAL);
8076 8166 }
8077 8167
8078 8168 dtrace_meta_pid = meta;
8079 8169 *idp = (dtrace_meta_provider_id_t)meta;
8080 8170
8081 8171 /*
8082 8172 * If there are providers and probes ready to go, pass them
8083 8173 * off to the new meta provider now.
8084 8174 */
8085 8175
8086 8176 help = dtrace_deferred_pid;
8087 8177 dtrace_deferred_pid = NULL;
8088 8178
8089 8179 mutex_exit(&dtrace_lock);
8090 8180
8091 8181 while (help != NULL) {
8092 8182 for (i = 0; i < help->dthps_nprovs; i++) {
8093 8183 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
8094 8184 help->dthps_pid);
8095 8185 }
8096 8186
8097 8187 next = help->dthps_next;
8098 8188 help->dthps_next = NULL;
8099 8189 help->dthps_prev = NULL;
8100 8190 help->dthps_deferred = 0;
8101 8191 help = next;
8102 8192 }
8103 8193
8104 8194 mutex_exit(&dtrace_meta_lock);
8105 8195
8106 8196 return (0);
8107 8197 }
8108 8198
8109 8199 int
8110 8200 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
8111 8201 {
8112 8202 dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
8113 8203
8114 8204 mutex_enter(&dtrace_meta_lock);
8115 8205 mutex_enter(&dtrace_lock);
8116 8206
8117 8207 if (old == dtrace_meta_pid) {
8118 8208 pp = &dtrace_meta_pid;
8119 8209 } else {
8120 8210 panic("attempt to unregister non-existent "
8121 8211 "dtrace meta-provider %p\n", (void *)old);
8122 8212 }
8123 8213
8124 8214 if (old->dtm_count != 0) {
8125 8215 mutex_exit(&dtrace_lock);
8126 8216 mutex_exit(&dtrace_meta_lock);
8127 8217 return (EBUSY);
8128 8218 }
8129 8219
8130 8220 *pp = NULL;
8131 8221
8132 8222 mutex_exit(&dtrace_lock);
8133 8223 mutex_exit(&dtrace_meta_lock);
8134 8224
8135 8225 kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
8136 8226 kmem_free(old, sizeof (dtrace_meta_t));
8137 8227
8138 8228 return (0);
8139 8229 }
8140 8230
8141 8231
8142 8232 /*
8143 8233 * DTrace DIF Object Functions
8144 8234 */
8145 8235 static int
8146 8236 dtrace_difo_err(uint_t pc, const char *format, ...)
8147 8237 {
8148 8238 if (dtrace_err_verbose) {
8149 8239 va_list alist;
8150 8240
8151 8241 (void) uprintf("dtrace DIF object error: [%u]: ", pc);
8152 8242 va_start(alist, format);
8153 8243 (void) vuprintf(format, alist);
8154 8244 va_end(alist);
8155 8245 }
8156 8246
8157 8247 #ifdef DTRACE_ERRDEBUG
8158 8248 dtrace_errdebug(format);
8159 8249 #endif
8160 8250 return (1);
8161 8251 }
8162 8252
8163 8253 /*
8164 8254 * Validate a DTrace DIF object by checking the IR instructions. The following
8165 8255 * rules are currently enforced by dtrace_difo_validate():
8166 8256 *
8167 8257 * 1. Each instruction must have a valid opcode
8168 8258 * 2. Each register, string, variable, or subroutine reference must be valid
8169 8259 * 3. No instruction can modify register %r0 (must be zero)
8170 8260 * 4. All instruction reserved bits must be set to zero
8171 8261 * 5. The last instruction must be a "ret" instruction
8172 8262 * 6. All branch targets must reference a valid instruction _after_ the branch
8173 8263 */
8174 8264 static int
8175 8265 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
8176 8266 cred_t *cr)
8177 8267 {
8178 8268 int err = 0, i;
8179 8269 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8180 8270 int kcheckload;
8181 8271 uint_t pc;
8182 8272
8183 8273 kcheckload = cr == NULL ||
8184 8274 (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0;
8185 8275
8186 8276 dp->dtdo_destructive = 0;
8187 8277
8188 8278 for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
8189 8279 dif_instr_t instr = dp->dtdo_buf[pc];
8190 8280
8191 8281 uint_t r1 = DIF_INSTR_R1(instr);
8192 8282 uint_t r2 = DIF_INSTR_R2(instr);
8193 8283 uint_t rd = DIF_INSTR_RD(instr);
8194 8284 uint_t rs = DIF_INSTR_RS(instr);
8195 8285 uint_t label = DIF_INSTR_LABEL(instr);
8196 8286 uint_t v = DIF_INSTR_VAR(instr);
8197 8287 uint_t subr = DIF_INSTR_SUBR(instr);
8198 8288 uint_t type = DIF_INSTR_TYPE(instr);
8199 8289 uint_t op = DIF_INSTR_OP(instr);
8200 8290
8201 8291 switch (op) {
8202 8292 case DIF_OP_OR:
8203 8293 case DIF_OP_XOR:
8204 8294 case DIF_OP_AND:
8205 8295 case DIF_OP_SLL:
8206 8296 case DIF_OP_SRL:
8207 8297 case DIF_OP_SRA:
8208 8298 case DIF_OP_SUB:
8209 8299 case DIF_OP_ADD:
8210 8300 case DIF_OP_MUL:
8211 8301 case DIF_OP_SDIV:
8212 8302 case DIF_OP_UDIV:
8213 8303 case DIF_OP_SREM:
8214 8304 case DIF_OP_UREM:
8215 8305 case DIF_OP_COPYS:
8216 8306 if (r1 >= nregs)
8217 8307 err += efunc(pc, "invalid register %u\n", r1);
8218 8308 if (r2 >= nregs)
8219 8309 err += efunc(pc, "invalid register %u\n", r2);
8220 8310 if (rd >= nregs)
8221 8311 err += efunc(pc, "invalid register %u\n", rd);
8222 8312 if (rd == 0)
8223 8313 err += efunc(pc, "cannot write to %r0\n");
8224 8314 break;
8225 8315 case DIF_OP_NOT:
8226 8316 case DIF_OP_MOV:
8227 8317 case DIF_OP_ALLOCS:
8228 8318 if (r1 >= nregs)
8229 8319 err += efunc(pc, "invalid register %u\n", r1);
8230 8320 if (r2 != 0)
8231 8321 err += efunc(pc, "non-zero reserved bits\n");
8232 8322 if (rd >= nregs)
8233 8323 err += efunc(pc, "invalid register %u\n", rd);
8234 8324 if (rd == 0)
8235 8325 err += efunc(pc, "cannot write to %r0\n");
8236 8326 break;
8237 8327 case DIF_OP_LDSB:
8238 8328 case DIF_OP_LDSH:
8239 8329 case DIF_OP_LDSW:
8240 8330 case DIF_OP_LDUB:
8241 8331 case DIF_OP_LDUH:
8242 8332 case DIF_OP_LDUW:
8243 8333 case DIF_OP_LDX:
8244 8334 if (r1 >= nregs)
8245 8335 err += efunc(pc, "invalid register %u\n", r1);
8246 8336 if (r2 != 0)
8247 8337 err += efunc(pc, "non-zero reserved bits\n");
8248 8338 if (rd >= nregs)
8249 8339 err += efunc(pc, "invalid register %u\n", rd);
8250 8340 if (rd == 0)
8251 8341 err += efunc(pc, "cannot write to %r0\n");
8252 8342 if (kcheckload)
8253 8343 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
8254 8344 DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
8255 8345 break;
8256 8346 case DIF_OP_RLDSB:
8257 8347 case DIF_OP_RLDSH:
8258 8348 case DIF_OP_RLDSW:
8259 8349 case DIF_OP_RLDUB:
8260 8350 case DIF_OP_RLDUH:
8261 8351 case DIF_OP_RLDUW:
8262 8352 case DIF_OP_RLDX:
8263 8353 if (r1 >= nregs)
8264 8354 err += efunc(pc, "invalid register %u\n", r1);
8265 8355 if (r2 != 0)
8266 8356 err += efunc(pc, "non-zero reserved bits\n");
8267 8357 if (rd >= nregs)
8268 8358 err += efunc(pc, "invalid register %u\n", rd);
8269 8359 if (rd == 0)
8270 8360 err += efunc(pc, "cannot write to %r0\n");
8271 8361 break;
8272 8362 case DIF_OP_ULDSB:
8273 8363 case DIF_OP_ULDSH:
8274 8364 case DIF_OP_ULDSW:
8275 8365 case DIF_OP_ULDUB:
8276 8366 case DIF_OP_ULDUH:
8277 8367 case DIF_OP_ULDUW:
8278 8368 case DIF_OP_ULDX:
8279 8369 if (r1 >= nregs)
8280 8370 err += efunc(pc, "invalid register %u\n", r1);
8281 8371 if (r2 != 0)
8282 8372 err += efunc(pc, "non-zero reserved bits\n");
8283 8373 if (rd >= nregs)
8284 8374 err += efunc(pc, "invalid register %u\n", rd);
8285 8375 if (rd == 0)
8286 8376 err += efunc(pc, "cannot write to %r0\n");
8287 8377 break;
8288 8378 case DIF_OP_STB:
8289 8379 case DIF_OP_STH:
8290 8380 case DIF_OP_STW:
8291 8381 case DIF_OP_STX:
8292 8382 if (r1 >= nregs)
8293 8383 err += efunc(pc, "invalid register %u\n", r1);
8294 8384 if (r2 != 0)
8295 8385 err += efunc(pc, "non-zero reserved bits\n");
8296 8386 if (rd >= nregs)
8297 8387 err += efunc(pc, "invalid register %u\n", rd);
8298 8388 if (rd == 0)
8299 8389 err += efunc(pc, "cannot write to 0 address\n");
8300 8390 break;
8301 8391 case DIF_OP_CMP:
8302 8392 case DIF_OP_SCMP:
8303 8393 if (r1 >= nregs)
8304 8394 err += efunc(pc, "invalid register %u\n", r1);
8305 8395 if (r2 >= nregs)
8306 8396 err += efunc(pc, "invalid register %u\n", r2);
8307 8397 if (rd != 0)
8308 8398 err += efunc(pc, "non-zero reserved bits\n");
8309 8399 break;
8310 8400 case DIF_OP_TST:
8311 8401 if (r1 >= nregs)
8312 8402 err += efunc(pc, "invalid register %u\n", r1);
8313 8403 if (r2 != 0 || rd != 0)
8314 8404 err += efunc(pc, "non-zero reserved bits\n");
8315 8405 break;
8316 8406 case DIF_OP_BA:
8317 8407 case DIF_OP_BE:
8318 8408 case DIF_OP_BNE:
8319 8409 case DIF_OP_BG:
8320 8410 case DIF_OP_BGU:
8321 8411 case DIF_OP_BGE:
8322 8412 case DIF_OP_BGEU:
8323 8413 case DIF_OP_BL:
8324 8414 case DIF_OP_BLU:
8325 8415 case DIF_OP_BLE:
8326 8416 case DIF_OP_BLEU:
8327 8417 if (label >= dp->dtdo_len) {
8328 8418 err += efunc(pc, "invalid branch target %u\n",
8329 8419 label);
8330 8420 }
8331 8421 if (label <= pc) {
8332 8422 err += efunc(pc, "backward branch to %u\n",
8333 8423 label);
8334 8424 }
8335 8425 break;
8336 8426 case DIF_OP_RET:
8337 8427 if (r1 != 0 || r2 != 0)
8338 8428 err += efunc(pc, "non-zero reserved bits\n");
8339 8429 if (rd >= nregs)
8340 8430 err += efunc(pc, "invalid register %u\n", rd);
8341 8431 break;
8342 8432 case DIF_OP_NOP:
8343 8433 case DIF_OP_POPTS:
8344 8434 case DIF_OP_FLUSHTS:
8345 8435 if (r1 != 0 || r2 != 0 || rd != 0)
8346 8436 err += efunc(pc, "non-zero reserved bits\n");
8347 8437 break;
8348 8438 case DIF_OP_SETX:
8349 8439 if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
8350 8440 err += efunc(pc, "invalid integer ref %u\n",
8351 8441 DIF_INSTR_INTEGER(instr));
8352 8442 }
8353 8443 if (rd >= nregs)
8354 8444 err += efunc(pc, "invalid register %u\n", rd);
8355 8445 if (rd == 0)
8356 8446 err += efunc(pc, "cannot write to %r0\n");
8357 8447 break;
8358 8448 case DIF_OP_SETS:
8359 8449 if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
8360 8450 err += efunc(pc, "invalid string ref %u\n",
8361 8451 DIF_INSTR_STRING(instr));
8362 8452 }
8363 8453 if (rd >= nregs)
8364 8454 err += efunc(pc, "invalid register %u\n", rd);
8365 8455 if (rd == 0)
8366 8456 err += efunc(pc, "cannot write to %r0\n");
8367 8457 break;
8368 8458 case DIF_OP_LDGA:
8369 8459 case DIF_OP_LDTA:
8370 8460 if (r1 > DIF_VAR_ARRAY_MAX)
8371 8461 err += efunc(pc, "invalid array %u\n", r1);
8372 8462 if (r2 >= nregs)
8373 8463 err += efunc(pc, "invalid register %u\n", r2);
8374 8464 if (rd >= nregs)
8375 8465 err += efunc(pc, "invalid register %u\n", rd);
8376 8466 if (rd == 0)
8377 8467 err += efunc(pc, "cannot write to %r0\n");
8378 8468 break;
8379 8469 case DIF_OP_LDGS:
8380 8470 case DIF_OP_LDTS:
8381 8471 case DIF_OP_LDLS:
8382 8472 case DIF_OP_LDGAA:
8383 8473 case DIF_OP_LDTAA:
8384 8474 if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
8385 8475 err += efunc(pc, "invalid variable %u\n", v);
8386 8476 if (rd >= nregs)
8387 8477 err += efunc(pc, "invalid register %u\n", rd);
8388 8478 if (rd == 0)
8389 8479 err += efunc(pc, "cannot write to %r0\n");
8390 8480 break;
8391 8481 case DIF_OP_STGS:
8392 8482 case DIF_OP_STTS:
8393 8483 case DIF_OP_STLS:
8394 8484 case DIF_OP_STGAA:
8395 8485 case DIF_OP_STTAA:
8396 8486 if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
8397 8487 err += efunc(pc, "invalid variable %u\n", v);
8398 8488 if (rs >= nregs)
8399 8489 err += efunc(pc, "invalid register %u\n", rd);
8400 8490 break;
8401 8491 case DIF_OP_CALL:
8402 8492 if (subr > DIF_SUBR_MAX)
|
↓ open down ↓ |
2458 lines elided |
↑ open up ↑ |
8403 8493 err += efunc(pc, "invalid subr %u\n", subr);
8404 8494 if (rd >= nregs)
8405 8495 err += efunc(pc, "invalid register %u\n", rd);
8406 8496 if (rd == 0)
8407 8497 err += efunc(pc, "cannot write to %r0\n");
8408 8498
8409 8499 if (subr == DIF_SUBR_COPYOUT ||
8410 8500 subr == DIF_SUBR_COPYOUTSTR) {
8411 8501 dp->dtdo_destructive = 1;
8412 8502 }
8503 +
8504 + if (subr == DIF_SUBR_GETF) {
8505 + /*
8506 + * If we have a getf() we need to record that
8507 + * in our state. Note that our state can be
8508 + * NULL if this is a helper -- but in that
8509 + * case, the call to getf() is itself illegal,
8510 + * and will be caught (slightly later) when
8511 + * the helper is validated.
8512 + */
8513 + if (vstate->dtvs_state != NULL)
8514 + vstate->dtvs_state->dts_getf++;
8515 + }
8516 +
8413 8517 break;
8414 8518 case DIF_OP_PUSHTR:
8415 8519 if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
8416 8520 err += efunc(pc, "invalid ref type %u\n", type);
8417 8521 if (r2 >= nregs)
8418 8522 err += efunc(pc, "invalid register %u\n", r2);
8419 8523 if (rs >= nregs)
8420 8524 err += efunc(pc, "invalid register %u\n", rs);
8421 8525 break;
8422 8526 case DIF_OP_PUSHTV:
8423 8527 if (type != DIF_TYPE_CTF)
8424 8528 err += efunc(pc, "invalid val type %u\n", type);
8425 8529 if (r2 >= nregs)
8426 8530 err += efunc(pc, "invalid register %u\n", r2);
8427 8531 if (rs >= nregs)
8428 8532 err += efunc(pc, "invalid register %u\n", rs);
8429 8533 break;
8430 8534 default:
8431 8535 err += efunc(pc, "invalid opcode %u\n",
8432 8536 DIF_INSTR_OP(instr));
8433 8537 }
8434 8538 }
8435 8539
8436 8540 if (dp->dtdo_len != 0 &&
8437 8541 DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
8438 8542 err += efunc(dp->dtdo_len - 1,
8439 8543 "expected 'ret' as last DIF instruction\n");
8440 8544 }
8441 8545
8442 8546 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
8443 8547 /*
8444 8548 * If we're not returning by reference, the size must be either
8445 8549 * 0 or the size of one of the base types.
8446 8550 */
8447 8551 switch (dp->dtdo_rtype.dtdt_size) {
8448 8552 case 0:
8449 8553 case sizeof (uint8_t):
8450 8554 case sizeof (uint16_t):
8451 8555 case sizeof (uint32_t):
8452 8556 case sizeof (uint64_t):
8453 8557 break;
8454 8558
8455 8559 default:
8456 8560 err += efunc(dp->dtdo_len - 1, "bad return size\n");
8457 8561 }
8458 8562 }
8459 8563
8460 8564 for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
8461 8565 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
8462 8566 dtrace_diftype_t *vt, *et;
8463 8567 uint_t id, ndx;
8464 8568
8465 8569 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
8466 8570 v->dtdv_scope != DIFV_SCOPE_THREAD &&
8467 8571 v->dtdv_scope != DIFV_SCOPE_LOCAL) {
8468 8572 err += efunc(i, "unrecognized variable scope %d\n",
8469 8573 v->dtdv_scope);
8470 8574 break;
8471 8575 }
8472 8576
8473 8577 if (v->dtdv_kind != DIFV_KIND_ARRAY &&
8474 8578 v->dtdv_kind != DIFV_KIND_SCALAR) {
8475 8579 err += efunc(i, "unrecognized variable type %d\n",
8476 8580 v->dtdv_kind);
8477 8581 break;
8478 8582 }
8479 8583
8480 8584 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
8481 8585 err += efunc(i, "%d exceeds variable id limit\n", id);
8482 8586 break;
8483 8587 }
8484 8588
8485 8589 if (id < DIF_VAR_OTHER_UBASE)
8486 8590 continue;
8487 8591
8488 8592 /*
8489 8593 * For user-defined variables, we need to check that this
8490 8594 * definition is identical to any previous definition that we
8491 8595 * encountered.
8492 8596 */
8493 8597 ndx = id - DIF_VAR_OTHER_UBASE;
8494 8598
8495 8599 switch (v->dtdv_scope) {
8496 8600 case DIFV_SCOPE_GLOBAL:
8497 8601 if (ndx < vstate->dtvs_nglobals) {
8498 8602 dtrace_statvar_t *svar;
8499 8603
8500 8604 if ((svar = vstate->dtvs_globals[ndx]) != NULL)
8501 8605 existing = &svar->dtsv_var;
8502 8606 }
8503 8607
8504 8608 break;
8505 8609
8506 8610 case DIFV_SCOPE_THREAD:
8507 8611 if (ndx < vstate->dtvs_ntlocals)
8508 8612 existing = &vstate->dtvs_tlocals[ndx];
8509 8613 break;
8510 8614
8511 8615 case DIFV_SCOPE_LOCAL:
8512 8616 if (ndx < vstate->dtvs_nlocals) {
8513 8617 dtrace_statvar_t *svar;
8514 8618
8515 8619 if ((svar = vstate->dtvs_locals[ndx]) != NULL)
8516 8620 existing = &svar->dtsv_var;
8517 8621 }
8518 8622
8519 8623 break;
8520 8624 }
8521 8625
8522 8626 vt = &v->dtdv_type;
8523 8627
8524 8628 if (vt->dtdt_flags & DIF_TF_BYREF) {
8525 8629 if (vt->dtdt_size == 0) {
8526 8630 err += efunc(i, "zero-sized variable\n");
8527 8631 break;
8528 8632 }
8529 8633
8530 8634 if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
8531 8635 vt->dtdt_size > dtrace_global_maxsize) {
8532 8636 err += efunc(i, "oversized by-ref global\n");
8533 8637 break;
8534 8638 }
8535 8639 }
8536 8640
8537 8641 if (existing == NULL || existing->dtdv_id == 0)
8538 8642 continue;
8539 8643
8540 8644 ASSERT(existing->dtdv_id == v->dtdv_id);
8541 8645 ASSERT(existing->dtdv_scope == v->dtdv_scope);
8542 8646
8543 8647 if (existing->dtdv_kind != v->dtdv_kind)
8544 8648 err += efunc(i, "%d changed variable kind\n", id);
8545 8649
8546 8650 et = &existing->dtdv_type;
8547 8651
8548 8652 if (vt->dtdt_flags != et->dtdt_flags) {
8549 8653 err += efunc(i, "%d changed variable type flags\n", id);
8550 8654 break;
8551 8655 }
8552 8656
8553 8657 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
8554 8658 err += efunc(i, "%d changed variable type size\n", id);
8555 8659 break;
8556 8660 }
8557 8661 }
8558 8662
8559 8663 return (err);
8560 8664 }
8561 8665
8562 8666 /*
8563 8667 * Validate a DTrace DIF object that it is to be used as a helper. Helpers
8564 8668 * are much more constrained than normal DIFOs. Specifically, they may
8565 8669 * not:
8566 8670 *
8567 8671 * 1. Make calls to subroutines other than copyin(), copyinstr() or
8568 8672 * miscellaneous string routines
8569 8673 * 2. Access DTrace variables other than the args[] array, and the
8570 8674 * curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
8571 8675 * 3. Have thread-local variables.
8572 8676 * 4. Have dynamic variables.
8573 8677 */
8574 8678 static int
8575 8679 dtrace_difo_validate_helper(dtrace_difo_t *dp)
8576 8680 {
8577 8681 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
8578 8682 int err = 0;
8579 8683 uint_t pc;
8580 8684
8581 8685 for (pc = 0; pc < dp->dtdo_len; pc++) {
8582 8686 dif_instr_t instr = dp->dtdo_buf[pc];
8583 8687
8584 8688 uint_t v = DIF_INSTR_VAR(instr);
8585 8689 uint_t subr = DIF_INSTR_SUBR(instr);
8586 8690 uint_t op = DIF_INSTR_OP(instr);
8587 8691
8588 8692 switch (op) {
8589 8693 case DIF_OP_OR:
8590 8694 case DIF_OP_XOR:
8591 8695 case DIF_OP_AND:
8592 8696 case DIF_OP_SLL:
8593 8697 case DIF_OP_SRL:
8594 8698 case DIF_OP_SRA:
8595 8699 case DIF_OP_SUB:
8596 8700 case DIF_OP_ADD:
8597 8701 case DIF_OP_MUL:
8598 8702 case DIF_OP_SDIV:
8599 8703 case DIF_OP_UDIV:
8600 8704 case DIF_OP_SREM:
8601 8705 case DIF_OP_UREM:
8602 8706 case DIF_OP_COPYS:
8603 8707 case DIF_OP_NOT:
8604 8708 case DIF_OP_MOV:
8605 8709 case DIF_OP_RLDSB:
8606 8710 case DIF_OP_RLDSH:
8607 8711 case DIF_OP_RLDSW:
8608 8712 case DIF_OP_RLDUB:
8609 8713 case DIF_OP_RLDUH:
8610 8714 case DIF_OP_RLDUW:
8611 8715 case DIF_OP_RLDX:
8612 8716 case DIF_OP_ULDSB:
8613 8717 case DIF_OP_ULDSH:
8614 8718 case DIF_OP_ULDSW:
8615 8719 case DIF_OP_ULDUB:
8616 8720 case DIF_OP_ULDUH:
8617 8721 case DIF_OP_ULDUW:
8618 8722 case DIF_OP_ULDX:
8619 8723 case DIF_OP_STB:
8620 8724 case DIF_OP_STH:
8621 8725 case DIF_OP_STW:
8622 8726 case DIF_OP_STX:
8623 8727 case DIF_OP_ALLOCS:
8624 8728 case DIF_OP_CMP:
8625 8729 case DIF_OP_SCMP:
8626 8730 case DIF_OP_TST:
8627 8731 case DIF_OP_BA:
8628 8732 case DIF_OP_BE:
8629 8733 case DIF_OP_BNE:
8630 8734 case DIF_OP_BG:
8631 8735 case DIF_OP_BGU:
8632 8736 case DIF_OP_BGE:
8633 8737 case DIF_OP_BGEU:
8634 8738 case DIF_OP_BL:
8635 8739 case DIF_OP_BLU:
8636 8740 case DIF_OP_BLE:
8637 8741 case DIF_OP_BLEU:
8638 8742 case DIF_OP_RET:
8639 8743 case DIF_OP_NOP:
8640 8744 case DIF_OP_POPTS:
8641 8745 case DIF_OP_FLUSHTS:
8642 8746 case DIF_OP_SETX:
8643 8747 case DIF_OP_SETS:
8644 8748 case DIF_OP_LDGA:
8645 8749 case DIF_OP_LDLS:
8646 8750 case DIF_OP_STGS:
8647 8751 case DIF_OP_STLS:
8648 8752 case DIF_OP_PUSHTR:
8649 8753 case DIF_OP_PUSHTV:
8650 8754 break;
8651 8755
8652 8756 case DIF_OP_LDGS:
8653 8757 if (v >= DIF_VAR_OTHER_UBASE)
8654 8758 break;
8655 8759
8656 8760 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
8657 8761 break;
8658 8762
8659 8763 if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
8660 8764 v == DIF_VAR_PPID || v == DIF_VAR_TID ||
8661 8765 v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
8662 8766 v == DIF_VAR_UID || v == DIF_VAR_GID)
8663 8767 break;
8664 8768
8665 8769 err += efunc(pc, "illegal variable %u\n", v);
8666 8770 break;
8667 8771
8668 8772 case DIF_OP_LDTA:
8669 8773 case DIF_OP_LDTS:
8670 8774 case DIF_OP_LDGAA:
8671 8775 case DIF_OP_LDTAA:
8672 8776 err += efunc(pc, "illegal dynamic variable load\n");
8673 8777 break;
8674 8778
8675 8779 case DIF_OP_STTS:
8676 8780 case DIF_OP_STGAA:
8677 8781 case DIF_OP_STTAA:
8678 8782 err += efunc(pc, "illegal dynamic variable store\n");
8679 8783 break;
8680 8784
8681 8785 case DIF_OP_CALL:
8682 8786 if (subr == DIF_SUBR_ALLOCA ||
8683 8787 subr == DIF_SUBR_BCOPY ||
8684 8788 subr == DIF_SUBR_COPYIN ||
8685 8789 subr == DIF_SUBR_COPYINTO ||
8686 8790 subr == DIF_SUBR_COPYINSTR ||
8687 8791 subr == DIF_SUBR_INDEX ||
8688 8792 subr == DIF_SUBR_INET_NTOA ||
8689 8793 subr == DIF_SUBR_INET_NTOA6 ||
8690 8794 subr == DIF_SUBR_INET_NTOP ||
8691 8795 subr == DIF_SUBR_LLTOSTR ||
8692 8796 subr == DIF_SUBR_RINDEX ||
8693 8797 subr == DIF_SUBR_STRCHR ||
8694 8798 subr == DIF_SUBR_STRJOIN ||
8695 8799 subr == DIF_SUBR_STRRCHR ||
8696 8800 subr == DIF_SUBR_STRSTR ||
8697 8801 subr == DIF_SUBR_HTONS ||
8698 8802 subr == DIF_SUBR_HTONL ||
8699 8803 subr == DIF_SUBR_HTONLL ||
8700 8804 subr == DIF_SUBR_NTOHS ||
8701 8805 subr == DIF_SUBR_NTOHL ||
8702 8806 subr == DIF_SUBR_NTOHLL)
8703 8807 break;
8704 8808
8705 8809 err += efunc(pc, "invalid subr %u\n", subr);
8706 8810 break;
8707 8811
8708 8812 default:
8709 8813 err += efunc(pc, "invalid opcode %u\n",
8710 8814 DIF_INSTR_OP(instr));
8711 8815 }
8712 8816 }
8713 8817
8714 8818 return (err);
8715 8819 }
8716 8820
8717 8821 /*
8718 8822 * Returns 1 if the expression in the DIF object can be cached on a per-thread
8719 8823 * basis; 0 if not.
8720 8824 */
8721 8825 static int
8722 8826 dtrace_difo_cacheable(dtrace_difo_t *dp)
8723 8827 {
8724 8828 int i;
8725 8829
8726 8830 if (dp == NULL)
8727 8831 return (0);
8728 8832
8729 8833 for (i = 0; i < dp->dtdo_varlen; i++) {
8730 8834 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8731 8835
8732 8836 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
8733 8837 continue;
8734 8838
8735 8839 switch (v->dtdv_id) {
8736 8840 case DIF_VAR_CURTHREAD:
8737 8841 case DIF_VAR_PID:
8738 8842 case DIF_VAR_TID:
8739 8843 case DIF_VAR_EXECNAME:
8740 8844 case DIF_VAR_ZONENAME:
8741 8845 break;
8742 8846
8743 8847 default:
8744 8848 return (0);
8745 8849 }
8746 8850 }
8747 8851
8748 8852 /*
8749 8853 * This DIF object may be cacheable. Now we need to look for any
8750 8854 * array loading instructions, any memory loading instructions, or
8751 8855 * any stores to thread-local variables.
8752 8856 */
8753 8857 for (i = 0; i < dp->dtdo_len; i++) {
8754 8858 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
8755 8859
8756 8860 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
8757 8861 (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
8758 8862 (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
8759 8863 op == DIF_OP_LDGA || op == DIF_OP_STTS)
8760 8864 return (0);
8761 8865 }
8762 8866
8763 8867 return (1);
8764 8868 }
8765 8869
8766 8870 static void
8767 8871 dtrace_difo_hold(dtrace_difo_t *dp)
8768 8872 {
8769 8873 int i;
8770 8874
8771 8875 ASSERT(MUTEX_HELD(&dtrace_lock));
8772 8876
8773 8877 dp->dtdo_refcnt++;
8774 8878 ASSERT(dp->dtdo_refcnt != 0);
8775 8879
8776 8880 /*
8777 8881 * We need to check this DIF object for references to the variable
8778 8882 * DIF_VAR_VTIMESTAMP.
8779 8883 */
8780 8884 for (i = 0; i < dp->dtdo_varlen; i++) {
8781 8885 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8782 8886
8783 8887 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
8784 8888 continue;
8785 8889
8786 8890 if (dtrace_vtime_references++ == 0)
8787 8891 dtrace_vtime_enable();
8788 8892 }
8789 8893 }
8790 8894
8791 8895 /*
8792 8896 * This routine calculates the dynamic variable chunksize for a given DIF
8793 8897 * object. The calculation is not fool-proof, and can probably be tricked by
8794 8898 * malicious DIF -- but it works for all compiler-generated DIF. Because this
8795 8899 * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
8796 8900 * if a dynamic variable size exceeds the chunksize.
8797 8901 */
8798 8902 static void
8799 8903 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
8800 8904 {
8801 8905 uint64_t sval;
8802 8906 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
8803 8907 const dif_instr_t *text = dp->dtdo_buf;
8804 8908 uint_t pc, srd = 0;
8805 8909 uint_t ttop = 0;
8806 8910 size_t size, ksize;
8807 8911 uint_t id, i;
8808 8912
8809 8913 for (pc = 0; pc < dp->dtdo_len; pc++) {
8810 8914 dif_instr_t instr = text[pc];
8811 8915 uint_t op = DIF_INSTR_OP(instr);
8812 8916 uint_t rd = DIF_INSTR_RD(instr);
8813 8917 uint_t r1 = DIF_INSTR_R1(instr);
8814 8918 uint_t nkeys = 0;
8815 8919 uchar_t scope;
8816 8920
8817 8921 dtrace_key_t *key = tupregs;
8818 8922
8819 8923 switch (op) {
8820 8924 case DIF_OP_SETX:
8821 8925 sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
8822 8926 srd = rd;
8823 8927 continue;
8824 8928
8825 8929 case DIF_OP_STTS:
8826 8930 key = &tupregs[DIF_DTR_NREGS];
8827 8931 key[0].dttk_size = 0;
8828 8932 key[1].dttk_size = 0;
8829 8933 nkeys = 2;
8830 8934 scope = DIFV_SCOPE_THREAD;
8831 8935 break;
8832 8936
8833 8937 case DIF_OP_STGAA:
8834 8938 case DIF_OP_STTAA:
8835 8939 nkeys = ttop;
8836 8940
8837 8941 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
8838 8942 key[nkeys++].dttk_size = 0;
8839 8943
8840 8944 key[nkeys++].dttk_size = 0;
8841 8945
8842 8946 if (op == DIF_OP_STTAA) {
8843 8947 scope = DIFV_SCOPE_THREAD;
8844 8948 } else {
8845 8949 scope = DIFV_SCOPE_GLOBAL;
8846 8950 }
8847 8951
8848 8952 break;
8849 8953
8850 8954 case DIF_OP_PUSHTR:
8851 8955 if (ttop == DIF_DTR_NREGS)
8852 8956 return;
8853 8957
8854 8958 if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
8855 8959 /*
8856 8960 * If the register for the size of the "pushtr"
8857 8961 * is %r0 (or the value is 0) and the type is
8858 8962 * a string, we'll use the system-wide default
8859 8963 * string size.
8860 8964 */
8861 8965 tupregs[ttop++].dttk_size =
8862 8966 dtrace_strsize_default;
8863 8967 } else {
8864 8968 if (srd == 0)
8865 8969 return;
8866 8970
8867 8971 tupregs[ttop++].dttk_size = sval;
8868 8972 }
8869 8973
8870 8974 break;
8871 8975
8872 8976 case DIF_OP_PUSHTV:
8873 8977 if (ttop == DIF_DTR_NREGS)
8874 8978 return;
8875 8979
8876 8980 tupregs[ttop++].dttk_size = 0;
8877 8981 break;
8878 8982
8879 8983 case DIF_OP_FLUSHTS:
8880 8984 ttop = 0;
8881 8985 break;
8882 8986
8883 8987 case DIF_OP_POPTS:
8884 8988 if (ttop != 0)
8885 8989 ttop--;
8886 8990 break;
8887 8991 }
8888 8992
8889 8993 sval = 0;
8890 8994 srd = 0;
8891 8995
8892 8996 if (nkeys == 0)
8893 8997 continue;
8894 8998
8895 8999 /*
8896 9000 * We have a dynamic variable allocation; calculate its size.
8897 9001 */
8898 9002 for (ksize = 0, i = 0; i < nkeys; i++)
8899 9003 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
8900 9004
8901 9005 size = sizeof (dtrace_dynvar_t);
8902 9006 size += sizeof (dtrace_key_t) * (nkeys - 1);
8903 9007 size += ksize;
8904 9008
8905 9009 /*
8906 9010 * Now we need to determine the size of the stored data.
8907 9011 */
8908 9012 id = DIF_INSTR_VAR(instr);
8909 9013
8910 9014 for (i = 0; i < dp->dtdo_varlen; i++) {
8911 9015 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8912 9016
8913 9017 if (v->dtdv_id == id && v->dtdv_scope == scope) {
8914 9018 size += v->dtdv_type.dtdt_size;
8915 9019 break;
8916 9020 }
8917 9021 }
8918 9022
8919 9023 if (i == dp->dtdo_varlen)
8920 9024 return;
8921 9025
8922 9026 /*
8923 9027 * We have the size. If this is larger than the chunk size
8924 9028 * for our dynamic variable state, reset the chunk size.
8925 9029 */
8926 9030 size = P2ROUNDUP(size, sizeof (uint64_t));
8927 9031
8928 9032 if (size > vstate->dtvs_dynvars.dtds_chunksize)
8929 9033 vstate->dtvs_dynvars.dtds_chunksize = size;
8930 9034 }
8931 9035 }
8932 9036
8933 9037 static void
8934 9038 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
8935 9039 {
8936 9040 int i, oldsvars, osz, nsz, otlocals, ntlocals;
8937 9041 uint_t id;
8938 9042
8939 9043 ASSERT(MUTEX_HELD(&dtrace_lock));
8940 9044 ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
8941 9045
8942 9046 for (i = 0; i < dp->dtdo_varlen; i++) {
8943 9047 dtrace_difv_t *v = &dp->dtdo_vartab[i];
8944 9048 dtrace_statvar_t *svar, ***svarp;
8945 9049 size_t dsize = 0;
8946 9050 uint8_t scope = v->dtdv_scope;
8947 9051 int *np;
8948 9052
8949 9053 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
8950 9054 continue;
8951 9055
8952 9056 id -= DIF_VAR_OTHER_UBASE;
8953 9057
8954 9058 switch (scope) {
8955 9059 case DIFV_SCOPE_THREAD:
8956 9060 while (id >= (otlocals = vstate->dtvs_ntlocals)) {
8957 9061 dtrace_difv_t *tlocals;
8958 9062
8959 9063 if ((ntlocals = (otlocals << 1)) == 0)
8960 9064 ntlocals = 1;
8961 9065
8962 9066 osz = otlocals * sizeof (dtrace_difv_t);
8963 9067 nsz = ntlocals * sizeof (dtrace_difv_t);
8964 9068
8965 9069 tlocals = kmem_zalloc(nsz, KM_SLEEP);
8966 9070
8967 9071 if (osz != 0) {
8968 9072 bcopy(vstate->dtvs_tlocals,
8969 9073 tlocals, osz);
8970 9074 kmem_free(vstate->dtvs_tlocals, osz);
8971 9075 }
8972 9076
8973 9077 vstate->dtvs_tlocals = tlocals;
8974 9078 vstate->dtvs_ntlocals = ntlocals;
8975 9079 }
8976 9080
8977 9081 vstate->dtvs_tlocals[id] = *v;
8978 9082 continue;
8979 9083
8980 9084 case DIFV_SCOPE_LOCAL:
8981 9085 np = &vstate->dtvs_nlocals;
8982 9086 svarp = &vstate->dtvs_locals;
8983 9087
8984 9088 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
8985 9089 dsize = NCPU * (v->dtdv_type.dtdt_size +
8986 9090 sizeof (uint64_t));
8987 9091 else
8988 9092 dsize = NCPU * sizeof (uint64_t);
8989 9093
8990 9094 break;
8991 9095
8992 9096 case DIFV_SCOPE_GLOBAL:
8993 9097 np = &vstate->dtvs_nglobals;
8994 9098 svarp = &vstate->dtvs_globals;
8995 9099
8996 9100 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
8997 9101 dsize = v->dtdv_type.dtdt_size +
8998 9102 sizeof (uint64_t);
8999 9103
9000 9104 break;
9001 9105
9002 9106 default:
9003 9107 ASSERT(0);
9004 9108 }
9005 9109
9006 9110 while (id >= (oldsvars = *np)) {
9007 9111 dtrace_statvar_t **statics;
9008 9112 int newsvars, oldsize, newsize;
9009 9113
9010 9114 if ((newsvars = (oldsvars << 1)) == 0)
9011 9115 newsvars = 1;
9012 9116
9013 9117 oldsize = oldsvars * sizeof (dtrace_statvar_t *);
9014 9118 newsize = newsvars * sizeof (dtrace_statvar_t *);
9015 9119
9016 9120 statics = kmem_zalloc(newsize, KM_SLEEP);
9017 9121
9018 9122 if (oldsize != 0) {
9019 9123 bcopy(*svarp, statics, oldsize);
9020 9124 kmem_free(*svarp, oldsize);
9021 9125 }
9022 9126
9023 9127 *svarp = statics;
9024 9128 *np = newsvars;
9025 9129 }
9026 9130
9027 9131 if ((svar = (*svarp)[id]) == NULL) {
9028 9132 svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
9029 9133 svar->dtsv_var = *v;
9030 9134
9031 9135 if ((svar->dtsv_size = dsize) != 0) {
9032 9136 svar->dtsv_data = (uint64_t)(uintptr_t)
9033 9137 kmem_zalloc(dsize, KM_SLEEP);
9034 9138 }
9035 9139
9036 9140 (*svarp)[id] = svar;
9037 9141 }
9038 9142
9039 9143 svar->dtsv_refcnt++;
9040 9144 }
9041 9145
9042 9146 dtrace_difo_chunksize(dp, vstate);
9043 9147 dtrace_difo_hold(dp);
9044 9148 }
9045 9149
9046 9150 static dtrace_difo_t *
9047 9151 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9048 9152 {
9049 9153 dtrace_difo_t *new;
9050 9154 size_t sz;
9051 9155
9052 9156 ASSERT(dp->dtdo_buf != NULL);
9053 9157 ASSERT(dp->dtdo_refcnt != 0);
9054 9158
9055 9159 new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
9056 9160
9057 9161 ASSERT(dp->dtdo_buf != NULL);
9058 9162 sz = dp->dtdo_len * sizeof (dif_instr_t);
9059 9163 new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
9060 9164 bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
9061 9165 new->dtdo_len = dp->dtdo_len;
9062 9166
9063 9167 if (dp->dtdo_strtab != NULL) {
9064 9168 ASSERT(dp->dtdo_strlen != 0);
9065 9169 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
9066 9170 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
9067 9171 new->dtdo_strlen = dp->dtdo_strlen;
9068 9172 }
9069 9173
9070 9174 if (dp->dtdo_inttab != NULL) {
9071 9175 ASSERT(dp->dtdo_intlen != 0);
9072 9176 sz = dp->dtdo_intlen * sizeof (uint64_t);
9073 9177 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
9074 9178 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
9075 9179 new->dtdo_intlen = dp->dtdo_intlen;
9076 9180 }
9077 9181
9078 9182 if (dp->dtdo_vartab != NULL) {
9079 9183 ASSERT(dp->dtdo_varlen != 0);
9080 9184 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
9081 9185 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
9082 9186 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
9083 9187 new->dtdo_varlen = dp->dtdo_varlen;
9084 9188 }
9085 9189
9086 9190 dtrace_difo_init(new, vstate);
9087 9191 return (new);
9088 9192 }
9089 9193
9090 9194 static void
9091 9195 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9092 9196 {
9093 9197 int i;
9094 9198
9095 9199 ASSERT(dp->dtdo_refcnt == 0);
9096 9200
9097 9201 for (i = 0; i < dp->dtdo_varlen; i++) {
9098 9202 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9099 9203 dtrace_statvar_t *svar, **svarp;
9100 9204 uint_t id;
9101 9205 uint8_t scope = v->dtdv_scope;
9102 9206 int *np;
9103 9207
9104 9208 switch (scope) {
9105 9209 case DIFV_SCOPE_THREAD:
9106 9210 continue;
9107 9211
9108 9212 case DIFV_SCOPE_LOCAL:
9109 9213 np = &vstate->dtvs_nlocals;
9110 9214 svarp = vstate->dtvs_locals;
9111 9215 break;
9112 9216
9113 9217 case DIFV_SCOPE_GLOBAL:
9114 9218 np = &vstate->dtvs_nglobals;
9115 9219 svarp = vstate->dtvs_globals;
9116 9220 break;
9117 9221
9118 9222 default:
9119 9223 ASSERT(0);
9120 9224 }
9121 9225
9122 9226 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
9123 9227 continue;
9124 9228
9125 9229 id -= DIF_VAR_OTHER_UBASE;
9126 9230 ASSERT(id < *np);
9127 9231
9128 9232 svar = svarp[id];
9129 9233 ASSERT(svar != NULL);
9130 9234 ASSERT(svar->dtsv_refcnt > 0);
9131 9235
9132 9236 if (--svar->dtsv_refcnt > 0)
9133 9237 continue;
9134 9238
9135 9239 if (svar->dtsv_size != 0) {
9136 9240 ASSERT(svar->dtsv_data != NULL);
9137 9241 kmem_free((void *)(uintptr_t)svar->dtsv_data,
9138 9242 svar->dtsv_size);
9139 9243 }
9140 9244
9141 9245 kmem_free(svar, sizeof (dtrace_statvar_t));
9142 9246 svarp[id] = NULL;
9143 9247 }
9144 9248
9145 9249 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
9146 9250 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
9147 9251 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
9148 9252 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
9149 9253
9150 9254 kmem_free(dp, sizeof (dtrace_difo_t));
9151 9255 }
9152 9256
9153 9257 static void
9154 9258 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
9155 9259 {
9156 9260 int i;
9157 9261
9158 9262 ASSERT(MUTEX_HELD(&dtrace_lock));
9159 9263 ASSERT(dp->dtdo_refcnt != 0);
9160 9264
9161 9265 for (i = 0; i < dp->dtdo_varlen; i++) {
9162 9266 dtrace_difv_t *v = &dp->dtdo_vartab[i];
9163 9267
9164 9268 if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
9165 9269 continue;
9166 9270
9167 9271 ASSERT(dtrace_vtime_references > 0);
9168 9272 if (--dtrace_vtime_references == 0)
9169 9273 dtrace_vtime_disable();
9170 9274 }
9171 9275
9172 9276 if (--dp->dtdo_refcnt == 0)
9173 9277 dtrace_difo_destroy(dp, vstate);
9174 9278 }
9175 9279
9176 9280 /*
9177 9281 * DTrace Format Functions
9178 9282 */
9179 9283 static uint16_t
9180 9284 dtrace_format_add(dtrace_state_t *state, char *str)
9181 9285 {
9182 9286 char *fmt, **new;
9183 9287 uint16_t ndx, len = strlen(str) + 1;
9184 9288
9185 9289 fmt = kmem_zalloc(len, KM_SLEEP);
9186 9290 bcopy(str, fmt, len);
9187 9291
9188 9292 for (ndx = 0; ndx < state->dts_nformats; ndx++) {
9189 9293 if (state->dts_formats[ndx] == NULL) {
9190 9294 state->dts_formats[ndx] = fmt;
9191 9295 return (ndx + 1);
9192 9296 }
9193 9297 }
9194 9298
9195 9299 if (state->dts_nformats == USHRT_MAX) {
9196 9300 /*
9197 9301 * This is only likely if a denial-of-service attack is being
9198 9302 * attempted. As such, it's okay to fail silently here.
9199 9303 */
9200 9304 kmem_free(fmt, len);
9201 9305 return (0);
9202 9306 }
9203 9307
9204 9308 /*
9205 9309 * For simplicity, we always resize the formats array to be exactly the
9206 9310 * number of formats.
9207 9311 */
9208 9312 ndx = state->dts_nformats++;
9209 9313 new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
9210 9314
9211 9315 if (state->dts_formats != NULL) {
9212 9316 ASSERT(ndx != 0);
9213 9317 bcopy(state->dts_formats, new, ndx * sizeof (char *));
9214 9318 kmem_free(state->dts_formats, ndx * sizeof (char *));
9215 9319 }
9216 9320
9217 9321 state->dts_formats = new;
9218 9322 state->dts_formats[ndx] = fmt;
9219 9323
9220 9324 return (ndx + 1);
9221 9325 }
9222 9326
9223 9327 static void
9224 9328 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
9225 9329 {
9226 9330 char *fmt;
9227 9331
9228 9332 ASSERT(state->dts_formats != NULL);
9229 9333 ASSERT(format <= state->dts_nformats);
9230 9334 ASSERT(state->dts_formats[format - 1] != NULL);
9231 9335
9232 9336 fmt = state->dts_formats[format - 1];
9233 9337 kmem_free(fmt, strlen(fmt) + 1);
9234 9338 state->dts_formats[format - 1] = NULL;
9235 9339 }
9236 9340
9237 9341 static void
9238 9342 dtrace_format_destroy(dtrace_state_t *state)
9239 9343 {
9240 9344 int i;
9241 9345
9242 9346 if (state->dts_nformats == 0) {
9243 9347 ASSERT(state->dts_formats == NULL);
9244 9348 return;
9245 9349 }
9246 9350
9247 9351 ASSERT(state->dts_formats != NULL);
9248 9352
9249 9353 for (i = 0; i < state->dts_nformats; i++) {
9250 9354 char *fmt = state->dts_formats[i];
9251 9355
9252 9356 if (fmt == NULL)
9253 9357 continue;
9254 9358
9255 9359 kmem_free(fmt, strlen(fmt) + 1);
9256 9360 }
9257 9361
9258 9362 kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
9259 9363 state->dts_nformats = 0;
9260 9364 state->dts_formats = NULL;
9261 9365 }
9262 9366
9263 9367 /*
9264 9368 * DTrace Predicate Functions
9265 9369 */
9266 9370 static dtrace_predicate_t *
9267 9371 dtrace_predicate_create(dtrace_difo_t *dp)
9268 9372 {
9269 9373 dtrace_predicate_t *pred;
9270 9374
9271 9375 ASSERT(MUTEX_HELD(&dtrace_lock));
9272 9376 ASSERT(dp->dtdo_refcnt != 0);
9273 9377
9274 9378 pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
9275 9379 pred->dtp_difo = dp;
9276 9380 pred->dtp_refcnt = 1;
9277 9381
9278 9382 if (!dtrace_difo_cacheable(dp))
9279 9383 return (pred);
9280 9384
9281 9385 if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
9282 9386 /*
9283 9387 * This is only theoretically possible -- we have had 2^32
9284 9388 * cacheable predicates on this machine. We cannot allow any
9285 9389 * more predicates to become cacheable: as unlikely as it is,
9286 9390 * there may be a thread caching a (now stale) predicate cache
9287 9391 * ID. (N.B.: the temptation is being successfully resisted to
9288 9392 * have this cmn_err() "Holy shit -- we executed this code!")
9289 9393 */
9290 9394 return (pred);
9291 9395 }
9292 9396
9293 9397 pred->dtp_cacheid = dtrace_predcache_id++;
9294 9398
9295 9399 return (pred);
9296 9400 }
9297 9401
9298 9402 static void
9299 9403 dtrace_predicate_hold(dtrace_predicate_t *pred)
9300 9404 {
9301 9405 ASSERT(MUTEX_HELD(&dtrace_lock));
9302 9406 ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
9303 9407 ASSERT(pred->dtp_refcnt > 0);
9304 9408
9305 9409 pred->dtp_refcnt++;
9306 9410 }
9307 9411
9308 9412 static void
9309 9413 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
9310 9414 {
9311 9415 dtrace_difo_t *dp = pred->dtp_difo;
9312 9416
9313 9417 ASSERT(MUTEX_HELD(&dtrace_lock));
9314 9418 ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
9315 9419 ASSERT(pred->dtp_refcnt > 0);
9316 9420
9317 9421 if (--pred->dtp_refcnt == 0) {
9318 9422 dtrace_difo_release(pred->dtp_difo, vstate);
9319 9423 kmem_free(pred, sizeof (dtrace_predicate_t));
9320 9424 }
9321 9425 }
9322 9426
9323 9427 /*
9324 9428 * DTrace Action Description Functions
9325 9429 */
9326 9430 static dtrace_actdesc_t *
9327 9431 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
9328 9432 uint64_t uarg, uint64_t arg)
9329 9433 {
9330 9434 dtrace_actdesc_t *act;
9331 9435
9332 9436 ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
9333 9437 arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
9334 9438
9335 9439 act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
9336 9440 act->dtad_kind = kind;
9337 9441 act->dtad_ntuple = ntuple;
9338 9442 act->dtad_uarg = uarg;
9339 9443 act->dtad_arg = arg;
9340 9444 act->dtad_refcnt = 1;
9341 9445
9342 9446 return (act);
9343 9447 }
9344 9448
9345 9449 static void
9346 9450 dtrace_actdesc_hold(dtrace_actdesc_t *act)
9347 9451 {
9348 9452 ASSERT(act->dtad_refcnt >= 1);
9349 9453 act->dtad_refcnt++;
9350 9454 }
9351 9455
9352 9456 static void
9353 9457 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
9354 9458 {
9355 9459 dtrace_actkind_t kind = act->dtad_kind;
9356 9460 dtrace_difo_t *dp;
9357 9461
9358 9462 ASSERT(act->dtad_refcnt >= 1);
9359 9463
9360 9464 if (--act->dtad_refcnt != 0)
9361 9465 return;
9362 9466
9363 9467 if ((dp = act->dtad_difo) != NULL)
9364 9468 dtrace_difo_release(dp, vstate);
9365 9469
9366 9470 if (DTRACEACT_ISPRINTFLIKE(kind)) {
9367 9471 char *str = (char *)(uintptr_t)act->dtad_arg;
9368 9472
9369 9473 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
9370 9474 (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
9371 9475
9372 9476 if (str != NULL)
9373 9477 kmem_free(str, strlen(str) + 1);
9374 9478 }
9375 9479
9376 9480 kmem_free(act, sizeof (dtrace_actdesc_t));
9377 9481 }
9378 9482
9379 9483 /*
9380 9484 * DTrace ECB Functions
9381 9485 */
9382 9486 static dtrace_ecb_t *
9383 9487 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
9384 9488 {
9385 9489 dtrace_ecb_t *ecb;
9386 9490 dtrace_epid_t epid;
9387 9491
9388 9492 ASSERT(MUTEX_HELD(&dtrace_lock));
9389 9493
9390 9494 ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
9391 9495 ecb->dte_predicate = NULL;
9392 9496 ecb->dte_probe = probe;
9393 9497
9394 9498 /*
9395 9499 * The default size is the size of the default action: recording
9396 9500 * the epid.
9397 9501 */
9398 9502 ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
9399 9503 ecb->dte_alignment = sizeof (dtrace_epid_t);
9400 9504
9401 9505 epid = state->dts_epid++;
9402 9506
9403 9507 if (epid - 1 >= state->dts_necbs) {
9404 9508 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
9405 9509 int necbs = state->dts_necbs << 1;
9406 9510
9407 9511 ASSERT(epid == state->dts_necbs + 1);
9408 9512
9409 9513 if (necbs == 0) {
9410 9514 ASSERT(oecbs == NULL);
9411 9515 necbs = 1;
9412 9516 }
9413 9517
9414 9518 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
9415 9519
9416 9520 if (oecbs != NULL)
9417 9521 bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
9418 9522
9419 9523 dtrace_membar_producer();
9420 9524 state->dts_ecbs = ecbs;
9421 9525
9422 9526 if (oecbs != NULL) {
9423 9527 /*
9424 9528 * If this state is active, we must dtrace_sync()
9425 9529 * before we can free the old dts_ecbs array: we're
9426 9530 * coming in hot, and there may be active ring
9427 9531 * buffer processing (which indexes into the dts_ecbs
9428 9532 * array) on another CPU.
9429 9533 */
9430 9534 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
9431 9535 dtrace_sync();
9432 9536
9433 9537 kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
9434 9538 }
9435 9539
9436 9540 dtrace_membar_producer();
9437 9541 state->dts_necbs = necbs;
9438 9542 }
9439 9543
9440 9544 ecb->dte_state = state;
9441 9545
9442 9546 ASSERT(state->dts_ecbs[epid - 1] == NULL);
9443 9547 dtrace_membar_producer();
9444 9548 state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
9445 9549
9446 9550 return (ecb);
9447 9551 }
9448 9552
9449 9553 static int
9450 9554 dtrace_ecb_enable(dtrace_ecb_t *ecb)
9451 9555 {
9452 9556 dtrace_probe_t *probe = ecb->dte_probe;
9453 9557
9454 9558 ASSERT(MUTEX_HELD(&cpu_lock));
9455 9559 ASSERT(MUTEX_HELD(&dtrace_lock));
9456 9560 ASSERT(ecb->dte_next == NULL);
9457 9561
9458 9562 if (probe == NULL) {
9459 9563 /*
9460 9564 * This is the NULL probe -- there's nothing to do.
9461 9565 */
9462 9566 return (0);
9463 9567 }
9464 9568
9465 9569 if (probe->dtpr_ecb == NULL) {
9466 9570 dtrace_provider_t *prov = probe->dtpr_provider;
9467 9571
9468 9572 /*
9469 9573 * We're the first ECB on this probe.
9470 9574 */
9471 9575 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
9472 9576
9473 9577 if (ecb->dte_predicate != NULL)
9474 9578 probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
9475 9579
9476 9580 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
9477 9581 probe->dtpr_id, probe->dtpr_arg));
9478 9582 } else {
9479 9583 /*
9480 9584 * This probe is already active. Swing the last pointer to
9481 9585 * point to the new ECB, and issue a dtrace_sync() to assure
9482 9586 * that all CPUs have seen the change.
9483 9587 */
9484 9588 ASSERT(probe->dtpr_ecb_last != NULL);
9485 9589 probe->dtpr_ecb_last->dte_next = ecb;
9486 9590 probe->dtpr_ecb_last = ecb;
9487 9591 probe->dtpr_predcache = 0;
9488 9592
9489 9593 dtrace_sync();
9490 9594 return (0);
9491 9595 }
9492 9596 }
9493 9597
9494 9598 static void
9495 9599 dtrace_ecb_resize(dtrace_ecb_t *ecb)
9496 9600 {
9497 9601 uint32_t maxalign = sizeof (dtrace_epid_t);
9498 9602 uint32_t align = sizeof (uint8_t), offs, diff;
9499 9603 dtrace_action_t *act;
9500 9604 int wastuple = 0;
9501 9605 uint32_t aggbase = UINT32_MAX;
9502 9606 dtrace_state_t *state = ecb->dte_state;
9503 9607
9504 9608 /*
9505 9609 * If we record anything, we always record the epid. (And we always
9506 9610 * record it first.)
9507 9611 */
9508 9612 offs = sizeof (dtrace_epid_t);
9509 9613 ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
9510 9614
9511 9615 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
9512 9616 dtrace_recdesc_t *rec = &act->dta_rec;
9513 9617
9514 9618 if ((align = rec->dtrd_alignment) > maxalign)
9515 9619 maxalign = align;
9516 9620
9517 9621 if (!wastuple && act->dta_intuple) {
9518 9622 /*
9519 9623 * This is the first record in a tuple. Align the
9520 9624 * offset to be at offset 4 in an 8-byte aligned
9521 9625 * block.
9522 9626 */
9523 9627 diff = offs + sizeof (dtrace_aggid_t);
9524 9628
9525 9629 if (diff = (diff & (sizeof (uint64_t) - 1)))
9526 9630 offs += sizeof (uint64_t) - diff;
9527 9631
9528 9632 aggbase = offs - sizeof (dtrace_aggid_t);
9529 9633 ASSERT(!(aggbase & (sizeof (uint64_t) - 1)));
9530 9634 }
9531 9635
9532 9636 /*LINTED*/
9533 9637 if (rec->dtrd_size != 0 && (diff = (offs & (align - 1)))) {
9534 9638 /*
9535 9639 * The current offset is not properly aligned; align it.
9536 9640 */
9537 9641 offs += align - diff;
9538 9642 }
9539 9643
9540 9644 rec->dtrd_offset = offs;
9541 9645
9542 9646 if (offs + rec->dtrd_size > ecb->dte_needed) {
9543 9647 ecb->dte_needed = offs + rec->dtrd_size;
9544 9648
9545 9649 if (ecb->dte_needed > state->dts_needed)
9546 9650 state->dts_needed = ecb->dte_needed;
9547 9651 }
9548 9652
9549 9653 if (DTRACEACT_ISAGG(act->dta_kind)) {
9550 9654 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
9551 9655 dtrace_action_t *first = agg->dtag_first, *prev;
9552 9656
9553 9657 ASSERT(rec->dtrd_size != 0 && first != NULL);
9554 9658 ASSERT(wastuple);
9555 9659 ASSERT(aggbase != UINT32_MAX);
9556 9660
9557 9661 agg->dtag_base = aggbase;
9558 9662
9559 9663 while ((prev = first->dta_prev) != NULL &&
9560 9664 DTRACEACT_ISAGG(prev->dta_kind)) {
9561 9665 agg = (dtrace_aggregation_t *)prev;
9562 9666 first = agg->dtag_first;
9563 9667 }
9564 9668
9565 9669 if (prev != NULL) {
9566 9670 offs = prev->dta_rec.dtrd_offset +
9567 9671 prev->dta_rec.dtrd_size;
9568 9672 } else {
9569 9673 offs = sizeof (dtrace_epid_t);
9570 9674 }
9571 9675 wastuple = 0;
9572 9676 } else {
9573 9677 if (!act->dta_intuple)
9574 9678 ecb->dte_size = offs + rec->dtrd_size;
9575 9679
9576 9680 offs += rec->dtrd_size;
9577 9681 }
9578 9682
9579 9683 wastuple = act->dta_intuple;
9580 9684 }
9581 9685
9582 9686 if ((act = ecb->dte_action) != NULL &&
9583 9687 !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
9584 9688 ecb->dte_size == sizeof (dtrace_epid_t)) {
9585 9689 /*
9586 9690 * If the size is still sizeof (dtrace_epid_t), then all
9587 9691 * actions store no data; set the size to 0.
9588 9692 */
9589 9693 ecb->dte_alignment = maxalign;
9590 9694 ecb->dte_size = 0;
9591 9695
9592 9696 /*
9593 9697 * If the needed space is still sizeof (dtrace_epid_t), then
9594 9698 * all actions need no additional space; set the needed
9595 9699 * size to 0.
9596 9700 */
9597 9701 if (ecb->dte_needed == sizeof (dtrace_epid_t))
9598 9702 ecb->dte_needed = 0;
9599 9703
9600 9704 return;
9601 9705 }
9602 9706
9603 9707 /*
9604 9708 * Set our alignment, and make sure that the dte_size and dte_needed
9605 9709 * are aligned to the size of an EPID.
9606 9710 */
9607 9711 ecb->dte_alignment = maxalign;
9608 9712 ecb->dte_size = (ecb->dte_size + (sizeof (dtrace_epid_t) - 1)) &
9609 9713 ~(sizeof (dtrace_epid_t) - 1);
9610 9714 ecb->dte_needed = (ecb->dte_needed + (sizeof (dtrace_epid_t) - 1)) &
9611 9715 ~(sizeof (dtrace_epid_t) - 1);
9612 9716 ASSERT(ecb->dte_size <= ecb->dte_needed);
9613 9717 }
9614 9718
9615 9719 static dtrace_action_t *
9616 9720 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
9617 9721 {
9618 9722 dtrace_aggregation_t *agg;
9619 9723 size_t size = sizeof (uint64_t);
9620 9724 int ntuple = desc->dtad_ntuple;
9621 9725 dtrace_action_t *act;
9622 9726 dtrace_recdesc_t *frec;
9623 9727 dtrace_aggid_t aggid;
9624 9728 dtrace_state_t *state = ecb->dte_state;
9625 9729
9626 9730 agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
9627 9731 agg->dtag_ecb = ecb;
9628 9732
9629 9733 ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
9630 9734
9631 9735 switch (desc->dtad_kind) {
9632 9736 case DTRACEAGG_MIN:
9633 9737 agg->dtag_initial = INT64_MAX;
9634 9738 agg->dtag_aggregate = dtrace_aggregate_min;
9635 9739 break;
9636 9740
9637 9741 case DTRACEAGG_MAX:
9638 9742 agg->dtag_initial = INT64_MIN;
9639 9743 agg->dtag_aggregate = dtrace_aggregate_max;
9640 9744 break;
9641 9745
9642 9746 case DTRACEAGG_COUNT:
9643 9747 agg->dtag_aggregate = dtrace_aggregate_count;
9644 9748 break;
9645 9749
9646 9750 case DTRACEAGG_QUANTIZE:
9647 9751 agg->dtag_aggregate = dtrace_aggregate_quantize;
9648 9752 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
9649 9753 sizeof (uint64_t);
9650 9754 break;
9651 9755
9652 9756 case DTRACEAGG_LQUANTIZE: {
9653 9757 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
9654 9758 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
9655 9759
9656 9760 agg->dtag_initial = desc->dtad_arg;
9657 9761 agg->dtag_aggregate = dtrace_aggregate_lquantize;
9658 9762
9659 9763 if (step == 0 || levels == 0)
9660 9764 goto err;
9661 9765
9662 9766 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
9663 9767 break;
9664 9768 }
9665 9769
9666 9770 case DTRACEAGG_LLQUANTIZE: {
9667 9771 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
9668 9772 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg);
9669 9773 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg);
9670 9774 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg);
9671 9775 int64_t v;
9672 9776
9673 9777 agg->dtag_initial = desc->dtad_arg;
9674 9778 agg->dtag_aggregate = dtrace_aggregate_llquantize;
9675 9779
9676 9780 if (factor < 2 || low >= high || nsteps < factor)
9677 9781 goto err;
9678 9782
9679 9783 /*
9680 9784 * Now check that the number of steps evenly divides a power
9681 9785 * of the factor. (This assures both integer bucket size and
9682 9786 * linearity within each magnitude.)
9683 9787 */
9684 9788 for (v = factor; v < nsteps; v *= factor)
9685 9789 continue;
9686 9790
9687 9791 if ((v % nsteps) || (nsteps % factor))
9688 9792 goto err;
9689 9793
9690 9794 size = (dtrace_aggregate_llquantize_bucket(factor,
9691 9795 low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t);
9692 9796 break;
9693 9797 }
9694 9798
9695 9799 case DTRACEAGG_AVG:
9696 9800 agg->dtag_aggregate = dtrace_aggregate_avg;
9697 9801 size = sizeof (uint64_t) * 2;
9698 9802 break;
9699 9803
9700 9804 case DTRACEAGG_STDDEV:
9701 9805 agg->dtag_aggregate = dtrace_aggregate_stddev;
9702 9806 size = sizeof (uint64_t) * 4;
9703 9807 break;
9704 9808
9705 9809 case DTRACEAGG_SUM:
9706 9810 agg->dtag_aggregate = dtrace_aggregate_sum;
9707 9811 break;
9708 9812
9709 9813 default:
9710 9814 goto err;
9711 9815 }
9712 9816
9713 9817 agg->dtag_action.dta_rec.dtrd_size = size;
9714 9818
9715 9819 if (ntuple == 0)
9716 9820 goto err;
9717 9821
9718 9822 /*
9719 9823 * We must make sure that we have enough actions for the n-tuple.
9720 9824 */
9721 9825 for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
9722 9826 if (DTRACEACT_ISAGG(act->dta_kind))
9723 9827 break;
9724 9828
9725 9829 if (--ntuple == 0) {
9726 9830 /*
9727 9831 * This is the action with which our n-tuple begins.
9728 9832 */
9729 9833 agg->dtag_first = act;
9730 9834 goto success;
9731 9835 }
9732 9836 }
9733 9837
9734 9838 /*
9735 9839 * This n-tuple is short by ntuple elements. Return failure.
9736 9840 */
9737 9841 ASSERT(ntuple != 0);
9738 9842 err:
9739 9843 kmem_free(agg, sizeof (dtrace_aggregation_t));
9740 9844 return (NULL);
9741 9845
9742 9846 success:
9743 9847 /*
9744 9848 * If the last action in the tuple has a size of zero, it's actually
9745 9849 * an expression argument for the aggregating action.
9746 9850 */
9747 9851 ASSERT(ecb->dte_action_last != NULL);
9748 9852 act = ecb->dte_action_last;
9749 9853
9750 9854 if (act->dta_kind == DTRACEACT_DIFEXPR) {
9751 9855 ASSERT(act->dta_difo != NULL);
9752 9856
9753 9857 if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
9754 9858 agg->dtag_hasarg = 1;
9755 9859 }
9756 9860
9757 9861 /*
9758 9862 * We need to allocate an id for this aggregation.
9759 9863 */
9760 9864 aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
9761 9865 VM_BESTFIT | VM_SLEEP);
9762 9866
9763 9867 if (aggid - 1 >= state->dts_naggregations) {
9764 9868 dtrace_aggregation_t **oaggs = state->dts_aggregations;
9765 9869 dtrace_aggregation_t **aggs;
9766 9870 int naggs = state->dts_naggregations << 1;
9767 9871 int onaggs = state->dts_naggregations;
9768 9872
9769 9873 ASSERT(aggid == state->dts_naggregations + 1);
9770 9874
9771 9875 if (naggs == 0) {
9772 9876 ASSERT(oaggs == NULL);
9773 9877 naggs = 1;
9774 9878 }
9775 9879
9776 9880 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
9777 9881
9778 9882 if (oaggs != NULL) {
9779 9883 bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
9780 9884 kmem_free(oaggs, onaggs * sizeof (*aggs));
9781 9885 }
9782 9886
9783 9887 state->dts_aggregations = aggs;
9784 9888 state->dts_naggregations = naggs;
9785 9889 }
9786 9890
9787 9891 ASSERT(state->dts_aggregations[aggid - 1] == NULL);
9788 9892 state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
9789 9893
9790 9894 frec = &agg->dtag_first->dta_rec;
9791 9895 if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
9792 9896 frec->dtrd_alignment = sizeof (dtrace_aggid_t);
9793 9897
9794 9898 for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
9795 9899 ASSERT(!act->dta_intuple);
9796 9900 act->dta_intuple = 1;
9797 9901 }
9798 9902
9799 9903 return (&agg->dtag_action);
9800 9904 }
9801 9905
9802 9906 static void
9803 9907 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
9804 9908 {
9805 9909 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
9806 9910 dtrace_state_t *state = ecb->dte_state;
9807 9911 dtrace_aggid_t aggid = agg->dtag_id;
9808 9912
9809 9913 ASSERT(DTRACEACT_ISAGG(act->dta_kind));
9810 9914 vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
9811 9915
9812 9916 ASSERT(state->dts_aggregations[aggid - 1] == agg);
9813 9917 state->dts_aggregations[aggid - 1] = NULL;
9814 9918
9815 9919 kmem_free(agg, sizeof (dtrace_aggregation_t));
9816 9920 }
9817 9921
9818 9922 static int
9819 9923 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
9820 9924 {
9821 9925 dtrace_action_t *action, *last;
9822 9926 dtrace_difo_t *dp = desc->dtad_difo;
9823 9927 uint32_t size = 0, align = sizeof (uint8_t), mask;
9824 9928 uint16_t format = 0;
9825 9929 dtrace_recdesc_t *rec;
9826 9930 dtrace_state_t *state = ecb->dte_state;
9827 9931 dtrace_optval_t *opt = state->dts_options, nframes, strsize;
9828 9932 uint64_t arg = desc->dtad_arg;
9829 9933
9830 9934 ASSERT(MUTEX_HELD(&dtrace_lock));
9831 9935 ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
9832 9936
9833 9937 if (DTRACEACT_ISAGG(desc->dtad_kind)) {
9834 9938 /*
9835 9939 * If this is an aggregating action, there must be neither
9836 9940 * a speculate nor a commit on the action chain.
9837 9941 */
9838 9942 dtrace_action_t *act;
9839 9943
9840 9944 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
9841 9945 if (act->dta_kind == DTRACEACT_COMMIT)
9842 9946 return (EINVAL);
9843 9947
9844 9948 if (act->dta_kind == DTRACEACT_SPECULATE)
9845 9949 return (EINVAL);
9846 9950 }
9847 9951
9848 9952 action = dtrace_ecb_aggregation_create(ecb, desc);
9849 9953
9850 9954 if (action == NULL)
9851 9955 return (EINVAL);
9852 9956 } else {
9853 9957 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
9854 9958 (desc->dtad_kind == DTRACEACT_DIFEXPR &&
9855 9959 dp != NULL && dp->dtdo_destructive)) {
9856 9960 state->dts_destructive = 1;
9857 9961 }
9858 9962
9859 9963 switch (desc->dtad_kind) {
9860 9964 case DTRACEACT_PRINTF:
9861 9965 case DTRACEACT_PRINTA:
9862 9966 case DTRACEACT_SYSTEM:
9863 9967 case DTRACEACT_FREOPEN:
9864 9968 case DTRACEACT_DIFEXPR:
9865 9969 /*
9866 9970 * We know that our arg is a string -- turn it into a
9867 9971 * format.
9868 9972 */
9869 9973 if (arg == NULL) {
9870 9974 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA ||
9871 9975 desc->dtad_kind == DTRACEACT_DIFEXPR);
9872 9976 format = 0;
9873 9977 } else {
9874 9978 ASSERT(arg != NULL);
9875 9979 ASSERT(arg > KERNELBASE);
9876 9980 format = dtrace_format_add(state,
9877 9981 (char *)(uintptr_t)arg);
9878 9982 }
9879 9983
9880 9984 /*FALLTHROUGH*/
9881 9985 case DTRACEACT_LIBACT:
9882 9986 case DTRACEACT_TRACEMEM:
9883 9987 case DTRACEACT_TRACEMEM_DYNSIZE:
9884 9988 if (dp == NULL)
9885 9989 return (EINVAL);
9886 9990
9887 9991 if ((size = dp->dtdo_rtype.dtdt_size) != 0)
9888 9992 break;
9889 9993
9890 9994 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
9891 9995 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9892 9996 return (EINVAL);
9893 9997
9894 9998 size = opt[DTRACEOPT_STRSIZE];
9895 9999 }
9896 10000
9897 10001 break;
9898 10002
9899 10003 case DTRACEACT_STACK:
9900 10004 if ((nframes = arg) == 0) {
9901 10005 nframes = opt[DTRACEOPT_STACKFRAMES];
9902 10006 ASSERT(nframes > 0);
9903 10007 arg = nframes;
9904 10008 }
9905 10009
9906 10010 size = nframes * sizeof (pc_t);
9907 10011 break;
9908 10012
9909 10013 case DTRACEACT_JSTACK:
9910 10014 if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
9911 10015 strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
9912 10016
9913 10017 if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
9914 10018 nframes = opt[DTRACEOPT_JSTACKFRAMES];
9915 10019
9916 10020 arg = DTRACE_USTACK_ARG(nframes, strsize);
9917 10021
9918 10022 /*FALLTHROUGH*/
9919 10023 case DTRACEACT_USTACK:
9920 10024 if (desc->dtad_kind != DTRACEACT_JSTACK &&
9921 10025 (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
9922 10026 strsize = DTRACE_USTACK_STRSIZE(arg);
9923 10027 nframes = opt[DTRACEOPT_USTACKFRAMES];
9924 10028 ASSERT(nframes > 0);
9925 10029 arg = DTRACE_USTACK_ARG(nframes, strsize);
9926 10030 }
9927 10031
9928 10032 /*
9929 10033 * Save a slot for the pid.
9930 10034 */
9931 10035 size = (nframes + 1) * sizeof (uint64_t);
9932 10036 size += DTRACE_USTACK_STRSIZE(arg);
9933 10037 size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
9934 10038
9935 10039 break;
9936 10040
9937 10041 case DTRACEACT_SYM:
9938 10042 case DTRACEACT_MOD:
9939 10043 if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
9940 10044 sizeof (uint64_t)) ||
9941 10045 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9942 10046 return (EINVAL);
9943 10047 break;
9944 10048
9945 10049 case DTRACEACT_USYM:
9946 10050 case DTRACEACT_UMOD:
9947 10051 case DTRACEACT_UADDR:
9948 10052 if (dp == NULL ||
9949 10053 (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
9950 10054 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9951 10055 return (EINVAL);
9952 10056
9953 10057 /*
9954 10058 * We have a slot for the pid, plus a slot for the
9955 10059 * argument. To keep things simple (aligned with
9956 10060 * bitness-neutral sizing), we store each as a 64-bit
9957 10061 * quantity.
9958 10062 */
9959 10063 size = 2 * sizeof (uint64_t);
9960 10064 break;
9961 10065
9962 10066 case DTRACEACT_STOP:
9963 10067 case DTRACEACT_BREAKPOINT:
9964 10068 case DTRACEACT_PANIC:
9965 10069 break;
9966 10070
9967 10071 case DTRACEACT_CHILL:
9968 10072 case DTRACEACT_DISCARD:
9969 10073 case DTRACEACT_RAISE:
9970 10074 if (dp == NULL)
9971 10075 return (EINVAL);
9972 10076 break;
9973 10077
9974 10078 case DTRACEACT_EXIT:
9975 10079 if (dp == NULL ||
9976 10080 (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
9977 10081 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
9978 10082 return (EINVAL);
9979 10083 break;
9980 10084
9981 10085 case DTRACEACT_SPECULATE:
9982 10086 if (ecb->dte_size > sizeof (dtrace_epid_t))
9983 10087 return (EINVAL);
9984 10088
9985 10089 if (dp == NULL)
9986 10090 return (EINVAL);
9987 10091
9988 10092 state->dts_speculates = 1;
9989 10093 break;
9990 10094
9991 10095 case DTRACEACT_COMMIT: {
9992 10096 dtrace_action_t *act = ecb->dte_action;
9993 10097
9994 10098 for (; act != NULL; act = act->dta_next) {
9995 10099 if (act->dta_kind == DTRACEACT_COMMIT)
9996 10100 return (EINVAL);
9997 10101 }
9998 10102
9999 10103 if (dp == NULL)
10000 10104 return (EINVAL);
10001 10105 break;
10002 10106 }
10003 10107
10004 10108 default:
10005 10109 return (EINVAL);
10006 10110 }
10007 10111
10008 10112 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
10009 10113 /*
10010 10114 * If this is a data-storing action or a speculate,
10011 10115 * we must be sure that there isn't a commit on the
10012 10116 * action chain.
10013 10117 */
10014 10118 dtrace_action_t *act = ecb->dte_action;
10015 10119
10016 10120 for (; act != NULL; act = act->dta_next) {
10017 10121 if (act->dta_kind == DTRACEACT_COMMIT)
10018 10122 return (EINVAL);
10019 10123 }
10020 10124 }
10021 10125
10022 10126 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
10023 10127 action->dta_rec.dtrd_size = size;
10024 10128 }
10025 10129
10026 10130 action->dta_refcnt = 1;
10027 10131 rec = &action->dta_rec;
10028 10132 size = rec->dtrd_size;
10029 10133
10030 10134 for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
10031 10135 if (!(size & mask)) {
10032 10136 align = mask + 1;
10033 10137 break;
10034 10138 }
10035 10139 }
10036 10140
10037 10141 action->dta_kind = desc->dtad_kind;
10038 10142
10039 10143 if ((action->dta_difo = dp) != NULL)
10040 10144 dtrace_difo_hold(dp);
10041 10145
10042 10146 rec->dtrd_action = action->dta_kind;
10043 10147 rec->dtrd_arg = arg;
10044 10148 rec->dtrd_uarg = desc->dtad_uarg;
10045 10149 rec->dtrd_alignment = (uint16_t)align;
10046 10150 rec->dtrd_format = format;
10047 10151
10048 10152 if ((last = ecb->dte_action_last) != NULL) {
10049 10153 ASSERT(ecb->dte_action != NULL);
10050 10154 action->dta_prev = last;
10051 10155 last->dta_next = action;
10052 10156 } else {
10053 10157 ASSERT(ecb->dte_action == NULL);
10054 10158 ecb->dte_action = action;
10055 10159 }
10056 10160
10057 10161 ecb->dte_action_last = action;
10058 10162
10059 10163 return (0);
10060 10164 }
10061 10165
10062 10166 static void
10063 10167 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
10064 10168 {
10065 10169 dtrace_action_t *act = ecb->dte_action, *next;
10066 10170 dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
10067 10171 dtrace_difo_t *dp;
10068 10172 uint16_t format;
10069 10173
10070 10174 if (act != NULL && act->dta_refcnt > 1) {
10071 10175 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
10072 10176 act->dta_refcnt--;
10073 10177 } else {
10074 10178 for (; act != NULL; act = next) {
10075 10179 next = act->dta_next;
10076 10180 ASSERT(next != NULL || act == ecb->dte_action_last);
10077 10181 ASSERT(act->dta_refcnt == 1);
10078 10182
10079 10183 if ((format = act->dta_rec.dtrd_format) != 0)
10080 10184 dtrace_format_remove(ecb->dte_state, format);
10081 10185
10082 10186 if ((dp = act->dta_difo) != NULL)
10083 10187 dtrace_difo_release(dp, vstate);
10084 10188
10085 10189 if (DTRACEACT_ISAGG(act->dta_kind)) {
10086 10190 dtrace_ecb_aggregation_destroy(ecb, act);
10087 10191 } else {
10088 10192 kmem_free(act, sizeof (dtrace_action_t));
10089 10193 }
10090 10194 }
10091 10195 }
10092 10196
10093 10197 ecb->dte_action = NULL;
10094 10198 ecb->dte_action_last = NULL;
10095 10199 ecb->dte_size = sizeof (dtrace_epid_t);
10096 10200 }
10097 10201
10098 10202 static void
10099 10203 dtrace_ecb_disable(dtrace_ecb_t *ecb)
10100 10204 {
10101 10205 /*
10102 10206 * We disable the ECB by removing it from its probe.
10103 10207 */
10104 10208 dtrace_ecb_t *pecb, *prev = NULL;
10105 10209 dtrace_probe_t *probe = ecb->dte_probe;
10106 10210
10107 10211 ASSERT(MUTEX_HELD(&dtrace_lock));
10108 10212
10109 10213 if (probe == NULL) {
10110 10214 /*
10111 10215 * This is the NULL probe; there is nothing to disable.
10112 10216 */
10113 10217 return;
10114 10218 }
10115 10219
10116 10220 for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
10117 10221 if (pecb == ecb)
10118 10222 break;
10119 10223 prev = pecb;
10120 10224 }
10121 10225
10122 10226 ASSERT(pecb != NULL);
10123 10227
10124 10228 if (prev == NULL) {
10125 10229 probe->dtpr_ecb = ecb->dte_next;
10126 10230 } else {
10127 10231 prev->dte_next = ecb->dte_next;
10128 10232 }
10129 10233
10130 10234 if (ecb == probe->dtpr_ecb_last) {
10131 10235 ASSERT(ecb->dte_next == NULL);
10132 10236 probe->dtpr_ecb_last = prev;
10133 10237 }
10134 10238
10135 10239 /*
10136 10240 * The ECB has been disconnected from the probe; now sync to assure
10137 10241 * that all CPUs have seen the change before returning.
10138 10242 */
10139 10243 dtrace_sync();
10140 10244
10141 10245 if (probe->dtpr_ecb == NULL) {
10142 10246 /*
10143 10247 * That was the last ECB on the probe; clear the predicate
10144 10248 * cache ID for the probe, disable it and sync one more time
10145 10249 * to assure that we'll never hit it again.
10146 10250 */
10147 10251 dtrace_provider_t *prov = probe->dtpr_provider;
10148 10252
10149 10253 ASSERT(ecb->dte_next == NULL);
10150 10254 ASSERT(probe->dtpr_ecb_last == NULL);
10151 10255 probe->dtpr_predcache = DTRACE_CACHEIDNONE;
10152 10256 prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
10153 10257 probe->dtpr_id, probe->dtpr_arg);
10154 10258 dtrace_sync();
10155 10259 } else {
10156 10260 /*
10157 10261 * There is at least one ECB remaining on the probe. If there
10158 10262 * is _exactly_ one, set the probe's predicate cache ID to be
10159 10263 * the predicate cache ID of the remaining ECB.
10160 10264 */
10161 10265 ASSERT(probe->dtpr_ecb_last != NULL);
10162 10266 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
10163 10267
10164 10268 if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
10165 10269 dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
10166 10270
10167 10271 ASSERT(probe->dtpr_ecb->dte_next == NULL);
10168 10272
10169 10273 if (p != NULL)
10170 10274 probe->dtpr_predcache = p->dtp_cacheid;
10171 10275 }
10172 10276
10173 10277 ecb->dte_next = NULL;
10174 10278 }
10175 10279 }
10176 10280
10177 10281 static void
10178 10282 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
10179 10283 {
10180 10284 dtrace_state_t *state = ecb->dte_state;
10181 10285 dtrace_vstate_t *vstate = &state->dts_vstate;
10182 10286 dtrace_predicate_t *pred;
10183 10287 dtrace_epid_t epid = ecb->dte_epid;
10184 10288
10185 10289 ASSERT(MUTEX_HELD(&dtrace_lock));
10186 10290 ASSERT(ecb->dte_next == NULL);
10187 10291 ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
10188 10292
10189 10293 if ((pred = ecb->dte_predicate) != NULL)
10190 10294 dtrace_predicate_release(pred, vstate);
10191 10295
10192 10296 dtrace_ecb_action_remove(ecb);
10193 10297
10194 10298 ASSERT(state->dts_ecbs[epid - 1] == ecb);
10195 10299 state->dts_ecbs[epid - 1] = NULL;
10196 10300
10197 10301 kmem_free(ecb, sizeof (dtrace_ecb_t));
10198 10302 }
10199 10303
10200 10304 static dtrace_ecb_t *
10201 10305 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
10202 10306 dtrace_enabling_t *enab)
10203 10307 {
10204 10308 dtrace_ecb_t *ecb;
10205 10309 dtrace_predicate_t *pred;
10206 10310 dtrace_actdesc_t *act;
10207 10311 dtrace_provider_t *prov;
10208 10312 dtrace_ecbdesc_t *desc = enab->dten_current;
10209 10313
10210 10314 ASSERT(MUTEX_HELD(&dtrace_lock));
10211 10315 ASSERT(state != NULL);
10212 10316
10213 10317 ecb = dtrace_ecb_add(state, probe);
10214 10318 ecb->dte_uarg = desc->dted_uarg;
10215 10319
10216 10320 if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
10217 10321 dtrace_predicate_hold(pred);
10218 10322 ecb->dte_predicate = pred;
10219 10323 }
10220 10324
10221 10325 if (probe != NULL) {
10222 10326 /*
10223 10327 * If the provider shows more leg than the consumer is old
10224 10328 * enough to see, we need to enable the appropriate implicit
10225 10329 * predicate bits to prevent the ecb from activating at
10226 10330 * revealing times.
10227 10331 *
10228 10332 * Providers specifying DTRACE_PRIV_USER at register time
10229 10333 * are stating that they need the /proc-style privilege
10230 10334 * model to be enforced, and this is what DTRACE_COND_OWNER
10231 10335 * and DTRACE_COND_ZONEOWNER will then do at probe time.
10232 10336 */
10233 10337 prov = probe->dtpr_provider;
10234 10338 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
10235 10339 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10236 10340 ecb->dte_cond |= DTRACE_COND_OWNER;
10237 10341
10238 10342 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
10239 10343 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
10240 10344 ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
10241 10345
10242 10346 /*
10243 10347 * If the provider shows us kernel innards and the user
10244 10348 * is lacking sufficient privilege, enable the
10245 10349 * DTRACE_COND_USERMODE implicit predicate.
10246 10350 */
10247 10351 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
10248 10352 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
10249 10353 ecb->dte_cond |= DTRACE_COND_USERMODE;
10250 10354 }
10251 10355
10252 10356 if (dtrace_ecb_create_cache != NULL) {
10253 10357 /*
10254 10358 * If we have a cached ecb, we'll use its action list instead
10255 10359 * of creating our own (saving both time and space).
10256 10360 */
10257 10361 dtrace_ecb_t *cached = dtrace_ecb_create_cache;
10258 10362 dtrace_action_t *act = cached->dte_action;
10259 10363
10260 10364 if (act != NULL) {
10261 10365 ASSERT(act->dta_refcnt > 0);
10262 10366 act->dta_refcnt++;
10263 10367 ecb->dte_action = act;
10264 10368 ecb->dte_action_last = cached->dte_action_last;
10265 10369 ecb->dte_needed = cached->dte_needed;
10266 10370 ecb->dte_size = cached->dte_size;
10267 10371 ecb->dte_alignment = cached->dte_alignment;
10268 10372 }
10269 10373
10270 10374 return (ecb);
10271 10375 }
10272 10376
10273 10377 for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
10274 10378 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
10275 10379 dtrace_ecb_destroy(ecb);
10276 10380 return (NULL);
10277 10381 }
10278 10382 }
10279 10383
10280 10384 dtrace_ecb_resize(ecb);
10281 10385
10282 10386 return (dtrace_ecb_create_cache = ecb);
10283 10387 }
10284 10388
10285 10389 static int
10286 10390 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
10287 10391 {
10288 10392 dtrace_ecb_t *ecb;
10289 10393 dtrace_enabling_t *enab = arg;
10290 10394 dtrace_state_t *state = enab->dten_vstate->dtvs_state;
10291 10395
10292 10396 ASSERT(state != NULL);
10293 10397
10294 10398 if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
10295 10399 /*
10296 10400 * This probe was created in a generation for which this
10297 10401 * enabling has previously created ECBs; we don't want to
10298 10402 * enable it again, so just kick out.
10299 10403 */
10300 10404 return (DTRACE_MATCH_NEXT);
10301 10405 }
10302 10406
10303 10407 if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
10304 10408 return (DTRACE_MATCH_DONE);
10305 10409
10306 10410 if (dtrace_ecb_enable(ecb) < 0)
10307 10411 return (DTRACE_MATCH_FAIL);
10308 10412
10309 10413 return (DTRACE_MATCH_NEXT);
10310 10414 }
10311 10415
10312 10416 static dtrace_ecb_t *
10313 10417 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
10314 10418 {
10315 10419 dtrace_ecb_t *ecb;
10316 10420
10317 10421 ASSERT(MUTEX_HELD(&dtrace_lock));
10318 10422
10319 10423 if (id == 0 || id > state->dts_necbs)
10320 10424 return (NULL);
10321 10425
10322 10426 ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
10323 10427 ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
10324 10428
10325 10429 return (state->dts_ecbs[id - 1]);
10326 10430 }
10327 10431
10328 10432 static dtrace_aggregation_t *
10329 10433 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
10330 10434 {
10331 10435 dtrace_aggregation_t *agg;
10332 10436
10333 10437 ASSERT(MUTEX_HELD(&dtrace_lock));
10334 10438
10335 10439 if (id == 0 || id > state->dts_naggregations)
10336 10440 return (NULL);
10337 10441
10338 10442 ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
10339 10443 ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
10340 10444 agg->dtag_id == id);
10341 10445
10342 10446 return (state->dts_aggregations[id - 1]);
10343 10447 }
10344 10448
10345 10449 /*
10346 10450 * DTrace Buffer Functions
10347 10451 *
10348 10452 * The following functions manipulate DTrace buffers. Most of these functions
10349 10453 * are called in the context of establishing or processing consumer state;
10350 10454 * exceptions are explicitly noted.
10351 10455 */
10352 10456
10353 10457 /*
10354 10458 * Note: called from cross call context. This function switches the two
10355 10459 * buffers on a given CPU. The atomicity of this operation is assured by
10356 10460 * disabling interrupts while the actual switch takes place; the disabling of
10357 10461 * interrupts serializes the execution with any execution of dtrace_probe() on
10358 10462 * the same CPU.
10359 10463 */
10360 10464 static void
10361 10465 dtrace_buffer_switch(dtrace_buffer_t *buf)
10362 10466 {
10363 10467 caddr_t tomax = buf->dtb_tomax;
10364 10468 caddr_t xamot = buf->dtb_xamot;
10365 10469 dtrace_icookie_t cookie;
10366 10470 hrtime_t now = dtrace_gethrtime();
10367 10471
10368 10472 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
10369 10473 ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
10370 10474
10371 10475 cookie = dtrace_interrupt_disable();
10372 10476 buf->dtb_tomax = xamot;
10373 10477 buf->dtb_xamot = tomax;
10374 10478 buf->dtb_xamot_drops = buf->dtb_drops;
10375 10479 buf->dtb_xamot_offset = buf->dtb_offset;
10376 10480 buf->dtb_xamot_errors = buf->dtb_errors;
10377 10481 buf->dtb_xamot_flags = buf->dtb_flags;
10378 10482 buf->dtb_offset = 0;
10379 10483 buf->dtb_drops = 0;
10380 10484 buf->dtb_errors = 0;
10381 10485 buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
10382 10486 buf->dtb_interval = now - buf->dtb_switched;
10383 10487 buf->dtb_switched = now;
10384 10488 dtrace_interrupt_enable(cookie);
10385 10489 }
10386 10490
10387 10491 /*
10388 10492 * Note: called from cross call context. This function activates a buffer
10389 10493 * on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation
10390 10494 * is guaranteed by the disabling of interrupts.
10391 10495 */
10392 10496 static void
10393 10497 dtrace_buffer_activate(dtrace_state_t *state)
10394 10498 {
10395 10499 dtrace_buffer_t *buf;
10396 10500 dtrace_icookie_t cookie = dtrace_interrupt_disable();
10397 10501
10398 10502 buf = &state->dts_buffer[CPU->cpu_id];
10399 10503
10400 10504 if (buf->dtb_tomax != NULL) {
10401 10505 /*
10402 10506 * We might like to assert that the buffer is marked inactive,
10403 10507 * but this isn't necessarily true: the buffer for the CPU
10404 10508 * that processes the BEGIN probe has its buffer activated
10405 10509 * manually. In this case, we take the (harmless) action
10406 10510 * re-clearing the bit INACTIVE bit.
10407 10511 */
10408 10512 buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
10409 10513 }
10410 10514
10411 10515 dtrace_interrupt_enable(cookie);
10412 10516 }
10413 10517
10414 10518 static int
10415 10519 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
10416 10520 processorid_t cpu, int *factor)
10417 10521 {
10418 10522 cpu_t *cp;
10419 10523 dtrace_buffer_t *buf;
10420 10524 int allocated = 0, desired = 0;
10421 10525
10422 10526 ASSERT(MUTEX_HELD(&cpu_lock));
10423 10527 ASSERT(MUTEX_HELD(&dtrace_lock));
10424 10528
10425 10529 *factor = 1;
10426 10530
10427 10531 if (size > dtrace_nonroot_maxsize &&
10428 10532 !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
10429 10533 return (EFBIG);
10430 10534
10431 10535 cp = cpu_list;
10432 10536
10433 10537 do {
10434 10538 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
10435 10539 continue;
10436 10540
10437 10541 buf = &bufs[cp->cpu_id];
10438 10542
10439 10543 /*
10440 10544 * If there is already a buffer allocated for this CPU, it
10441 10545 * is only possible that this is a DR event. In this case,
10442 10546 * the buffer size must match our specified size.
10443 10547 */
10444 10548 if (buf->dtb_tomax != NULL) {
10445 10549 ASSERT(buf->dtb_size == size);
10446 10550 continue;
10447 10551 }
10448 10552
10449 10553 ASSERT(buf->dtb_xamot == NULL);
10450 10554
10451 10555 if ((buf->dtb_tomax = kmem_zalloc(size,
10452 10556 KM_NOSLEEP | KM_NORMALPRI)) == NULL)
10453 10557 goto err;
10454 10558
10455 10559 buf->dtb_size = size;
10456 10560 buf->dtb_flags = flags;
10457 10561 buf->dtb_offset = 0;
10458 10562 buf->dtb_drops = 0;
10459 10563
10460 10564 if (flags & DTRACEBUF_NOSWITCH)
10461 10565 continue;
10462 10566
10463 10567 if ((buf->dtb_xamot = kmem_zalloc(size,
10464 10568 KM_NOSLEEP | KM_NORMALPRI)) == NULL)
10465 10569 goto err;
10466 10570 } while ((cp = cp->cpu_next) != cpu_list);
10467 10571
10468 10572 return (0);
10469 10573
10470 10574 err:
10471 10575 cp = cpu_list;
10472 10576
10473 10577 do {
10474 10578 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
10475 10579 continue;
10476 10580
10477 10581 buf = &bufs[cp->cpu_id];
10478 10582 desired += 2;
10479 10583
10480 10584 if (buf->dtb_xamot != NULL) {
10481 10585 ASSERT(buf->dtb_tomax != NULL);
10482 10586 ASSERT(buf->dtb_size == size);
10483 10587 kmem_free(buf->dtb_xamot, size);
10484 10588 allocated++;
10485 10589 }
10486 10590
10487 10591 if (buf->dtb_tomax != NULL) {
10488 10592 ASSERT(buf->dtb_size == size);
10489 10593 kmem_free(buf->dtb_tomax, size);
10490 10594 allocated++;
10491 10595 }
10492 10596
10493 10597 buf->dtb_tomax = NULL;
10494 10598 buf->dtb_xamot = NULL;
10495 10599 buf->dtb_size = 0;
10496 10600 } while ((cp = cp->cpu_next) != cpu_list);
10497 10601
10498 10602 *factor = desired / (allocated > 0 ? allocated : 1);
10499 10603
10500 10604 return (ENOMEM);
10501 10605 }
10502 10606
10503 10607 /*
10504 10608 * Note: called from probe context. This function just increments the drop
10505 10609 * count on a buffer. It has been made a function to allow for the
10506 10610 * possibility of understanding the source of mysterious drop counts. (A
10507 10611 * problem for which one may be particularly disappointed that DTrace cannot
10508 10612 * be used to understand DTrace.)
10509 10613 */
10510 10614 static void
10511 10615 dtrace_buffer_drop(dtrace_buffer_t *buf)
10512 10616 {
10513 10617 buf->dtb_drops++;
10514 10618 }
10515 10619
10516 10620 /*
10517 10621 * Note: called from probe context. This function is called to reserve space
10518 10622 * in a buffer. If mstate is non-NULL, sets the scratch base and size in the
10519 10623 * mstate. Returns the new offset in the buffer, or a negative value if an
10520 10624 * error has occurred.
10521 10625 */
10522 10626 static intptr_t
10523 10627 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
10524 10628 dtrace_state_t *state, dtrace_mstate_t *mstate)
10525 10629 {
10526 10630 intptr_t offs = buf->dtb_offset, soffs;
10527 10631 intptr_t woffs;
10528 10632 caddr_t tomax;
10529 10633 size_t total;
10530 10634
10531 10635 if (buf->dtb_flags & DTRACEBUF_INACTIVE)
10532 10636 return (-1);
10533 10637
10534 10638 if ((tomax = buf->dtb_tomax) == NULL) {
10535 10639 dtrace_buffer_drop(buf);
10536 10640 return (-1);
10537 10641 }
10538 10642
10539 10643 if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
10540 10644 while (offs & (align - 1)) {
10541 10645 /*
10542 10646 * Assert that our alignment is off by a number which
10543 10647 * is itself sizeof (uint32_t) aligned.
10544 10648 */
10545 10649 ASSERT(!((align - (offs & (align - 1))) &
10546 10650 (sizeof (uint32_t) - 1)));
10547 10651 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
10548 10652 offs += sizeof (uint32_t);
10549 10653 }
10550 10654
10551 10655 if ((soffs = offs + needed) > buf->dtb_size) {
10552 10656 dtrace_buffer_drop(buf);
10553 10657 return (-1);
10554 10658 }
10555 10659
10556 10660 if (mstate == NULL)
10557 10661 return (offs);
10558 10662
10559 10663 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
10560 10664 mstate->dtms_scratch_size = buf->dtb_size - soffs;
10561 10665 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
10562 10666
10563 10667 return (offs);
10564 10668 }
10565 10669
10566 10670 if (buf->dtb_flags & DTRACEBUF_FILL) {
10567 10671 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
10568 10672 (buf->dtb_flags & DTRACEBUF_FULL))
10569 10673 return (-1);
10570 10674 goto out;
10571 10675 }
10572 10676
10573 10677 total = needed + (offs & (align - 1));
10574 10678
10575 10679 /*
10576 10680 * For a ring buffer, life is quite a bit more complicated. Before
10577 10681 * we can store any padding, we need to adjust our wrapping offset.
10578 10682 * (If we've never before wrapped or we're not about to, no adjustment
10579 10683 * is required.)
10580 10684 */
10581 10685 if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
10582 10686 offs + total > buf->dtb_size) {
10583 10687 woffs = buf->dtb_xamot_offset;
10584 10688
10585 10689 if (offs + total > buf->dtb_size) {
10586 10690 /*
10587 10691 * We can't fit in the end of the buffer. First, a
10588 10692 * sanity check that we can fit in the buffer at all.
10589 10693 */
10590 10694 if (total > buf->dtb_size) {
10591 10695 dtrace_buffer_drop(buf);
10592 10696 return (-1);
10593 10697 }
10594 10698
10595 10699 /*
10596 10700 * We're going to be storing at the top of the buffer,
10597 10701 * so now we need to deal with the wrapped offset. We
10598 10702 * only reset our wrapped offset to 0 if it is
10599 10703 * currently greater than the current offset. If it
10600 10704 * is less than the current offset, it is because a
10601 10705 * previous allocation induced a wrap -- but the
10602 10706 * allocation didn't subsequently take the space due
10603 10707 * to an error or false predicate evaluation. In this
10604 10708 * case, we'll just leave the wrapped offset alone: if
10605 10709 * the wrapped offset hasn't been advanced far enough
10606 10710 * for this allocation, it will be adjusted in the
10607 10711 * lower loop.
10608 10712 */
10609 10713 if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
10610 10714 if (woffs >= offs)
10611 10715 woffs = 0;
10612 10716 } else {
10613 10717 woffs = 0;
10614 10718 }
10615 10719
10616 10720 /*
10617 10721 * Now we know that we're going to be storing to the
10618 10722 * top of the buffer and that there is room for us
10619 10723 * there. We need to clear the buffer from the current
10620 10724 * offset to the end (there may be old gunk there).
10621 10725 */
10622 10726 while (offs < buf->dtb_size)
10623 10727 tomax[offs++] = 0;
10624 10728
10625 10729 /*
10626 10730 * We need to set our offset to zero. And because we
10627 10731 * are wrapping, we need to set the bit indicating as
10628 10732 * much. We can also adjust our needed space back
10629 10733 * down to the space required by the ECB -- we know
10630 10734 * that the top of the buffer is aligned.
10631 10735 */
10632 10736 offs = 0;
10633 10737 total = needed;
10634 10738 buf->dtb_flags |= DTRACEBUF_WRAPPED;
10635 10739 } else {
10636 10740 /*
10637 10741 * There is room for us in the buffer, so we simply
10638 10742 * need to check the wrapped offset.
10639 10743 */
10640 10744 if (woffs < offs) {
10641 10745 /*
10642 10746 * The wrapped offset is less than the offset.
10643 10747 * This can happen if we allocated buffer space
10644 10748 * that induced a wrap, but then we didn't
10645 10749 * subsequently take the space due to an error
10646 10750 * or false predicate evaluation. This is
10647 10751 * okay; we know that _this_ allocation isn't
10648 10752 * going to induce a wrap. We still can't
10649 10753 * reset the wrapped offset to be zero,
10650 10754 * however: the space may have been trashed in
10651 10755 * the previous failed probe attempt. But at
10652 10756 * least the wrapped offset doesn't need to
10653 10757 * be adjusted at all...
10654 10758 */
10655 10759 goto out;
10656 10760 }
10657 10761 }
10658 10762
10659 10763 while (offs + total > woffs) {
10660 10764 dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
10661 10765 size_t size;
10662 10766
10663 10767 if (epid == DTRACE_EPIDNONE) {
10664 10768 size = sizeof (uint32_t);
10665 10769 } else {
10666 10770 ASSERT(epid <= state->dts_necbs);
10667 10771 ASSERT(state->dts_ecbs[epid - 1] != NULL);
10668 10772
10669 10773 size = state->dts_ecbs[epid - 1]->dte_size;
10670 10774 }
10671 10775
10672 10776 ASSERT(woffs + size <= buf->dtb_size);
10673 10777 ASSERT(size != 0);
10674 10778
10675 10779 if (woffs + size == buf->dtb_size) {
10676 10780 /*
10677 10781 * We've reached the end of the buffer; we want
10678 10782 * to set the wrapped offset to 0 and break
10679 10783 * out. However, if the offs is 0, then we're
10680 10784 * in a strange edge-condition: the amount of
10681 10785 * space that we want to reserve plus the size
10682 10786 * of the record that we're overwriting is
10683 10787 * greater than the size of the buffer. This
10684 10788 * is problematic because if we reserve the
10685 10789 * space but subsequently don't consume it (due
10686 10790 * to a failed predicate or error) the wrapped
10687 10791 * offset will be 0 -- yet the EPID at offset 0
10688 10792 * will not be committed. This situation is
10689 10793 * relatively easy to deal with: if we're in
10690 10794 * this case, the buffer is indistinguishable
10691 10795 * from one that hasn't wrapped; we need only
10692 10796 * finish the job by clearing the wrapped bit,
10693 10797 * explicitly setting the offset to be 0, and
10694 10798 * zero'ing out the old data in the buffer.
10695 10799 */
10696 10800 if (offs == 0) {
10697 10801 buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
10698 10802 buf->dtb_offset = 0;
10699 10803 woffs = total;
10700 10804
10701 10805 while (woffs < buf->dtb_size)
10702 10806 tomax[woffs++] = 0;
10703 10807 }
10704 10808
10705 10809 woffs = 0;
10706 10810 break;
10707 10811 }
10708 10812
10709 10813 woffs += size;
10710 10814 }
10711 10815
10712 10816 /*
10713 10817 * We have a wrapped offset. It may be that the wrapped offset
10714 10818 * has become zero -- that's okay.
10715 10819 */
10716 10820 buf->dtb_xamot_offset = woffs;
10717 10821 }
10718 10822
10719 10823 out:
10720 10824 /*
10721 10825 * Now we can plow the buffer with any necessary padding.
10722 10826 */
10723 10827 while (offs & (align - 1)) {
10724 10828 /*
10725 10829 * Assert that our alignment is off by a number which
10726 10830 * is itself sizeof (uint32_t) aligned.
10727 10831 */
10728 10832 ASSERT(!((align - (offs & (align - 1))) &
10729 10833 (sizeof (uint32_t) - 1)));
10730 10834 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
10731 10835 offs += sizeof (uint32_t);
10732 10836 }
10733 10837
10734 10838 if (buf->dtb_flags & DTRACEBUF_FILL) {
10735 10839 if (offs + needed > buf->dtb_size - state->dts_reserve) {
10736 10840 buf->dtb_flags |= DTRACEBUF_FULL;
10737 10841 return (-1);
10738 10842 }
10739 10843 }
10740 10844
10741 10845 if (mstate == NULL)
10742 10846 return (offs);
10743 10847
10744 10848 /*
10745 10849 * For ring buffers and fill buffers, the scratch space is always
10746 10850 * the inactive buffer.
10747 10851 */
10748 10852 mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
10749 10853 mstate->dtms_scratch_size = buf->dtb_size;
10750 10854 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
10751 10855
10752 10856 return (offs);
10753 10857 }
10754 10858
10755 10859 static void
10756 10860 dtrace_buffer_polish(dtrace_buffer_t *buf)
10757 10861 {
10758 10862 ASSERT(buf->dtb_flags & DTRACEBUF_RING);
10759 10863 ASSERT(MUTEX_HELD(&dtrace_lock));
10760 10864
10761 10865 if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
10762 10866 return;
10763 10867
10764 10868 /*
10765 10869 * We need to polish the ring buffer. There are three cases:
10766 10870 *
10767 10871 * - The first (and presumably most common) is that there is no gap
10768 10872 * between the buffer offset and the wrapped offset. In this case,
10769 10873 * there is nothing in the buffer that isn't valid data; we can
10770 10874 * mark the buffer as polished and return.
10771 10875 *
10772 10876 * - The second (less common than the first but still more common
10773 10877 * than the third) is that there is a gap between the buffer offset
10774 10878 * and the wrapped offset, and the wrapped offset is larger than the
10775 10879 * buffer offset. This can happen because of an alignment issue, or
10776 10880 * can happen because of a call to dtrace_buffer_reserve() that
10777 10881 * didn't subsequently consume the buffer space. In this case,
10778 10882 * we need to zero the data from the buffer offset to the wrapped
10779 10883 * offset.
10780 10884 *
10781 10885 * - The third (and least common) is that there is a gap between the
10782 10886 * buffer offset and the wrapped offset, but the wrapped offset is
10783 10887 * _less_ than the buffer offset. This can only happen because a
10784 10888 * call to dtrace_buffer_reserve() induced a wrap, but the space
10785 10889 * was not subsequently consumed. In this case, we need to zero the
10786 10890 * space from the offset to the end of the buffer _and_ from the
10787 10891 * top of the buffer to the wrapped offset.
10788 10892 */
10789 10893 if (buf->dtb_offset < buf->dtb_xamot_offset) {
10790 10894 bzero(buf->dtb_tomax + buf->dtb_offset,
10791 10895 buf->dtb_xamot_offset - buf->dtb_offset);
10792 10896 }
10793 10897
10794 10898 if (buf->dtb_offset > buf->dtb_xamot_offset) {
10795 10899 bzero(buf->dtb_tomax + buf->dtb_offset,
10796 10900 buf->dtb_size - buf->dtb_offset);
10797 10901 bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
10798 10902 }
10799 10903 }
10800 10904
10801 10905 /*
10802 10906 * This routine determines if data generated at the specified time has likely
10803 10907 * been entirely consumed at user-level. This routine is called to determine
10804 10908 * if an ECB on a defunct probe (but for an active enabling) can be safely
10805 10909 * disabled and destroyed.
10806 10910 */
10807 10911 static int
10808 10912 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when)
10809 10913 {
10810 10914 int i;
10811 10915
10812 10916 for (i = 0; i < NCPU; i++) {
10813 10917 dtrace_buffer_t *buf = &bufs[i];
10814 10918
10815 10919 if (buf->dtb_size == 0)
10816 10920 continue;
10817 10921
10818 10922 if (buf->dtb_flags & DTRACEBUF_RING)
10819 10923 return (0);
10820 10924
10821 10925 if (!buf->dtb_switched && buf->dtb_offset != 0)
10822 10926 return (0);
10823 10927
10824 10928 if (buf->dtb_switched - buf->dtb_interval < when)
10825 10929 return (0);
10826 10930 }
10827 10931
10828 10932 return (1);
10829 10933 }
10830 10934
10831 10935 static void
10832 10936 dtrace_buffer_free(dtrace_buffer_t *bufs)
10833 10937 {
10834 10938 int i;
10835 10939
10836 10940 for (i = 0; i < NCPU; i++) {
10837 10941 dtrace_buffer_t *buf = &bufs[i];
10838 10942
10839 10943 if (buf->dtb_tomax == NULL) {
10840 10944 ASSERT(buf->dtb_xamot == NULL);
10841 10945 ASSERT(buf->dtb_size == 0);
10842 10946 continue;
10843 10947 }
10844 10948
10845 10949 if (buf->dtb_xamot != NULL) {
10846 10950 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
10847 10951 kmem_free(buf->dtb_xamot, buf->dtb_size);
10848 10952 }
10849 10953
10850 10954 kmem_free(buf->dtb_tomax, buf->dtb_size);
10851 10955 buf->dtb_size = 0;
10852 10956 buf->dtb_tomax = NULL;
10853 10957 buf->dtb_xamot = NULL;
10854 10958 }
10855 10959 }
10856 10960
10857 10961 /*
10858 10962 * DTrace Enabling Functions
10859 10963 */
10860 10964 static dtrace_enabling_t *
10861 10965 dtrace_enabling_create(dtrace_vstate_t *vstate)
10862 10966 {
10863 10967 dtrace_enabling_t *enab;
10864 10968
10865 10969 enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
10866 10970 enab->dten_vstate = vstate;
10867 10971
10868 10972 return (enab);
10869 10973 }
10870 10974
10871 10975 static void
10872 10976 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
10873 10977 {
10874 10978 dtrace_ecbdesc_t **ndesc;
10875 10979 size_t osize, nsize;
10876 10980
10877 10981 /*
10878 10982 * We can't add to enablings after we've enabled them, or after we've
10879 10983 * retained them.
10880 10984 */
10881 10985 ASSERT(enab->dten_probegen == 0);
10882 10986 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
10883 10987
10884 10988 if (enab->dten_ndesc < enab->dten_maxdesc) {
10885 10989 enab->dten_desc[enab->dten_ndesc++] = ecb;
10886 10990 return;
10887 10991 }
10888 10992
10889 10993 osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
10890 10994
10891 10995 if (enab->dten_maxdesc == 0) {
10892 10996 enab->dten_maxdesc = 1;
10893 10997 } else {
10894 10998 enab->dten_maxdesc <<= 1;
10895 10999 }
10896 11000
10897 11001 ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
10898 11002
10899 11003 nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
10900 11004 ndesc = kmem_zalloc(nsize, KM_SLEEP);
10901 11005 bcopy(enab->dten_desc, ndesc, osize);
10902 11006 kmem_free(enab->dten_desc, osize);
10903 11007
10904 11008 enab->dten_desc = ndesc;
10905 11009 enab->dten_desc[enab->dten_ndesc++] = ecb;
10906 11010 }
10907 11011
10908 11012 static void
10909 11013 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
10910 11014 dtrace_probedesc_t *pd)
10911 11015 {
10912 11016 dtrace_ecbdesc_t *new;
10913 11017 dtrace_predicate_t *pred;
10914 11018 dtrace_actdesc_t *act;
10915 11019
10916 11020 /*
10917 11021 * We're going to create a new ECB description that matches the
10918 11022 * specified ECB in every way, but has the specified probe description.
10919 11023 */
10920 11024 new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
10921 11025
10922 11026 if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
10923 11027 dtrace_predicate_hold(pred);
10924 11028
10925 11029 for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
10926 11030 dtrace_actdesc_hold(act);
10927 11031
10928 11032 new->dted_action = ecb->dted_action;
10929 11033 new->dted_pred = ecb->dted_pred;
10930 11034 new->dted_probe = *pd;
10931 11035 new->dted_uarg = ecb->dted_uarg;
10932 11036
10933 11037 dtrace_enabling_add(enab, new);
10934 11038 }
10935 11039
10936 11040 static void
10937 11041 dtrace_enabling_dump(dtrace_enabling_t *enab)
10938 11042 {
10939 11043 int i;
10940 11044
10941 11045 for (i = 0; i < enab->dten_ndesc; i++) {
10942 11046 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
10943 11047
10944 11048 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
10945 11049 desc->dtpd_provider, desc->dtpd_mod,
10946 11050 desc->dtpd_func, desc->dtpd_name);
10947 11051 }
10948 11052 }
10949 11053
10950 11054 static void
10951 11055 dtrace_enabling_destroy(dtrace_enabling_t *enab)
10952 11056 {
10953 11057 int i;
10954 11058 dtrace_ecbdesc_t *ep;
10955 11059 dtrace_vstate_t *vstate = enab->dten_vstate;
10956 11060
10957 11061 ASSERT(MUTEX_HELD(&dtrace_lock));
10958 11062
10959 11063 for (i = 0; i < enab->dten_ndesc; i++) {
10960 11064 dtrace_actdesc_t *act, *next;
10961 11065 dtrace_predicate_t *pred;
10962 11066
10963 11067 ep = enab->dten_desc[i];
10964 11068
10965 11069 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
10966 11070 dtrace_predicate_release(pred, vstate);
10967 11071
10968 11072 for (act = ep->dted_action; act != NULL; act = next) {
10969 11073 next = act->dtad_next;
10970 11074 dtrace_actdesc_release(act, vstate);
10971 11075 }
10972 11076
10973 11077 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
10974 11078 }
10975 11079
10976 11080 kmem_free(enab->dten_desc,
10977 11081 enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
10978 11082
10979 11083 /*
10980 11084 * If this was a retained enabling, decrement the dts_nretained count
10981 11085 * and take it off of the dtrace_retained list.
10982 11086 */
10983 11087 if (enab->dten_prev != NULL || enab->dten_next != NULL ||
10984 11088 dtrace_retained == enab) {
10985 11089 ASSERT(enab->dten_vstate->dtvs_state != NULL);
10986 11090 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
10987 11091 enab->dten_vstate->dtvs_state->dts_nretained--;
10988 11092 dtrace_retained_gen++;
10989 11093 }
10990 11094
10991 11095 if (enab->dten_prev == NULL) {
10992 11096 if (dtrace_retained == enab) {
10993 11097 dtrace_retained = enab->dten_next;
10994 11098
10995 11099 if (dtrace_retained != NULL)
10996 11100 dtrace_retained->dten_prev = NULL;
10997 11101 }
10998 11102 } else {
10999 11103 ASSERT(enab != dtrace_retained);
11000 11104 ASSERT(dtrace_retained != NULL);
11001 11105 enab->dten_prev->dten_next = enab->dten_next;
11002 11106 }
11003 11107
11004 11108 if (enab->dten_next != NULL) {
11005 11109 ASSERT(dtrace_retained != NULL);
11006 11110 enab->dten_next->dten_prev = enab->dten_prev;
11007 11111 }
11008 11112
11009 11113 kmem_free(enab, sizeof (dtrace_enabling_t));
11010 11114 }
11011 11115
11012 11116 static int
11013 11117 dtrace_enabling_retain(dtrace_enabling_t *enab)
11014 11118 {
11015 11119 dtrace_state_t *state;
11016 11120
11017 11121 ASSERT(MUTEX_HELD(&dtrace_lock));
11018 11122 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
11019 11123 ASSERT(enab->dten_vstate != NULL);
11020 11124
11021 11125 state = enab->dten_vstate->dtvs_state;
11022 11126 ASSERT(state != NULL);
11023 11127
11024 11128 /*
11025 11129 * We only allow each state to retain dtrace_retain_max enablings.
11026 11130 */
11027 11131 if (state->dts_nretained >= dtrace_retain_max)
11028 11132 return (ENOSPC);
11029 11133
11030 11134 state->dts_nretained++;
11031 11135 dtrace_retained_gen++;
11032 11136
11033 11137 if (dtrace_retained == NULL) {
11034 11138 dtrace_retained = enab;
11035 11139 return (0);
11036 11140 }
11037 11141
11038 11142 enab->dten_next = dtrace_retained;
11039 11143 dtrace_retained->dten_prev = enab;
11040 11144 dtrace_retained = enab;
11041 11145
11042 11146 return (0);
11043 11147 }
11044 11148
11045 11149 static int
11046 11150 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
11047 11151 dtrace_probedesc_t *create)
11048 11152 {
11049 11153 dtrace_enabling_t *new, *enab;
11050 11154 int found = 0, err = ENOENT;
11051 11155
11052 11156 ASSERT(MUTEX_HELD(&dtrace_lock));
11053 11157 ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
11054 11158 ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
11055 11159 ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
11056 11160 ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
11057 11161
11058 11162 new = dtrace_enabling_create(&state->dts_vstate);
11059 11163
11060 11164 /*
11061 11165 * Iterate over all retained enablings, looking for enablings that
11062 11166 * match the specified state.
11063 11167 */
11064 11168 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11065 11169 int i;
11066 11170
11067 11171 /*
11068 11172 * dtvs_state can only be NULL for helper enablings -- and
11069 11173 * helper enablings can't be retained.
11070 11174 */
11071 11175 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11072 11176
11073 11177 if (enab->dten_vstate->dtvs_state != state)
11074 11178 continue;
11075 11179
11076 11180 /*
11077 11181 * Now iterate over each probe description; we're looking for
11078 11182 * an exact match to the specified probe description.
11079 11183 */
11080 11184 for (i = 0; i < enab->dten_ndesc; i++) {
11081 11185 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11082 11186 dtrace_probedesc_t *pd = &ep->dted_probe;
11083 11187
11084 11188 if (strcmp(pd->dtpd_provider, match->dtpd_provider))
11085 11189 continue;
11086 11190
11087 11191 if (strcmp(pd->dtpd_mod, match->dtpd_mod))
11088 11192 continue;
11089 11193
11090 11194 if (strcmp(pd->dtpd_func, match->dtpd_func))
11091 11195 continue;
11092 11196
11093 11197 if (strcmp(pd->dtpd_name, match->dtpd_name))
11094 11198 continue;
11095 11199
11096 11200 /*
11097 11201 * We have a winning probe! Add it to our growing
11098 11202 * enabling.
11099 11203 */
11100 11204 found = 1;
11101 11205 dtrace_enabling_addlike(new, ep, create);
11102 11206 }
11103 11207 }
11104 11208
11105 11209 if (!found || (err = dtrace_enabling_retain(new)) != 0) {
11106 11210 dtrace_enabling_destroy(new);
11107 11211 return (err);
11108 11212 }
11109 11213
11110 11214 return (0);
11111 11215 }
11112 11216
11113 11217 static void
11114 11218 dtrace_enabling_retract(dtrace_state_t *state)
11115 11219 {
11116 11220 dtrace_enabling_t *enab, *next;
11117 11221
11118 11222 ASSERT(MUTEX_HELD(&dtrace_lock));
11119 11223
11120 11224 /*
11121 11225 * Iterate over all retained enablings, destroy the enablings retained
11122 11226 * for the specified state.
11123 11227 */
11124 11228 for (enab = dtrace_retained; enab != NULL; enab = next) {
11125 11229 next = enab->dten_next;
11126 11230
11127 11231 /*
11128 11232 * dtvs_state can only be NULL for helper enablings -- and
11129 11233 * helper enablings can't be retained.
11130 11234 */
11131 11235 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11132 11236
11133 11237 if (enab->dten_vstate->dtvs_state == state) {
11134 11238 ASSERT(state->dts_nretained > 0);
11135 11239 dtrace_enabling_destroy(enab);
11136 11240 }
11137 11241 }
11138 11242
11139 11243 ASSERT(state->dts_nretained == 0);
11140 11244 }
11141 11245
11142 11246 static int
11143 11247 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
11144 11248 {
11145 11249 int i = 0;
11146 11250 int total_matched = 0, matched = 0;
11147 11251
11148 11252 ASSERT(MUTEX_HELD(&cpu_lock));
11149 11253 ASSERT(MUTEX_HELD(&dtrace_lock));
11150 11254
11151 11255 for (i = 0; i < enab->dten_ndesc; i++) {
11152 11256 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
11153 11257
11154 11258 enab->dten_current = ep;
11155 11259 enab->dten_error = 0;
11156 11260
11157 11261 /*
11158 11262 * If a provider failed to enable a probe then get out and
11159 11263 * let the consumer know we failed.
11160 11264 */
11161 11265 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
11162 11266 return (EBUSY);
11163 11267
11164 11268 total_matched += matched;
11165 11269
11166 11270 if (enab->dten_error != 0) {
11167 11271 /*
11168 11272 * If we get an error half-way through enabling the
11169 11273 * probes, we kick out -- perhaps with some number of
11170 11274 * them enabled. Leaving enabled probes enabled may
11171 11275 * be slightly confusing for user-level, but we expect
11172 11276 * that no one will attempt to actually drive on in
11173 11277 * the face of such errors. If this is an anonymous
11174 11278 * enabling (indicated with a NULL nmatched pointer),
11175 11279 * we cmn_err() a message. We aren't expecting to
11176 11280 * get such an error -- such as it can exist at all,
11177 11281 * it would be a result of corrupted DOF in the driver
11178 11282 * properties.
11179 11283 */
11180 11284 if (nmatched == NULL) {
11181 11285 cmn_err(CE_WARN, "dtrace_enabling_match() "
11182 11286 "error on %p: %d", (void *)ep,
11183 11287 enab->dten_error);
11184 11288 }
11185 11289
11186 11290 return (enab->dten_error);
11187 11291 }
11188 11292 }
11189 11293
11190 11294 enab->dten_probegen = dtrace_probegen;
11191 11295 if (nmatched != NULL)
11192 11296 *nmatched = total_matched;
11193 11297
11194 11298 return (0);
11195 11299 }
11196 11300
11197 11301 static void
11198 11302 dtrace_enabling_matchall(void)
11199 11303 {
11200 11304 dtrace_enabling_t *enab;
11201 11305
11202 11306 mutex_enter(&cpu_lock);
11203 11307 mutex_enter(&dtrace_lock);
11204 11308
11205 11309 /*
11206 11310 * Iterate over all retained enablings to see if any probes match
11207 11311 * against them. We only perform this operation on enablings for which
11208 11312 * we have sufficient permissions by virtue of being in the global zone
11209 11313 * or in the same zone as the DTrace client. Because we can be called
11210 11314 * after dtrace_detach() has been called, we cannot assert that there
11211 11315 * are retained enablings. We can safely load from dtrace_retained,
11212 11316 * however: the taskq_destroy() at the end of dtrace_detach() will
11213 11317 * block pending our completion.
11214 11318 */
11215 11319 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11216 11320 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred;
11217 11321 cred_t *cr = dcr->dcr_cred;
11218 11322 zoneid_t zone = cr != NULL ? crgetzoneid(cr) : 0;
11219 11323
11220 11324 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL &&
11221 11325 (zone == GLOBAL_ZONEID || getzoneid() == zone)))
11222 11326 (void) dtrace_enabling_match(enab, NULL);
11223 11327 }
11224 11328
11225 11329 mutex_exit(&dtrace_lock);
11226 11330 mutex_exit(&cpu_lock);
11227 11331 }
11228 11332
11229 11333 /*
11230 11334 * If an enabling is to be enabled without having matched probes (that is, if
11231 11335 * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
11232 11336 * enabling must be _primed_ by creating an ECB for every ECB description.
11233 11337 * This must be done to assure that we know the number of speculations, the
11234 11338 * number of aggregations, the minimum buffer size needed, etc. before we
11235 11339 * transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually
11236 11340 * enabling any probes, we create ECBs for every ECB decription, but with a
11237 11341 * NULL probe -- which is exactly what this function does.
11238 11342 */
11239 11343 static void
11240 11344 dtrace_enabling_prime(dtrace_state_t *state)
11241 11345 {
11242 11346 dtrace_enabling_t *enab;
11243 11347 int i;
11244 11348
11245 11349 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
11246 11350 ASSERT(enab->dten_vstate->dtvs_state != NULL);
11247 11351
11248 11352 if (enab->dten_vstate->dtvs_state != state)
11249 11353 continue;
11250 11354
11251 11355 /*
11252 11356 * We don't want to prime an enabling more than once, lest
11253 11357 * we allow a malicious user to induce resource exhaustion.
11254 11358 * (The ECBs that result from priming an enabling aren't
11255 11359 * leaked -- but they also aren't deallocated until the
11256 11360 * consumer state is destroyed.)
11257 11361 */
11258 11362 if (enab->dten_primed)
11259 11363 continue;
11260 11364
11261 11365 for (i = 0; i < enab->dten_ndesc; i++) {
11262 11366 enab->dten_current = enab->dten_desc[i];
11263 11367 (void) dtrace_probe_enable(NULL, enab);
11264 11368 }
11265 11369
11266 11370 enab->dten_primed = 1;
11267 11371 }
11268 11372 }
11269 11373
11270 11374 /*
11271 11375 * Called to indicate that probes should be provided due to retained
11272 11376 * enablings. This is implemented in terms of dtrace_probe_provide(), but it
11273 11377 * must take an initial lap through the enabling calling the dtps_provide()
11274 11378 * entry point explicitly to allow for autocreated probes.
11275 11379 */
11276 11380 static void
11277 11381 dtrace_enabling_provide(dtrace_provider_t *prv)
11278 11382 {
11279 11383 int i, all = 0;
11280 11384 dtrace_probedesc_t desc;
11281 11385 dtrace_genid_t gen;
11282 11386
11283 11387 ASSERT(MUTEX_HELD(&dtrace_lock));
11284 11388 ASSERT(MUTEX_HELD(&dtrace_provider_lock));
11285 11389
11286 11390 if (prv == NULL) {
11287 11391 all = 1;
11288 11392 prv = dtrace_provider;
11289 11393 }
11290 11394
11291 11395 do {
11292 11396 dtrace_enabling_t *enab;
11293 11397 void *parg = prv->dtpv_arg;
11294 11398
11295 11399 retry:
11296 11400 gen = dtrace_retained_gen;
11297 11401 for (enab = dtrace_retained; enab != NULL;
11298 11402 enab = enab->dten_next) {
11299 11403 for (i = 0; i < enab->dten_ndesc; i++) {
11300 11404 desc = enab->dten_desc[i]->dted_probe;
11301 11405 mutex_exit(&dtrace_lock);
11302 11406 prv->dtpv_pops.dtps_provide(parg, &desc);
11303 11407 mutex_enter(&dtrace_lock);
11304 11408 /*
11305 11409 * Process the retained enablings again if
11306 11410 * they have changed while we weren't holding
11307 11411 * dtrace_lock.
11308 11412 */
11309 11413 if (gen != dtrace_retained_gen)
11310 11414 goto retry;
11311 11415 }
11312 11416 }
11313 11417 } while (all && (prv = prv->dtpv_next) != NULL);
11314 11418
11315 11419 mutex_exit(&dtrace_lock);
11316 11420 dtrace_probe_provide(NULL, all ? NULL : prv);
11317 11421 mutex_enter(&dtrace_lock);
11318 11422 }
11319 11423
11320 11424 /*
11321 11425 * Called to reap ECBs that are attached to probes from defunct providers.
11322 11426 */
11323 11427 static void
11324 11428 dtrace_enabling_reap(void)
11325 11429 {
11326 11430 dtrace_provider_t *prov;
11327 11431 dtrace_probe_t *probe;
11328 11432 dtrace_ecb_t *ecb;
11329 11433 hrtime_t when;
11330 11434 int i;
11331 11435
11332 11436 mutex_enter(&cpu_lock);
11333 11437 mutex_enter(&dtrace_lock);
11334 11438
11335 11439 for (i = 0; i < dtrace_nprobes; i++) {
11336 11440 if ((probe = dtrace_probes[i]) == NULL)
11337 11441 continue;
11338 11442
11339 11443 if (probe->dtpr_ecb == NULL)
11340 11444 continue;
11341 11445
11342 11446 prov = probe->dtpr_provider;
11343 11447
11344 11448 if ((when = prov->dtpv_defunct) == 0)
11345 11449 continue;
11346 11450
11347 11451 /*
11348 11452 * We have ECBs on a defunct provider: we want to reap these
11349 11453 * ECBs to allow the provider to unregister. The destruction
11350 11454 * of these ECBs must be done carefully: if we destroy the ECB
11351 11455 * and the consumer later wishes to consume an EPID that
11352 11456 * corresponds to the destroyed ECB (and if the EPID metadata
11353 11457 * has not been previously consumed), the consumer will abort
11354 11458 * processing on the unknown EPID. To reduce (but not, sadly,
11355 11459 * eliminate) the possibility of this, we will only destroy an
11356 11460 * ECB for a defunct provider if, for the state that
11357 11461 * corresponds to the ECB:
11358 11462 *
11359 11463 * (a) There is no speculative tracing (which can effectively
11360 11464 * cache an EPID for an arbitrary amount of time).
11361 11465 *
11362 11466 * (b) The principal buffers have been switched twice since the
11363 11467 * provider became defunct.
11364 11468 *
11365 11469 * (c) The aggregation buffers are of zero size or have been
11366 11470 * switched twice since the provider became defunct.
11367 11471 *
11368 11472 * We use dts_speculates to determine (a) and call a function
11369 11473 * (dtrace_buffer_consumed()) to determine (b) and (c). Note
11370 11474 * that as soon as we've been unable to destroy one of the ECBs
11371 11475 * associated with the probe, we quit trying -- reaping is only
11372 11476 * fruitful in as much as we can destroy all ECBs associated
11373 11477 * with the defunct provider's probes.
11374 11478 */
11375 11479 while ((ecb = probe->dtpr_ecb) != NULL) {
11376 11480 dtrace_state_t *state = ecb->dte_state;
11377 11481 dtrace_buffer_t *buf = state->dts_buffer;
11378 11482 dtrace_buffer_t *aggbuf = state->dts_aggbuffer;
11379 11483
11380 11484 if (state->dts_speculates)
11381 11485 break;
11382 11486
11383 11487 if (!dtrace_buffer_consumed(buf, when))
11384 11488 break;
11385 11489
11386 11490 if (!dtrace_buffer_consumed(aggbuf, when))
11387 11491 break;
11388 11492
11389 11493 dtrace_ecb_disable(ecb);
11390 11494 ASSERT(probe->dtpr_ecb != ecb);
11391 11495 dtrace_ecb_destroy(ecb);
11392 11496 }
11393 11497 }
11394 11498
11395 11499 mutex_exit(&dtrace_lock);
11396 11500 mutex_exit(&cpu_lock);
11397 11501 }
11398 11502
11399 11503 /*
11400 11504 * DTrace DOF Functions
11401 11505 */
11402 11506 /*ARGSUSED*/
11403 11507 static void
11404 11508 dtrace_dof_error(dof_hdr_t *dof, const char *str)
11405 11509 {
11406 11510 if (dtrace_err_verbose)
11407 11511 cmn_err(CE_WARN, "failed to process DOF: %s", str);
11408 11512
11409 11513 #ifdef DTRACE_ERRDEBUG
11410 11514 dtrace_errdebug(str);
11411 11515 #endif
11412 11516 }
11413 11517
11414 11518 /*
11415 11519 * Create DOF out of a currently enabled state. Right now, we only create
11416 11520 * DOF containing the run-time options -- but this could be expanded to create
11417 11521 * complete DOF representing the enabled state.
11418 11522 */
11419 11523 static dof_hdr_t *
11420 11524 dtrace_dof_create(dtrace_state_t *state)
11421 11525 {
11422 11526 dof_hdr_t *dof;
11423 11527 dof_sec_t *sec;
11424 11528 dof_optdesc_t *opt;
11425 11529 int i, len = sizeof (dof_hdr_t) +
11426 11530 roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
11427 11531 sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
11428 11532
11429 11533 ASSERT(MUTEX_HELD(&dtrace_lock));
11430 11534
11431 11535 dof = kmem_zalloc(len, KM_SLEEP);
11432 11536 dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
11433 11537 dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
11434 11538 dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
11435 11539 dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
11436 11540
11437 11541 dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
11438 11542 dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
11439 11543 dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
11440 11544 dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
11441 11545 dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
11442 11546 dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
11443 11547
11444 11548 dof->dofh_flags = 0;
11445 11549 dof->dofh_hdrsize = sizeof (dof_hdr_t);
11446 11550 dof->dofh_secsize = sizeof (dof_sec_t);
11447 11551 dof->dofh_secnum = 1; /* only DOF_SECT_OPTDESC */
11448 11552 dof->dofh_secoff = sizeof (dof_hdr_t);
11449 11553 dof->dofh_loadsz = len;
11450 11554 dof->dofh_filesz = len;
11451 11555 dof->dofh_pad = 0;
11452 11556
11453 11557 /*
11454 11558 * Fill in the option section header...
11455 11559 */
11456 11560 sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
11457 11561 sec->dofs_type = DOF_SECT_OPTDESC;
11458 11562 sec->dofs_align = sizeof (uint64_t);
11459 11563 sec->dofs_flags = DOF_SECF_LOAD;
11460 11564 sec->dofs_entsize = sizeof (dof_optdesc_t);
11461 11565
11462 11566 opt = (dof_optdesc_t *)((uintptr_t)sec +
11463 11567 roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
11464 11568
11465 11569 sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
11466 11570 sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
11467 11571
11468 11572 for (i = 0; i < DTRACEOPT_MAX; i++) {
11469 11573 opt[i].dofo_option = i;
11470 11574 opt[i].dofo_strtab = DOF_SECIDX_NONE;
11471 11575 opt[i].dofo_value = state->dts_options[i];
11472 11576 }
11473 11577
11474 11578 return (dof);
11475 11579 }
11476 11580
11477 11581 static dof_hdr_t *
11478 11582 dtrace_dof_copyin(uintptr_t uarg, int *errp)
11479 11583 {
11480 11584 dof_hdr_t hdr, *dof;
11481 11585
11482 11586 ASSERT(!MUTEX_HELD(&dtrace_lock));
11483 11587
11484 11588 /*
11485 11589 * First, we're going to copyin() the sizeof (dof_hdr_t).
11486 11590 */
11487 11591 if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
11488 11592 dtrace_dof_error(NULL, "failed to copyin DOF header");
11489 11593 *errp = EFAULT;
11490 11594 return (NULL);
11491 11595 }
11492 11596
11493 11597 /*
11494 11598 * Now we'll allocate the entire DOF and copy it in -- provided
11495 11599 * that the length isn't outrageous.
11496 11600 */
11497 11601 if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
11498 11602 dtrace_dof_error(&hdr, "load size exceeds maximum");
11499 11603 *errp = E2BIG;
11500 11604 return (NULL);
11501 11605 }
11502 11606
11503 11607 if (hdr.dofh_loadsz < sizeof (hdr)) {
11504 11608 dtrace_dof_error(&hdr, "invalid load size");
11505 11609 *errp = EINVAL;
11506 11610 return (NULL);
11507 11611 }
11508 11612
11509 11613 dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
11510 11614
11511 11615 if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 ||
11512 11616 dof->dofh_loadsz != hdr.dofh_loadsz) {
11513 11617 kmem_free(dof, hdr.dofh_loadsz);
11514 11618 *errp = EFAULT;
11515 11619 return (NULL);
11516 11620 }
11517 11621
11518 11622 return (dof);
11519 11623 }
11520 11624
11521 11625 static dof_hdr_t *
11522 11626 dtrace_dof_property(const char *name)
11523 11627 {
11524 11628 uchar_t *buf;
11525 11629 uint64_t loadsz;
11526 11630 unsigned int len, i;
11527 11631 dof_hdr_t *dof;
11528 11632
11529 11633 /*
11530 11634 * Unfortunately, array of values in .conf files are always (and
11531 11635 * only) interpreted to be integer arrays. We must read our DOF
11532 11636 * as an integer array, and then squeeze it into a byte array.
11533 11637 */
11534 11638 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
11535 11639 (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
11536 11640 return (NULL);
11537 11641
11538 11642 for (i = 0; i < len; i++)
11539 11643 buf[i] = (uchar_t)(((int *)buf)[i]);
11540 11644
11541 11645 if (len < sizeof (dof_hdr_t)) {
11542 11646 ddi_prop_free(buf);
11543 11647 dtrace_dof_error(NULL, "truncated header");
11544 11648 return (NULL);
11545 11649 }
11546 11650
11547 11651 if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
11548 11652 ddi_prop_free(buf);
11549 11653 dtrace_dof_error(NULL, "truncated DOF");
11550 11654 return (NULL);
11551 11655 }
11552 11656
11553 11657 if (loadsz >= dtrace_dof_maxsize) {
11554 11658 ddi_prop_free(buf);
11555 11659 dtrace_dof_error(NULL, "oversized DOF");
11556 11660 return (NULL);
11557 11661 }
11558 11662
11559 11663 dof = kmem_alloc(loadsz, KM_SLEEP);
11560 11664 bcopy(buf, dof, loadsz);
11561 11665 ddi_prop_free(buf);
11562 11666
11563 11667 return (dof);
11564 11668 }
11565 11669
11566 11670 static void
11567 11671 dtrace_dof_destroy(dof_hdr_t *dof)
11568 11672 {
11569 11673 kmem_free(dof, dof->dofh_loadsz);
11570 11674 }
11571 11675
11572 11676 /*
11573 11677 * Return the dof_sec_t pointer corresponding to a given section index. If the
11574 11678 * index is not valid, dtrace_dof_error() is called and NULL is returned. If
11575 11679 * a type other than DOF_SECT_NONE is specified, the header is checked against
11576 11680 * this type and NULL is returned if the types do not match.
11577 11681 */
11578 11682 static dof_sec_t *
11579 11683 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
11580 11684 {
11581 11685 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
11582 11686 ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
11583 11687
11584 11688 if (i >= dof->dofh_secnum) {
11585 11689 dtrace_dof_error(dof, "referenced section index is invalid");
11586 11690 return (NULL);
11587 11691 }
11588 11692
11589 11693 if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
11590 11694 dtrace_dof_error(dof, "referenced section is not loadable");
11591 11695 return (NULL);
11592 11696 }
11593 11697
11594 11698 if (type != DOF_SECT_NONE && type != sec->dofs_type) {
11595 11699 dtrace_dof_error(dof, "referenced section is the wrong type");
11596 11700 return (NULL);
11597 11701 }
11598 11702
11599 11703 return (sec);
11600 11704 }
11601 11705
11602 11706 static dtrace_probedesc_t *
11603 11707 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
11604 11708 {
11605 11709 dof_probedesc_t *probe;
11606 11710 dof_sec_t *strtab;
11607 11711 uintptr_t daddr = (uintptr_t)dof;
11608 11712 uintptr_t str;
11609 11713 size_t size;
11610 11714
11611 11715 if (sec->dofs_type != DOF_SECT_PROBEDESC) {
11612 11716 dtrace_dof_error(dof, "invalid probe section");
11613 11717 return (NULL);
11614 11718 }
11615 11719
11616 11720 if (sec->dofs_align != sizeof (dof_secidx_t)) {
11617 11721 dtrace_dof_error(dof, "bad alignment in probe description");
11618 11722 return (NULL);
11619 11723 }
11620 11724
11621 11725 if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
11622 11726 dtrace_dof_error(dof, "truncated probe description");
11623 11727 return (NULL);
11624 11728 }
11625 11729
11626 11730 probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
11627 11731 strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
11628 11732
11629 11733 if (strtab == NULL)
11630 11734 return (NULL);
11631 11735
11632 11736 str = daddr + strtab->dofs_offset;
11633 11737 size = strtab->dofs_size;
11634 11738
11635 11739 if (probe->dofp_provider >= strtab->dofs_size) {
11636 11740 dtrace_dof_error(dof, "corrupt probe provider");
11637 11741 return (NULL);
11638 11742 }
11639 11743
11640 11744 (void) strncpy(desc->dtpd_provider,
11641 11745 (char *)(str + probe->dofp_provider),
11642 11746 MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
11643 11747
11644 11748 if (probe->dofp_mod >= strtab->dofs_size) {
11645 11749 dtrace_dof_error(dof, "corrupt probe module");
11646 11750 return (NULL);
11647 11751 }
11648 11752
11649 11753 (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
11650 11754 MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
11651 11755
11652 11756 if (probe->dofp_func >= strtab->dofs_size) {
11653 11757 dtrace_dof_error(dof, "corrupt probe function");
11654 11758 return (NULL);
11655 11759 }
11656 11760
11657 11761 (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
11658 11762 MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
11659 11763
11660 11764 if (probe->dofp_name >= strtab->dofs_size) {
11661 11765 dtrace_dof_error(dof, "corrupt probe name");
11662 11766 return (NULL);
11663 11767 }
11664 11768
11665 11769 (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
11666 11770 MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
11667 11771
11668 11772 return (desc);
11669 11773 }
11670 11774
11671 11775 static dtrace_difo_t *
11672 11776 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11673 11777 cred_t *cr)
11674 11778 {
11675 11779 dtrace_difo_t *dp;
11676 11780 size_t ttl = 0;
11677 11781 dof_difohdr_t *dofd;
11678 11782 uintptr_t daddr = (uintptr_t)dof;
11679 11783 size_t max = dtrace_difo_maxsize;
11680 11784 int i, l, n;
11681 11785
11682 11786 static const struct {
11683 11787 int section;
11684 11788 int bufoffs;
11685 11789 int lenoffs;
11686 11790 int entsize;
11687 11791 int align;
11688 11792 const char *msg;
11689 11793 } difo[] = {
11690 11794 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
11691 11795 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
11692 11796 sizeof (dif_instr_t), "multiple DIF sections" },
11693 11797
11694 11798 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
11695 11799 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
11696 11800 sizeof (uint64_t), "multiple integer tables" },
11697 11801
11698 11802 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
11699 11803 offsetof(dtrace_difo_t, dtdo_strlen), 0,
11700 11804 sizeof (char), "multiple string tables" },
11701 11805
11702 11806 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
11703 11807 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
11704 11808 sizeof (uint_t), "multiple variable tables" },
11705 11809
11706 11810 { DOF_SECT_NONE, 0, 0, 0, NULL }
11707 11811 };
11708 11812
11709 11813 if (sec->dofs_type != DOF_SECT_DIFOHDR) {
11710 11814 dtrace_dof_error(dof, "invalid DIFO header section");
11711 11815 return (NULL);
11712 11816 }
11713 11817
11714 11818 if (sec->dofs_align != sizeof (dof_secidx_t)) {
11715 11819 dtrace_dof_error(dof, "bad alignment in DIFO header");
11716 11820 return (NULL);
11717 11821 }
11718 11822
11719 11823 if (sec->dofs_size < sizeof (dof_difohdr_t) ||
11720 11824 sec->dofs_size % sizeof (dof_secidx_t)) {
11721 11825 dtrace_dof_error(dof, "bad size in DIFO header");
11722 11826 return (NULL);
11723 11827 }
11724 11828
11725 11829 dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
11726 11830 n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
11727 11831
11728 11832 dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
11729 11833 dp->dtdo_rtype = dofd->dofd_rtype;
11730 11834
11731 11835 for (l = 0; l < n; l++) {
11732 11836 dof_sec_t *subsec;
11733 11837 void **bufp;
11734 11838 uint32_t *lenp;
11735 11839
11736 11840 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
11737 11841 dofd->dofd_links[l])) == NULL)
11738 11842 goto err; /* invalid section link */
11739 11843
11740 11844 if (ttl + subsec->dofs_size > max) {
11741 11845 dtrace_dof_error(dof, "exceeds maximum size");
11742 11846 goto err;
11743 11847 }
11744 11848
11745 11849 ttl += subsec->dofs_size;
11746 11850
11747 11851 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
11748 11852 if (subsec->dofs_type != difo[i].section)
11749 11853 continue;
11750 11854
11751 11855 if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
11752 11856 dtrace_dof_error(dof, "section not loaded");
11753 11857 goto err;
11754 11858 }
11755 11859
11756 11860 if (subsec->dofs_align != difo[i].align) {
11757 11861 dtrace_dof_error(dof, "bad alignment");
11758 11862 goto err;
11759 11863 }
11760 11864
11761 11865 bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
11762 11866 lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
11763 11867
11764 11868 if (*bufp != NULL) {
11765 11869 dtrace_dof_error(dof, difo[i].msg);
11766 11870 goto err;
11767 11871 }
11768 11872
11769 11873 if (difo[i].entsize != subsec->dofs_entsize) {
11770 11874 dtrace_dof_error(dof, "entry size mismatch");
11771 11875 goto err;
11772 11876 }
11773 11877
11774 11878 if (subsec->dofs_entsize != 0 &&
11775 11879 (subsec->dofs_size % subsec->dofs_entsize) != 0) {
11776 11880 dtrace_dof_error(dof, "corrupt entry size");
11777 11881 goto err;
11778 11882 }
11779 11883
11780 11884 *lenp = subsec->dofs_size;
11781 11885 *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
11782 11886 bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
11783 11887 *bufp, subsec->dofs_size);
11784 11888
11785 11889 if (subsec->dofs_entsize != 0)
11786 11890 *lenp /= subsec->dofs_entsize;
11787 11891
11788 11892 break;
11789 11893 }
11790 11894
11791 11895 /*
11792 11896 * If we encounter a loadable DIFO sub-section that is not
11793 11897 * known to us, assume this is a broken program and fail.
11794 11898 */
11795 11899 if (difo[i].section == DOF_SECT_NONE &&
11796 11900 (subsec->dofs_flags & DOF_SECF_LOAD)) {
11797 11901 dtrace_dof_error(dof, "unrecognized DIFO subsection");
11798 11902 goto err;
11799 11903 }
11800 11904 }
11801 11905
11802 11906 if (dp->dtdo_buf == NULL) {
11803 11907 /*
11804 11908 * We can't have a DIF object without DIF text.
11805 11909 */
11806 11910 dtrace_dof_error(dof, "missing DIF text");
11807 11911 goto err;
11808 11912 }
11809 11913
11810 11914 /*
11811 11915 * Before we validate the DIF object, run through the variable table
11812 11916 * looking for the strings -- if any of their size are under, we'll set
11813 11917 * their size to be the system-wide default string size. Note that
11814 11918 * this should _not_ happen if the "strsize" option has been set --
11815 11919 * in this case, the compiler should have set the size to reflect the
11816 11920 * setting of the option.
11817 11921 */
11818 11922 for (i = 0; i < dp->dtdo_varlen; i++) {
11819 11923 dtrace_difv_t *v = &dp->dtdo_vartab[i];
11820 11924 dtrace_diftype_t *t = &v->dtdv_type;
11821 11925
11822 11926 if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
11823 11927 continue;
11824 11928
11825 11929 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
11826 11930 t->dtdt_size = dtrace_strsize_default;
11827 11931 }
11828 11932
11829 11933 if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
11830 11934 goto err;
11831 11935
11832 11936 dtrace_difo_init(dp, vstate);
11833 11937 return (dp);
11834 11938
11835 11939 err:
11836 11940 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
11837 11941 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
11838 11942 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
11839 11943 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
11840 11944
11841 11945 kmem_free(dp, sizeof (dtrace_difo_t));
11842 11946 return (NULL);
11843 11947 }
11844 11948
11845 11949 static dtrace_predicate_t *
11846 11950 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11847 11951 cred_t *cr)
11848 11952 {
11849 11953 dtrace_difo_t *dp;
11850 11954
11851 11955 if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
11852 11956 return (NULL);
11853 11957
11854 11958 return (dtrace_predicate_create(dp));
11855 11959 }
11856 11960
11857 11961 static dtrace_actdesc_t *
11858 11962 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11859 11963 cred_t *cr)
11860 11964 {
11861 11965 dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
11862 11966 dof_actdesc_t *desc;
11863 11967 dof_sec_t *difosec;
11864 11968 size_t offs;
11865 11969 uintptr_t daddr = (uintptr_t)dof;
11866 11970 uint64_t arg;
11867 11971 dtrace_actkind_t kind;
11868 11972
11869 11973 if (sec->dofs_type != DOF_SECT_ACTDESC) {
11870 11974 dtrace_dof_error(dof, "invalid action section");
11871 11975 return (NULL);
11872 11976 }
11873 11977
11874 11978 if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
11875 11979 dtrace_dof_error(dof, "truncated action description");
11876 11980 return (NULL);
11877 11981 }
11878 11982
11879 11983 if (sec->dofs_align != sizeof (uint64_t)) {
11880 11984 dtrace_dof_error(dof, "bad alignment in action description");
11881 11985 return (NULL);
11882 11986 }
11883 11987
11884 11988 if (sec->dofs_size < sec->dofs_entsize) {
11885 11989 dtrace_dof_error(dof, "section entry size exceeds total size");
11886 11990 return (NULL);
11887 11991 }
11888 11992
11889 11993 if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
11890 11994 dtrace_dof_error(dof, "bad entry size in action description");
11891 11995 return (NULL);
11892 11996 }
11893 11997
11894 11998 if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
11895 11999 dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
11896 12000 return (NULL);
11897 12001 }
11898 12002
11899 12003 for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
11900 12004 desc = (dof_actdesc_t *)(daddr +
11901 12005 (uintptr_t)sec->dofs_offset + offs);
11902 12006 kind = (dtrace_actkind_t)desc->dofa_kind;
11903 12007
11904 12008 if ((DTRACEACT_ISPRINTFLIKE(kind) &&
11905 12009 (kind != DTRACEACT_PRINTA ||
11906 12010 desc->dofa_strtab != DOF_SECIDX_NONE)) ||
11907 12011 (kind == DTRACEACT_DIFEXPR &&
11908 12012 desc->dofa_strtab != DOF_SECIDX_NONE)) {
11909 12013 dof_sec_t *strtab;
11910 12014 char *str, *fmt;
11911 12015 uint64_t i;
11912 12016
11913 12017 /*
11914 12018 * The argument to these actions is an index into the
11915 12019 * DOF string table. For printf()-like actions, this
11916 12020 * is the format string. For print(), this is the
11917 12021 * CTF type of the expression result.
11918 12022 */
11919 12023 if ((strtab = dtrace_dof_sect(dof,
11920 12024 DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
11921 12025 goto err;
11922 12026
11923 12027 str = (char *)((uintptr_t)dof +
11924 12028 (uintptr_t)strtab->dofs_offset);
11925 12029
11926 12030 for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
11927 12031 if (str[i] == '\0')
11928 12032 break;
11929 12033 }
11930 12034
11931 12035 if (i >= strtab->dofs_size) {
11932 12036 dtrace_dof_error(dof, "bogus format string");
11933 12037 goto err;
11934 12038 }
11935 12039
11936 12040 if (i == desc->dofa_arg) {
11937 12041 dtrace_dof_error(dof, "empty format string");
11938 12042 goto err;
11939 12043 }
11940 12044
11941 12045 i -= desc->dofa_arg;
11942 12046 fmt = kmem_alloc(i + 1, KM_SLEEP);
11943 12047 bcopy(&str[desc->dofa_arg], fmt, i + 1);
11944 12048 arg = (uint64_t)(uintptr_t)fmt;
11945 12049 } else {
11946 12050 if (kind == DTRACEACT_PRINTA) {
11947 12051 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
11948 12052 arg = 0;
11949 12053 } else {
11950 12054 arg = desc->dofa_arg;
11951 12055 }
11952 12056 }
11953 12057
11954 12058 act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
11955 12059 desc->dofa_uarg, arg);
11956 12060
11957 12061 if (last != NULL) {
11958 12062 last->dtad_next = act;
11959 12063 } else {
11960 12064 first = act;
11961 12065 }
11962 12066
11963 12067 last = act;
11964 12068
11965 12069 if (desc->dofa_difo == DOF_SECIDX_NONE)
11966 12070 continue;
11967 12071
11968 12072 if ((difosec = dtrace_dof_sect(dof,
11969 12073 DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
11970 12074 goto err;
11971 12075
11972 12076 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
11973 12077
11974 12078 if (act->dtad_difo == NULL)
11975 12079 goto err;
11976 12080 }
11977 12081
11978 12082 ASSERT(first != NULL);
11979 12083 return (first);
11980 12084
11981 12085 err:
11982 12086 for (act = first; act != NULL; act = next) {
11983 12087 next = act->dtad_next;
11984 12088 dtrace_actdesc_release(act, vstate);
11985 12089 }
11986 12090
11987 12091 return (NULL);
11988 12092 }
11989 12093
11990 12094 static dtrace_ecbdesc_t *
11991 12095 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
11992 12096 cred_t *cr)
11993 12097 {
11994 12098 dtrace_ecbdesc_t *ep;
11995 12099 dof_ecbdesc_t *ecb;
11996 12100 dtrace_probedesc_t *desc;
11997 12101 dtrace_predicate_t *pred = NULL;
11998 12102
11999 12103 if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
12000 12104 dtrace_dof_error(dof, "truncated ECB description");
12001 12105 return (NULL);
12002 12106 }
12003 12107
12004 12108 if (sec->dofs_align != sizeof (uint64_t)) {
12005 12109 dtrace_dof_error(dof, "bad alignment in ECB description");
12006 12110 return (NULL);
12007 12111 }
12008 12112
12009 12113 ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
12010 12114 sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
12011 12115
12012 12116 if (sec == NULL)
12013 12117 return (NULL);
12014 12118
12015 12119 ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
12016 12120 ep->dted_uarg = ecb->dofe_uarg;
12017 12121 desc = &ep->dted_probe;
12018 12122
12019 12123 if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
12020 12124 goto err;
12021 12125
12022 12126 if (ecb->dofe_pred != DOF_SECIDX_NONE) {
12023 12127 if ((sec = dtrace_dof_sect(dof,
12024 12128 DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
12025 12129 goto err;
12026 12130
12027 12131 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
12028 12132 goto err;
12029 12133
12030 12134 ep->dted_pred.dtpdd_predicate = pred;
12031 12135 }
12032 12136
12033 12137 if (ecb->dofe_actions != DOF_SECIDX_NONE) {
12034 12138 if ((sec = dtrace_dof_sect(dof,
12035 12139 DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
12036 12140 goto err;
12037 12141
12038 12142 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
12039 12143
12040 12144 if (ep->dted_action == NULL)
12041 12145 goto err;
12042 12146 }
12043 12147
12044 12148 return (ep);
12045 12149
12046 12150 err:
12047 12151 if (pred != NULL)
12048 12152 dtrace_predicate_release(pred, vstate);
12049 12153 kmem_free(ep, sizeof (dtrace_ecbdesc_t));
12050 12154 return (NULL);
12051 12155 }
12052 12156
12053 12157 /*
12054 12158 * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
12055 12159 * specified DOF. At present, this amounts to simply adding 'ubase' to the
12056 12160 * site of any user SETX relocations to account for load object base address.
12057 12161 * In the future, if we need other relocations, this function can be extended.
12058 12162 */
12059 12163 static int
12060 12164 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
12061 12165 {
12062 12166 uintptr_t daddr = (uintptr_t)dof;
12063 12167 dof_relohdr_t *dofr =
12064 12168 (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
12065 12169 dof_sec_t *ss, *rs, *ts;
12066 12170 dof_relodesc_t *r;
12067 12171 uint_t i, n;
12068 12172
12069 12173 if (sec->dofs_size < sizeof (dof_relohdr_t) ||
12070 12174 sec->dofs_align != sizeof (dof_secidx_t)) {
12071 12175 dtrace_dof_error(dof, "invalid relocation header");
12072 12176 return (-1);
12073 12177 }
12074 12178
12075 12179 ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
12076 12180 rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
12077 12181 ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
12078 12182
12079 12183 if (ss == NULL || rs == NULL || ts == NULL)
12080 12184 return (-1); /* dtrace_dof_error() has been called already */
12081 12185
12082 12186 if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
12083 12187 rs->dofs_align != sizeof (uint64_t)) {
12084 12188 dtrace_dof_error(dof, "invalid relocation section");
12085 12189 return (-1);
12086 12190 }
12087 12191
12088 12192 r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
12089 12193 n = rs->dofs_size / rs->dofs_entsize;
12090 12194
12091 12195 for (i = 0; i < n; i++) {
12092 12196 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
12093 12197
12094 12198 switch (r->dofr_type) {
12095 12199 case DOF_RELO_NONE:
12096 12200 break;
12097 12201 case DOF_RELO_SETX:
12098 12202 if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
12099 12203 sizeof (uint64_t) > ts->dofs_size) {
12100 12204 dtrace_dof_error(dof, "bad relocation offset");
12101 12205 return (-1);
12102 12206 }
12103 12207
12104 12208 if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
12105 12209 dtrace_dof_error(dof, "misaligned setx relo");
12106 12210 return (-1);
12107 12211 }
12108 12212
12109 12213 *(uint64_t *)taddr += ubase;
12110 12214 break;
12111 12215 default:
12112 12216 dtrace_dof_error(dof, "invalid relocation type");
12113 12217 return (-1);
12114 12218 }
12115 12219
12116 12220 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
12117 12221 }
12118 12222
12119 12223 return (0);
12120 12224 }
12121 12225
12122 12226 /*
12123 12227 * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
12124 12228 * header: it should be at the front of a memory region that is at least
12125 12229 * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
12126 12230 * size. It need not be validated in any other way.
12127 12231 */
12128 12232 static int
12129 12233 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
12130 12234 dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
12131 12235 {
12132 12236 uint64_t len = dof->dofh_loadsz, seclen;
12133 12237 uintptr_t daddr = (uintptr_t)dof;
12134 12238 dtrace_ecbdesc_t *ep;
12135 12239 dtrace_enabling_t *enab;
12136 12240 uint_t i;
12137 12241
12138 12242 ASSERT(MUTEX_HELD(&dtrace_lock));
12139 12243 ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
12140 12244
12141 12245 /*
12142 12246 * Check the DOF header identification bytes. In addition to checking
12143 12247 * valid settings, we also verify that unused bits/bytes are zeroed so
12144 12248 * we can use them later without fear of regressing existing binaries.
12145 12249 */
12146 12250 if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
12147 12251 DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
12148 12252 dtrace_dof_error(dof, "DOF magic string mismatch");
12149 12253 return (-1);
12150 12254 }
12151 12255
12152 12256 if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
12153 12257 dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
12154 12258 dtrace_dof_error(dof, "DOF has invalid data model");
12155 12259 return (-1);
12156 12260 }
12157 12261
12158 12262 if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
12159 12263 dtrace_dof_error(dof, "DOF encoding mismatch");
12160 12264 return (-1);
12161 12265 }
12162 12266
12163 12267 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
12164 12268 dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
12165 12269 dtrace_dof_error(dof, "DOF version mismatch");
12166 12270 return (-1);
12167 12271 }
12168 12272
12169 12273 if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
12170 12274 dtrace_dof_error(dof, "DOF uses unsupported instruction set");
12171 12275 return (-1);
12172 12276 }
12173 12277
12174 12278 if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
12175 12279 dtrace_dof_error(dof, "DOF uses too many integer registers");
12176 12280 return (-1);
12177 12281 }
12178 12282
12179 12283 if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
12180 12284 dtrace_dof_error(dof, "DOF uses too many tuple registers");
12181 12285 return (-1);
12182 12286 }
12183 12287
12184 12288 for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
12185 12289 if (dof->dofh_ident[i] != 0) {
12186 12290 dtrace_dof_error(dof, "DOF has invalid ident byte set");
12187 12291 return (-1);
12188 12292 }
12189 12293 }
12190 12294
12191 12295 if (dof->dofh_flags & ~DOF_FL_VALID) {
12192 12296 dtrace_dof_error(dof, "DOF has invalid flag bits set");
12193 12297 return (-1);
12194 12298 }
12195 12299
12196 12300 if (dof->dofh_secsize == 0) {
12197 12301 dtrace_dof_error(dof, "zero section header size");
12198 12302 return (-1);
12199 12303 }
12200 12304
12201 12305 /*
12202 12306 * Check that the section headers don't exceed the amount of DOF
12203 12307 * data. Note that we cast the section size and number of sections
12204 12308 * to uint64_t's to prevent possible overflow in the multiplication.
12205 12309 */
12206 12310 seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
12207 12311
12208 12312 if (dof->dofh_secoff > len || seclen > len ||
12209 12313 dof->dofh_secoff + seclen > len) {
12210 12314 dtrace_dof_error(dof, "truncated section headers");
12211 12315 return (-1);
12212 12316 }
12213 12317
12214 12318 if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
12215 12319 dtrace_dof_error(dof, "misaligned section headers");
12216 12320 return (-1);
12217 12321 }
12218 12322
12219 12323 if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
12220 12324 dtrace_dof_error(dof, "misaligned section size");
12221 12325 return (-1);
12222 12326 }
12223 12327
12224 12328 /*
12225 12329 * Take an initial pass through the section headers to be sure that
12226 12330 * the headers don't have stray offsets. If the 'noprobes' flag is
12227 12331 * set, do not permit sections relating to providers, probes, or args.
12228 12332 */
12229 12333 for (i = 0; i < dof->dofh_secnum; i++) {
12230 12334 dof_sec_t *sec = (dof_sec_t *)(daddr +
12231 12335 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12232 12336
12233 12337 if (noprobes) {
12234 12338 switch (sec->dofs_type) {
12235 12339 case DOF_SECT_PROVIDER:
12236 12340 case DOF_SECT_PROBES:
12237 12341 case DOF_SECT_PRARGS:
12238 12342 case DOF_SECT_PROFFS:
12239 12343 dtrace_dof_error(dof, "illegal sections "
12240 12344 "for enabling");
12241 12345 return (-1);
12242 12346 }
12243 12347 }
12244 12348
12245 12349 if (DOF_SEC_ISLOADABLE(sec->dofs_type) &&
12246 12350 !(sec->dofs_flags & DOF_SECF_LOAD)) {
12247 12351 dtrace_dof_error(dof, "loadable section with load "
12248 12352 "flag unset");
12249 12353 return (-1);
12250 12354 }
12251 12355
12252 12356 if (!(sec->dofs_flags & DOF_SECF_LOAD))
12253 12357 continue; /* just ignore non-loadable sections */
12254 12358
12255 12359 if (sec->dofs_align & (sec->dofs_align - 1)) {
12256 12360 dtrace_dof_error(dof, "bad section alignment");
12257 12361 return (-1);
12258 12362 }
12259 12363
12260 12364 if (sec->dofs_offset & (sec->dofs_align - 1)) {
12261 12365 dtrace_dof_error(dof, "misaligned section");
12262 12366 return (-1);
12263 12367 }
12264 12368
12265 12369 if (sec->dofs_offset > len || sec->dofs_size > len ||
12266 12370 sec->dofs_offset + sec->dofs_size > len) {
12267 12371 dtrace_dof_error(dof, "corrupt section header");
12268 12372 return (-1);
12269 12373 }
12270 12374
12271 12375 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
12272 12376 sec->dofs_offset + sec->dofs_size - 1) != '\0') {
12273 12377 dtrace_dof_error(dof, "non-terminating string table");
12274 12378 return (-1);
12275 12379 }
12276 12380 }
12277 12381
12278 12382 /*
12279 12383 * Take a second pass through the sections and locate and perform any
12280 12384 * relocations that are present. We do this after the first pass to
12281 12385 * be sure that all sections have had their headers validated.
12282 12386 */
12283 12387 for (i = 0; i < dof->dofh_secnum; i++) {
12284 12388 dof_sec_t *sec = (dof_sec_t *)(daddr +
12285 12389 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12286 12390
12287 12391 if (!(sec->dofs_flags & DOF_SECF_LOAD))
12288 12392 continue; /* skip sections that are not loadable */
12289 12393
12290 12394 switch (sec->dofs_type) {
12291 12395 case DOF_SECT_URELHDR:
12292 12396 if (dtrace_dof_relocate(dof, sec, ubase) != 0)
12293 12397 return (-1);
12294 12398 break;
12295 12399 }
12296 12400 }
12297 12401
12298 12402 if ((enab = *enabp) == NULL)
12299 12403 enab = *enabp = dtrace_enabling_create(vstate);
12300 12404
12301 12405 for (i = 0; i < dof->dofh_secnum; i++) {
12302 12406 dof_sec_t *sec = (dof_sec_t *)(daddr +
12303 12407 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12304 12408
12305 12409 if (sec->dofs_type != DOF_SECT_ECBDESC)
12306 12410 continue;
12307 12411
12308 12412 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
12309 12413 dtrace_enabling_destroy(enab);
12310 12414 *enabp = NULL;
12311 12415 return (-1);
12312 12416 }
12313 12417
12314 12418 dtrace_enabling_add(enab, ep);
12315 12419 }
12316 12420
12317 12421 return (0);
12318 12422 }
12319 12423
12320 12424 /*
12321 12425 * Process DOF for any options. This routine assumes that the DOF has been
12322 12426 * at least processed by dtrace_dof_slurp().
12323 12427 */
12324 12428 static int
12325 12429 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
12326 12430 {
12327 12431 int i, rval;
12328 12432 uint32_t entsize;
12329 12433 size_t offs;
12330 12434 dof_optdesc_t *desc;
12331 12435
12332 12436 for (i = 0; i < dof->dofh_secnum; i++) {
12333 12437 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
12334 12438 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
12335 12439
12336 12440 if (sec->dofs_type != DOF_SECT_OPTDESC)
12337 12441 continue;
12338 12442
12339 12443 if (sec->dofs_align != sizeof (uint64_t)) {
12340 12444 dtrace_dof_error(dof, "bad alignment in "
12341 12445 "option description");
12342 12446 return (EINVAL);
12343 12447 }
12344 12448
12345 12449 if ((entsize = sec->dofs_entsize) == 0) {
12346 12450 dtrace_dof_error(dof, "zeroed option entry size");
12347 12451 return (EINVAL);
12348 12452 }
12349 12453
12350 12454 if (entsize < sizeof (dof_optdesc_t)) {
12351 12455 dtrace_dof_error(dof, "bad option entry size");
12352 12456 return (EINVAL);
12353 12457 }
12354 12458
12355 12459 for (offs = 0; offs < sec->dofs_size; offs += entsize) {
12356 12460 desc = (dof_optdesc_t *)((uintptr_t)dof +
12357 12461 (uintptr_t)sec->dofs_offset + offs);
12358 12462
12359 12463 if (desc->dofo_strtab != DOF_SECIDX_NONE) {
12360 12464 dtrace_dof_error(dof, "non-zero option string");
12361 12465 return (EINVAL);
12362 12466 }
12363 12467
12364 12468 if (desc->dofo_value == DTRACEOPT_UNSET) {
12365 12469 dtrace_dof_error(dof, "unset option");
12366 12470 return (EINVAL);
12367 12471 }
12368 12472
12369 12473 if ((rval = dtrace_state_option(state,
12370 12474 desc->dofo_option, desc->dofo_value)) != 0) {
12371 12475 dtrace_dof_error(dof, "rejected option");
12372 12476 return (rval);
12373 12477 }
12374 12478 }
12375 12479 }
12376 12480
12377 12481 return (0);
12378 12482 }
12379 12483
12380 12484 /*
12381 12485 * DTrace Consumer State Functions
12382 12486 */
12383 12487 int
12384 12488 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
12385 12489 {
12386 12490 size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
12387 12491 void *base;
12388 12492 uintptr_t limit;
12389 12493 dtrace_dynvar_t *dvar, *next, *start;
12390 12494 int i;
12391 12495
12392 12496 ASSERT(MUTEX_HELD(&dtrace_lock));
12393 12497 ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
12394 12498
12395 12499 bzero(dstate, sizeof (dtrace_dstate_t));
12396 12500
12397 12501 if ((dstate->dtds_chunksize = chunksize) == 0)
12398 12502 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
12399 12503
12400 12504 if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
12401 12505 size = min;
12402 12506
12403 12507 if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL)
12404 12508 return (ENOMEM);
12405 12509
12406 12510 dstate->dtds_size = size;
12407 12511 dstate->dtds_base = base;
12408 12512 dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
12409 12513 bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
12410 12514
12411 12515 hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
12412 12516
12413 12517 if (hashsize != 1 && (hashsize & 1))
12414 12518 hashsize--;
12415 12519
12416 12520 dstate->dtds_hashsize = hashsize;
12417 12521 dstate->dtds_hash = dstate->dtds_base;
12418 12522
12419 12523 /*
12420 12524 * Set all of our hash buckets to point to the single sink, and (if
12421 12525 * it hasn't already been set), set the sink's hash value to be the
12422 12526 * sink sentinel value. The sink is needed for dynamic variable
12423 12527 * lookups to know that they have iterated over an entire, valid hash
12424 12528 * chain.
12425 12529 */
12426 12530 for (i = 0; i < hashsize; i++)
12427 12531 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
12428 12532
12429 12533 if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
12430 12534 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
12431 12535
12432 12536 /*
12433 12537 * Determine number of active CPUs. Divide free list evenly among
12434 12538 * active CPUs.
12435 12539 */
12436 12540 start = (dtrace_dynvar_t *)
12437 12541 ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
12438 12542 limit = (uintptr_t)base + size;
12439 12543
12440 12544 maxper = (limit - (uintptr_t)start) / NCPU;
12441 12545 maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
12442 12546
12443 12547 for (i = 0; i < NCPU; i++) {
12444 12548 dstate->dtds_percpu[i].dtdsc_free = dvar = start;
12445 12549
12446 12550 /*
12447 12551 * If we don't even have enough chunks to make it once through
12448 12552 * NCPUs, we're just going to allocate everything to the first
12449 12553 * CPU. And if we're on the last CPU, we're going to allocate
12450 12554 * whatever is left over. In either case, we set the limit to
12451 12555 * be the limit of the dynamic variable space.
12452 12556 */
12453 12557 if (maxper == 0 || i == NCPU - 1) {
12454 12558 limit = (uintptr_t)base + size;
12455 12559 start = NULL;
12456 12560 } else {
12457 12561 limit = (uintptr_t)start + maxper;
12458 12562 start = (dtrace_dynvar_t *)limit;
12459 12563 }
12460 12564
12461 12565 ASSERT(limit <= (uintptr_t)base + size);
12462 12566
12463 12567 for (;;) {
12464 12568 next = (dtrace_dynvar_t *)((uintptr_t)dvar +
12465 12569 dstate->dtds_chunksize);
12466 12570
12467 12571 if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
12468 12572 break;
12469 12573
12470 12574 dvar->dtdv_next = next;
12471 12575 dvar = next;
12472 12576 }
12473 12577
12474 12578 if (maxper == 0)
12475 12579 break;
12476 12580 }
12477 12581
12478 12582 return (0);
12479 12583 }
12480 12584
12481 12585 void
12482 12586 dtrace_dstate_fini(dtrace_dstate_t *dstate)
12483 12587 {
12484 12588 ASSERT(MUTEX_HELD(&cpu_lock));
12485 12589
12486 12590 if (dstate->dtds_base == NULL)
12487 12591 return;
12488 12592
12489 12593 kmem_free(dstate->dtds_base, dstate->dtds_size);
12490 12594 kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
12491 12595 }
12492 12596
12493 12597 static void
12494 12598 dtrace_vstate_fini(dtrace_vstate_t *vstate)
12495 12599 {
12496 12600 /*
12497 12601 * Logical XOR, where are you?
12498 12602 */
12499 12603 ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
12500 12604
12501 12605 if (vstate->dtvs_nglobals > 0) {
12502 12606 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
12503 12607 sizeof (dtrace_statvar_t *));
12504 12608 }
12505 12609
12506 12610 if (vstate->dtvs_ntlocals > 0) {
12507 12611 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
12508 12612 sizeof (dtrace_difv_t));
12509 12613 }
12510 12614
12511 12615 ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
12512 12616
12513 12617 if (vstate->dtvs_nlocals > 0) {
12514 12618 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
12515 12619 sizeof (dtrace_statvar_t *));
12516 12620 }
12517 12621 }
12518 12622
12519 12623 static void
12520 12624 dtrace_state_clean(dtrace_state_t *state)
12521 12625 {
12522 12626 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
12523 12627 return;
12524 12628
12525 12629 dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
12526 12630 dtrace_speculation_clean(state);
12527 12631 }
12528 12632
12529 12633 static void
12530 12634 dtrace_state_deadman(dtrace_state_t *state)
12531 12635 {
12532 12636 hrtime_t now;
12533 12637
12534 12638 dtrace_sync();
12535 12639
12536 12640 now = dtrace_gethrtime();
12537 12641
12538 12642 if (state != dtrace_anon.dta_state &&
12539 12643 now - state->dts_laststatus >= dtrace_deadman_user)
12540 12644 return;
12541 12645
12542 12646 /*
12543 12647 * We must be sure that dts_alive never appears to be less than the
12544 12648 * value upon entry to dtrace_state_deadman(), and because we lack a
12545 12649 * dtrace_cas64(), we cannot store to it atomically. We thus instead
12546 12650 * store INT64_MAX to it, followed by a memory barrier, followed by
12547 12651 * the new value. This assures that dts_alive never appears to be
12548 12652 * less than its true value, regardless of the order in which the
12549 12653 * stores to the underlying storage are issued.
12550 12654 */
12551 12655 state->dts_alive = INT64_MAX;
12552 12656 dtrace_membar_producer();
12553 12657 state->dts_alive = now;
12554 12658 }
12555 12659
12556 12660 dtrace_state_t *
12557 12661 dtrace_state_create(dev_t *devp, cred_t *cr)
12558 12662 {
12559 12663 minor_t minor;
12560 12664 major_t major;
12561 12665 char c[30];
12562 12666 dtrace_state_t *state;
12563 12667 dtrace_optval_t *opt;
12564 12668 int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
12565 12669
12566 12670 ASSERT(MUTEX_HELD(&dtrace_lock));
12567 12671 ASSERT(MUTEX_HELD(&cpu_lock));
12568 12672
12569 12673 minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
12570 12674 VM_BESTFIT | VM_SLEEP);
12571 12675
12572 12676 if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
12573 12677 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
12574 12678 return (NULL);
12575 12679 }
12576 12680
12577 12681 state = ddi_get_soft_state(dtrace_softstate, minor);
12578 12682 state->dts_epid = DTRACE_EPIDNONE + 1;
12579 12683
12580 12684 (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
12581 12685 state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
12582 12686 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
12583 12687
12584 12688 if (devp != NULL) {
12585 12689 major = getemajor(*devp);
12586 12690 } else {
12587 12691 major = ddi_driver_major(dtrace_devi);
12588 12692 }
12589 12693
12590 12694 state->dts_dev = makedevice(major, minor);
12591 12695
12592 12696 if (devp != NULL)
12593 12697 *devp = state->dts_dev;
12594 12698
12595 12699 /*
12596 12700 * We allocate NCPU buffers. On the one hand, this can be quite
12597 12701 * a bit of memory per instance (nearly 36K on a Starcat). On the
12598 12702 * other hand, it saves an additional memory reference in the probe
12599 12703 * path.
12600 12704 */
12601 12705 state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
12602 12706 state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
12603 12707 state->dts_cleaner = CYCLIC_NONE;
12604 12708 state->dts_deadman = CYCLIC_NONE;
12605 12709 state->dts_vstate.dtvs_state = state;
12606 12710
12607 12711 for (i = 0; i < DTRACEOPT_MAX; i++)
12608 12712 state->dts_options[i] = DTRACEOPT_UNSET;
12609 12713
12610 12714 /*
12611 12715 * Set the default options.
12612 12716 */
12613 12717 opt = state->dts_options;
12614 12718 opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
12615 12719 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
12616 12720 opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
12617 12721 opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
12618 12722 opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
12619 12723 opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
12620 12724 opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
12621 12725 opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
12622 12726 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
12623 12727 opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
12624 12728 opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
12625 12729 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
12626 12730 opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
12627 12731 opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
12628 12732
12629 12733 state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
12630 12734
12631 12735 /*
12632 12736 * Depending on the user credentials, we set flag bits which alter probe
12633 12737 * visibility or the amount of destructiveness allowed. In the case of
12634 12738 * actual anonymous tracing, or the possession of all privileges, all of
12635 12739 * the normal checks are bypassed.
12636 12740 */
12637 12741 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
12638 12742 state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
12639 12743 state->dts_cred.dcr_action = DTRACE_CRA_ALL;
12640 12744 } else {
12641 12745 /*
12642 12746 * Set up the credentials for this instantiation. We take a
12643 12747 * hold on the credential to prevent it from disappearing on
12644 12748 * us; this in turn prevents the zone_t referenced by this
12645 12749 * credential from disappearing. This means that we can
12646 12750 * examine the credential and the zone from probe context.
12647 12751 */
12648 12752 crhold(cr);
12649 12753 state->dts_cred.dcr_cred = cr;
12650 12754
12651 12755 /*
12652 12756 * CRA_PROC means "we have *some* privilege for dtrace" and
12653 12757 * unlocks the use of variables like pid, zonename, etc.
12654 12758 */
12655 12759 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
12656 12760 PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
12657 12761 state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
12658 12762 }
12659 12763
12660 12764 /*
12661 12765 * dtrace_user allows use of syscall and profile providers.
12662 12766 * If the user also has proc_owner and/or proc_zone, we
12663 12767 * extend the scope to include additional visibility and
12664 12768 * destructive power.
12665 12769 */
12666 12770 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
12667 12771 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
12668 12772 state->dts_cred.dcr_visible |=
12669 12773 DTRACE_CRV_ALLPROC;
12670 12774
12671 12775 state->dts_cred.dcr_action |=
12672 12776 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12673 12777 }
12674 12778
12675 12779 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
12676 12780 state->dts_cred.dcr_visible |=
12677 12781 DTRACE_CRV_ALLZONE;
12678 12782
12679 12783 state->dts_cred.dcr_action |=
12680 12784 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12681 12785 }
12682 12786
12683 12787 /*
12684 12788 * If we have all privs in whatever zone this is,
12685 12789 * we can do destructive things to processes which
12686 12790 * have altered credentials.
12687 12791 */
12688 12792 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
12689 12793 cr->cr_zone->zone_privset)) {
12690 12794 state->dts_cred.dcr_action |=
12691 12795 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
12692 12796 }
12693 12797 }
12694 12798
12695 12799 /*
12696 12800 * Holding the dtrace_kernel privilege also implies that
12697 12801 * the user has the dtrace_user privilege from a visibility
12698 12802 * perspective. But without further privileges, some
12699 12803 * destructive actions are not available.
12700 12804 */
12701 12805 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
12702 12806 /*
12703 12807 * Make all probes in all zones visible. However,
12704 12808 * this doesn't mean that all actions become available
12705 12809 * to all zones.
12706 12810 */
12707 12811 state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
12708 12812 DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
12709 12813
12710 12814 state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
12711 12815 DTRACE_CRA_PROC;
12712 12816 /*
12713 12817 * Holding proc_owner means that destructive actions
12714 12818 * for *this* zone are allowed.
12715 12819 */
12716 12820 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
12717 12821 state->dts_cred.dcr_action |=
12718 12822 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12719 12823
12720 12824 /*
12721 12825 * Holding proc_zone means that destructive actions
12722 12826 * for this user/group ID in all zones is allowed.
12723 12827 */
12724 12828 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
12725 12829 state->dts_cred.dcr_action |=
12726 12830 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12727 12831
12728 12832 /*
12729 12833 * If we have all privs in whatever zone this is,
12730 12834 * we can do destructive things to processes which
12731 12835 * have altered credentials.
12732 12836 */
12733 12837 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
12734 12838 cr->cr_zone->zone_privset)) {
12735 12839 state->dts_cred.dcr_action |=
12736 12840 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
12737 12841 }
12738 12842 }
12739 12843
12740 12844 /*
12741 12845 * Holding the dtrace_proc privilege gives control over fasttrap
12742 12846 * and pid providers. We need to grant wider destructive
12743 12847 * privileges in the event that the user has proc_owner and/or
12744 12848 * proc_zone.
12745 12849 */
12746 12850 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
12747 12851 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
12748 12852 state->dts_cred.dcr_action |=
12749 12853 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
12750 12854
12751 12855 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
12752 12856 state->dts_cred.dcr_action |=
12753 12857 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
12754 12858 }
12755 12859 }
12756 12860
12757 12861 return (state);
12758 12862 }
12759 12863
12760 12864 static int
12761 12865 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
12762 12866 {
12763 12867 dtrace_optval_t *opt = state->dts_options, size;
12764 12868 processorid_t cpu;
12765 12869 int flags = 0, rval, factor, divisor = 1;
12766 12870
12767 12871 ASSERT(MUTEX_HELD(&dtrace_lock));
12768 12872 ASSERT(MUTEX_HELD(&cpu_lock));
12769 12873 ASSERT(which < DTRACEOPT_MAX);
12770 12874 ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
12771 12875 (state == dtrace_anon.dta_state &&
12772 12876 state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
12773 12877
12774 12878 if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
12775 12879 return (0);
12776 12880
12777 12881 if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
12778 12882 cpu = opt[DTRACEOPT_CPU];
12779 12883
12780 12884 if (which == DTRACEOPT_SPECSIZE)
12781 12885 flags |= DTRACEBUF_NOSWITCH;
12782 12886
12783 12887 if (which == DTRACEOPT_BUFSIZE) {
12784 12888 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
12785 12889 flags |= DTRACEBUF_RING;
12786 12890
12787 12891 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
12788 12892 flags |= DTRACEBUF_FILL;
12789 12893
12790 12894 if (state != dtrace_anon.dta_state ||
12791 12895 state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
12792 12896 flags |= DTRACEBUF_INACTIVE;
12793 12897 }
12794 12898
12795 12899 for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) {
12796 12900 /*
12797 12901 * The size must be 8-byte aligned. If the size is not 8-byte
12798 12902 * aligned, drop it down by the difference.
12799 12903 */
12800 12904 if (size & (sizeof (uint64_t) - 1))
12801 12905 size -= size & (sizeof (uint64_t) - 1);
12802 12906
12803 12907 if (size < state->dts_reserve) {
12804 12908 /*
12805 12909 * Buffers always must be large enough to accommodate
12806 12910 * their prereserved space. We return E2BIG instead
12807 12911 * of ENOMEM in this case to allow for user-level
12808 12912 * software to differentiate the cases.
12809 12913 */
12810 12914 return (E2BIG);
12811 12915 }
12812 12916
12813 12917 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor);
12814 12918
12815 12919 if (rval != ENOMEM) {
12816 12920 opt[which] = size;
12817 12921 return (rval);
12818 12922 }
12819 12923
12820 12924 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
12821 12925 return (rval);
12822 12926
12823 12927 for (divisor = 2; divisor < factor; divisor <<= 1)
12824 12928 continue;
12825 12929 }
12826 12930
12827 12931 return (ENOMEM);
12828 12932 }
12829 12933
12830 12934 static int
12831 12935 dtrace_state_buffers(dtrace_state_t *state)
12832 12936 {
12833 12937 dtrace_speculation_t *spec = state->dts_speculations;
12834 12938 int rval, i;
12835 12939
12836 12940 if ((rval = dtrace_state_buffer(state, state->dts_buffer,
12837 12941 DTRACEOPT_BUFSIZE)) != 0)
12838 12942 return (rval);
12839 12943
12840 12944 if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
12841 12945 DTRACEOPT_AGGSIZE)) != 0)
12842 12946 return (rval);
12843 12947
12844 12948 for (i = 0; i < state->dts_nspeculations; i++) {
12845 12949 if ((rval = dtrace_state_buffer(state,
12846 12950 spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
12847 12951 return (rval);
12848 12952 }
12849 12953
12850 12954 return (0);
12851 12955 }
12852 12956
12853 12957 static void
12854 12958 dtrace_state_prereserve(dtrace_state_t *state)
12855 12959 {
12856 12960 dtrace_ecb_t *ecb;
12857 12961 dtrace_probe_t *probe;
12858 12962
12859 12963 state->dts_reserve = 0;
12860 12964
12861 12965 if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
12862 12966 return;
12863 12967
12864 12968 /*
12865 12969 * If our buffer policy is a "fill" buffer policy, we need to set the
12866 12970 * prereserved space to be the space required by the END probes.
12867 12971 */
12868 12972 probe = dtrace_probes[dtrace_probeid_end - 1];
12869 12973 ASSERT(probe != NULL);
12870 12974
12871 12975 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
12872 12976 if (ecb->dte_state != state)
12873 12977 continue;
12874 12978
12875 12979 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
12876 12980 }
12877 12981 }
12878 12982
12879 12983 static int
12880 12984 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
12881 12985 {
12882 12986 dtrace_optval_t *opt = state->dts_options, sz, nspec;
12883 12987 dtrace_speculation_t *spec;
12884 12988 dtrace_buffer_t *buf;
12885 12989 cyc_handler_t hdlr;
12886 12990 cyc_time_t when;
12887 12991 int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
12888 12992 dtrace_icookie_t cookie;
12889 12993
12890 12994 mutex_enter(&cpu_lock);
12891 12995 mutex_enter(&dtrace_lock);
12892 12996
12893 12997 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
12894 12998 rval = EBUSY;
12895 12999 goto out;
12896 13000 }
12897 13001
12898 13002 /*
12899 13003 * Before we can perform any checks, we must prime all of the
12900 13004 * retained enablings that correspond to this state.
12901 13005 */
12902 13006 dtrace_enabling_prime(state);
12903 13007
12904 13008 if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
12905 13009 rval = EACCES;
12906 13010 goto out;
12907 13011 }
12908 13012
12909 13013 dtrace_state_prereserve(state);
12910 13014
12911 13015 /*
12912 13016 * Now we want to do is try to allocate our speculations.
12913 13017 * We do not automatically resize the number of speculations; if
12914 13018 * this fails, we will fail the operation.
12915 13019 */
12916 13020 nspec = opt[DTRACEOPT_NSPEC];
12917 13021 ASSERT(nspec != DTRACEOPT_UNSET);
12918 13022
12919 13023 if (nspec > INT_MAX) {
12920 13024 rval = ENOMEM;
12921 13025 goto out;
12922 13026 }
12923 13027
12924 13028 spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t),
12925 13029 KM_NOSLEEP | KM_NORMALPRI);
12926 13030
12927 13031 if (spec == NULL) {
12928 13032 rval = ENOMEM;
12929 13033 goto out;
12930 13034 }
12931 13035
12932 13036 state->dts_speculations = spec;
12933 13037 state->dts_nspeculations = (int)nspec;
12934 13038
12935 13039 for (i = 0; i < nspec; i++) {
12936 13040 if ((buf = kmem_zalloc(bufsize,
12937 13041 KM_NOSLEEP | KM_NORMALPRI)) == NULL) {
12938 13042 rval = ENOMEM;
12939 13043 goto err;
12940 13044 }
12941 13045
12942 13046 spec[i].dtsp_buffer = buf;
12943 13047 }
12944 13048
12945 13049 if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
12946 13050 if (dtrace_anon.dta_state == NULL) {
12947 13051 rval = ENOENT;
12948 13052 goto out;
12949 13053 }
12950 13054
12951 13055 if (state->dts_necbs != 0) {
12952 13056 rval = EALREADY;
12953 13057 goto out;
12954 13058 }
12955 13059
12956 13060 state->dts_anon = dtrace_anon_grab();
12957 13061 ASSERT(state->dts_anon != NULL);
12958 13062 state = state->dts_anon;
12959 13063
12960 13064 /*
12961 13065 * We want "grabanon" to be set in the grabbed state, so we'll
12962 13066 * copy that option value from the grabbing state into the
12963 13067 * grabbed state.
12964 13068 */
12965 13069 state->dts_options[DTRACEOPT_GRABANON] =
12966 13070 opt[DTRACEOPT_GRABANON];
12967 13071
12968 13072 *cpu = dtrace_anon.dta_beganon;
12969 13073
12970 13074 /*
12971 13075 * If the anonymous state is active (as it almost certainly
12972 13076 * is if the anonymous enabling ultimately matched anything),
12973 13077 * we don't allow any further option processing -- but we
12974 13078 * don't return failure.
12975 13079 */
12976 13080 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
12977 13081 goto out;
12978 13082 }
12979 13083
12980 13084 if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
12981 13085 opt[DTRACEOPT_AGGSIZE] != 0) {
12982 13086 if (state->dts_aggregations == NULL) {
12983 13087 /*
12984 13088 * We're not going to create an aggregation buffer
12985 13089 * because we don't have any ECBs that contain
12986 13090 * aggregations -- set this option to 0.
12987 13091 */
12988 13092 opt[DTRACEOPT_AGGSIZE] = 0;
12989 13093 } else {
12990 13094 /*
12991 13095 * If we have an aggregation buffer, we must also have
12992 13096 * a buffer to use as scratch.
12993 13097 */
12994 13098 if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
12995 13099 opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
12996 13100 opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
12997 13101 }
12998 13102 }
12999 13103 }
13000 13104
13001 13105 if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
13002 13106 opt[DTRACEOPT_SPECSIZE] != 0) {
13003 13107 if (!state->dts_speculates) {
13004 13108 /*
13005 13109 * We're not going to create speculation buffers
13006 13110 * because we don't have any ECBs that actually
13007 13111 * speculate -- set the speculation size to 0.
13008 13112 */
13009 13113 opt[DTRACEOPT_SPECSIZE] = 0;
13010 13114 }
13011 13115 }
13012 13116
13013 13117 /*
13014 13118 * The bare minimum size for any buffer that we're actually going to
13015 13119 * do anything to is sizeof (uint64_t).
13016 13120 */
13017 13121 sz = sizeof (uint64_t);
13018 13122
13019 13123 if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
13020 13124 (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
13021 13125 (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
13022 13126 /*
13023 13127 * A buffer size has been explicitly set to 0 (or to a size
13024 13128 * that will be adjusted to 0) and we need the space -- we
13025 13129 * need to return failure. We return ENOSPC to differentiate
13026 13130 * it from failing to allocate a buffer due to failure to meet
13027 13131 * the reserve (for which we return E2BIG).
13028 13132 */
13029 13133 rval = ENOSPC;
13030 13134 goto out;
13031 13135 }
13032 13136
13033 13137 if ((rval = dtrace_state_buffers(state)) != 0)
13034 13138 goto err;
13035 13139
13036 13140 if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
13037 13141 sz = dtrace_dstate_defsize;
13038 13142
13039 13143 do {
13040 13144 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
13041 13145
13042 13146 if (rval == 0)
13043 13147 break;
13044 13148
13045 13149 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
13046 13150 goto err;
13047 13151 } while (sz >>= 1);
13048 13152
13049 13153 opt[DTRACEOPT_DYNVARSIZE] = sz;
13050 13154
13051 13155 if (rval != 0)
13052 13156 goto err;
13053 13157
13054 13158 if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
13055 13159 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
13056 13160
13057 13161 if (opt[DTRACEOPT_CLEANRATE] == 0)
13058 13162 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13059 13163
13060 13164 if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
13061 13165 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
13062 13166
13063 13167 if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
13064 13168 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
13065 13169
13066 13170 hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
13067 13171 hdlr.cyh_arg = state;
13068 13172 hdlr.cyh_level = CY_LOW_LEVEL;
13069 13173
13070 13174 when.cyt_when = 0;
13071 13175 when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
13072 13176
13073 13177 state->dts_cleaner = cyclic_add(&hdlr, &when);
13074 13178
13075 13179 hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
13076 13180 hdlr.cyh_arg = state;
|
↓ open down ↓ |
4654 lines elided |
↑ open up ↑ |
13077 13181 hdlr.cyh_level = CY_LOW_LEVEL;
13078 13182
13079 13183 when.cyt_when = 0;
13080 13184 when.cyt_interval = dtrace_deadman_interval;
13081 13185
13082 13186 state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
13083 13187 state->dts_deadman = cyclic_add(&hdlr, &when);
13084 13188
13085 13189 state->dts_activity = DTRACE_ACTIVITY_WARMUP;
13086 13190
13191 + if (state->dts_getf != 0 &&
13192 + !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
13193 + /*
13194 + * We don't have kernel privs but we have at least one call
13195 + * to getf(); we need to bump our zone's count, and (if
13196 + * this is the first enabling to have an unprivileged call
13197 + * to getf()) we need to hook into closef().
13198 + */
13199 + state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++;
13200 +
13201 + if (dtrace_getf++ == 0) {
13202 + ASSERT(dtrace_closef == NULL);
13203 + dtrace_closef = dtrace_getf_barrier;
13204 + }
13205 + }
13206 +
13087 13207 /*
13088 13208 * Now it's time to actually fire the BEGIN probe. We need to disable
13089 13209 * interrupts here both to record the CPU on which we fired the BEGIN
13090 13210 * probe (the data from this CPU will be processed first at user
13091 13211 * level) and to manually activate the buffer for this CPU.
13092 13212 */
13093 13213 cookie = dtrace_interrupt_disable();
13094 13214 *cpu = CPU->cpu_id;
13095 13215 ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
13096 13216 state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
13097 13217
13098 13218 dtrace_probe(dtrace_probeid_begin,
13099 13219 (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13100 13220 dtrace_interrupt_enable(cookie);
13101 13221 /*
13102 13222 * We may have had an exit action from a BEGIN probe; only change our
13103 13223 * state to ACTIVE if we're still in WARMUP.
13104 13224 */
13105 13225 ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
13106 13226 state->dts_activity == DTRACE_ACTIVITY_DRAINING);
13107 13227
13108 13228 if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
13109 13229 state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
13110 13230
13111 13231 /*
13112 13232 * Regardless of whether or not now we're in ACTIVE or DRAINING, we
13113 13233 * want each CPU to transition its principal buffer out of the
13114 13234 * INACTIVE state. Doing this assures that no CPU will suddenly begin
13115 13235 * processing an ECB halfway down a probe's ECB chain; all CPUs will
13116 13236 * atomically transition from processing none of a state's ECBs to
13117 13237 * processing all of them.
13118 13238 */
13119 13239 dtrace_xcall(DTRACE_CPUALL,
13120 13240 (dtrace_xcall_t)dtrace_buffer_activate, state);
13121 13241 goto out;
13122 13242
13123 13243 err:
13124 13244 dtrace_buffer_free(state->dts_buffer);
13125 13245 dtrace_buffer_free(state->dts_aggbuffer);
13126 13246
13127 13247 if ((nspec = state->dts_nspeculations) == 0) {
13128 13248 ASSERT(state->dts_speculations == NULL);
13129 13249 goto out;
13130 13250 }
13131 13251
13132 13252 spec = state->dts_speculations;
13133 13253 ASSERT(spec != NULL);
13134 13254
13135 13255 for (i = 0; i < state->dts_nspeculations; i++) {
13136 13256 if ((buf = spec[i].dtsp_buffer) == NULL)
13137 13257 break;
13138 13258
13139 13259 dtrace_buffer_free(buf);
13140 13260 kmem_free(buf, bufsize);
13141 13261 }
13142 13262
13143 13263 kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13144 13264 state->dts_nspeculations = 0;
13145 13265 state->dts_speculations = NULL;
13146 13266
13147 13267 out:
13148 13268 mutex_exit(&dtrace_lock);
13149 13269 mutex_exit(&cpu_lock);
13150 13270
13151 13271 return (rval);
13152 13272 }
13153 13273
13154 13274 static int
13155 13275 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
13156 13276 {
13157 13277 dtrace_icookie_t cookie;
13158 13278
13159 13279 ASSERT(MUTEX_HELD(&dtrace_lock));
13160 13280
13161 13281 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
13162 13282 state->dts_activity != DTRACE_ACTIVITY_DRAINING)
13163 13283 return (EINVAL);
13164 13284
13165 13285 /*
13166 13286 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
13167 13287 * to be sure that every CPU has seen it. See below for the details
13168 13288 * on why this is done.
13169 13289 */
13170 13290 state->dts_activity = DTRACE_ACTIVITY_DRAINING;
13171 13291 dtrace_sync();
13172 13292
13173 13293 /*
13174 13294 * By this point, it is impossible for any CPU to be still processing
13175 13295 * with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to
13176 13296 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
13177 13297 * other CPU in dtrace_buffer_reserve(). This allows dtrace_probe()
13178 13298 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
13179 13299 * iff we're in the END probe.
13180 13300 */
13181 13301 state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
13182 13302 dtrace_sync();
13183 13303 ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
13184 13304
13185 13305 /*
13186 13306 * Finally, we can release the reserve and call the END probe. We
13187 13307 * disable interrupts across calling the END probe to allow us to
13188 13308 * return the CPU on which we actually called the END probe. This
13189 13309 * allows user-land to be sure that this CPU's principal buffer is
13190 13310 * processed last.
13191 13311 */
13192 13312 state->dts_reserve = 0;
|
↓ open down ↓ |
96 lines elided |
↑ open up ↑ |
13193 13313
13194 13314 cookie = dtrace_interrupt_disable();
13195 13315 *cpu = CPU->cpu_id;
13196 13316 dtrace_probe(dtrace_probeid_end,
13197 13317 (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
13198 13318 dtrace_interrupt_enable(cookie);
13199 13319
13200 13320 state->dts_activity = DTRACE_ACTIVITY_STOPPED;
13201 13321 dtrace_sync();
13202 13322
13323 + if (state->dts_getf != 0 &&
13324 + !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) {
13325 + /*
13326 + * We don't have kernel privs but we have at least one call
13327 + * to getf(); we need to lower our zone's count, and (if
13328 + * this is the last enabling to have an unprivileged call
13329 + * to getf()) we need to clear the closef() hook.
13330 + */
13331 + ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0);
13332 + ASSERT(dtrace_closef == dtrace_getf_barrier);
13333 + ASSERT(dtrace_getf > 0);
13334 +
13335 + state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--;
13336 +
13337 + if (--dtrace_getf == 0)
13338 + dtrace_closef = NULL;
13339 + }
13340 +
13203 13341 return (0);
13204 13342 }
13205 13343
13206 13344 static int
13207 13345 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
13208 13346 dtrace_optval_t val)
13209 13347 {
13210 13348 ASSERT(MUTEX_HELD(&dtrace_lock));
13211 13349
13212 13350 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
13213 13351 return (EBUSY);
13214 13352
13215 13353 if (option >= DTRACEOPT_MAX)
13216 13354 return (EINVAL);
13217 13355
13218 13356 if (option != DTRACEOPT_CPU && val < 0)
13219 13357 return (EINVAL);
13220 13358
13221 13359 switch (option) {
13222 13360 case DTRACEOPT_DESTRUCTIVE:
13223 13361 if (dtrace_destructive_disallow)
13224 13362 return (EACCES);
13225 13363
13226 13364 state->dts_cred.dcr_destructive = 1;
13227 13365 break;
13228 13366
13229 13367 case DTRACEOPT_BUFSIZE:
13230 13368 case DTRACEOPT_DYNVARSIZE:
13231 13369 case DTRACEOPT_AGGSIZE:
13232 13370 case DTRACEOPT_SPECSIZE:
13233 13371 case DTRACEOPT_STRSIZE:
13234 13372 if (val < 0)
13235 13373 return (EINVAL);
13236 13374
13237 13375 if (val >= LONG_MAX) {
13238 13376 /*
13239 13377 * If this is an otherwise negative value, set it to
13240 13378 * the highest multiple of 128m less than LONG_MAX.
13241 13379 * Technically, we're adjusting the size without
13242 13380 * regard to the buffer resizing policy, but in fact,
13243 13381 * this has no effect -- if we set the buffer size to
13244 13382 * ~LONG_MAX and the buffer policy is ultimately set to
13245 13383 * be "manual", the buffer allocation is guaranteed to
13246 13384 * fail, if only because the allocation requires two
13247 13385 * buffers. (We set the the size to the highest
13248 13386 * multiple of 128m because it ensures that the size
13249 13387 * will remain a multiple of a megabyte when
13250 13388 * repeatedly halved -- all the way down to 15m.)
13251 13389 */
13252 13390 val = LONG_MAX - (1 << 27) + 1;
13253 13391 }
13254 13392 }
13255 13393
13256 13394 state->dts_options[option] = val;
13257 13395
13258 13396 return (0);
13259 13397 }
13260 13398
13261 13399 static void
13262 13400 dtrace_state_destroy(dtrace_state_t *state)
13263 13401 {
13264 13402 dtrace_ecb_t *ecb;
13265 13403 dtrace_vstate_t *vstate = &state->dts_vstate;
13266 13404 minor_t minor = getminor(state->dts_dev);
13267 13405 int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
13268 13406 dtrace_speculation_t *spec = state->dts_speculations;
13269 13407 int nspec = state->dts_nspeculations;
13270 13408 uint32_t match;
13271 13409
13272 13410 ASSERT(MUTEX_HELD(&dtrace_lock));
13273 13411 ASSERT(MUTEX_HELD(&cpu_lock));
13274 13412
13275 13413 /*
13276 13414 * First, retract any retained enablings for this state.
13277 13415 */
13278 13416 dtrace_enabling_retract(state);
13279 13417 ASSERT(state->dts_nretained == 0);
13280 13418
13281 13419 if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
13282 13420 state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
13283 13421 /*
13284 13422 * We have managed to come into dtrace_state_destroy() on a
13285 13423 * hot enabling -- almost certainly because of a disorderly
13286 13424 * shutdown of a consumer. (That is, a consumer that is
13287 13425 * exiting without having called dtrace_stop().) In this case,
13288 13426 * we're going to set our activity to be KILLED, and then
13289 13427 * issue a sync to be sure that everyone is out of probe
13290 13428 * context before we start blowing away ECBs.
13291 13429 */
13292 13430 state->dts_activity = DTRACE_ACTIVITY_KILLED;
13293 13431 dtrace_sync();
13294 13432 }
13295 13433
13296 13434 /*
13297 13435 * Release the credential hold we took in dtrace_state_create().
13298 13436 */
13299 13437 if (state->dts_cred.dcr_cred != NULL)
13300 13438 crfree(state->dts_cred.dcr_cred);
13301 13439
13302 13440 /*
13303 13441 * Now we can safely disable and destroy any enabled probes. Because
13304 13442 * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
13305 13443 * (especially if they're all enabled), we take two passes through the
13306 13444 * ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and
13307 13445 * in the second we disable whatever is left over.
13308 13446 */
13309 13447 for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
13310 13448 for (i = 0; i < state->dts_necbs; i++) {
13311 13449 if ((ecb = state->dts_ecbs[i]) == NULL)
13312 13450 continue;
13313 13451
13314 13452 if (match && ecb->dte_probe != NULL) {
13315 13453 dtrace_probe_t *probe = ecb->dte_probe;
13316 13454 dtrace_provider_t *prov = probe->dtpr_provider;
13317 13455
13318 13456 if (!(prov->dtpv_priv.dtpp_flags & match))
13319 13457 continue;
13320 13458 }
13321 13459
13322 13460 dtrace_ecb_disable(ecb);
13323 13461 dtrace_ecb_destroy(ecb);
13324 13462 }
13325 13463
13326 13464 if (!match)
13327 13465 break;
13328 13466 }
13329 13467
13330 13468 /*
13331 13469 * Before we free the buffers, perform one more sync to assure that
13332 13470 * every CPU is out of probe context.
13333 13471 */
13334 13472 dtrace_sync();
13335 13473
13336 13474 dtrace_buffer_free(state->dts_buffer);
13337 13475 dtrace_buffer_free(state->dts_aggbuffer);
13338 13476
13339 13477 for (i = 0; i < nspec; i++)
13340 13478 dtrace_buffer_free(spec[i].dtsp_buffer);
13341 13479
13342 13480 if (state->dts_cleaner != CYCLIC_NONE)
13343 13481 cyclic_remove(state->dts_cleaner);
13344 13482
13345 13483 if (state->dts_deadman != CYCLIC_NONE)
13346 13484 cyclic_remove(state->dts_deadman);
13347 13485
13348 13486 dtrace_dstate_fini(&vstate->dtvs_dynvars);
13349 13487 dtrace_vstate_fini(vstate);
13350 13488 kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
13351 13489
13352 13490 if (state->dts_aggregations != NULL) {
13353 13491 #ifdef DEBUG
13354 13492 for (i = 0; i < state->dts_naggregations; i++)
13355 13493 ASSERT(state->dts_aggregations[i] == NULL);
13356 13494 #endif
13357 13495 ASSERT(state->dts_naggregations > 0);
13358 13496 kmem_free(state->dts_aggregations,
13359 13497 state->dts_naggregations * sizeof (dtrace_aggregation_t *));
13360 13498 }
13361 13499
13362 13500 kmem_free(state->dts_buffer, bufsize);
13363 13501 kmem_free(state->dts_aggbuffer, bufsize);
13364 13502
13365 13503 for (i = 0; i < nspec; i++)
13366 13504 kmem_free(spec[i].dtsp_buffer, bufsize);
13367 13505
13368 13506 kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
13369 13507
13370 13508 dtrace_format_destroy(state);
13371 13509
13372 13510 vmem_destroy(state->dts_aggid_arena);
13373 13511 ddi_soft_state_free(dtrace_softstate, minor);
13374 13512 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
13375 13513 }
13376 13514
13377 13515 /*
13378 13516 * DTrace Anonymous Enabling Functions
13379 13517 */
13380 13518 static dtrace_state_t *
13381 13519 dtrace_anon_grab(void)
13382 13520 {
13383 13521 dtrace_state_t *state;
13384 13522
13385 13523 ASSERT(MUTEX_HELD(&dtrace_lock));
13386 13524
13387 13525 if ((state = dtrace_anon.dta_state) == NULL) {
13388 13526 ASSERT(dtrace_anon.dta_enabling == NULL);
13389 13527 return (NULL);
13390 13528 }
13391 13529
13392 13530 ASSERT(dtrace_anon.dta_enabling != NULL);
13393 13531 ASSERT(dtrace_retained != NULL);
13394 13532
13395 13533 dtrace_enabling_destroy(dtrace_anon.dta_enabling);
13396 13534 dtrace_anon.dta_enabling = NULL;
13397 13535 dtrace_anon.dta_state = NULL;
13398 13536
13399 13537 return (state);
13400 13538 }
13401 13539
13402 13540 static void
13403 13541 dtrace_anon_property(void)
13404 13542 {
13405 13543 int i, rv;
13406 13544 dtrace_state_t *state;
13407 13545 dof_hdr_t *dof;
13408 13546 char c[32]; /* enough for "dof-data-" + digits */
13409 13547
13410 13548 ASSERT(MUTEX_HELD(&dtrace_lock));
13411 13549 ASSERT(MUTEX_HELD(&cpu_lock));
13412 13550
13413 13551 for (i = 0; ; i++) {
13414 13552 (void) snprintf(c, sizeof (c), "dof-data-%d", i);
13415 13553
13416 13554 dtrace_err_verbose = 1;
13417 13555
13418 13556 if ((dof = dtrace_dof_property(c)) == NULL) {
13419 13557 dtrace_err_verbose = 0;
13420 13558 break;
13421 13559 }
13422 13560
13423 13561 /*
13424 13562 * We want to create anonymous state, so we need to transition
13425 13563 * the kernel debugger to indicate that DTrace is active. If
13426 13564 * this fails (e.g. because the debugger has modified text in
13427 13565 * some way), we won't continue with the processing.
13428 13566 */
13429 13567 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
13430 13568 cmn_err(CE_NOTE, "kernel debugger active; anonymous "
13431 13569 "enabling ignored.");
13432 13570 dtrace_dof_destroy(dof);
13433 13571 break;
13434 13572 }
13435 13573
13436 13574 /*
13437 13575 * If we haven't allocated an anonymous state, we'll do so now.
13438 13576 */
13439 13577 if ((state = dtrace_anon.dta_state) == NULL) {
13440 13578 state = dtrace_state_create(NULL, NULL);
13441 13579 dtrace_anon.dta_state = state;
13442 13580
13443 13581 if (state == NULL) {
13444 13582 /*
13445 13583 * This basically shouldn't happen: the only
13446 13584 * failure mode from dtrace_state_create() is a
13447 13585 * failure of ddi_soft_state_zalloc() that
13448 13586 * itself should never happen. Still, the
13449 13587 * interface allows for a failure mode, and
13450 13588 * we want to fail as gracefully as possible:
13451 13589 * we'll emit an error message and cease
13452 13590 * processing anonymous state in this case.
13453 13591 */
13454 13592 cmn_err(CE_WARN, "failed to create "
13455 13593 "anonymous state");
13456 13594 dtrace_dof_destroy(dof);
13457 13595 break;
13458 13596 }
13459 13597 }
13460 13598
13461 13599 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
13462 13600 &dtrace_anon.dta_enabling, 0, B_TRUE);
13463 13601
13464 13602 if (rv == 0)
13465 13603 rv = dtrace_dof_options(dof, state);
13466 13604
13467 13605 dtrace_err_verbose = 0;
13468 13606 dtrace_dof_destroy(dof);
13469 13607
13470 13608 if (rv != 0) {
13471 13609 /*
13472 13610 * This is malformed DOF; chuck any anonymous state
13473 13611 * that we created.
13474 13612 */
13475 13613 ASSERT(dtrace_anon.dta_enabling == NULL);
13476 13614 dtrace_state_destroy(state);
13477 13615 dtrace_anon.dta_state = NULL;
13478 13616 break;
13479 13617 }
13480 13618
13481 13619 ASSERT(dtrace_anon.dta_enabling != NULL);
13482 13620 }
13483 13621
13484 13622 if (dtrace_anon.dta_enabling != NULL) {
13485 13623 int rval;
13486 13624
13487 13625 /*
13488 13626 * dtrace_enabling_retain() can only fail because we are
13489 13627 * trying to retain more enablings than are allowed -- but
13490 13628 * we only have one anonymous enabling, and we are guaranteed
13491 13629 * to be allowed at least one retained enabling; we assert
13492 13630 * that dtrace_enabling_retain() returns success.
13493 13631 */
13494 13632 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
13495 13633 ASSERT(rval == 0);
13496 13634
13497 13635 dtrace_enabling_dump(dtrace_anon.dta_enabling);
13498 13636 }
13499 13637 }
13500 13638
13501 13639 /*
13502 13640 * DTrace Helper Functions
13503 13641 */
13504 13642 static void
13505 13643 dtrace_helper_trace(dtrace_helper_action_t *helper,
13506 13644 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
13507 13645 {
13508 13646 uint32_t size, next, nnext, i;
13509 13647 dtrace_helptrace_t *ent;
13510 13648 uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
13511 13649
13512 13650 if (!dtrace_helptrace_enabled)
13513 13651 return;
13514 13652
13515 13653 ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
13516 13654
13517 13655 /*
13518 13656 * What would a tracing framework be without its own tracing
13519 13657 * framework? (Well, a hell of a lot simpler, for starters...)
13520 13658 */
13521 13659 size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
13522 13660 sizeof (uint64_t) - sizeof (uint64_t);
13523 13661
13524 13662 /*
13525 13663 * Iterate until we can allocate a slot in the trace buffer.
13526 13664 */
13527 13665 do {
13528 13666 next = dtrace_helptrace_next;
13529 13667
13530 13668 if (next + size < dtrace_helptrace_bufsize) {
13531 13669 nnext = next + size;
13532 13670 } else {
13533 13671 nnext = size;
13534 13672 }
13535 13673 } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
13536 13674
13537 13675 /*
13538 13676 * We have our slot; fill it in.
13539 13677 */
13540 13678 if (nnext == size)
13541 13679 next = 0;
13542 13680
13543 13681 ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
13544 13682 ent->dtht_helper = helper;
13545 13683 ent->dtht_where = where;
13546 13684 ent->dtht_nlocals = vstate->dtvs_nlocals;
13547 13685
13548 13686 ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
13549 13687 mstate->dtms_fltoffs : -1;
13550 13688 ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
13551 13689 ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
13552 13690
13553 13691 for (i = 0; i < vstate->dtvs_nlocals; i++) {
13554 13692 dtrace_statvar_t *svar;
13555 13693
13556 13694 if ((svar = vstate->dtvs_locals[i]) == NULL)
13557 13695 continue;
13558 13696
13559 13697 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
13560 13698 ent->dtht_locals[i] =
13561 13699 ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
13562 13700 }
13563 13701 }
13564 13702
13565 13703 static uint64_t
13566 13704 dtrace_helper(int which, dtrace_mstate_t *mstate,
13567 13705 dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
13568 13706 {
13569 13707 uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
13570 13708 uint64_t sarg0 = mstate->dtms_arg[0];
13571 13709 uint64_t sarg1 = mstate->dtms_arg[1];
13572 13710 uint64_t rval;
13573 13711 dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
13574 13712 dtrace_helper_action_t *helper;
13575 13713 dtrace_vstate_t *vstate;
13576 13714 dtrace_difo_t *pred;
13577 13715 int i, trace = dtrace_helptrace_enabled;
13578 13716
13579 13717 ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
13580 13718
13581 13719 if (helpers == NULL)
13582 13720 return (0);
13583 13721
13584 13722 if ((helper = helpers->dthps_actions[which]) == NULL)
13585 13723 return (0);
13586 13724
13587 13725 vstate = &helpers->dthps_vstate;
13588 13726 mstate->dtms_arg[0] = arg0;
13589 13727 mstate->dtms_arg[1] = arg1;
13590 13728
13591 13729 /*
13592 13730 * Now iterate over each helper. If its predicate evaluates to 'true',
13593 13731 * we'll call the corresponding actions. Note that the below calls
13594 13732 * to dtrace_dif_emulate() may set faults in machine state. This is
13595 13733 * okay: our caller (the outer dtrace_dif_emulate()) will simply plow
13596 13734 * the stored DIF offset with its own (which is the desired behavior).
13597 13735 * Also, note the calls to dtrace_dif_emulate() may allocate scratch
13598 13736 * from machine state; this is okay, too.
13599 13737 */
13600 13738 for (; helper != NULL; helper = helper->dtha_next) {
13601 13739 if ((pred = helper->dtha_predicate) != NULL) {
13602 13740 if (trace)
13603 13741 dtrace_helper_trace(helper, mstate, vstate, 0);
13604 13742
13605 13743 if (!dtrace_dif_emulate(pred, mstate, vstate, state))
13606 13744 goto next;
13607 13745
13608 13746 if (*flags & CPU_DTRACE_FAULT)
13609 13747 goto err;
13610 13748 }
13611 13749
13612 13750 for (i = 0; i < helper->dtha_nactions; i++) {
13613 13751 if (trace)
13614 13752 dtrace_helper_trace(helper,
13615 13753 mstate, vstate, i + 1);
13616 13754
13617 13755 rval = dtrace_dif_emulate(helper->dtha_actions[i],
13618 13756 mstate, vstate, state);
13619 13757
13620 13758 if (*flags & CPU_DTRACE_FAULT)
13621 13759 goto err;
13622 13760 }
13623 13761
13624 13762 next:
13625 13763 if (trace)
13626 13764 dtrace_helper_trace(helper, mstate, vstate,
13627 13765 DTRACE_HELPTRACE_NEXT);
13628 13766 }
13629 13767
13630 13768 if (trace)
13631 13769 dtrace_helper_trace(helper, mstate, vstate,
13632 13770 DTRACE_HELPTRACE_DONE);
13633 13771
13634 13772 /*
13635 13773 * Restore the arg0 that we saved upon entry.
13636 13774 */
13637 13775 mstate->dtms_arg[0] = sarg0;
13638 13776 mstate->dtms_arg[1] = sarg1;
13639 13777
13640 13778 return (rval);
13641 13779
13642 13780 err:
13643 13781 if (trace)
13644 13782 dtrace_helper_trace(helper, mstate, vstate,
13645 13783 DTRACE_HELPTRACE_ERR);
13646 13784
13647 13785 /*
13648 13786 * Restore the arg0 that we saved upon entry.
13649 13787 */
13650 13788 mstate->dtms_arg[0] = sarg0;
13651 13789 mstate->dtms_arg[1] = sarg1;
13652 13790
13653 13791 return (NULL);
13654 13792 }
13655 13793
13656 13794 static void
13657 13795 dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
13658 13796 dtrace_vstate_t *vstate)
13659 13797 {
13660 13798 int i;
13661 13799
13662 13800 if (helper->dtha_predicate != NULL)
13663 13801 dtrace_difo_release(helper->dtha_predicate, vstate);
13664 13802
13665 13803 for (i = 0; i < helper->dtha_nactions; i++) {
13666 13804 ASSERT(helper->dtha_actions[i] != NULL);
13667 13805 dtrace_difo_release(helper->dtha_actions[i], vstate);
13668 13806 }
13669 13807
13670 13808 kmem_free(helper->dtha_actions,
13671 13809 helper->dtha_nactions * sizeof (dtrace_difo_t *));
13672 13810 kmem_free(helper, sizeof (dtrace_helper_action_t));
13673 13811 }
13674 13812
13675 13813 static int
13676 13814 dtrace_helper_destroygen(int gen)
13677 13815 {
13678 13816 proc_t *p = curproc;
13679 13817 dtrace_helpers_t *help = p->p_dtrace_helpers;
13680 13818 dtrace_vstate_t *vstate;
13681 13819 int i;
13682 13820
13683 13821 ASSERT(MUTEX_HELD(&dtrace_lock));
13684 13822
13685 13823 if (help == NULL || gen > help->dthps_generation)
13686 13824 return (EINVAL);
13687 13825
13688 13826 vstate = &help->dthps_vstate;
13689 13827
13690 13828 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
13691 13829 dtrace_helper_action_t *last = NULL, *h, *next;
13692 13830
13693 13831 for (h = help->dthps_actions[i]; h != NULL; h = next) {
13694 13832 next = h->dtha_next;
13695 13833
13696 13834 if (h->dtha_generation == gen) {
13697 13835 if (last != NULL) {
13698 13836 last->dtha_next = next;
13699 13837 } else {
13700 13838 help->dthps_actions[i] = next;
13701 13839 }
13702 13840
13703 13841 dtrace_helper_action_destroy(h, vstate);
13704 13842 } else {
13705 13843 last = h;
13706 13844 }
13707 13845 }
13708 13846 }
13709 13847
13710 13848 /*
13711 13849 * Interate until we've cleared out all helper providers with the
13712 13850 * given generation number.
13713 13851 */
13714 13852 for (;;) {
13715 13853 dtrace_helper_provider_t *prov;
13716 13854
13717 13855 /*
13718 13856 * Look for a helper provider with the right generation. We
13719 13857 * have to start back at the beginning of the list each time
13720 13858 * because we drop dtrace_lock. It's unlikely that we'll make
13721 13859 * more than two passes.
13722 13860 */
13723 13861 for (i = 0; i < help->dthps_nprovs; i++) {
13724 13862 prov = help->dthps_provs[i];
13725 13863
13726 13864 if (prov->dthp_generation == gen)
13727 13865 break;
13728 13866 }
13729 13867
13730 13868 /*
13731 13869 * If there were no matches, we're done.
13732 13870 */
13733 13871 if (i == help->dthps_nprovs)
13734 13872 break;
13735 13873
13736 13874 /*
13737 13875 * Move the last helper provider into this slot.
13738 13876 */
13739 13877 help->dthps_nprovs--;
13740 13878 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
13741 13879 help->dthps_provs[help->dthps_nprovs] = NULL;
13742 13880
13743 13881 mutex_exit(&dtrace_lock);
13744 13882
13745 13883 /*
13746 13884 * If we have a meta provider, remove this helper provider.
13747 13885 */
13748 13886 mutex_enter(&dtrace_meta_lock);
13749 13887 if (dtrace_meta_pid != NULL) {
13750 13888 ASSERT(dtrace_deferred_pid == NULL);
13751 13889 dtrace_helper_provider_remove(&prov->dthp_prov,
13752 13890 p->p_pid);
13753 13891 }
13754 13892 mutex_exit(&dtrace_meta_lock);
13755 13893
13756 13894 dtrace_helper_provider_destroy(prov);
13757 13895
13758 13896 mutex_enter(&dtrace_lock);
13759 13897 }
13760 13898
13761 13899 return (0);
13762 13900 }
13763 13901
13764 13902 static int
13765 13903 dtrace_helper_validate(dtrace_helper_action_t *helper)
13766 13904 {
13767 13905 int err = 0, i;
13768 13906 dtrace_difo_t *dp;
13769 13907
13770 13908 if ((dp = helper->dtha_predicate) != NULL)
13771 13909 err += dtrace_difo_validate_helper(dp);
13772 13910
13773 13911 for (i = 0; i < helper->dtha_nactions; i++)
13774 13912 err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
13775 13913
13776 13914 return (err == 0);
13777 13915 }
13778 13916
13779 13917 static int
13780 13918 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
13781 13919 {
13782 13920 dtrace_helpers_t *help;
13783 13921 dtrace_helper_action_t *helper, *last;
13784 13922 dtrace_actdesc_t *act;
13785 13923 dtrace_vstate_t *vstate;
13786 13924 dtrace_predicate_t *pred;
13787 13925 int count = 0, nactions = 0, i;
13788 13926
13789 13927 if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
13790 13928 return (EINVAL);
13791 13929
13792 13930 help = curproc->p_dtrace_helpers;
13793 13931 last = help->dthps_actions[which];
13794 13932 vstate = &help->dthps_vstate;
13795 13933
13796 13934 for (count = 0; last != NULL; last = last->dtha_next) {
13797 13935 count++;
13798 13936 if (last->dtha_next == NULL)
13799 13937 break;
13800 13938 }
13801 13939
13802 13940 /*
13803 13941 * If we already have dtrace_helper_actions_max helper actions for this
13804 13942 * helper action type, we'll refuse to add a new one.
13805 13943 */
13806 13944 if (count >= dtrace_helper_actions_max)
13807 13945 return (ENOSPC);
13808 13946
13809 13947 helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
13810 13948 helper->dtha_generation = help->dthps_generation;
13811 13949
13812 13950 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
13813 13951 ASSERT(pred->dtp_difo != NULL);
13814 13952 dtrace_difo_hold(pred->dtp_difo);
13815 13953 helper->dtha_predicate = pred->dtp_difo;
13816 13954 }
13817 13955
13818 13956 for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
13819 13957 if (act->dtad_kind != DTRACEACT_DIFEXPR)
13820 13958 goto err;
13821 13959
13822 13960 if (act->dtad_difo == NULL)
13823 13961 goto err;
13824 13962
13825 13963 nactions++;
13826 13964 }
13827 13965
13828 13966 helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
13829 13967 (helper->dtha_nactions = nactions), KM_SLEEP);
13830 13968
13831 13969 for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
13832 13970 dtrace_difo_hold(act->dtad_difo);
13833 13971 helper->dtha_actions[i++] = act->dtad_difo;
13834 13972 }
13835 13973
13836 13974 if (!dtrace_helper_validate(helper))
13837 13975 goto err;
13838 13976
13839 13977 if (last == NULL) {
13840 13978 help->dthps_actions[which] = helper;
13841 13979 } else {
13842 13980 last->dtha_next = helper;
13843 13981 }
13844 13982
13845 13983 if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
13846 13984 dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
13847 13985 dtrace_helptrace_next = 0;
13848 13986 }
13849 13987
13850 13988 return (0);
13851 13989 err:
13852 13990 dtrace_helper_action_destroy(helper, vstate);
13853 13991 return (EINVAL);
13854 13992 }
13855 13993
13856 13994 static void
13857 13995 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
13858 13996 dof_helper_t *dofhp)
13859 13997 {
13860 13998 ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
13861 13999
13862 14000 mutex_enter(&dtrace_meta_lock);
13863 14001 mutex_enter(&dtrace_lock);
13864 14002
13865 14003 if (!dtrace_attached() || dtrace_meta_pid == NULL) {
13866 14004 /*
13867 14005 * If the dtrace module is loaded but not attached, or if
13868 14006 * there aren't isn't a meta provider registered to deal with
13869 14007 * these provider descriptions, we need to postpone creating
13870 14008 * the actual providers until later.
13871 14009 */
13872 14010
13873 14011 if (help->dthps_next == NULL && help->dthps_prev == NULL &&
13874 14012 dtrace_deferred_pid != help) {
13875 14013 help->dthps_deferred = 1;
13876 14014 help->dthps_pid = p->p_pid;
13877 14015 help->dthps_next = dtrace_deferred_pid;
13878 14016 help->dthps_prev = NULL;
13879 14017 if (dtrace_deferred_pid != NULL)
13880 14018 dtrace_deferred_pid->dthps_prev = help;
13881 14019 dtrace_deferred_pid = help;
13882 14020 }
13883 14021
13884 14022 mutex_exit(&dtrace_lock);
13885 14023
13886 14024 } else if (dofhp != NULL) {
13887 14025 /*
13888 14026 * If the dtrace module is loaded and we have a particular
13889 14027 * helper provider description, pass that off to the
13890 14028 * meta provider.
13891 14029 */
13892 14030
13893 14031 mutex_exit(&dtrace_lock);
13894 14032
13895 14033 dtrace_helper_provide(dofhp, p->p_pid);
13896 14034
13897 14035 } else {
13898 14036 /*
13899 14037 * Otherwise, just pass all the helper provider descriptions
13900 14038 * off to the meta provider.
13901 14039 */
13902 14040
13903 14041 int i;
13904 14042 mutex_exit(&dtrace_lock);
13905 14043
13906 14044 for (i = 0; i < help->dthps_nprovs; i++) {
13907 14045 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
13908 14046 p->p_pid);
13909 14047 }
13910 14048 }
13911 14049
13912 14050 mutex_exit(&dtrace_meta_lock);
13913 14051 }
13914 14052
13915 14053 static int
13916 14054 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
13917 14055 {
13918 14056 dtrace_helpers_t *help;
13919 14057 dtrace_helper_provider_t *hprov, **tmp_provs;
13920 14058 uint_t tmp_maxprovs, i;
13921 14059
13922 14060 ASSERT(MUTEX_HELD(&dtrace_lock));
13923 14061
13924 14062 help = curproc->p_dtrace_helpers;
13925 14063 ASSERT(help != NULL);
13926 14064
13927 14065 /*
13928 14066 * If we already have dtrace_helper_providers_max helper providers,
13929 14067 * we're refuse to add a new one.
13930 14068 */
13931 14069 if (help->dthps_nprovs >= dtrace_helper_providers_max)
13932 14070 return (ENOSPC);
13933 14071
13934 14072 /*
13935 14073 * Check to make sure this isn't a duplicate.
13936 14074 */
13937 14075 for (i = 0; i < help->dthps_nprovs; i++) {
13938 14076 if (dofhp->dofhp_addr ==
13939 14077 help->dthps_provs[i]->dthp_prov.dofhp_addr)
13940 14078 return (EALREADY);
13941 14079 }
13942 14080
13943 14081 hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
13944 14082 hprov->dthp_prov = *dofhp;
13945 14083 hprov->dthp_ref = 1;
13946 14084 hprov->dthp_generation = gen;
13947 14085
13948 14086 /*
13949 14087 * Allocate a bigger table for helper providers if it's already full.
13950 14088 */
13951 14089 if (help->dthps_maxprovs == help->dthps_nprovs) {
13952 14090 tmp_maxprovs = help->dthps_maxprovs;
13953 14091 tmp_provs = help->dthps_provs;
13954 14092
13955 14093 if (help->dthps_maxprovs == 0)
13956 14094 help->dthps_maxprovs = 2;
13957 14095 else
13958 14096 help->dthps_maxprovs *= 2;
13959 14097 if (help->dthps_maxprovs > dtrace_helper_providers_max)
13960 14098 help->dthps_maxprovs = dtrace_helper_providers_max;
13961 14099
13962 14100 ASSERT(tmp_maxprovs < help->dthps_maxprovs);
13963 14101
13964 14102 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
13965 14103 sizeof (dtrace_helper_provider_t *), KM_SLEEP);
13966 14104
13967 14105 if (tmp_provs != NULL) {
13968 14106 bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
13969 14107 sizeof (dtrace_helper_provider_t *));
13970 14108 kmem_free(tmp_provs, tmp_maxprovs *
13971 14109 sizeof (dtrace_helper_provider_t *));
13972 14110 }
13973 14111 }
13974 14112
13975 14113 help->dthps_provs[help->dthps_nprovs] = hprov;
13976 14114 help->dthps_nprovs++;
13977 14115
13978 14116 return (0);
13979 14117 }
13980 14118
13981 14119 static void
13982 14120 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
13983 14121 {
13984 14122 mutex_enter(&dtrace_lock);
13985 14123
13986 14124 if (--hprov->dthp_ref == 0) {
13987 14125 dof_hdr_t *dof;
13988 14126 mutex_exit(&dtrace_lock);
13989 14127 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
13990 14128 dtrace_dof_destroy(dof);
13991 14129 kmem_free(hprov, sizeof (dtrace_helper_provider_t));
13992 14130 } else {
13993 14131 mutex_exit(&dtrace_lock);
13994 14132 }
13995 14133 }
13996 14134
13997 14135 static int
13998 14136 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
13999 14137 {
14000 14138 uintptr_t daddr = (uintptr_t)dof;
14001 14139 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
14002 14140 dof_provider_t *provider;
14003 14141 dof_probe_t *probe;
14004 14142 uint8_t *arg;
14005 14143 char *strtab, *typestr;
14006 14144 dof_stridx_t typeidx;
14007 14145 size_t typesz;
14008 14146 uint_t nprobes, j, k;
14009 14147
14010 14148 ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
14011 14149
14012 14150 if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
14013 14151 dtrace_dof_error(dof, "misaligned section offset");
14014 14152 return (-1);
14015 14153 }
14016 14154
14017 14155 /*
14018 14156 * The section needs to be large enough to contain the DOF provider
14019 14157 * structure appropriate for the given version.
14020 14158 */
14021 14159 if (sec->dofs_size <
14022 14160 ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
14023 14161 offsetof(dof_provider_t, dofpv_prenoffs) :
14024 14162 sizeof (dof_provider_t))) {
14025 14163 dtrace_dof_error(dof, "provider section too small");
14026 14164 return (-1);
14027 14165 }
14028 14166
14029 14167 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
14030 14168 str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
14031 14169 prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
14032 14170 arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
14033 14171 off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
14034 14172
14035 14173 if (str_sec == NULL || prb_sec == NULL ||
14036 14174 arg_sec == NULL || off_sec == NULL)
14037 14175 return (-1);
14038 14176
14039 14177 enoff_sec = NULL;
14040 14178
14041 14179 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
14042 14180 provider->dofpv_prenoffs != DOF_SECT_NONE &&
14043 14181 (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
14044 14182 provider->dofpv_prenoffs)) == NULL)
14045 14183 return (-1);
14046 14184
14047 14185 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
14048 14186
14049 14187 if (provider->dofpv_name >= str_sec->dofs_size ||
14050 14188 strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
14051 14189 dtrace_dof_error(dof, "invalid provider name");
14052 14190 return (-1);
14053 14191 }
14054 14192
14055 14193 if (prb_sec->dofs_entsize == 0 ||
14056 14194 prb_sec->dofs_entsize > prb_sec->dofs_size) {
14057 14195 dtrace_dof_error(dof, "invalid entry size");
14058 14196 return (-1);
14059 14197 }
14060 14198
14061 14199 if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
14062 14200 dtrace_dof_error(dof, "misaligned entry size");
14063 14201 return (-1);
14064 14202 }
14065 14203
14066 14204 if (off_sec->dofs_entsize != sizeof (uint32_t)) {
14067 14205 dtrace_dof_error(dof, "invalid entry size");
14068 14206 return (-1);
14069 14207 }
14070 14208
14071 14209 if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
14072 14210 dtrace_dof_error(dof, "misaligned section offset");
14073 14211 return (-1);
14074 14212 }
14075 14213
14076 14214 if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
14077 14215 dtrace_dof_error(dof, "invalid entry size");
14078 14216 return (-1);
14079 14217 }
14080 14218
14081 14219 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
14082 14220
14083 14221 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
14084 14222
14085 14223 /*
14086 14224 * Take a pass through the probes to check for errors.
14087 14225 */
14088 14226 for (j = 0; j < nprobes; j++) {
14089 14227 probe = (dof_probe_t *)(uintptr_t)(daddr +
14090 14228 prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
14091 14229
14092 14230 if (probe->dofpr_func >= str_sec->dofs_size) {
14093 14231 dtrace_dof_error(dof, "invalid function name");
14094 14232 return (-1);
14095 14233 }
14096 14234
14097 14235 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
14098 14236 dtrace_dof_error(dof, "function name too long");
14099 14237 return (-1);
14100 14238 }
14101 14239
14102 14240 if (probe->dofpr_name >= str_sec->dofs_size ||
14103 14241 strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
14104 14242 dtrace_dof_error(dof, "invalid probe name");
14105 14243 return (-1);
14106 14244 }
14107 14245
14108 14246 /*
14109 14247 * The offset count must not wrap the index, and the offsets
14110 14248 * must also not overflow the section's data.
14111 14249 */
14112 14250 if (probe->dofpr_offidx + probe->dofpr_noffs <
14113 14251 probe->dofpr_offidx ||
14114 14252 (probe->dofpr_offidx + probe->dofpr_noffs) *
14115 14253 off_sec->dofs_entsize > off_sec->dofs_size) {
14116 14254 dtrace_dof_error(dof, "invalid probe offset");
14117 14255 return (-1);
14118 14256 }
14119 14257
14120 14258 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
14121 14259 /*
14122 14260 * If there's no is-enabled offset section, make sure
14123 14261 * there aren't any is-enabled offsets. Otherwise
14124 14262 * perform the same checks as for probe offsets
14125 14263 * (immediately above).
14126 14264 */
14127 14265 if (enoff_sec == NULL) {
14128 14266 if (probe->dofpr_enoffidx != 0 ||
14129 14267 probe->dofpr_nenoffs != 0) {
14130 14268 dtrace_dof_error(dof, "is-enabled "
14131 14269 "offsets with null section");
14132 14270 return (-1);
14133 14271 }
14134 14272 } else if (probe->dofpr_enoffidx +
14135 14273 probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
14136 14274 (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
14137 14275 enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
14138 14276 dtrace_dof_error(dof, "invalid is-enabled "
14139 14277 "offset");
14140 14278 return (-1);
14141 14279 }
14142 14280
14143 14281 if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
14144 14282 dtrace_dof_error(dof, "zero probe and "
14145 14283 "is-enabled offsets");
14146 14284 return (-1);
14147 14285 }
14148 14286 } else if (probe->dofpr_noffs == 0) {
14149 14287 dtrace_dof_error(dof, "zero probe offsets");
14150 14288 return (-1);
14151 14289 }
14152 14290
14153 14291 if (probe->dofpr_argidx + probe->dofpr_xargc <
14154 14292 probe->dofpr_argidx ||
14155 14293 (probe->dofpr_argidx + probe->dofpr_xargc) *
14156 14294 arg_sec->dofs_entsize > arg_sec->dofs_size) {
14157 14295 dtrace_dof_error(dof, "invalid args");
14158 14296 return (-1);
14159 14297 }
14160 14298
14161 14299 typeidx = probe->dofpr_nargv;
14162 14300 typestr = strtab + probe->dofpr_nargv;
14163 14301 for (k = 0; k < probe->dofpr_nargc; k++) {
14164 14302 if (typeidx >= str_sec->dofs_size) {
14165 14303 dtrace_dof_error(dof, "bad "
14166 14304 "native argument type");
14167 14305 return (-1);
14168 14306 }
14169 14307
14170 14308 typesz = strlen(typestr) + 1;
14171 14309 if (typesz > DTRACE_ARGTYPELEN) {
14172 14310 dtrace_dof_error(dof, "native "
14173 14311 "argument type too long");
14174 14312 return (-1);
14175 14313 }
14176 14314 typeidx += typesz;
14177 14315 typestr += typesz;
14178 14316 }
14179 14317
14180 14318 typeidx = probe->dofpr_xargv;
14181 14319 typestr = strtab + probe->dofpr_xargv;
14182 14320 for (k = 0; k < probe->dofpr_xargc; k++) {
14183 14321 if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
14184 14322 dtrace_dof_error(dof, "bad "
14185 14323 "native argument index");
14186 14324 return (-1);
14187 14325 }
14188 14326
14189 14327 if (typeidx >= str_sec->dofs_size) {
14190 14328 dtrace_dof_error(dof, "bad "
14191 14329 "translated argument type");
14192 14330 return (-1);
14193 14331 }
14194 14332
14195 14333 typesz = strlen(typestr) + 1;
14196 14334 if (typesz > DTRACE_ARGTYPELEN) {
14197 14335 dtrace_dof_error(dof, "translated argument "
14198 14336 "type too long");
14199 14337 return (-1);
14200 14338 }
14201 14339
14202 14340 typeidx += typesz;
14203 14341 typestr += typesz;
14204 14342 }
14205 14343 }
14206 14344
14207 14345 return (0);
14208 14346 }
14209 14347
14210 14348 static int
14211 14349 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
14212 14350 {
14213 14351 dtrace_helpers_t *help;
14214 14352 dtrace_vstate_t *vstate;
14215 14353 dtrace_enabling_t *enab = NULL;
14216 14354 int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
14217 14355 uintptr_t daddr = (uintptr_t)dof;
14218 14356
14219 14357 ASSERT(MUTEX_HELD(&dtrace_lock));
14220 14358
14221 14359 if ((help = curproc->p_dtrace_helpers) == NULL)
14222 14360 help = dtrace_helpers_create(curproc);
14223 14361
14224 14362 vstate = &help->dthps_vstate;
14225 14363
14226 14364 if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
14227 14365 dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
14228 14366 dtrace_dof_destroy(dof);
14229 14367 return (rv);
14230 14368 }
14231 14369
14232 14370 /*
14233 14371 * Look for helper providers and validate their descriptions.
14234 14372 */
14235 14373 if (dhp != NULL) {
14236 14374 for (i = 0; i < dof->dofh_secnum; i++) {
14237 14375 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
14238 14376 dof->dofh_secoff + i * dof->dofh_secsize);
14239 14377
14240 14378 if (sec->dofs_type != DOF_SECT_PROVIDER)
14241 14379 continue;
14242 14380
14243 14381 if (dtrace_helper_provider_validate(dof, sec) != 0) {
14244 14382 dtrace_enabling_destroy(enab);
14245 14383 dtrace_dof_destroy(dof);
14246 14384 return (-1);
14247 14385 }
14248 14386
14249 14387 nprovs++;
14250 14388 }
14251 14389 }
14252 14390
14253 14391 /*
14254 14392 * Now we need to walk through the ECB descriptions in the enabling.
14255 14393 */
14256 14394 for (i = 0; i < enab->dten_ndesc; i++) {
14257 14395 dtrace_ecbdesc_t *ep = enab->dten_desc[i];
14258 14396 dtrace_probedesc_t *desc = &ep->dted_probe;
14259 14397
14260 14398 if (strcmp(desc->dtpd_provider, "dtrace") != 0)
14261 14399 continue;
14262 14400
14263 14401 if (strcmp(desc->dtpd_mod, "helper") != 0)
14264 14402 continue;
14265 14403
14266 14404 if (strcmp(desc->dtpd_func, "ustack") != 0)
14267 14405 continue;
14268 14406
14269 14407 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
14270 14408 ep)) != 0) {
14271 14409 /*
14272 14410 * Adding this helper action failed -- we are now going
14273 14411 * to rip out the entire generation and return failure.
14274 14412 */
14275 14413 (void) dtrace_helper_destroygen(help->dthps_generation);
14276 14414 dtrace_enabling_destroy(enab);
14277 14415 dtrace_dof_destroy(dof);
14278 14416 return (-1);
14279 14417 }
14280 14418
14281 14419 nhelpers++;
14282 14420 }
14283 14421
14284 14422 if (nhelpers < enab->dten_ndesc)
14285 14423 dtrace_dof_error(dof, "unmatched helpers");
14286 14424
14287 14425 gen = help->dthps_generation++;
14288 14426 dtrace_enabling_destroy(enab);
14289 14427
14290 14428 if (dhp != NULL && nprovs > 0) {
14291 14429 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
14292 14430 if (dtrace_helper_provider_add(dhp, gen) == 0) {
14293 14431 mutex_exit(&dtrace_lock);
14294 14432 dtrace_helper_provider_register(curproc, help, dhp);
14295 14433 mutex_enter(&dtrace_lock);
14296 14434
14297 14435 destroy = 0;
14298 14436 }
14299 14437 }
14300 14438
14301 14439 if (destroy)
14302 14440 dtrace_dof_destroy(dof);
14303 14441
14304 14442 return (gen);
14305 14443 }
14306 14444
14307 14445 static dtrace_helpers_t *
14308 14446 dtrace_helpers_create(proc_t *p)
14309 14447 {
14310 14448 dtrace_helpers_t *help;
14311 14449
14312 14450 ASSERT(MUTEX_HELD(&dtrace_lock));
14313 14451 ASSERT(p->p_dtrace_helpers == NULL);
14314 14452
14315 14453 help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
14316 14454 help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
14317 14455 DTRACE_NHELPER_ACTIONS, KM_SLEEP);
14318 14456
14319 14457 p->p_dtrace_helpers = help;
14320 14458 dtrace_helpers++;
14321 14459
14322 14460 return (help);
14323 14461 }
14324 14462
14325 14463 static void
14326 14464 dtrace_helpers_destroy(void)
14327 14465 {
14328 14466 dtrace_helpers_t *help;
14329 14467 dtrace_vstate_t *vstate;
14330 14468 proc_t *p = curproc;
14331 14469 int i;
14332 14470
14333 14471 mutex_enter(&dtrace_lock);
14334 14472
14335 14473 ASSERT(p->p_dtrace_helpers != NULL);
14336 14474 ASSERT(dtrace_helpers > 0);
14337 14475
14338 14476 help = p->p_dtrace_helpers;
14339 14477 vstate = &help->dthps_vstate;
14340 14478
14341 14479 /*
14342 14480 * We're now going to lose the help from this process.
14343 14481 */
14344 14482 p->p_dtrace_helpers = NULL;
14345 14483 dtrace_sync();
14346 14484
14347 14485 /*
14348 14486 * Destory the helper actions.
14349 14487 */
14350 14488 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14351 14489 dtrace_helper_action_t *h, *next;
14352 14490
14353 14491 for (h = help->dthps_actions[i]; h != NULL; h = next) {
14354 14492 next = h->dtha_next;
14355 14493 dtrace_helper_action_destroy(h, vstate);
14356 14494 h = next;
14357 14495 }
14358 14496 }
14359 14497
14360 14498 mutex_exit(&dtrace_lock);
14361 14499
14362 14500 /*
14363 14501 * Destroy the helper providers.
14364 14502 */
14365 14503 if (help->dthps_maxprovs > 0) {
14366 14504 mutex_enter(&dtrace_meta_lock);
14367 14505 if (dtrace_meta_pid != NULL) {
14368 14506 ASSERT(dtrace_deferred_pid == NULL);
14369 14507
14370 14508 for (i = 0; i < help->dthps_nprovs; i++) {
14371 14509 dtrace_helper_provider_remove(
14372 14510 &help->dthps_provs[i]->dthp_prov, p->p_pid);
14373 14511 }
14374 14512 } else {
14375 14513 mutex_enter(&dtrace_lock);
14376 14514 ASSERT(help->dthps_deferred == 0 ||
14377 14515 help->dthps_next != NULL ||
14378 14516 help->dthps_prev != NULL ||
14379 14517 help == dtrace_deferred_pid);
14380 14518
14381 14519 /*
14382 14520 * Remove the helper from the deferred list.
14383 14521 */
14384 14522 if (help->dthps_next != NULL)
14385 14523 help->dthps_next->dthps_prev = help->dthps_prev;
14386 14524 if (help->dthps_prev != NULL)
14387 14525 help->dthps_prev->dthps_next = help->dthps_next;
14388 14526 if (dtrace_deferred_pid == help) {
14389 14527 dtrace_deferred_pid = help->dthps_next;
14390 14528 ASSERT(help->dthps_prev == NULL);
14391 14529 }
14392 14530
14393 14531 mutex_exit(&dtrace_lock);
14394 14532 }
14395 14533
14396 14534 mutex_exit(&dtrace_meta_lock);
14397 14535
14398 14536 for (i = 0; i < help->dthps_nprovs; i++) {
14399 14537 dtrace_helper_provider_destroy(help->dthps_provs[i]);
14400 14538 }
14401 14539
14402 14540 kmem_free(help->dthps_provs, help->dthps_maxprovs *
14403 14541 sizeof (dtrace_helper_provider_t *));
14404 14542 }
14405 14543
14406 14544 mutex_enter(&dtrace_lock);
14407 14545
14408 14546 dtrace_vstate_fini(&help->dthps_vstate);
14409 14547 kmem_free(help->dthps_actions,
14410 14548 sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
14411 14549 kmem_free(help, sizeof (dtrace_helpers_t));
14412 14550
14413 14551 --dtrace_helpers;
14414 14552 mutex_exit(&dtrace_lock);
14415 14553 }
14416 14554
14417 14555 static void
14418 14556 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
14419 14557 {
14420 14558 dtrace_helpers_t *help, *newhelp;
14421 14559 dtrace_helper_action_t *helper, *new, *last;
14422 14560 dtrace_difo_t *dp;
14423 14561 dtrace_vstate_t *vstate;
14424 14562 int i, j, sz, hasprovs = 0;
14425 14563
14426 14564 mutex_enter(&dtrace_lock);
14427 14565 ASSERT(from->p_dtrace_helpers != NULL);
14428 14566 ASSERT(dtrace_helpers > 0);
14429 14567
14430 14568 help = from->p_dtrace_helpers;
14431 14569 newhelp = dtrace_helpers_create(to);
14432 14570 ASSERT(to->p_dtrace_helpers != NULL);
14433 14571
14434 14572 newhelp->dthps_generation = help->dthps_generation;
14435 14573 vstate = &newhelp->dthps_vstate;
14436 14574
14437 14575 /*
14438 14576 * Duplicate the helper actions.
14439 14577 */
14440 14578 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
14441 14579 if ((helper = help->dthps_actions[i]) == NULL)
14442 14580 continue;
14443 14581
14444 14582 for (last = NULL; helper != NULL; helper = helper->dtha_next) {
14445 14583 new = kmem_zalloc(sizeof (dtrace_helper_action_t),
14446 14584 KM_SLEEP);
14447 14585 new->dtha_generation = helper->dtha_generation;
14448 14586
14449 14587 if ((dp = helper->dtha_predicate) != NULL) {
14450 14588 dp = dtrace_difo_duplicate(dp, vstate);
14451 14589 new->dtha_predicate = dp;
14452 14590 }
14453 14591
14454 14592 new->dtha_nactions = helper->dtha_nactions;
14455 14593 sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
14456 14594 new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
14457 14595
14458 14596 for (j = 0; j < new->dtha_nactions; j++) {
14459 14597 dtrace_difo_t *dp = helper->dtha_actions[j];
14460 14598
14461 14599 ASSERT(dp != NULL);
14462 14600 dp = dtrace_difo_duplicate(dp, vstate);
14463 14601 new->dtha_actions[j] = dp;
14464 14602 }
14465 14603
14466 14604 if (last != NULL) {
14467 14605 last->dtha_next = new;
14468 14606 } else {
14469 14607 newhelp->dthps_actions[i] = new;
14470 14608 }
14471 14609
14472 14610 last = new;
14473 14611 }
14474 14612 }
14475 14613
14476 14614 /*
14477 14615 * Duplicate the helper providers and register them with the
14478 14616 * DTrace framework.
14479 14617 */
14480 14618 if (help->dthps_nprovs > 0) {
14481 14619 newhelp->dthps_nprovs = help->dthps_nprovs;
14482 14620 newhelp->dthps_maxprovs = help->dthps_nprovs;
14483 14621 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
14484 14622 sizeof (dtrace_helper_provider_t *), KM_SLEEP);
14485 14623 for (i = 0; i < newhelp->dthps_nprovs; i++) {
14486 14624 newhelp->dthps_provs[i] = help->dthps_provs[i];
14487 14625 newhelp->dthps_provs[i]->dthp_ref++;
14488 14626 }
14489 14627
14490 14628 hasprovs = 1;
14491 14629 }
14492 14630
14493 14631 mutex_exit(&dtrace_lock);
14494 14632
14495 14633 if (hasprovs)
14496 14634 dtrace_helper_provider_register(to, newhelp, NULL);
14497 14635 }
14498 14636
14499 14637 /*
14500 14638 * DTrace Hook Functions
14501 14639 */
14502 14640 static void
14503 14641 dtrace_module_loaded(struct modctl *ctl)
14504 14642 {
14505 14643 dtrace_provider_t *prv;
14506 14644
14507 14645 mutex_enter(&dtrace_provider_lock);
14508 14646 mutex_enter(&mod_lock);
14509 14647
14510 14648 ASSERT(ctl->mod_busy);
14511 14649
14512 14650 /*
14513 14651 * We're going to call each providers per-module provide operation
14514 14652 * specifying only this module.
14515 14653 */
14516 14654 for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
14517 14655 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
14518 14656
14519 14657 mutex_exit(&mod_lock);
14520 14658 mutex_exit(&dtrace_provider_lock);
14521 14659
14522 14660 /*
14523 14661 * If we have any retained enablings, we need to match against them.
14524 14662 * Enabling probes requires that cpu_lock be held, and we cannot hold
14525 14663 * cpu_lock here -- it is legal for cpu_lock to be held when loading a
14526 14664 * module. (In particular, this happens when loading scheduling
14527 14665 * classes.) So if we have any retained enablings, we need to dispatch
14528 14666 * our task queue to do the match for us.
14529 14667 */
14530 14668 mutex_enter(&dtrace_lock);
14531 14669
14532 14670 if (dtrace_retained == NULL) {
14533 14671 mutex_exit(&dtrace_lock);
14534 14672 return;
14535 14673 }
14536 14674
14537 14675 (void) taskq_dispatch(dtrace_taskq,
14538 14676 (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
14539 14677
14540 14678 mutex_exit(&dtrace_lock);
14541 14679
14542 14680 /*
14543 14681 * And now, for a little heuristic sleaze: in general, we want to
14544 14682 * match modules as soon as they load. However, we cannot guarantee
14545 14683 * this, because it would lead us to the lock ordering violation
14546 14684 * outlined above. The common case, of course, is that cpu_lock is
14547 14685 * _not_ held -- so we delay here for a clock tick, hoping that that's
14548 14686 * long enough for the task queue to do its work. If it's not, it's
14549 14687 * not a serious problem -- it just means that the module that we
14550 14688 * just loaded may not be immediately instrumentable.
14551 14689 */
14552 14690 delay(1);
14553 14691 }
14554 14692
14555 14693 static void
14556 14694 dtrace_module_unloaded(struct modctl *ctl)
14557 14695 {
14558 14696 dtrace_probe_t template, *probe, *first, *next;
14559 14697 dtrace_provider_t *prov;
14560 14698
14561 14699 template.dtpr_mod = ctl->mod_modname;
14562 14700
14563 14701 mutex_enter(&dtrace_provider_lock);
14564 14702 mutex_enter(&mod_lock);
14565 14703 mutex_enter(&dtrace_lock);
14566 14704
14567 14705 if (dtrace_bymod == NULL) {
14568 14706 /*
14569 14707 * The DTrace module is loaded (obviously) but not attached;
14570 14708 * we don't have any work to do.
14571 14709 */
14572 14710 mutex_exit(&dtrace_provider_lock);
14573 14711 mutex_exit(&mod_lock);
14574 14712 mutex_exit(&dtrace_lock);
14575 14713 return;
14576 14714 }
14577 14715
14578 14716 for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
14579 14717 probe != NULL; probe = probe->dtpr_nextmod) {
14580 14718 if (probe->dtpr_ecb != NULL) {
14581 14719 mutex_exit(&dtrace_provider_lock);
14582 14720 mutex_exit(&mod_lock);
14583 14721 mutex_exit(&dtrace_lock);
14584 14722
14585 14723 /*
14586 14724 * This shouldn't _actually_ be possible -- we're
14587 14725 * unloading a module that has an enabled probe in it.
14588 14726 * (It's normally up to the provider to make sure that
14589 14727 * this can't happen.) However, because dtps_enable()
14590 14728 * doesn't have a failure mode, there can be an
14591 14729 * enable/unload race. Upshot: we don't want to
14592 14730 * assert, but we're not going to disable the
14593 14731 * probe, either.
14594 14732 */
14595 14733 if (dtrace_err_verbose) {
14596 14734 cmn_err(CE_WARN, "unloaded module '%s' had "
14597 14735 "enabled probes", ctl->mod_modname);
14598 14736 }
14599 14737
14600 14738 return;
14601 14739 }
14602 14740 }
14603 14741
14604 14742 probe = first;
14605 14743
14606 14744 for (first = NULL; probe != NULL; probe = next) {
14607 14745 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
14608 14746
14609 14747 dtrace_probes[probe->dtpr_id - 1] = NULL;
14610 14748
14611 14749 next = probe->dtpr_nextmod;
14612 14750 dtrace_hash_remove(dtrace_bymod, probe);
14613 14751 dtrace_hash_remove(dtrace_byfunc, probe);
14614 14752 dtrace_hash_remove(dtrace_byname, probe);
14615 14753
14616 14754 if (first == NULL) {
14617 14755 first = probe;
14618 14756 probe->dtpr_nextmod = NULL;
14619 14757 } else {
14620 14758 probe->dtpr_nextmod = first;
14621 14759 first = probe;
14622 14760 }
14623 14761 }
14624 14762
14625 14763 /*
14626 14764 * We've removed all of the module's probes from the hash chains and
14627 14765 * from the probe array. Now issue a dtrace_sync() to be sure that
14628 14766 * everyone has cleared out from any probe array processing.
14629 14767 */
14630 14768 dtrace_sync();
14631 14769
14632 14770 for (probe = first; probe != NULL; probe = first) {
14633 14771 first = probe->dtpr_nextmod;
14634 14772 prov = probe->dtpr_provider;
14635 14773 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
14636 14774 probe->dtpr_arg);
14637 14775 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
14638 14776 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
14639 14777 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
14640 14778 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
14641 14779 kmem_free(probe, sizeof (dtrace_probe_t));
14642 14780 }
14643 14781
14644 14782 mutex_exit(&dtrace_lock);
14645 14783 mutex_exit(&mod_lock);
14646 14784 mutex_exit(&dtrace_provider_lock);
14647 14785 }
14648 14786
14649 14787 void
14650 14788 dtrace_suspend(void)
14651 14789 {
14652 14790 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
14653 14791 }
14654 14792
14655 14793 void
14656 14794 dtrace_resume(void)
14657 14795 {
14658 14796 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
14659 14797 }
14660 14798
14661 14799 static int
14662 14800 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
14663 14801 {
14664 14802 ASSERT(MUTEX_HELD(&cpu_lock));
14665 14803 mutex_enter(&dtrace_lock);
14666 14804
14667 14805 switch (what) {
14668 14806 case CPU_CONFIG: {
14669 14807 dtrace_state_t *state;
14670 14808 dtrace_optval_t *opt, rs, c;
14671 14809
14672 14810 /*
14673 14811 * For now, we only allocate a new buffer for anonymous state.
14674 14812 */
14675 14813 if ((state = dtrace_anon.dta_state) == NULL)
14676 14814 break;
14677 14815
14678 14816 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
14679 14817 break;
14680 14818
14681 14819 opt = state->dts_options;
14682 14820 c = opt[DTRACEOPT_CPU];
14683 14821
14684 14822 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
14685 14823 break;
14686 14824
14687 14825 /*
14688 14826 * Regardless of what the actual policy is, we're going to
14689 14827 * temporarily set our resize policy to be manual. We're
14690 14828 * also going to temporarily set our CPU option to denote
14691 14829 * the newly configured CPU.
14692 14830 */
14693 14831 rs = opt[DTRACEOPT_BUFRESIZE];
14694 14832 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
14695 14833 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
14696 14834
14697 14835 (void) dtrace_state_buffers(state);
14698 14836
14699 14837 opt[DTRACEOPT_BUFRESIZE] = rs;
14700 14838 opt[DTRACEOPT_CPU] = c;
14701 14839
14702 14840 break;
14703 14841 }
14704 14842
14705 14843 case CPU_UNCONFIG:
14706 14844 /*
14707 14845 * We don't free the buffer in the CPU_UNCONFIG case. (The
14708 14846 * buffer will be freed when the consumer exits.)
14709 14847 */
14710 14848 break;
14711 14849
14712 14850 default:
14713 14851 break;
14714 14852 }
14715 14853
14716 14854 mutex_exit(&dtrace_lock);
14717 14855 return (0);
14718 14856 }
14719 14857
14720 14858 static void
14721 14859 dtrace_cpu_setup_initial(processorid_t cpu)
14722 14860 {
14723 14861 (void) dtrace_cpu_setup(CPU_CONFIG, cpu);
14724 14862 }
14725 14863
14726 14864 static void
14727 14865 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
14728 14866 {
14729 14867 if (dtrace_toxranges >= dtrace_toxranges_max) {
14730 14868 int osize, nsize;
14731 14869 dtrace_toxrange_t *range;
14732 14870
14733 14871 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
14734 14872
14735 14873 if (osize == 0) {
14736 14874 ASSERT(dtrace_toxrange == NULL);
14737 14875 ASSERT(dtrace_toxranges_max == 0);
14738 14876 dtrace_toxranges_max = 1;
14739 14877 } else {
14740 14878 dtrace_toxranges_max <<= 1;
14741 14879 }
14742 14880
14743 14881 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
14744 14882 range = kmem_zalloc(nsize, KM_SLEEP);
14745 14883
14746 14884 if (dtrace_toxrange != NULL) {
14747 14885 ASSERT(osize != 0);
14748 14886 bcopy(dtrace_toxrange, range, osize);
14749 14887 kmem_free(dtrace_toxrange, osize);
14750 14888 }
14751 14889
14752 14890 dtrace_toxrange = range;
|
↓ open down ↓ |
1540 lines elided |
↑ open up ↑ |
14753 14891 }
14754 14892
14755 14893 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
14756 14894 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
14757 14895
14758 14896 dtrace_toxrange[dtrace_toxranges].dtt_base = base;
14759 14897 dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
14760 14898 dtrace_toxranges++;
14761 14899 }
14762 14900
14901 +static void
14902 +dtrace_getf_barrier()
14903 +{
14904 + /*
14905 + * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings
14906 + * that contain calls to getf(), this routine will be called on every
14907 + * closef() before either the underlying vnode is released or the
14908 + * file_t itself is freed. By the time we are here, it is essential
14909 + * that the file_t can no longer be accessed from a call to getf()
14910 + * in probe context -- that assures that a dtrace_sync() can be used
14911 + * to clear out any enablings referring to the old structures.
14912 + */
14913 + if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 ||
14914 + kcred->cr_zone->zone_dtrace_getf != 0)
14915 + dtrace_sync();
14916 +}
14917 +
14763 14918 /*
14764 14919 * DTrace Driver Cookbook Functions
14765 14920 */
14766 14921 /*ARGSUSED*/
14767 14922 static int
14768 14923 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
14769 14924 {
14770 14925 dtrace_provider_id_t id;
14771 14926 dtrace_state_t *state = NULL;
14772 14927 dtrace_enabling_t *enab;
14773 14928
14774 14929 mutex_enter(&cpu_lock);
14775 14930 mutex_enter(&dtrace_provider_lock);
14776 14931 mutex_enter(&dtrace_lock);
14777 14932
14778 14933 if (ddi_soft_state_init(&dtrace_softstate,
14779 14934 sizeof (dtrace_state_t), 0) != 0) {
14780 14935 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
14781 14936 mutex_exit(&cpu_lock);
14782 14937 mutex_exit(&dtrace_provider_lock);
14783 14938 mutex_exit(&dtrace_lock);
14784 14939 return (DDI_FAILURE);
14785 14940 }
14786 14941
14787 14942 if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
14788 14943 DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
14789 14944 ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
14790 14945 DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
14791 14946 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
14792 14947 ddi_remove_minor_node(devi, NULL);
14793 14948 ddi_soft_state_fini(&dtrace_softstate);
14794 14949 mutex_exit(&cpu_lock);
14795 14950 mutex_exit(&dtrace_provider_lock);
14796 14951 mutex_exit(&dtrace_lock);
14797 14952 return (DDI_FAILURE);
14798 14953 }
14799 14954
14800 14955 ddi_report_dev(devi);
14801 14956 dtrace_devi = devi;
14802 14957
14803 14958 dtrace_modload = dtrace_module_loaded;
14804 14959 dtrace_modunload = dtrace_module_unloaded;
14805 14960 dtrace_cpu_init = dtrace_cpu_setup_initial;
14806 14961 dtrace_helpers_cleanup = dtrace_helpers_destroy;
14807 14962 dtrace_helpers_fork = dtrace_helpers_duplicate;
14808 14963 dtrace_cpustart_init = dtrace_suspend;
14809 14964 dtrace_cpustart_fini = dtrace_resume;
14810 14965 dtrace_debugger_init = dtrace_suspend;
14811 14966 dtrace_debugger_fini = dtrace_resume;
14812 14967
14813 14968 register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
14814 14969
14815 14970 ASSERT(MUTEX_HELD(&cpu_lock));
14816 14971
14817 14972 dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
14818 14973 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
14819 14974 dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
14820 14975 UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
14821 14976 VM_SLEEP | VMC_IDENTIFIER);
14822 14977 dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
14823 14978 1, INT_MAX, 0);
14824 14979
14825 14980 dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
14826 14981 sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
14827 14982 NULL, NULL, NULL, NULL, NULL, 0);
14828 14983
14829 14984 ASSERT(MUTEX_HELD(&cpu_lock));
14830 14985 dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
14831 14986 offsetof(dtrace_probe_t, dtpr_nextmod),
14832 14987 offsetof(dtrace_probe_t, dtpr_prevmod));
14833 14988
14834 14989 dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
14835 14990 offsetof(dtrace_probe_t, dtpr_nextfunc),
14836 14991 offsetof(dtrace_probe_t, dtpr_prevfunc));
14837 14992
14838 14993 dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
14839 14994 offsetof(dtrace_probe_t, dtpr_nextname),
14840 14995 offsetof(dtrace_probe_t, dtpr_prevname));
14841 14996
14842 14997 if (dtrace_retain_max < 1) {
14843 14998 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
14844 14999 "setting to 1", dtrace_retain_max);
14845 15000 dtrace_retain_max = 1;
14846 15001 }
14847 15002
14848 15003 /*
14849 15004 * Now discover our toxic ranges.
14850 15005 */
14851 15006 dtrace_toxic_ranges(dtrace_toxrange_add);
14852 15007
14853 15008 /*
14854 15009 * Before we register ourselves as a provider to our own framework,
14855 15010 * we would like to assert that dtrace_provider is NULL -- but that's
14856 15011 * not true if we were loaded as a dependency of a DTrace provider.
14857 15012 * Once we've registered, we can assert that dtrace_provider is our
14858 15013 * pseudo provider.
14859 15014 */
14860 15015 (void) dtrace_register("dtrace", &dtrace_provider_attr,
14861 15016 DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
14862 15017
14863 15018 ASSERT(dtrace_provider != NULL);
14864 15019 ASSERT((dtrace_provider_id_t)dtrace_provider == id);
14865 15020
14866 15021 dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
14867 15022 dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
14868 15023 dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
14869 15024 dtrace_provider, NULL, NULL, "END", 0, NULL);
14870 15025 dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
14871 15026 dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
14872 15027
14873 15028 dtrace_anon_property();
14874 15029 mutex_exit(&cpu_lock);
14875 15030
14876 15031 /*
14877 15032 * If DTrace helper tracing is enabled, we need to allocate the
14878 15033 * trace buffer and initialize the values.
14879 15034 */
14880 15035 if (dtrace_helptrace_enabled) {
14881 15036 ASSERT(dtrace_helptrace_buffer == NULL);
14882 15037 dtrace_helptrace_buffer =
14883 15038 kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
14884 15039 dtrace_helptrace_next = 0;
14885 15040 }
14886 15041
14887 15042 /*
14888 15043 * If there are already providers, we must ask them to provide their
14889 15044 * probes, and then match any anonymous enabling against them. Note
14890 15045 * that there should be no other retained enablings at this time:
14891 15046 * the only retained enablings at this time should be the anonymous
14892 15047 * enabling.
14893 15048 */
14894 15049 if (dtrace_anon.dta_enabling != NULL) {
14895 15050 ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
14896 15051
14897 15052 dtrace_enabling_provide(NULL);
14898 15053 state = dtrace_anon.dta_state;
14899 15054
14900 15055 /*
14901 15056 * We couldn't hold cpu_lock across the above call to
14902 15057 * dtrace_enabling_provide(), but we must hold it to actually
14903 15058 * enable the probes. We have to drop all of our locks, pick
14904 15059 * up cpu_lock, and regain our locks before matching the
14905 15060 * retained anonymous enabling.
14906 15061 */
14907 15062 mutex_exit(&dtrace_lock);
14908 15063 mutex_exit(&dtrace_provider_lock);
14909 15064
14910 15065 mutex_enter(&cpu_lock);
14911 15066 mutex_enter(&dtrace_provider_lock);
14912 15067 mutex_enter(&dtrace_lock);
14913 15068
14914 15069 if ((enab = dtrace_anon.dta_enabling) != NULL)
14915 15070 (void) dtrace_enabling_match(enab, NULL);
14916 15071
14917 15072 mutex_exit(&cpu_lock);
14918 15073 }
14919 15074
14920 15075 mutex_exit(&dtrace_lock);
14921 15076 mutex_exit(&dtrace_provider_lock);
14922 15077
14923 15078 if (state != NULL) {
14924 15079 /*
14925 15080 * If we created any anonymous state, set it going now.
14926 15081 */
14927 15082 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
14928 15083 }
14929 15084
14930 15085 return (DDI_SUCCESS);
14931 15086 }
14932 15087
14933 15088 /*ARGSUSED*/
14934 15089 static int
14935 15090 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
14936 15091 {
14937 15092 dtrace_state_t *state;
14938 15093 uint32_t priv;
14939 15094 uid_t uid;
14940 15095 zoneid_t zoneid;
14941 15096
14942 15097 if (getminor(*devp) == DTRACEMNRN_HELPER)
14943 15098 return (0);
14944 15099
14945 15100 /*
14946 15101 * If this wasn't an open with the "helper" minor, then it must be
14947 15102 * the "dtrace" minor.
14948 15103 */
14949 15104 if (getminor(*devp) != DTRACEMNRN_DTRACE)
14950 15105 return (ENXIO);
14951 15106
14952 15107 /*
14953 15108 * If no DTRACE_PRIV_* bits are set in the credential, then the
14954 15109 * caller lacks sufficient permission to do anything with DTrace.
14955 15110 */
14956 15111 dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
14957 15112 if (priv == DTRACE_PRIV_NONE)
14958 15113 return (EACCES);
14959 15114
14960 15115 /*
14961 15116 * Ask all providers to provide all their probes.
14962 15117 */
14963 15118 mutex_enter(&dtrace_provider_lock);
14964 15119 dtrace_probe_provide(NULL, NULL);
14965 15120 mutex_exit(&dtrace_provider_lock);
14966 15121
14967 15122 mutex_enter(&cpu_lock);
14968 15123 mutex_enter(&dtrace_lock);
14969 15124 dtrace_opens++;
14970 15125 dtrace_membar_producer();
14971 15126
14972 15127 /*
14973 15128 * If the kernel debugger is active (that is, if the kernel debugger
14974 15129 * modified text in some way), we won't allow the open.
14975 15130 */
14976 15131 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
14977 15132 dtrace_opens--;
14978 15133 mutex_exit(&cpu_lock);
14979 15134 mutex_exit(&dtrace_lock);
14980 15135 return (EBUSY);
14981 15136 }
14982 15137
14983 15138 state = dtrace_state_create(devp, cred_p);
14984 15139 mutex_exit(&cpu_lock);
14985 15140
14986 15141 if (state == NULL) {
14987 15142 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
14988 15143 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
14989 15144 mutex_exit(&dtrace_lock);
14990 15145 return (EAGAIN);
14991 15146 }
14992 15147
14993 15148 mutex_exit(&dtrace_lock);
14994 15149
14995 15150 return (0);
14996 15151 }
14997 15152
14998 15153 /*ARGSUSED*/
14999 15154 static int
15000 15155 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
15001 15156 {
15002 15157 minor_t minor = getminor(dev);
15003 15158 dtrace_state_t *state;
15004 15159
15005 15160 if (minor == DTRACEMNRN_HELPER)
15006 15161 return (0);
15007 15162
15008 15163 state = ddi_get_soft_state(dtrace_softstate, minor);
15009 15164
15010 15165 mutex_enter(&cpu_lock);
15011 15166 mutex_enter(&dtrace_lock);
15012 15167
15013 15168 if (state->dts_anon) {
15014 15169 /*
15015 15170 * There is anonymous state. Destroy that first.
15016 15171 */
15017 15172 ASSERT(dtrace_anon.dta_state == NULL);
15018 15173 dtrace_state_destroy(state->dts_anon);
15019 15174 }
15020 15175
15021 15176 dtrace_state_destroy(state);
15022 15177 ASSERT(dtrace_opens > 0);
15023 15178
15024 15179 /*
15025 15180 * Only relinquish control of the kernel debugger interface when there
15026 15181 * are no consumers and no anonymous enablings.
15027 15182 */
15028 15183 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL)
15029 15184 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15030 15185
15031 15186 mutex_exit(&dtrace_lock);
15032 15187 mutex_exit(&cpu_lock);
15033 15188
15034 15189 return (0);
15035 15190 }
15036 15191
15037 15192 /*ARGSUSED*/
15038 15193 static int
15039 15194 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
15040 15195 {
15041 15196 int rval;
15042 15197 dof_helper_t help, *dhp = NULL;
15043 15198
15044 15199 switch (cmd) {
15045 15200 case DTRACEHIOC_ADDDOF:
15046 15201 if (copyin((void *)arg, &help, sizeof (help)) != 0) {
15047 15202 dtrace_dof_error(NULL, "failed to copyin DOF helper");
15048 15203 return (EFAULT);
15049 15204 }
15050 15205
15051 15206 dhp = &help;
15052 15207 arg = (intptr_t)help.dofhp_dof;
15053 15208 /*FALLTHROUGH*/
15054 15209
15055 15210 case DTRACEHIOC_ADD: {
15056 15211 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
15057 15212
15058 15213 if (dof == NULL)
15059 15214 return (rval);
15060 15215
15061 15216 mutex_enter(&dtrace_lock);
15062 15217
15063 15218 /*
15064 15219 * dtrace_helper_slurp() takes responsibility for the dof --
15065 15220 * it may free it now or it may save it and free it later.
15066 15221 */
15067 15222 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
15068 15223 *rv = rval;
15069 15224 rval = 0;
15070 15225 } else {
15071 15226 rval = EINVAL;
15072 15227 }
15073 15228
15074 15229 mutex_exit(&dtrace_lock);
15075 15230 return (rval);
15076 15231 }
15077 15232
15078 15233 case DTRACEHIOC_REMOVE: {
15079 15234 mutex_enter(&dtrace_lock);
15080 15235 rval = dtrace_helper_destroygen(arg);
15081 15236 mutex_exit(&dtrace_lock);
15082 15237
15083 15238 return (rval);
15084 15239 }
15085 15240
15086 15241 default:
15087 15242 break;
15088 15243 }
15089 15244
15090 15245 return (ENOTTY);
15091 15246 }
15092 15247
15093 15248 /*ARGSUSED*/
15094 15249 static int
15095 15250 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
15096 15251 {
15097 15252 minor_t minor = getminor(dev);
15098 15253 dtrace_state_t *state;
15099 15254 int rval;
15100 15255
15101 15256 if (minor == DTRACEMNRN_HELPER)
15102 15257 return (dtrace_ioctl_helper(cmd, arg, rv));
15103 15258
15104 15259 state = ddi_get_soft_state(dtrace_softstate, minor);
15105 15260
15106 15261 if (state->dts_anon) {
15107 15262 ASSERT(dtrace_anon.dta_state == NULL);
15108 15263 state = state->dts_anon;
15109 15264 }
15110 15265
15111 15266 switch (cmd) {
15112 15267 case DTRACEIOC_PROVIDER: {
15113 15268 dtrace_providerdesc_t pvd;
15114 15269 dtrace_provider_t *pvp;
15115 15270
15116 15271 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
15117 15272 return (EFAULT);
15118 15273
15119 15274 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
15120 15275 mutex_enter(&dtrace_provider_lock);
15121 15276
15122 15277 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
15123 15278 if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
15124 15279 break;
15125 15280 }
15126 15281
15127 15282 mutex_exit(&dtrace_provider_lock);
15128 15283
15129 15284 if (pvp == NULL)
15130 15285 return (ESRCH);
15131 15286
15132 15287 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
15133 15288 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
15134 15289 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
15135 15290 return (EFAULT);
15136 15291
15137 15292 return (0);
15138 15293 }
15139 15294
15140 15295 case DTRACEIOC_EPROBE: {
15141 15296 dtrace_eprobedesc_t epdesc;
15142 15297 dtrace_ecb_t *ecb;
15143 15298 dtrace_action_t *act;
15144 15299 void *buf;
15145 15300 size_t size;
15146 15301 uintptr_t dest;
15147 15302 int nrecs;
15148 15303
15149 15304 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
15150 15305 return (EFAULT);
15151 15306
15152 15307 mutex_enter(&dtrace_lock);
15153 15308
15154 15309 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
15155 15310 mutex_exit(&dtrace_lock);
15156 15311 return (EINVAL);
15157 15312 }
15158 15313
15159 15314 if (ecb->dte_probe == NULL) {
15160 15315 mutex_exit(&dtrace_lock);
15161 15316 return (EINVAL);
15162 15317 }
15163 15318
15164 15319 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
15165 15320 epdesc.dtepd_uarg = ecb->dte_uarg;
15166 15321 epdesc.dtepd_size = ecb->dte_size;
15167 15322
15168 15323 nrecs = epdesc.dtepd_nrecs;
15169 15324 epdesc.dtepd_nrecs = 0;
15170 15325 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15171 15326 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15172 15327 continue;
15173 15328
15174 15329 epdesc.dtepd_nrecs++;
15175 15330 }
15176 15331
15177 15332 /*
15178 15333 * Now that we have the size, we need to allocate a temporary
15179 15334 * buffer in which to store the complete description. We need
15180 15335 * the temporary buffer to be able to drop dtrace_lock()
15181 15336 * across the copyout(), below.
15182 15337 */
15183 15338 size = sizeof (dtrace_eprobedesc_t) +
15184 15339 (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
15185 15340
15186 15341 buf = kmem_alloc(size, KM_SLEEP);
15187 15342 dest = (uintptr_t)buf;
15188 15343
15189 15344 bcopy(&epdesc, (void *)dest, sizeof (epdesc));
15190 15345 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
15191 15346
15192 15347 for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
15193 15348 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
15194 15349 continue;
15195 15350
15196 15351 if (nrecs-- == 0)
15197 15352 break;
15198 15353
15199 15354 bcopy(&act->dta_rec, (void *)dest,
15200 15355 sizeof (dtrace_recdesc_t));
15201 15356 dest += sizeof (dtrace_recdesc_t);
15202 15357 }
15203 15358
15204 15359 mutex_exit(&dtrace_lock);
15205 15360
15206 15361 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
15207 15362 kmem_free(buf, size);
15208 15363 return (EFAULT);
15209 15364 }
15210 15365
15211 15366 kmem_free(buf, size);
15212 15367 return (0);
15213 15368 }
15214 15369
15215 15370 case DTRACEIOC_AGGDESC: {
15216 15371 dtrace_aggdesc_t aggdesc;
15217 15372 dtrace_action_t *act;
15218 15373 dtrace_aggregation_t *agg;
15219 15374 int nrecs;
15220 15375 uint32_t offs;
15221 15376 dtrace_recdesc_t *lrec;
15222 15377 void *buf;
15223 15378 size_t size;
15224 15379 uintptr_t dest;
15225 15380
15226 15381 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
15227 15382 return (EFAULT);
15228 15383
15229 15384 mutex_enter(&dtrace_lock);
15230 15385
15231 15386 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
15232 15387 mutex_exit(&dtrace_lock);
15233 15388 return (EINVAL);
15234 15389 }
15235 15390
15236 15391 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
15237 15392
15238 15393 nrecs = aggdesc.dtagd_nrecs;
15239 15394 aggdesc.dtagd_nrecs = 0;
15240 15395
15241 15396 offs = agg->dtag_base;
15242 15397 lrec = &agg->dtag_action.dta_rec;
15243 15398 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
15244 15399
15245 15400 for (act = agg->dtag_first; ; act = act->dta_next) {
15246 15401 ASSERT(act->dta_intuple ||
15247 15402 DTRACEACT_ISAGG(act->dta_kind));
15248 15403
15249 15404 /*
15250 15405 * If this action has a record size of zero, it
15251 15406 * denotes an argument to the aggregating action.
15252 15407 * Because the presence of this record doesn't (or
15253 15408 * shouldn't) affect the way the data is interpreted,
15254 15409 * we don't copy it out to save user-level the
15255 15410 * confusion of dealing with a zero-length record.
15256 15411 */
15257 15412 if (act->dta_rec.dtrd_size == 0) {
15258 15413 ASSERT(agg->dtag_hasarg);
15259 15414 continue;
15260 15415 }
15261 15416
15262 15417 aggdesc.dtagd_nrecs++;
15263 15418
15264 15419 if (act == &agg->dtag_action)
15265 15420 break;
15266 15421 }
15267 15422
15268 15423 /*
15269 15424 * Now that we have the size, we need to allocate a temporary
15270 15425 * buffer in which to store the complete description. We need
15271 15426 * the temporary buffer to be able to drop dtrace_lock()
15272 15427 * across the copyout(), below.
15273 15428 */
15274 15429 size = sizeof (dtrace_aggdesc_t) +
15275 15430 (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
15276 15431
15277 15432 buf = kmem_alloc(size, KM_SLEEP);
15278 15433 dest = (uintptr_t)buf;
15279 15434
15280 15435 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
15281 15436 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
15282 15437
15283 15438 for (act = agg->dtag_first; ; act = act->dta_next) {
15284 15439 dtrace_recdesc_t rec = act->dta_rec;
15285 15440
15286 15441 /*
15287 15442 * See the comment in the above loop for why we pass
15288 15443 * over zero-length records.
15289 15444 */
15290 15445 if (rec.dtrd_size == 0) {
15291 15446 ASSERT(agg->dtag_hasarg);
15292 15447 continue;
15293 15448 }
15294 15449
15295 15450 if (nrecs-- == 0)
15296 15451 break;
15297 15452
15298 15453 rec.dtrd_offset -= offs;
15299 15454 bcopy(&rec, (void *)dest, sizeof (rec));
15300 15455 dest += sizeof (dtrace_recdesc_t);
15301 15456
15302 15457 if (act == &agg->dtag_action)
15303 15458 break;
15304 15459 }
15305 15460
15306 15461 mutex_exit(&dtrace_lock);
15307 15462
15308 15463 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
15309 15464 kmem_free(buf, size);
15310 15465 return (EFAULT);
15311 15466 }
15312 15467
15313 15468 kmem_free(buf, size);
15314 15469 return (0);
15315 15470 }
15316 15471
15317 15472 case DTRACEIOC_ENABLE: {
15318 15473 dof_hdr_t *dof;
15319 15474 dtrace_enabling_t *enab = NULL;
15320 15475 dtrace_vstate_t *vstate;
15321 15476 int err = 0;
15322 15477
15323 15478 *rv = 0;
15324 15479
15325 15480 /*
15326 15481 * If a NULL argument has been passed, we take this as our
15327 15482 * cue to reevaluate our enablings.
15328 15483 */
15329 15484 if (arg == NULL) {
15330 15485 dtrace_enabling_matchall();
15331 15486
15332 15487 return (0);
15333 15488 }
15334 15489
15335 15490 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
15336 15491 return (rval);
15337 15492
15338 15493 mutex_enter(&cpu_lock);
15339 15494 mutex_enter(&dtrace_lock);
15340 15495 vstate = &state->dts_vstate;
15341 15496
15342 15497 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
15343 15498 mutex_exit(&dtrace_lock);
15344 15499 mutex_exit(&cpu_lock);
15345 15500 dtrace_dof_destroy(dof);
15346 15501 return (EBUSY);
15347 15502 }
15348 15503
15349 15504 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
15350 15505 mutex_exit(&dtrace_lock);
15351 15506 mutex_exit(&cpu_lock);
15352 15507 dtrace_dof_destroy(dof);
15353 15508 return (EINVAL);
15354 15509 }
15355 15510
15356 15511 if ((rval = dtrace_dof_options(dof, state)) != 0) {
15357 15512 dtrace_enabling_destroy(enab);
15358 15513 mutex_exit(&dtrace_lock);
15359 15514 mutex_exit(&cpu_lock);
15360 15515 dtrace_dof_destroy(dof);
15361 15516 return (rval);
15362 15517 }
15363 15518
15364 15519 if ((err = dtrace_enabling_match(enab, rv)) == 0) {
15365 15520 err = dtrace_enabling_retain(enab);
15366 15521 } else {
15367 15522 dtrace_enabling_destroy(enab);
15368 15523 }
15369 15524
15370 15525 mutex_exit(&cpu_lock);
15371 15526 mutex_exit(&dtrace_lock);
15372 15527 dtrace_dof_destroy(dof);
15373 15528
15374 15529 return (err);
15375 15530 }
15376 15531
15377 15532 case DTRACEIOC_REPLICATE: {
15378 15533 dtrace_repldesc_t desc;
15379 15534 dtrace_probedesc_t *match = &desc.dtrpd_match;
15380 15535 dtrace_probedesc_t *create = &desc.dtrpd_create;
15381 15536 int err;
15382 15537
15383 15538 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15384 15539 return (EFAULT);
15385 15540
15386 15541 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15387 15542 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15388 15543 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15389 15544 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15390 15545
15391 15546 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15392 15547 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15393 15548 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15394 15549 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15395 15550
15396 15551 mutex_enter(&dtrace_lock);
15397 15552 err = dtrace_enabling_replicate(state, match, create);
15398 15553 mutex_exit(&dtrace_lock);
15399 15554
15400 15555 return (err);
15401 15556 }
15402 15557
15403 15558 case DTRACEIOC_PROBEMATCH:
15404 15559 case DTRACEIOC_PROBES: {
15405 15560 dtrace_probe_t *probe = NULL;
15406 15561 dtrace_probedesc_t desc;
15407 15562 dtrace_probekey_t pkey;
15408 15563 dtrace_id_t i;
15409 15564 int m = 0;
15410 15565 uint32_t priv;
15411 15566 uid_t uid;
15412 15567 zoneid_t zoneid;
15413 15568
15414 15569 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15415 15570 return (EFAULT);
15416 15571
15417 15572 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
15418 15573 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
15419 15574 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
15420 15575 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
15421 15576
15422 15577 /*
15423 15578 * Before we attempt to match this probe, we want to give
15424 15579 * all providers the opportunity to provide it.
15425 15580 */
15426 15581 if (desc.dtpd_id == DTRACE_IDNONE) {
15427 15582 mutex_enter(&dtrace_provider_lock);
15428 15583 dtrace_probe_provide(&desc, NULL);
15429 15584 mutex_exit(&dtrace_provider_lock);
15430 15585 desc.dtpd_id++;
15431 15586 }
15432 15587
15433 15588 if (cmd == DTRACEIOC_PROBEMATCH) {
15434 15589 dtrace_probekey(&desc, &pkey);
15435 15590 pkey.dtpk_id = DTRACE_IDNONE;
15436 15591 }
15437 15592
15438 15593 dtrace_cred2priv(cr, &priv, &uid, &zoneid);
15439 15594
15440 15595 mutex_enter(&dtrace_lock);
15441 15596
15442 15597 if (cmd == DTRACEIOC_PROBEMATCH) {
15443 15598 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
15444 15599 if ((probe = dtrace_probes[i - 1]) != NULL &&
15445 15600 (m = dtrace_match_probe(probe, &pkey,
15446 15601 priv, uid, zoneid)) != 0)
15447 15602 break;
15448 15603 }
15449 15604
15450 15605 if (m < 0) {
15451 15606 mutex_exit(&dtrace_lock);
15452 15607 return (EINVAL);
15453 15608 }
15454 15609
15455 15610 } else {
15456 15611 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
15457 15612 if ((probe = dtrace_probes[i - 1]) != NULL &&
15458 15613 dtrace_match_priv(probe, priv, uid, zoneid))
15459 15614 break;
15460 15615 }
15461 15616 }
15462 15617
15463 15618 if (probe == NULL) {
15464 15619 mutex_exit(&dtrace_lock);
15465 15620 return (ESRCH);
15466 15621 }
15467 15622
15468 15623 dtrace_probe_description(probe, &desc);
15469 15624 mutex_exit(&dtrace_lock);
15470 15625
15471 15626 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15472 15627 return (EFAULT);
15473 15628
15474 15629 return (0);
15475 15630 }
15476 15631
15477 15632 case DTRACEIOC_PROBEARG: {
15478 15633 dtrace_argdesc_t desc;
15479 15634 dtrace_probe_t *probe;
15480 15635 dtrace_provider_t *prov;
15481 15636
15482 15637 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15483 15638 return (EFAULT);
15484 15639
15485 15640 if (desc.dtargd_id == DTRACE_IDNONE)
15486 15641 return (EINVAL);
15487 15642
15488 15643 if (desc.dtargd_ndx == DTRACE_ARGNONE)
15489 15644 return (EINVAL);
15490 15645
15491 15646 mutex_enter(&dtrace_provider_lock);
15492 15647 mutex_enter(&mod_lock);
15493 15648 mutex_enter(&dtrace_lock);
15494 15649
15495 15650 if (desc.dtargd_id > dtrace_nprobes) {
15496 15651 mutex_exit(&dtrace_lock);
15497 15652 mutex_exit(&mod_lock);
15498 15653 mutex_exit(&dtrace_provider_lock);
15499 15654 return (EINVAL);
15500 15655 }
15501 15656
15502 15657 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
15503 15658 mutex_exit(&dtrace_lock);
15504 15659 mutex_exit(&mod_lock);
15505 15660 mutex_exit(&dtrace_provider_lock);
15506 15661 return (EINVAL);
15507 15662 }
15508 15663
15509 15664 mutex_exit(&dtrace_lock);
15510 15665
15511 15666 prov = probe->dtpr_provider;
15512 15667
15513 15668 if (prov->dtpv_pops.dtps_getargdesc == NULL) {
15514 15669 /*
15515 15670 * There isn't any typed information for this probe.
15516 15671 * Set the argument number to DTRACE_ARGNONE.
15517 15672 */
15518 15673 desc.dtargd_ndx = DTRACE_ARGNONE;
15519 15674 } else {
15520 15675 desc.dtargd_native[0] = '\0';
15521 15676 desc.dtargd_xlate[0] = '\0';
15522 15677 desc.dtargd_mapping = desc.dtargd_ndx;
15523 15678
15524 15679 prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
15525 15680 probe->dtpr_id, probe->dtpr_arg, &desc);
15526 15681 }
15527 15682
15528 15683 mutex_exit(&mod_lock);
15529 15684 mutex_exit(&dtrace_provider_lock);
15530 15685
15531 15686 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15532 15687 return (EFAULT);
15533 15688
15534 15689 return (0);
15535 15690 }
15536 15691
15537 15692 case DTRACEIOC_GO: {
15538 15693 processorid_t cpuid;
15539 15694 rval = dtrace_state_go(state, &cpuid);
15540 15695
15541 15696 if (rval != 0)
15542 15697 return (rval);
15543 15698
15544 15699 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
15545 15700 return (EFAULT);
15546 15701
15547 15702 return (0);
15548 15703 }
15549 15704
15550 15705 case DTRACEIOC_STOP: {
15551 15706 processorid_t cpuid;
15552 15707
15553 15708 mutex_enter(&dtrace_lock);
15554 15709 rval = dtrace_state_stop(state, &cpuid);
15555 15710 mutex_exit(&dtrace_lock);
15556 15711
15557 15712 if (rval != 0)
15558 15713 return (rval);
15559 15714
15560 15715 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
15561 15716 return (EFAULT);
15562 15717
15563 15718 return (0);
15564 15719 }
15565 15720
15566 15721 case DTRACEIOC_DOFGET: {
15567 15722 dof_hdr_t hdr, *dof;
15568 15723 uint64_t len;
15569 15724
15570 15725 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
15571 15726 return (EFAULT);
15572 15727
15573 15728 mutex_enter(&dtrace_lock);
15574 15729 dof = dtrace_dof_create(state);
15575 15730 mutex_exit(&dtrace_lock);
15576 15731
15577 15732 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
15578 15733 rval = copyout(dof, (void *)arg, len);
15579 15734 dtrace_dof_destroy(dof);
15580 15735
15581 15736 return (rval == 0 ? 0 : EFAULT);
15582 15737 }
15583 15738
15584 15739 case DTRACEIOC_AGGSNAP:
15585 15740 case DTRACEIOC_BUFSNAP: {
15586 15741 dtrace_bufdesc_t desc;
15587 15742 caddr_t cached;
15588 15743 dtrace_buffer_t *buf;
15589 15744
15590 15745 if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
15591 15746 return (EFAULT);
15592 15747
15593 15748 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
15594 15749 return (EINVAL);
15595 15750
15596 15751 mutex_enter(&dtrace_lock);
15597 15752
15598 15753 if (cmd == DTRACEIOC_BUFSNAP) {
15599 15754 buf = &state->dts_buffer[desc.dtbd_cpu];
15600 15755 } else {
15601 15756 buf = &state->dts_aggbuffer[desc.dtbd_cpu];
15602 15757 }
15603 15758
15604 15759 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
15605 15760 size_t sz = buf->dtb_offset;
15606 15761
15607 15762 if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
15608 15763 mutex_exit(&dtrace_lock);
15609 15764 return (EBUSY);
15610 15765 }
15611 15766
15612 15767 /*
15613 15768 * If this buffer has already been consumed, we're
15614 15769 * going to indicate that there's nothing left here
15615 15770 * to consume.
15616 15771 */
15617 15772 if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
15618 15773 mutex_exit(&dtrace_lock);
15619 15774
15620 15775 desc.dtbd_size = 0;
15621 15776 desc.dtbd_drops = 0;
15622 15777 desc.dtbd_errors = 0;
15623 15778 desc.dtbd_oldest = 0;
15624 15779 sz = sizeof (desc);
15625 15780
15626 15781 if (copyout(&desc, (void *)arg, sz) != 0)
15627 15782 return (EFAULT);
15628 15783
15629 15784 return (0);
15630 15785 }
15631 15786
15632 15787 /*
15633 15788 * If this is a ring buffer that has wrapped, we want
15634 15789 * to copy the whole thing out.
15635 15790 */
15636 15791 if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
15637 15792 dtrace_buffer_polish(buf);
15638 15793 sz = buf->dtb_size;
15639 15794 }
15640 15795
15641 15796 if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
15642 15797 mutex_exit(&dtrace_lock);
15643 15798 return (EFAULT);
15644 15799 }
15645 15800
15646 15801 desc.dtbd_size = sz;
15647 15802 desc.dtbd_drops = buf->dtb_drops;
15648 15803 desc.dtbd_errors = buf->dtb_errors;
15649 15804 desc.dtbd_oldest = buf->dtb_xamot_offset;
15650 15805
15651 15806 mutex_exit(&dtrace_lock);
15652 15807
15653 15808 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15654 15809 return (EFAULT);
15655 15810
15656 15811 buf->dtb_flags |= DTRACEBUF_CONSUMED;
15657 15812
15658 15813 return (0);
15659 15814 }
15660 15815
15661 15816 if (buf->dtb_tomax == NULL) {
15662 15817 ASSERT(buf->dtb_xamot == NULL);
15663 15818 mutex_exit(&dtrace_lock);
15664 15819 return (ENOENT);
15665 15820 }
15666 15821
15667 15822 cached = buf->dtb_tomax;
15668 15823 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
15669 15824
15670 15825 dtrace_xcall(desc.dtbd_cpu,
15671 15826 (dtrace_xcall_t)dtrace_buffer_switch, buf);
15672 15827
15673 15828 state->dts_errors += buf->dtb_xamot_errors;
15674 15829
15675 15830 /*
15676 15831 * If the buffers did not actually switch, then the cross call
15677 15832 * did not take place -- presumably because the given CPU is
15678 15833 * not in the ready set. If this is the case, we'll return
15679 15834 * ENOENT.
15680 15835 */
15681 15836 if (buf->dtb_tomax == cached) {
15682 15837 ASSERT(buf->dtb_xamot != cached);
15683 15838 mutex_exit(&dtrace_lock);
15684 15839 return (ENOENT);
15685 15840 }
15686 15841
15687 15842 ASSERT(cached == buf->dtb_xamot);
15688 15843
15689 15844 /*
15690 15845 * We have our snapshot; now copy it out.
15691 15846 */
15692 15847 if (copyout(buf->dtb_xamot, desc.dtbd_data,
15693 15848 buf->dtb_xamot_offset) != 0) {
15694 15849 mutex_exit(&dtrace_lock);
15695 15850 return (EFAULT);
15696 15851 }
15697 15852
15698 15853 desc.dtbd_size = buf->dtb_xamot_offset;
15699 15854 desc.dtbd_drops = buf->dtb_xamot_drops;
15700 15855 desc.dtbd_errors = buf->dtb_xamot_errors;
15701 15856 desc.dtbd_oldest = 0;
15702 15857
15703 15858 mutex_exit(&dtrace_lock);
15704 15859
15705 15860 /*
15706 15861 * Finally, copy out the buffer description.
15707 15862 */
15708 15863 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
15709 15864 return (EFAULT);
15710 15865
15711 15866 return (0);
15712 15867 }
15713 15868
15714 15869 case DTRACEIOC_CONF: {
15715 15870 dtrace_conf_t conf;
15716 15871
15717 15872 bzero(&conf, sizeof (conf));
15718 15873 conf.dtc_difversion = DIF_VERSION;
15719 15874 conf.dtc_difintregs = DIF_DIR_NREGS;
15720 15875 conf.dtc_diftupregs = DIF_DTR_NREGS;
15721 15876 conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
15722 15877
15723 15878 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
15724 15879 return (EFAULT);
15725 15880
15726 15881 return (0);
15727 15882 }
15728 15883
15729 15884 case DTRACEIOC_STATUS: {
15730 15885 dtrace_status_t stat;
15731 15886 dtrace_dstate_t *dstate;
15732 15887 int i, j;
15733 15888 uint64_t nerrs;
15734 15889
15735 15890 /*
15736 15891 * See the comment in dtrace_state_deadman() for the reason
15737 15892 * for setting dts_laststatus to INT64_MAX before setting
15738 15893 * it to the correct value.
15739 15894 */
15740 15895 state->dts_laststatus = INT64_MAX;
15741 15896 dtrace_membar_producer();
15742 15897 state->dts_laststatus = dtrace_gethrtime();
15743 15898
15744 15899 bzero(&stat, sizeof (stat));
15745 15900
15746 15901 mutex_enter(&dtrace_lock);
15747 15902
15748 15903 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
15749 15904 mutex_exit(&dtrace_lock);
15750 15905 return (ENOENT);
15751 15906 }
15752 15907
15753 15908 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
15754 15909 stat.dtst_exiting = 1;
15755 15910
15756 15911 nerrs = state->dts_errors;
15757 15912 dstate = &state->dts_vstate.dtvs_dynvars;
15758 15913
15759 15914 for (i = 0; i < NCPU; i++) {
15760 15915 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
15761 15916
15762 15917 stat.dtst_dyndrops += dcpu->dtdsc_drops;
15763 15918 stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
15764 15919 stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
15765 15920
15766 15921 if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
15767 15922 stat.dtst_filled++;
15768 15923
15769 15924 nerrs += state->dts_buffer[i].dtb_errors;
15770 15925
15771 15926 for (j = 0; j < state->dts_nspeculations; j++) {
15772 15927 dtrace_speculation_t *spec;
15773 15928 dtrace_buffer_t *buf;
15774 15929
15775 15930 spec = &state->dts_speculations[j];
15776 15931 buf = &spec->dtsp_buffer[i];
15777 15932 stat.dtst_specdrops += buf->dtb_xamot_drops;
15778 15933 }
15779 15934 }
15780 15935
15781 15936 stat.dtst_specdrops_busy = state->dts_speculations_busy;
15782 15937 stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
15783 15938 stat.dtst_stkstroverflows = state->dts_stkstroverflows;
15784 15939 stat.dtst_dblerrors = state->dts_dblerrors;
15785 15940 stat.dtst_killed =
15786 15941 (state->dts_activity == DTRACE_ACTIVITY_KILLED);
15787 15942 stat.dtst_errors = nerrs;
15788 15943
15789 15944 mutex_exit(&dtrace_lock);
15790 15945
15791 15946 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
15792 15947 return (EFAULT);
15793 15948
15794 15949 return (0);
15795 15950 }
15796 15951
15797 15952 case DTRACEIOC_FORMAT: {
15798 15953 dtrace_fmtdesc_t fmt;
15799 15954 char *str;
15800 15955 int len;
15801 15956
15802 15957 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
15803 15958 return (EFAULT);
15804 15959
15805 15960 mutex_enter(&dtrace_lock);
15806 15961
15807 15962 if (fmt.dtfd_format == 0 ||
15808 15963 fmt.dtfd_format > state->dts_nformats) {
15809 15964 mutex_exit(&dtrace_lock);
15810 15965 return (EINVAL);
15811 15966 }
15812 15967
15813 15968 /*
15814 15969 * Format strings are allocated contiguously and they are
15815 15970 * never freed; if a format index is less than the number
15816 15971 * of formats, we can assert that the format map is non-NULL
15817 15972 * and that the format for the specified index is non-NULL.
15818 15973 */
15819 15974 ASSERT(state->dts_formats != NULL);
15820 15975 str = state->dts_formats[fmt.dtfd_format - 1];
15821 15976 ASSERT(str != NULL);
15822 15977
15823 15978 len = strlen(str) + 1;
15824 15979
15825 15980 if (len > fmt.dtfd_length) {
15826 15981 fmt.dtfd_length = len;
15827 15982
15828 15983 if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
15829 15984 mutex_exit(&dtrace_lock);
15830 15985 return (EINVAL);
15831 15986 }
15832 15987 } else {
15833 15988 if (copyout(str, fmt.dtfd_string, len) != 0) {
15834 15989 mutex_exit(&dtrace_lock);
15835 15990 return (EINVAL);
15836 15991 }
15837 15992 }
15838 15993
15839 15994 mutex_exit(&dtrace_lock);
15840 15995 return (0);
15841 15996 }
15842 15997
15843 15998 default:
15844 15999 break;
15845 16000 }
15846 16001
15847 16002 return (ENOTTY);
15848 16003 }
15849 16004
15850 16005 /*ARGSUSED*/
15851 16006 static int
15852 16007 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
15853 16008 {
15854 16009 dtrace_state_t *state;
15855 16010
15856 16011 switch (cmd) {
15857 16012 case DDI_DETACH:
15858 16013 break;
15859 16014
15860 16015 case DDI_SUSPEND:
15861 16016 return (DDI_SUCCESS);
15862 16017
15863 16018 default:
15864 16019 return (DDI_FAILURE);
15865 16020 }
15866 16021
15867 16022 mutex_enter(&cpu_lock);
15868 16023 mutex_enter(&dtrace_provider_lock);
15869 16024 mutex_enter(&dtrace_lock);
15870 16025
15871 16026 ASSERT(dtrace_opens == 0);
15872 16027
15873 16028 if (dtrace_helpers > 0) {
15874 16029 mutex_exit(&dtrace_provider_lock);
15875 16030 mutex_exit(&dtrace_lock);
15876 16031 mutex_exit(&cpu_lock);
15877 16032 return (DDI_FAILURE);
15878 16033 }
15879 16034
15880 16035 if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
15881 16036 mutex_exit(&dtrace_provider_lock);
15882 16037 mutex_exit(&dtrace_lock);
15883 16038 mutex_exit(&cpu_lock);
15884 16039 return (DDI_FAILURE);
15885 16040 }
15886 16041
15887 16042 dtrace_provider = NULL;
15888 16043
15889 16044 if ((state = dtrace_anon_grab()) != NULL) {
15890 16045 /*
15891 16046 * If there were ECBs on this state, the provider should
15892 16047 * have not been allowed to detach; assert that there is
15893 16048 * none.
15894 16049 */
15895 16050 ASSERT(state->dts_necbs == 0);
15896 16051 dtrace_state_destroy(state);
15897 16052
15898 16053 /*
15899 16054 * If we're being detached with anonymous state, we need to
15900 16055 * indicate to the kernel debugger that DTrace is now inactive.
15901 16056 */
15902 16057 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
15903 16058 }
15904 16059
15905 16060 bzero(&dtrace_anon, sizeof (dtrace_anon_t));
15906 16061 unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
|
↓ open down ↓ |
1134 lines elided |
↑ open up ↑ |
15907 16062 dtrace_cpu_init = NULL;
15908 16063 dtrace_helpers_cleanup = NULL;
15909 16064 dtrace_helpers_fork = NULL;
15910 16065 dtrace_cpustart_init = NULL;
15911 16066 dtrace_cpustart_fini = NULL;
15912 16067 dtrace_debugger_init = NULL;
15913 16068 dtrace_debugger_fini = NULL;
15914 16069 dtrace_modload = NULL;
15915 16070 dtrace_modunload = NULL;
15916 16071
16072 + ASSERT(dtrace_getf == 0);
16073 + ASSERT(dtrace_closef == NULL);
16074 +
15917 16075 mutex_exit(&cpu_lock);
15918 16076
15919 16077 if (dtrace_helptrace_enabled) {
15920 16078 kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
15921 16079 dtrace_helptrace_buffer = NULL;
15922 16080 }
15923 16081
15924 16082 kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
15925 16083 dtrace_probes = NULL;
15926 16084 dtrace_nprobes = 0;
15927 16085
15928 16086 dtrace_hash_destroy(dtrace_bymod);
15929 16087 dtrace_hash_destroy(dtrace_byfunc);
15930 16088 dtrace_hash_destroy(dtrace_byname);
15931 16089 dtrace_bymod = NULL;
15932 16090 dtrace_byfunc = NULL;
15933 16091 dtrace_byname = NULL;
15934 16092
15935 16093 kmem_cache_destroy(dtrace_state_cache);
15936 16094 vmem_destroy(dtrace_minor);
15937 16095 vmem_destroy(dtrace_arena);
15938 16096
15939 16097 if (dtrace_toxrange != NULL) {
15940 16098 kmem_free(dtrace_toxrange,
15941 16099 dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
15942 16100 dtrace_toxrange = NULL;
15943 16101 dtrace_toxranges = 0;
15944 16102 dtrace_toxranges_max = 0;
15945 16103 }
15946 16104
15947 16105 ddi_remove_minor_node(dtrace_devi, NULL);
15948 16106 dtrace_devi = NULL;
15949 16107
15950 16108 ddi_soft_state_fini(&dtrace_softstate);
15951 16109
15952 16110 ASSERT(dtrace_vtime_references == 0);
15953 16111 ASSERT(dtrace_opens == 0);
15954 16112 ASSERT(dtrace_retained == NULL);
15955 16113
15956 16114 mutex_exit(&dtrace_lock);
15957 16115 mutex_exit(&dtrace_provider_lock);
15958 16116
15959 16117 /*
15960 16118 * We don't destroy the task queue until after we have dropped our
15961 16119 * locks (taskq_destroy() may block on running tasks). To prevent
15962 16120 * attempting to do work after we have effectively detached but before
15963 16121 * the task queue has been destroyed, all tasks dispatched via the
15964 16122 * task queue must check that DTrace is still attached before
15965 16123 * performing any operation.
15966 16124 */
15967 16125 taskq_destroy(dtrace_taskq);
15968 16126 dtrace_taskq = NULL;
15969 16127
15970 16128 return (DDI_SUCCESS);
15971 16129 }
15972 16130
15973 16131 /*ARGSUSED*/
15974 16132 static int
15975 16133 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
15976 16134 {
15977 16135 int error;
15978 16136
15979 16137 switch (infocmd) {
15980 16138 case DDI_INFO_DEVT2DEVINFO:
15981 16139 *result = (void *)dtrace_devi;
15982 16140 error = DDI_SUCCESS;
15983 16141 break;
15984 16142 case DDI_INFO_DEVT2INSTANCE:
15985 16143 *result = (void *)0;
15986 16144 error = DDI_SUCCESS;
15987 16145 break;
15988 16146 default:
15989 16147 error = DDI_FAILURE;
15990 16148 }
15991 16149 return (error);
15992 16150 }
15993 16151
15994 16152 static struct cb_ops dtrace_cb_ops = {
15995 16153 dtrace_open, /* open */
15996 16154 dtrace_close, /* close */
15997 16155 nulldev, /* strategy */
15998 16156 nulldev, /* print */
15999 16157 nodev, /* dump */
16000 16158 nodev, /* read */
16001 16159 nodev, /* write */
16002 16160 dtrace_ioctl, /* ioctl */
16003 16161 nodev, /* devmap */
16004 16162 nodev, /* mmap */
16005 16163 nodev, /* segmap */
16006 16164 nochpoll, /* poll */
16007 16165 ddi_prop_op, /* cb_prop_op */
16008 16166 0, /* streamtab */
16009 16167 D_NEW | D_MP /* Driver compatibility flag */
16010 16168 };
16011 16169
16012 16170 static struct dev_ops dtrace_ops = {
16013 16171 DEVO_REV, /* devo_rev */
16014 16172 0, /* refcnt */
16015 16173 dtrace_info, /* get_dev_info */
16016 16174 nulldev, /* identify */
16017 16175 nulldev, /* probe */
16018 16176 dtrace_attach, /* attach */
16019 16177 dtrace_detach, /* detach */
16020 16178 nodev, /* reset */
16021 16179 &dtrace_cb_ops, /* driver operations */
16022 16180 NULL, /* bus operations */
16023 16181 nodev, /* dev power */
16024 16182 ddi_quiesce_not_needed, /* quiesce */
16025 16183 };
16026 16184
16027 16185 static struct modldrv modldrv = {
16028 16186 &mod_driverops, /* module type (this is a pseudo driver) */
16029 16187 "Dynamic Tracing", /* name of module */
16030 16188 &dtrace_ops, /* driver ops */
16031 16189 };
16032 16190
16033 16191 static struct modlinkage modlinkage = {
16034 16192 MODREV_1,
16035 16193 (void *)&modldrv,
16036 16194 NULL
16037 16195 };
16038 16196
16039 16197 int
16040 16198 _init(void)
16041 16199 {
16042 16200 return (mod_install(&modlinkage));
16043 16201 }
16044 16202
16045 16203 int
16046 16204 _info(struct modinfo *modinfop)
16047 16205 {
16048 16206 return (mod_info(&modlinkage, modinfop));
16049 16207 }
16050 16208
16051 16209 int
16052 16210 _fini(void)
16053 16211 {
16054 16212 return (mod_remove(&modlinkage));
16055 16213 }
|
↓ open down ↓ |
129 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX