Print this page
2915 DTrace in a zone should see "cpu", "curpsinfo", et al
2916 DTrace in a zone should be able to access fds[]
2917 DTrace in a zone should have limited provider access
Reviewed by: Joshua M. Clulow <josh@sysmgr.org>
Reviewed by: Adam Leventhal <ahl@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/sys/dtrace_impl.h
+++ new/usr/src/uts/common/sys/dtrace_impl.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
29 29 * Copyright (c) 2012 by Delphix. All rights reserved.
30 30 */
31 31
32 32 #ifndef _SYS_DTRACE_IMPL_H
33 33 #define _SYS_DTRACE_IMPL_H
34 34
35 35 #ifdef __cplusplus
36 36 extern "C" {
37 37 #endif
38 38
39 39 /*
40 40 * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
41 41 *
42 42 * Note: The contents of this file are private to the implementation of the
43 43 * Solaris system and DTrace subsystem and are subject to change at any time
44 44 * without notice. Applications and drivers using these interfaces will fail
45 45 * to run on future releases. These interfaces should not be used for any
46 46 * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
47 47 * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
48 48 */
49 49
50 50 #include <sys/dtrace.h>
51 51
52 52 /*
53 53 * DTrace Implementation Constants and Typedefs
54 54 */
55 55 #define DTRACE_MAXPROPLEN 128
56 56 #define DTRACE_DYNVAR_CHUNKSIZE 256
57 57
58 58 struct dtrace_probe;
59 59 struct dtrace_ecb;
60 60 struct dtrace_predicate;
61 61 struct dtrace_action;
62 62 struct dtrace_provider;
63 63 struct dtrace_state;
64 64
65 65 typedef struct dtrace_probe dtrace_probe_t;
66 66 typedef struct dtrace_ecb dtrace_ecb_t;
67 67 typedef struct dtrace_predicate dtrace_predicate_t;
68 68 typedef struct dtrace_action dtrace_action_t;
69 69 typedef struct dtrace_provider dtrace_provider_t;
70 70 typedef struct dtrace_meta dtrace_meta_t;
71 71 typedef struct dtrace_state dtrace_state_t;
72 72 typedef uint32_t dtrace_optid_t;
73 73 typedef uint32_t dtrace_specid_t;
74 74 typedef uint64_t dtrace_genid_t;
75 75
76 76 /*
77 77 * DTrace Probes
78 78 *
79 79 * The probe is the fundamental unit of the DTrace architecture. Probes are
80 80 * created by DTrace providers, and managed by the DTrace framework. A probe
81 81 * is identified by a unique <provider, module, function, name> tuple, and has
82 82 * a unique probe identifier assigned to it. (Some probes are not associated
83 83 * with a specific point in text; these are called _unanchored probes_ and have
84 84 * no module or function associated with them.) Probes are represented as a
85 85 * dtrace_probe structure. To allow quick lookups based on each element of the
86 86 * probe tuple, probes are hashed by each of provider, module, function and
87 87 * name. (If a lookup is performed based on a regular expression, a
88 88 * dtrace_probekey is prepared, and a linear search is performed.) Each probe
89 89 * is additionally pointed to by a linear array indexed by its identifier. The
90 90 * identifier is the provider's mechanism for indicating to the DTrace
91 91 * framework that a probe has fired: the identifier is passed as the first
92 92 * argument to dtrace_probe(), where it is then mapped into the corresponding
93 93 * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can
94 94 * iterate over the probe's list of enabling control blocks; see "DTrace
95 95 * Enabling Control Blocks", below.)
96 96 */
97 97 struct dtrace_probe {
98 98 dtrace_id_t dtpr_id; /* probe identifier */
99 99 dtrace_ecb_t *dtpr_ecb; /* ECB list; see below */
100 100 dtrace_ecb_t *dtpr_ecb_last; /* last ECB in list */
101 101 void *dtpr_arg; /* provider argument */
102 102 dtrace_cacheid_t dtpr_predcache; /* predicate cache ID */
103 103 int dtpr_aframes; /* artificial frames */
104 104 dtrace_provider_t *dtpr_provider; /* pointer to provider */
105 105 char *dtpr_mod; /* probe's module name */
106 106 char *dtpr_func; /* probe's function name */
107 107 char *dtpr_name; /* probe's name */
108 108 dtrace_probe_t *dtpr_nextmod; /* next in module hash */
109 109 dtrace_probe_t *dtpr_prevmod; /* previous in module hash */
110 110 dtrace_probe_t *dtpr_nextfunc; /* next in function hash */
111 111 dtrace_probe_t *dtpr_prevfunc; /* previous in function hash */
112 112 dtrace_probe_t *dtpr_nextname; /* next in name hash */
113 113 dtrace_probe_t *dtpr_prevname; /* previous in name hash */
114 114 dtrace_genid_t dtpr_gen; /* probe generation ID */
115 115 };
116 116
117 117 typedef int dtrace_probekey_f(const char *, const char *, int);
118 118
119 119 typedef struct dtrace_probekey {
120 120 const char *dtpk_prov; /* provider name to match */
121 121 dtrace_probekey_f *dtpk_pmatch; /* provider matching function */
122 122 const char *dtpk_mod; /* module name to match */
123 123 dtrace_probekey_f *dtpk_mmatch; /* module matching function */
124 124 const char *dtpk_func; /* func name to match */
125 125 dtrace_probekey_f *dtpk_fmatch; /* func matching function */
126 126 const char *dtpk_name; /* name to match */
127 127 dtrace_probekey_f *dtpk_nmatch; /* name matching function */
128 128 dtrace_id_t dtpk_id; /* identifier to match */
129 129 } dtrace_probekey_t;
130 130
131 131 typedef struct dtrace_hashbucket {
132 132 struct dtrace_hashbucket *dthb_next; /* next on hash chain */
133 133 dtrace_probe_t *dthb_chain; /* chain of probes */
134 134 int dthb_len; /* number of probes here */
135 135 } dtrace_hashbucket_t;
136 136
137 137 typedef struct dtrace_hash {
138 138 dtrace_hashbucket_t **dth_tab; /* hash table */
139 139 int dth_size; /* size of hash table */
140 140 int dth_mask; /* mask to index into table */
141 141 int dth_nbuckets; /* total number of buckets */
142 142 uintptr_t dth_nextoffs; /* offset of next in probe */
143 143 uintptr_t dth_prevoffs; /* offset of prev in probe */
144 144 uintptr_t dth_stroffs; /* offset of str in probe */
145 145 } dtrace_hash_t;
146 146
147 147 /*
148 148 * DTrace Enabling Control Blocks
149 149 *
150 150 * When a provider wishes to fire a probe, it calls into dtrace_probe(),
151 151 * passing the probe identifier as the first argument. As described above,
152 152 * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
153 153 * structure. This structure contains information about the probe, and a
154 154 * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to
155 155 * DTrace consumer state, and contains an optional predicate, and a list of
156 156 * actions. (Shown schematically below.) The ECB abstraction allows a single
157 157 * probe to be multiplexed across disjoint consumers, or across disjoint
158 158 * enablings of a single probe within one consumer.
159 159 *
160 160 * Enabling Control Block
161 161 * dtrace_ecb_t
162 162 * +------------------------+
163 163 * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
164 164 * | dtrace_state_t * ------+--------------> State associated with this ECB
165 165 * | dtrace_predicate_t * --+---------+
166 166 * | dtrace_action_t * -----+----+ |
167 167 * | dtrace_ecb_t * ---+ | | | Predicate (if any)
168 168 * +-------------------+----+ | | dtrace_predicate_t
169 169 * | | +---> +--------------------+
170 170 * | | | dtrace_difo_t * ---+----> DIFO
171 171 * | | +--------------------+
172 172 * | |
173 173 * Next ECB | | Action
174 174 * (if any) | | dtrace_action_t
175 175 * : +--> +-------------------+
176 176 * : | dtrace_actkind_t -+------> kind
177 177 * v | dtrace_difo_t * --+------> DIFO (if any)
178 178 * | dtrace_recdesc_t -+------> record descr.
179 179 * | dtrace_action_t * +------+
180 180 * +-------------------+ |
181 181 * | Next action
182 182 * +-------------------------------+ (if any)
183 183 * |
184 184 * | Action
185 185 * | dtrace_action_t
186 186 * +--> +-------------------+
187 187 * | dtrace_actkind_t -+------> kind
188 188 * | dtrace_difo_t * --+------> DIFO (if any)
189 189 * | dtrace_action_t * +------+
190 190 * +-------------------+ |
191 191 * | Next action
192 192 * +-------------------------------+ (if any)
193 193 * |
194 194 * :
195 195 * v
196 196 *
197 197 *
198 198 * dtrace_probe() iterates over the ECB list. If the ECB needs less space
199 199 * than is available in the principal buffer, the ECB is processed: if the
200 200 * predicate is non-NULL, the DIF object is executed. If the result is
201 201 * non-zero, the action list is processed, with each action being executed
202 202 * accordingly. When the action list has been completely executed, processing
203 203 * advances to the next ECB. The ECB abstraction allows disjoint consumers
204 204 * to multiplex on single probes.
205 205 *
206 206 * Execution of the ECB results in consuming dte_size bytes in the buffer
207 207 * to record data. During execution, dte_needed bytes must be available in
208 208 * the buffer. This space is used for both recorded data and tuple data.
209 209 */
210 210 struct dtrace_ecb {
211 211 dtrace_epid_t dte_epid; /* enabled probe ID */
212 212 uint32_t dte_alignment; /* required alignment */
213 213 size_t dte_needed; /* space needed for execution */
214 214 size_t dte_size; /* size of recorded payload */
215 215 dtrace_predicate_t *dte_predicate; /* predicate, if any */
216 216 dtrace_action_t *dte_action; /* actions, if any */
217 217 dtrace_ecb_t *dte_next; /* next ECB on probe */
218 218 dtrace_state_t *dte_state; /* pointer to state */
219 219 uint32_t dte_cond; /* security condition */
220 220 dtrace_probe_t *dte_probe; /* pointer to probe */
221 221 dtrace_action_t *dte_action_last; /* last action on ECB */
222 222 uint64_t dte_uarg; /* library argument */
223 223 };
224 224
225 225 struct dtrace_predicate {
226 226 dtrace_difo_t *dtp_difo; /* DIF object */
227 227 dtrace_cacheid_t dtp_cacheid; /* cache identifier */
228 228 int dtp_refcnt; /* reference count */
229 229 };
230 230
231 231 struct dtrace_action {
232 232 dtrace_actkind_t dta_kind; /* kind of action */
233 233 uint16_t dta_intuple; /* boolean: in aggregation */
234 234 uint32_t dta_refcnt; /* reference count */
235 235 dtrace_difo_t *dta_difo; /* pointer to DIFO */
236 236 dtrace_recdesc_t dta_rec; /* record description */
237 237 dtrace_action_t *dta_prev; /* previous action */
238 238 dtrace_action_t *dta_next; /* next action */
239 239 };
240 240
241 241 typedef struct dtrace_aggregation {
242 242 dtrace_action_t dtag_action; /* action; must be first */
243 243 dtrace_aggid_t dtag_id; /* identifier */
244 244 dtrace_ecb_t *dtag_ecb; /* corresponding ECB */
245 245 dtrace_action_t *dtag_first; /* first action in tuple */
246 246 uint32_t dtag_base; /* base of aggregation */
247 247 uint8_t dtag_hasarg; /* boolean: has argument */
248 248 uint64_t dtag_initial; /* initial value */
249 249 void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t);
250 250 } dtrace_aggregation_t;
251 251
252 252 /*
253 253 * DTrace Buffers
254 254 *
255 255 * Principal buffers, aggregation buffers, and speculative buffers are all
256 256 * managed with the dtrace_buffer structure. By default, this structure
257 257 * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
258 258 * active and passive buffers, respectively. For speculative buffers,
259 259 * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
260 260 * to a scratch buffer. For all buffer types, the dtrace_buffer structure is
261 261 * always allocated on a per-CPU basis; a single dtrace_buffer structure is
262 262 * never shared among CPUs. (That is, there is never true sharing of the
263 263 * dtrace_buffer structure; to prevent false sharing of the structure, it must
264 264 * always be aligned to the coherence granularity -- generally 64 bytes.)
265 265 *
266 266 * One of the critical design decisions of DTrace is that a given ECB always
267 267 * stores the same quantity and type of data. This is done to assure that the
268 268 * only metadata required for an ECB's traced data is the EPID. That is, from
269 269 * the EPID, the consumer can determine the data layout. (The data buffer
270 270 * layout is shown schematically below.) By assuring that one can determine
271 271 * data layout from the EPID, the metadata stream can be separated from the
272 272 * data stream -- simplifying the data stream enormously. The ECB always
273 273 * proceeds the recorded data as part of the dtrace_rechdr_t structure that
274 274 * includes the EPID and a high-resolution timestamp used for output ordering
275 275 * consistency.
276 276 *
277 277 * base of data buffer ---> +--------+--------------------+--------+
278 278 * | rechdr | data | rechdr |
279 279 * +--------+------+--------+----+--------+
280 280 * | data | rechdr | data |
281 281 * +---------------+--------+-------------+
282 282 * | data, cont. |
283 283 * +--------+--------------------+--------+
284 284 * | rechdr | data | |
285 285 * +--------+--------------------+ |
286 286 * | || |
287 287 * | || |
288 288 * | \/ |
289 289 * : :
290 290 * . .
291 291 * . .
292 292 * . .
293 293 * : :
294 294 * | |
295 295 * limit of data buffer ---> +--------------------------------------+
296 296 *
297 297 * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
298 298 * principal buffer (both scratch and payload) exceed the available space. If
299 299 * the ECB's needs exceed available space (and if the principal buffer policy
300 300 * is the default "switch" policy), the ECB is dropped, the buffer's drop count
301 301 * is incremented, and processing advances to the next ECB. If the ECB's needs
302 302 * can be met with the available space, the ECB is processed, but the offset in
303 303 * the principal buffer is only advanced if the ECB completes processing
304 304 * without error.
305 305 *
306 306 * When a buffer is to be switched (either because the buffer is the principal
307 307 * buffer with a "switch" policy or because it is an aggregation buffer), a
308 308 * cross call is issued to the CPU associated with the buffer. In the cross
309 309 * call context, interrupts are disabled, and the active and the inactive
310 310 * buffers are atomically switched. This involves switching the data pointers,
311 311 * copying the various state fields (offset, drops, errors, etc.) into their
312 312 * inactive equivalents, and clearing the state fields. Because interrupts are
313 313 * disabled during this procedure, the switch is guaranteed to appear atomic to
314 314 * dtrace_probe().
315 315 *
316 316 * DTrace Ring Buffering
317 317 *
318 318 * To process a ring buffer correctly, one must know the oldest valid record.
319 319 * Processing starts at the oldest record in the buffer and continues until
320 320 * the end of the buffer is reached. Processing then resumes starting with
321 321 * the record stored at offset 0 in the buffer, and continues until the
322 322 * youngest record is processed. If trace records are of a fixed-length,
323 323 * determining the oldest record is trivial:
324 324 *
325 325 * - If the ring buffer has not wrapped, the oldest record is the record
326 326 * stored at offset 0.
327 327 *
328 328 * - If the ring buffer has wrapped, the oldest record is the record stored
329 329 * at the current offset.
330 330 *
331 331 * With variable length records, however, just knowing the current offset
332 332 * doesn't suffice for determining the oldest valid record: assuming that one
333 333 * allows for arbitrary data, one has no way of searching forward from the
334 334 * current offset to find the oldest valid record. (That is, one has no way
335 335 * of separating data from metadata.) It would be possible to simply refuse to
336 336 * process any data in the ring buffer between the current offset and the
337 337 * limit, but this leaves (potentially) an enormous amount of otherwise valid
338 338 * data unprocessed.
339 339 *
340 340 * To effect ring buffering, we track two offsets in the buffer: the current
341 341 * offset and the _wrapped_ offset. If a request is made to reserve some
342 342 * amount of data, and the buffer has wrapped, the wrapped offset is
343 343 * incremented until the wrapped offset minus the current offset is greater
344 344 * than or equal to the reserve request. This is done by repeatedly looking
345 345 * up the ECB corresponding to the EPID at the current wrapped offset, and
346 346 * incrementing the wrapped offset by the size of the data payload
347 347 * corresponding to that ECB. If this offset is greater than or equal to the
348 348 * limit of the data buffer, the wrapped offset is set to 0. Thus, the
349 349 * current offset effectively "chases" the wrapped offset around the buffer.
350 350 * Schematically:
351 351 *
352 352 * base of data buffer ---> +------+--------------------+------+
353 353 * | EPID | data | EPID |
354 354 * +------+--------+------+----+------+
355 355 * | data | EPID | data |
356 356 * +---------------+------+-----------+
357 357 * | data, cont. |
358 358 * +------+---------------------------+
359 359 * | EPID | data |
360 360 * current offset ---> +------+---------------------------+
361 361 * | invalid data |
362 362 * wrapped offset ---> +------+--------------------+------+
363 363 * | EPID | data | EPID |
364 364 * +------+--------+------+----+------+
365 365 * | data | EPID | data |
366 366 * +---------------+------+-----------+
367 367 * : :
368 368 * . .
369 369 * . ... valid data ... .
370 370 * . .
371 371 * : :
372 372 * +------+-------------+------+------+
373 373 * | EPID | data | EPID | data |
374 374 * +------+------------++------+------+
375 375 * | data, cont. | leftover |
376 376 * limit of data buffer ---> +-------------------+--------------+
377 377 *
378 378 * If the amount of requested buffer space exceeds the amount of space
379 379 * available between the current offset and the end of the buffer:
380 380 *
381 381 * (1) all words in the data buffer between the current offset and the limit
382 382 * of the data buffer (marked "leftover", above) are set to
383 383 * DTRACE_EPIDNONE
384 384 *
385 385 * (2) the wrapped offset is set to zero
386 386 *
387 387 * (3) the iteration process described above occurs until the wrapped offset
388 388 * is greater than the amount of desired space.
389 389 *
390 390 * The wrapped offset is implemented by (re-)using the inactive offset.
391 391 * In a "switch" buffer policy, the inactive offset stores the offset in
392 392 * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
393 393 * offset.
394 394 *
395 395 * DTrace Scratch Buffering
396 396 *
397 397 * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
398 398 * To accommodate such requests easily, scratch memory may be allocated in
399 399 * the buffer beyond the current offset plus the needed memory of the current
400 400 * ECB. If there isn't sufficient room in the buffer for the requested amount
401 401 * of scratch space, the allocation fails and an error is generated. Scratch
402 402 * memory is tracked in the dtrace_mstate_t and is automatically freed when
403 403 * the ECB ceases processing. Note that ring buffers cannot allocate their
404 404 * scratch from the principal buffer -- lest they needlessly overwrite older,
405 405 * valid data. Ring buffers therefore have their own dedicated scratch buffer
406 406 * from which scratch is allocated.
407 407 */
408 408 #define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */
409 409 #define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */
410 410 #define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */
411 411 #define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */
412 412 #define DTRACEBUF_DROPPED 0x0010 /* drops occurred */
413 413 #define DTRACEBUF_ERROR 0x0020 /* errors occurred */
414 414 #define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */
415 415 #define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */
416 416 #define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */
417 417
418 418 typedef struct dtrace_buffer {
419 419 uint64_t dtb_offset; /* current offset in buffer */
420 420 uint64_t dtb_size; /* size of buffer */
421 421 uint32_t dtb_flags; /* flags */
422 422 uint32_t dtb_drops; /* number of drops */
423 423 caddr_t dtb_tomax; /* active buffer */
424 424 caddr_t dtb_xamot; /* inactive buffer */
425 425 uint32_t dtb_xamot_flags; /* inactive flags */
426 426 uint32_t dtb_xamot_drops; /* drops in inactive buffer */
427 427 uint64_t dtb_xamot_offset; /* offset in inactive buffer */
428 428 uint32_t dtb_errors; /* number of errors */
429 429 uint32_t dtb_xamot_errors; /* errors in inactive buffer */
430 430 #ifndef _LP64
431 431 uint64_t dtb_pad1; /* pad out to 64 bytes */
432 432 #endif
433 433 uint64_t dtb_switched; /* time of last switch */
434 434 uint64_t dtb_interval; /* observed switch interval */
435 435 uint64_t dtb_pad2[6]; /* pad to avoid false sharing */
436 436 } dtrace_buffer_t;
437 437
438 438 /*
439 439 * DTrace Aggregation Buffers
440 440 *
441 441 * Aggregation buffers use much of the same mechanism as described above
442 442 * ("DTrace Buffers"). However, because an aggregation is fundamentally a
443 443 * hash, there exists dynamic metadata associated with an aggregation buffer
444 444 * that is not associated with other kinds of buffers. This aggregation
445 445 * metadata is _only_ relevant for the in-kernel implementation of
446 446 * aggregations; it is not actually relevant to user-level consumers. To do
447 447 * this, we allocate dynamic aggregation data (hash keys and hash buckets)
448 448 * starting below the _limit_ of the buffer, and we allocate data from the
449 449 * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the
450 450 * data is copied out; the metadata is simply discarded. Schematically,
451 451 * aggregation buffers look like:
452 452 *
453 453 * base of data buffer ---> +-------+------+-----------+-------+
454 454 * | aggid | key | value | aggid |
455 455 * +-------+------+-----------+-------+
456 456 * | key |
457 457 * +-------+-------+-----+------------+
458 458 * | value | aggid | key | value |
459 459 * +-------+------++-----+------+-----+
460 460 * | aggid | key | value | |
461 461 * +-------+------+-------------+ |
462 462 * | || |
463 463 * | || |
464 464 * | \/ |
465 465 * : :
466 466 * . .
467 467 * . .
468 468 * . .
469 469 * : :
470 470 * | /\ |
471 471 * | || +------------+
472 472 * | || | |
473 473 * +---------------------+ |
474 474 * | hash keys |
475 475 * | (dtrace_aggkey structures) |
476 476 * | |
477 477 * +----------------------------------+
478 478 * | hash buckets |
479 479 * | (dtrace_aggbuffer structure) |
480 480 * | |
481 481 * limit of data buffer ---> +----------------------------------+
482 482 *
483 483 *
484 484 * As implied above, just as we assure that ECBs always store a constant
485 485 * amount of data, we assure that a given aggregation -- identified by its
486 486 * aggregation ID -- always stores data of a constant quantity and type.
487 487 * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
488 488 * given record.
489 489 *
490 490 * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
491 491 * aligned. (If this the structure changes such that this becomes false, an
492 492 * assertion will fail in dtrace_aggregate().)
493 493 */
494 494 typedef struct dtrace_aggkey {
495 495 uint32_t dtak_hashval; /* hash value */
496 496 uint32_t dtak_action:4; /* action -- 4 bits */
497 497 uint32_t dtak_size:28; /* size -- 28 bits */
498 498 caddr_t dtak_data; /* data pointer */
499 499 struct dtrace_aggkey *dtak_next; /* next in hash chain */
500 500 } dtrace_aggkey_t;
501 501
502 502 typedef struct dtrace_aggbuffer {
503 503 uintptr_t dtagb_hashsize; /* number of buckets */
504 504 uintptr_t dtagb_free; /* free list of keys */
505 505 dtrace_aggkey_t **dtagb_hash; /* hash table */
506 506 } dtrace_aggbuffer_t;
507 507
508 508 /*
509 509 * DTrace Speculations
510 510 *
511 511 * Speculations have a per-CPU buffer and a global state. Once a speculation
512 512 * buffer has been comitted or discarded, it cannot be reused until all CPUs
513 513 * have taken the same action (commit or discard) on their respective
514 514 * speculative buffer. However, because DTrace probes may execute in arbitrary
515 515 * context, other CPUs cannot simply be cross-called at probe firing time to
516 516 * perform the necessary commit or discard. The speculation states thus
517 517 * optimize for the case that a speculative buffer is only active on one CPU at
518 518 * the time of a commit() or discard() -- for if this is the case, other CPUs
519 519 * need not take action, and the speculation is immediately available for
520 520 * reuse. If the speculation is active on multiple CPUs, it must be
521 521 * asynchronously cleaned -- potentially leading to a higher rate of dirty
522 522 * speculative drops. The speculation states are as follows:
523 523 *
524 524 * DTRACESPEC_INACTIVE <= Initial state; inactive speculation
525 525 * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to
526 526 * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU
527 527 * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU
528 528 * DTRACESPEC_COMMITTING <= Currently being commited on one CPU
529 529 * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
530 530 * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs
531 531 *
532 532 * The state transition diagram is as follows:
533 533 *
534 534 * +----------------------------------------------------------+
535 535 * | |
536 536 * | +------------+ |
537 537 * | +-------------------| COMMITTING |<-----------------+ |
538 538 * | | +------------+ | |
539 539 * | | copied spec. ^ commit() on | | discard() on
540 540 * | | into principal | active CPU | | active CPU
541 541 * | | | commit() | |
542 542 * V V | | |
543 543 * +----------+ +--------+ +-----------+
544 544 * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
545 545 * +----------+ speculation() +--------+ speculate() +-----------+
546 546 * ^ ^ | | |
547 547 * | | | discard() | |
548 548 * | | asynchronously | discard() on | | speculate()
549 549 * | | cleaned V inactive CPU | | on inactive
550 550 * | | +------------+ | | CPU
551 551 * | +-------------------| DISCARDING |<-----------------+ |
552 552 * | +------------+ |
553 553 * | asynchronously ^ |
554 554 * | copied spec. | discard() |
555 555 * | into principal +------------------------+ |
556 556 * | | V
557 557 * +----------------+ commit() +------------+
558 558 * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
559 559 * +----------------+ +------------+
560 560 */
561 561 typedef enum dtrace_speculation_state {
562 562 DTRACESPEC_INACTIVE = 0,
563 563 DTRACESPEC_ACTIVE,
564 564 DTRACESPEC_ACTIVEONE,
565 565 DTRACESPEC_ACTIVEMANY,
566 566 DTRACESPEC_COMMITTING,
567 567 DTRACESPEC_COMMITTINGMANY,
568 568 DTRACESPEC_DISCARDING
569 569 } dtrace_speculation_state_t;
570 570
571 571 typedef struct dtrace_speculation {
572 572 dtrace_speculation_state_t dtsp_state; /* current speculation state */
573 573 int dtsp_cleaning; /* non-zero if being cleaned */
574 574 dtrace_buffer_t *dtsp_buffer; /* speculative buffer */
575 575 } dtrace_speculation_t;
576 576
577 577 /*
578 578 * DTrace Dynamic Variables
579 579 *
580 580 * The dynamic variable problem is obviously decomposed into two subproblems:
581 581 * allocating new dynamic storage, and freeing old dynamic storage. The
582 582 * presence of the second problem makes the first much more complicated -- or
583 583 * rather, the absence of the second renders the first trivial. This is the
584 584 * case with aggregations, for which there is effectively no deallocation of
585 585 * dynamic storage. (Or more accurately, all dynamic storage is deallocated
586 586 * when a snapshot is taken of the aggregation.) As DTrace dynamic variables
587 587 * allow for both dynamic allocation and dynamic deallocation, the
588 588 * implementation of dynamic variables is quite a bit more complicated than
589 589 * that of their aggregation kin.
590 590 *
591 591 * We observe that allocating new dynamic storage is tricky only because the
592 592 * size can vary -- the allocation problem is much easier if allocation sizes
593 593 * are uniform. We further observe that in D, the size of dynamic variables is
594 594 * actually _not_ dynamic -- dynamic variable sizes may be determined by static
595 595 * analysis of DIF text. (This is true even of putatively dynamically-sized
596 596 * objects like strings and stacks, the sizes of which are dictated by the
597 597 * "stringsize" and "stackframes" variables, respectively.) We exploit this by
598 598 * performing this analysis on all DIF before enabling any probes. For each
599 599 * dynamic load or store, we calculate the dynamically-allocated size plus the
600 600 * size of the dtrace_dynvar structure plus the storage required to key the
601 601 * data. For all DIF, we take the largest value and dub it the _chunksize_.
602 602 * We then divide dynamic memory into two parts: a hash table that is wide
603 603 * enough to have every chunk in its own bucket, and a larger region of equal
604 604 * chunksize units. Whenever we wish to dynamically allocate a variable, we
605 605 * always allocate a single chunk of memory. Depending on the uniformity of
606 606 * allocation, this will waste some amount of memory -- but it eliminates the
607 607 * non-determinism inherent in traditional heap fragmentation.
608 608 *
609 609 * Dynamic objects are allocated by storing a non-zero value to them; they are
610 610 * deallocated by storing a zero value to them. Dynamic variables are
611 611 * complicated enormously by being shared between CPUs. In particular,
612 612 * consider the following scenario:
613 613 *
614 614 * CPU A CPU B
615 615 * +---------------------------------+ +---------------------------------+
616 616 * | | | |
617 617 * | allocates dynamic object a[123] | | |
618 618 * | by storing the value 345 to it | | |
619 619 * | ---------> |
620 620 * | | | wishing to load from object |
621 621 * | | | a[123], performs lookup in |
622 622 * | | | dynamic variable space |
623 623 * | <--------- |
624 624 * | deallocates object a[123] by | | |
625 625 * | storing 0 to it | | |
626 626 * | | | |
627 627 * | allocates dynamic object b[567] | | performs load from a[123] |
628 628 * | by storing the value 789 to it | | |
629 629 * : : : :
630 630 * . . . .
631 631 *
632 632 * This is obviously a race in the D program, but there are nonetheless only
633 633 * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly,
634 634 * CPU B may _not_ see the value 789 for a[123].
635 635 *
636 636 * There are essentially two ways to deal with this:
637 637 *
638 638 * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load
639 639 * from a[123], it needs to lock a[123] and hold the lock for the
640 640 * duration that it wishes to manipulate it.
641 641 *
642 642 * (2) Avoid reusing freed chunks until it is known that no CPU is referring
643 643 * to them.
644 644 *
645 645 * The implementation of (1) is rife with complexity, because it requires the
646 646 * user of a dynamic variable to explicitly decree when they are done using it.
647 647 * Were all variables by value, this perhaps wouldn't be debilitating -- but
648 648 * dynamic variables of non-scalar types are tracked by reference. That is, if
649 649 * a dynamic variable is, say, a string, and that variable is to be traced to,
650 650 * say, the principal buffer, the DIF emulation code returns to the main
651 651 * dtrace_probe() loop a pointer to the underlying storage, not the contents of
652 652 * the storage. Further, code calling on DIF emulation would have to be aware
653 653 * that the DIF emulation has returned a reference to a dynamic variable that
654 654 * has been potentially locked. The variable would have to be unlocked after
655 655 * the main dtrace_probe() loop is finished with the variable, and the main
656 656 * dtrace_probe() loop would have to be careful to not call any further DIF
657 657 * emulation while the variable is locked to avoid deadlock. More generally,
658 658 * if one were to implement (1), DIF emulation code dealing with dynamic
659 659 * variables could only deal with one dynamic variable at a time (lest deadlock
660 660 * result). To sum, (1) exports too much subtlety to the users of dynamic
661 661 * variables -- increasing maintenance burden and imposing serious constraints
662 662 * on future DTrace development.
663 663 *
664 664 * The implementation of (2) is also complex, but the complexity is more
665 665 * manageable. We need to be sure that when a variable is deallocated, it is
666 666 * not placed on a traditional free list, but rather on a _dirty_ list. Once a
667 667 * variable is on a dirty list, it cannot be found by CPUs performing a
668 668 * subsequent lookup of the variable -- but it may still be in use by other
669 669 * CPUs. To assure that all CPUs that may be seeing the old variable have
670 670 * cleared out of probe context, a dtrace_sync() can be issued. Once the
671 671 * dtrace_sync() has completed, it can be known that all CPUs are done
672 672 * manipulating the dynamic variable -- the dirty list can be atomically
673 673 * appended to the free list. Unfortunately, there's a slight hiccup in this
674 674 * mechanism: dtrace_sync() may not be issued from probe context. The
675 675 * dtrace_sync() must be therefore issued asynchronously from non-probe
676 676 * context. For this we rely on the DTrace cleaner, a cyclic that runs at the
677 677 * "cleanrate" frequency. To ease this implementation, we define several chunk
678 678 * lists:
679 679 *
680 680 * - Dirty. Deallocated chunks, not yet cleaned. Not available.
681 681 *
682 682 * - Rinsing. Formerly dirty chunks that are currently being asynchronously
683 683 * cleaned. Not available, but will be shortly. Dynamic variable
684 684 * allocation may not spin or block for availability, however.
685 685 *
686 686 * - Clean. Clean chunks, ready for allocation -- but not on the free list.
687 687 *
688 688 * - Free. Available for allocation.
689 689 *
690 690 * Moreover, to avoid absurd contention, _each_ of these lists is implemented
691 691 * on a per-CPU basis. This is only for performance, not correctness; chunks
692 692 * may be allocated from another CPU's free list. The algorithm for allocation
693 693 * then is this:
694 694 *
695 695 * (1) Attempt to atomically allocate from current CPU's free list. If list
696 696 * is non-empty and allocation is successful, allocation is complete.
697 697 *
698 698 * (2) If the clean list is non-empty, atomically move it to the free list,
699 699 * and reattempt (1).
700 700 *
701 701 * (3) If the dynamic variable space is in the CLEAN state, look for free
702 702 * and clean lists on other CPUs by setting the current CPU to the next
703 703 * CPU, and reattempting (1). If the next CPU is the current CPU (that
704 704 * is, if all CPUs have been checked), atomically switch the state of
705 705 * the dynamic variable space based on the following:
706 706 *
707 707 * - If no free chunks were found and no dirty chunks were found,
708 708 * atomically set the state to EMPTY.
709 709 *
710 710 * - If dirty chunks were found, atomically set the state to DIRTY.
711 711 *
712 712 * - If rinsing chunks were found, atomically set the state to RINSING.
713 713 *
714 714 * (4) Based on state of dynamic variable space state, increment appropriate
715 715 * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
716 716 * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
717 717 * RINSING state). Fail the allocation.
718 718 *
719 719 * The cleaning cyclic operates with the following algorithm: for all CPUs
720 720 * with a non-empty dirty list, atomically move the dirty list to the rinsing
721 721 * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list,
722 722 * atomically move the rinsing list to the clean list. Perform another
723 723 * dtrace_sync(). By this point, all CPUs have seen the new clean list; the
724 724 * state of the dynamic variable space can be restored to CLEAN.
725 725 *
726 726 * There exist two final races that merit explanation. The first is a simple
727 727 * allocation race:
728 728 *
729 729 * CPU A CPU B
730 730 * +---------------------------------+ +---------------------------------+
731 731 * | | | |
732 732 * | allocates dynamic object a[123] | | allocates dynamic object a[123] |
733 733 * | by storing the value 345 to it | | by storing the value 567 to it |
734 734 * | | | |
735 735 * : : : :
736 736 * . . . .
737 737 *
738 738 * Again, this is a race in the D program. It can be resolved by having a[123]
739 739 * hold the value 345 or a[123] hold the value 567 -- but it must be true that
740 740 * a[123] have only _one_ of these values. (That is, the racing CPUs may not
741 741 * put the same element twice on the same hash chain.) This is resolved
742 742 * simply: before the allocation is undertaken, the start of the new chunk's
743 743 * hash chain is noted. Later, after the allocation is complete, the hash
744 744 * chain is atomically switched to point to the new element. If this fails
745 745 * (because of either concurrent allocations or an allocation concurrent with a
746 746 * deletion), the newly allocated chunk is deallocated to the dirty list, and
747 747 * the whole process of looking up (and potentially allocating) the dynamic
748 748 * variable is reattempted.
749 749 *
750 750 * The final race is a simple deallocation race:
751 751 *
752 752 * CPU A CPU B
753 753 * +---------------------------------+ +---------------------------------+
754 754 * | | | |
755 755 * | deallocates dynamic object | | deallocates dynamic object |
756 756 * | a[123] by storing the value 0 | | a[123] by storing the value 0 |
757 757 * | to it | | to it |
758 758 * | | | |
759 759 * : : : :
760 760 * . . . .
761 761 *
762 762 * Once again, this is a race in the D program, but it is one that we must
763 763 * handle without corrupting the underlying data structures. Because
764 764 * deallocations require the deletion of a chunk from the middle of a hash
765 765 * chain, we cannot use a single-word atomic operation to remove it. For this,
766 766 * we add a spin lock to the hash buckets that is _only_ used for deallocations
767 767 * (allocation races are handled as above). Further, this spin lock is _only_
768 768 * held for the duration of the delete; before control is returned to the DIF
769 769 * emulation code, the hash bucket is unlocked.
770 770 */
771 771 typedef struct dtrace_key {
772 772 uint64_t dttk_value; /* data value or data pointer */
773 773 uint64_t dttk_size; /* 0 if by-val, >0 if by-ref */
774 774 } dtrace_key_t;
775 775
776 776 typedef struct dtrace_tuple {
777 777 uint32_t dtt_nkeys; /* number of keys in tuple */
778 778 uint32_t dtt_pad; /* padding */
779 779 dtrace_key_t dtt_key[1]; /* array of tuple keys */
780 780 } dtrace_tuple_t;
781 781
782 782 typedef struct dtrace_dynvar {
783 783 uint64_t dtdv_hashval; /* hash value -- 0 if free */
784 784 struct dtrace_dynvar *dtdv_next; /* next on list or hash chain */
785 785 void *dtdv_data; /* pointer to data */
786 786 dtrace_tuple_t dtdv_tuple; /* tuple key */
787 787 } dtrace_dynvar_t;
788 788
789 789 typedef enum dtrace_dynvar_op {
790 790 DTRACE_DYNVAR_ALLOC,
791 791 DTRACE_DYNVAR_NOALLOC,
792 792 DTRACE_DYNVAR_DEALLOC
793 793 } dtrace_dynvar_op_t;
794 794
795 795 typedef struct dtrace_dynhash {
796 796 dtrace_dynvar_t *dtdh_chain; /* hash chain for this bucket */
797 797 uintptr_t dtdh_lock; /* deallocation lock */
798 798 #ifdef _LP64
799 799 uintptr_t dtdh_pad[6]; /* pad to avoid false sharing */
800 800 #else
801 801 uintptr_t dtdh_pad[14]; /* pad to avoid false sharing */
802 802 #endif
803 803 } dtrace_dynhash_t;
804 804
805 805 typedef struct dtrace_dstate_percpu {
806 806 dtrace_dynvar_t *dtdsc_free; /* free list for this CPU */
807 807 dtrace_dynvar_t *dtdsc_dirty; /* dirty list for this CPU */
808 808 dtrace_dynvar_t *dtdsc_rinsing; /* rinsing list for this CPU */
809 809 dtrace_dynvar_t *dtdsc_clean; /* clean list for this CPU */
810 810 uint64_t dtdsc_drops; /* number of capacity drops */
811 811 uint64_t dtdsc_dirty_drops; /* number of dirty drops */
812 812 uint64_t dtdsc_rinsing_drops; /* number of rinsing drops */
813 813 #ifdef _LP64
814 814 uint64_t dtdsc_pad; /* pad to avoid false sharing */
815 815 #else
816 816 uint64_t dtdsc_pad[2]; /* pad to avoid false sharing */
817 817 #endif
818 818 } dtrace_dstate_percpu_t;
819 819
820 820 typedef enum dtrace_dstate_state {
821 821 DTRACE_DSTATE_CLEAN = 0,
822 822 DTRACE_DSTATE_EMPTY,
823 823 DTRACE_DSTATE_DIRTY,
824 824 DTRACE_DSTATE_RINSING
825 825 } dtrace_dstate_state_t;
826 826
827 827 typedef struct dtrace_dstate {
828 828 void *dtds_base; /* base of dynamic var. space */
829 829 size_t dtds_size; /* size of dynamic var. space */
830 830 size_t dtds_hashsize; /* number of buckets in hash */
831 831 size_t dtds_chunksize; /* size of each chunk */
832 832 dtrace_dynhash_t *dtds_hash; /* pointer to hash table */
833 833 dtrace_dstate_state_t dtds_state; /* current dynamic var. state */
834 834 dtrace_dstate_percpu_t *dtds_percpu; /* per-CPU dyn. var. state */
835 835 } dtrace_dstate_t;
836 836
837 837 /*
838 838 * DTrace Variable State
839 839 *
840 840 * The DTrace variable state tracks user-defined variables in its dtrace_vstate
841 841 * structure. Each DTrace consumer has exactly one dtrace_vstate structure,
842 842 * but some dtrace_vstate structures may exist without a corresponding DTrace
843 843 * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>,
844 844 * user-defined variables can have one of three scopes:
845 845 *
846 846 * DIFV_SCOPE_GLOBAL => global scope
847 847 * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables)
848 848 * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables)
849 849 *
850 850 * The variable state tracks variables by both their scope and their allocation
851 851 * type:
852 852 *
853 853 * - The dtvs_globals and dtvs_locals members each point to an array of
854 854 * dtrace_statvar structures. These structures contain both the variable
855 855 * metadata (dtrace_difv structures) and the underlying storage for all
856 856 * statically allocated variables, including statically allocated
857 857 * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
858 858 *
859 859 * - The dtvs_tlocals member points to an array of dtrace_difv structures for
860 860 * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the
861 861 * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
862 862 * is allocated out of the dynamic variable space.
863 863 *
864 864 * - The dtvs_dynvars member is the dynamic variable state associated with the
865 865 * variable state. The dynamic variable state (described in "DTrace Dynamic
866 866 * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
867 867 * dynamically-allocated DIFV_SCOPE_GLOBAL variables.
868 868 */
869 869 typedef struct dtrace_statvar {
870 870 uint64_t dtsv_data; /* data or pointer to it */
871 871 size_t dtsv_size; /* size of pointed-to data */
872 872 int dtsv_refcnt; /* reference count */
873 873 dtrace_difv_t dtsv_var; /* variable metadata */
874 874 } dtrace_statvar_t;
875 875
876 876 typedef struct dtrace_vstate {
877 877 dtrace_state_t *dtvs_state; /* back pointer to state */
878 878 dtrace_statvar_t **dtvs_globals; /* statically-allocated glbls */
879 879 int dtvs_nglobals; /* number of globals */
880 880 dtrace_difv_t *dtvs_tlocals; /* thread-local metadata */
881 881 int dtvs_ntlocals; /* number of thread-locals */
882 882 dtrace_statvar_t **dtvs_locals; /* clause-local data */
883 883 int dtvs_nlocals; /* number of clause-locals */
884 884 dtrace_dstate_t dtvs_dynvars; /* dynamic variable state */
885 885 } dtrace_vstate_t;
886 886
887 887 /*
888 888 * DTrace Machine State
889 889 *
890 890 * In the process of processing a fired probe, DTrace needs to track and/or
891 891 * cache some per-CPU state associated with that particular firing. This is
892 892 * state that is always discarded after the probe firing has completed, and
893 893 * much of it is not specific to any DTrace consumer, remaining valid across
894 894 * all ECBs. This state is tracked in the dtrace_mstate structure.
895 895 */
896 896 #define DTRACE_MSTATE_ARGS 0x00000001
897 897 #define DTRACE_MSTATE_PROBE 0x00000002
898 898 #define DTRACE_MSTATE_EPID 0x00000004
899 899 #define DTRACE_MSTATE_TIMESTAMP 0x00000008
900 900 #define DTRACE_MSTATE_STACKDEPTH 0x00000010
901 901 #define DTRACE_MSTATE_CALLER 0x00000020
902 902 #define DTRACE_MSTATE_IPL 0x00000040
903 903 #define DTRACE_MSTATE_FLTOFFS 0x00000080
904 904 #define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100
905 905 #define DTRACE_MSTATE_USTACKDEPTH 0x00000200
906 906 #define DTRACE_MSTATE_UCALLER 0x00000400
907 907
908 908 typedef struct dtrace_mstate {
909 909 uintptr_t dtms_scratch_base; /* base of scratch space */
910 910 uintptr_t dtms_scratch_ptr; /* current scratch pointer */
911 911 size_t dtms_scratch_size; /* scratch size */
912 912 uint32_t dtms_present; /* variables that are present */
913 913 uint64_t dtms_arg[5]; /* cached arguments */
914 914 dtrace_epid_t dtms_epid; /* current EPID */
915 915 uint64_t dtms_timestamp; /* cached timestamp */
916 916 hrtime_t dtms_walltimestamp; /* cached wall timestamp */
|
↓ open down ↓ |
916 lines elided |
↑ open up ↑ |
917 917 int dtms_stackdepth; /* cached stackdepth */
918 918 int dtms_ustackdepth; /* cached ustackdepth */
919 919 struct dtrace_probe *dtms_probe; /* current probe */
920 920 uintptr_t dtms_caller; /* cached caller */
921 921 uint64_t dtms_ucaller; /* cached user-level caller */
922 922 int dtms_ipl; /* cached interrupt pri lev */
923 923 int dtms_fltoffs; /* faulting DIFO offset */
924 924 uintptr_t dtms_strtok; /* saved strtok() pointer */
925 925 uint32_t dtms_access; /* memory access rights */
926 926 dtrace_difo_t *dtms_difo; /* current dif object */
927 + file_t *dtms_getf; /* cached rval of getf() */
927 928 } dtrace_mstate_t;
928 929
929 930 #define DTRACE_COND_OWNER 0x1
930 931 #define DTRACE_COND_USERMODE 0x2
931 932 #define DTRACE_COND_ZONEOWNER 0x4
932 933
933 934 #define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */
934 935
935 936 /*
936 937 * Access flag used by dtrace_mstate.dtms_access.
937 938 */
938 939 #define DTRACE_ACCESS_KERNEL 0x1 /* the priv to read kmem */
939 940 #define DTRACE_ACCESS_PROC 0x2 /* the priv for proc state */
940 941 #define DTRACE_ACCESS_ARGS 0x4 /* the priv to examine args */
941 942
942 943 /*
943 944 * DTrace Activity
944 945 *
945 946 * Each DTrace consumer is in one of several states, which (for purposes of
946 947 * avoiding yet-another overloading of the noun "state") we call the current
947 948 * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on
948 949 * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may
949 950 * only transition in one direction; the activity transition diagram is a
950 951 * directed acyclic graph. The activity transition diagram is as follows:
951 952 *
952 953 *
953 954 * +----------+ +--------+ +--------+
954 955 * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
955 956 * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+
956 957 * before BEGIN | after BEGIN | | |
957 958 * | | | |
958 959 * exit() action | | | |
959 960 * from BEGIN ECB | | | |
960 961 * | | | |
961 962 * v | | |
962 963 * +----------+ exit() action | | |
963 964 * +-----------------------------| DRAINING |<-------------------+ | |
964 965 * | +----------+ | |
965 966 * | | | |
966 967 * | dtrace_stop(), | | |
967 968 * | before END | | |
968 969 * | | | |
969 970 * | v | |
970 971 * | +---------+ +----------+ | |
971 972 * | | STOPPED |<----------------| COOLDOWN |<----------------------+ |
972 973 * | +---------+ dtrace_stop(), +----------+ dtrace_stop(), |
973 974 * | after END before END |
974 975 * | |
975 976 * | +--------+ |
976 977 * +----------------------------->| KILLED |<--------------------------+
977 978 * deadman timeout or +--------+ deadman timeout or
978 979 * killed consumer killed consumer
979 980 *
980 981 * Note that once a DTrace consumer has stopped tracing, there is no way to
981 982 * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
982 983 * the DTrace pseudodevice.
983 984 */
984 985 typedef enum dtrace_activity {
985 986 DTRACE_ACTIVITY_INACTIVE = 0, /* not yet running */
986 987 DTRACE_ACTIVITY_WARMUP, /* while starting */
987 988 DTRACE_ACTIVITY_ACTIVE, /* running */
988 989 DTRACE_ACTIVITY_DRAINING, /* before stopping */
989 990 DTRACE_ACTIVITY_COOLDOWN, /* while stopping */
990 991 DTRACE_ACTIVITY_STOPPED, /* after stopping */
991 992 DTRACE_ACTIVITY_KILLED /* killed */
992 993 } dtrace_activity_t;
993 994
994 995 /*
995 996 * DTrace Helper Implementation
996 997 *
997 998 * A description of the helper architecture may be found in <sys/dtrace.h>.
998 999 * Each process contains a pointer to its helpers in its p_dtrace_helpers
999 1000 * member. This is a pointer to a dtrace_helpers structure, which contains an
1000 1001 * array of pointers to dtrace_helper structures, helper variable state (shared
1001 1002 * among a process's helpers) and a generation count. (The generation count is
1002 1003 * used to provide an identifier when a helper is added so that it may be
1003 1004 * subsequently removed.) The dtrace_helper structure is self-explanatory,
1004 1005 * containing pointers to the objects needed to execute the helper. Note that
1005 1006 * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more
1006 1007 * than dtrace_helpers_max are allowed per-process.
1007 1008 */
1008 1009 #define DTRACE_HELPER_ACTION_USTACK 0
1009 1010 #define DTRACE_NHELPER_ACTIONS 1
1010 1011
1011 1012 typedef struct dtrace_helper_action {
1012 1013 int dtha_generation; /* helper action generation */
1013 1014 int dtha_nactions; /* number of actions */
1014 1015 dtrace_difo_t *dtha_predicate; /* helper action predicate */
1015 1016 dtrace_difo_t **dtha_actions; /* array of actions */
1016 1017 struct dtrace_helper_action *dtha_next; /* next helper action */
1017 1018 } dtrace_helper_action_t;
1018 1019
1019 1020 typedef struct dtrace_helper_provider {
1020 1021 int dthp_generation; /* helper provider generation */
1021 1022 uint32_t dthp_ref; /* reference count */
1022 1023 dof_helper_t dthp_prov; /* DOF w/ provider and probes */
1023 1024 } dtrace_helper_provider_t;
1024 1025
1025 1026 typedef struct dtrace_helpers {
1026 1027 dtrace_helper_action_t **dthps_actions; /* array of helper actions */
1027 1028 dtrace_vstate_t dthps_vstate; /* helper action var. state */
1028 1029 dtrace_helper_provider_t **dthps_provs; /* array of providers */
1029 1030 uint_t dthps_nprovs; /* count of providers */
1030 1031 uint_t dthps_maxprovs; /* provider array size */
1031 1032 int dthps_generation; /* current generation */
1032 1033 pid_t dthps_pid; /* pid of associated proc */
1033 1034 int dthps_deferred; /* helper in deferred list */
1034 1035 struct dtrace_helpers *dthps_next; /* next pointer */
1035 1036 struct dtrace_helpers *dthps_prev; /* prev pointer */
1036 1037 } dtrace_helpers_t;
1037 1038
1038 1039 /*
1039 1040 * DTrace Helper Action Tracing
1040 1041 *
1041 1042 * Debugging helper actions can be arduous. To ease the development and
1042 1043 * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
1043 1044 * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which
1044 1045 * it is by default on DEBUG kernels), all helper activity will be traced to a
1045 1046 * global, in-kernel ring buffer. Each entry includes a pointer to the specific
1046 1047 * helper, the location within the helper, and a trace of all local variables.
1047 1048 * The ring buffer may be displayed in a human-readable format with the
1048 1049 * ::dtrace_helptrace mdb(1) dcmd.
1049 1050 */
1050 1051 #define DTRACE_HELPTRACE_NEXT (-1)
1051 1052 #define DTRACE_HELPTRACE_DONE (-2)
1052 1053 #define DTRACE_HELPTRACE_ERR (-3)
1053 1054
1054 1055 typedef struct dtrace_helptrace {
1055 1056 dtrace_helper_action_t *dtht_helper; /* helper action */
1056 1057 int dtht_where; /* where in helper action */
1057 1058 int dtht_nlocals; /* number of locals */
1058 1059 int dtht_fault; /* type of fault (if any) */
1059 1060 int dtht_fltoffs; /* DIF offset */
1060 1061 uint64_t dtht_illval; /* faulting value */
1061 1062 uint64_t dtht_locals[1]; /* local variables */
1062 1063 } dtrace_helptrace_t;
1063 1064
1064 1065 /*
1065 1066 * DTrace Credentials
1066 1067 *
1067 1068 * In probe context, we have limited flexibility to examine the credentials
1068 1069 * of the DTrace consumer that created a particular enabling. We use
1069 1070 * the Least Privilege interfaces to cache the consumer's cred pointer and
1070 1071 * some facts about that credential in a dtrace_cred_t structure. These
1071 1072 * can limit the consumer's breadth of visibility and what actions the
1072 1073 * consumer may take.
1073 1074 */
1074 1075 #define DTRACE_CRV_ALLPROC 0x01
1075 1076 #define DTRACE_CRV_KERNEL 0x02
1076 1077 #define DTRACE_CRV_ALLZONE 0x04
1077 1078
1078 1079 #define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
1079 1080 DTRACE_CRV_ALLZONE)
1080 1081
1081 1082 #define DTRACE_CRA_PROC 0x0001
1082 1083 #define DTRACE_CRA_PROC_CONTROL 0x0002
1083 1084 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER 0x0004
1084 1085 #define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE 0x0008
1085 1086 #define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG 0x0010
1086 1087 #define DTRACE_CRA_KERNEL 0x0020
1087 1088 #define DTRACE_CRA_KERNEL_DESTRUCTIVE 0x0040
1088 1089
1089 1090 #define DTRACE_CRA_ALL (DTRACE_CRA_PROC | \
1090 1091 DTRACE_CRA_PROC_CONTROL | \
1091 1092 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
1092 1093 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
1093 1094 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
1094 1095 DTRACE_CRA_KERNEL | \
1095 1096 DTRACE_CRA_KERNEL_DESTRUCTIVE)
1096 1097
1097 1098 typedef struct dtrace_cred {
1098 1099 cred_t *dcr_cred;
1099 1100 uint8_t dcr_destructive;
1100 1101 uint8_t dcr_visible;
1101 1102 uint16_t dcr_action;
1102 1103 } dtrace_cred_t;
1103 1104
1104 1105 /*
1105 1106 * DTrace Consumer State
1106 1107 *
1107 1108 * Each DTrace consumer has an associated dtrace_state structure that contains
1108 1109 * its in-kernel DTrace state -- including options, credentials, statistics and
1109 1110 * pointers to ECBs, buffers, speculations and formats. A dtrace_state
1110 1111 * structure is also allocated for anonymous enablings. When anonymous state
1111 1112 * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
1112 1113 * dtrace_state structure.
1113 1114 */
1114 1115 struct dtrace_state {
1115 1116 dev_t dts_dev; /* device */
1116 1117 int dts_necbs; /* total number of ECBs */
1117 1118 dtrace_ecb_t **dts_ecbs; /* array of ECBs */
1118 1119 dtrace_epid_t dts_epid; /* next EPID to allocate */
1119 1120 size_t dts_needed; /* greatest needed space */
1120 1121 struct dtrace_state *dts_anon; /* anon. state, if grabbed */
1121 1122 dtrace_activity_t dts_activity; /* current activity */
1122 1123 dtrace_vstate_t dts_vstate; /* variable state */
1123 1124 dtrace_buffer_t *dts_buffer; /* principal buffer */
1124 1125 dtrace_buffer_t *dts_aggbuffer; /* aggregation buffer */
1125 1126 dtrace_speculation_t *dts_speculations; /* speculation array */
1126 1127 int dts_nspeculations; /* number of speculations */
1127 1128 int dts_naggregations; /* number of aggregations */
1128 1129 dtrace_aggregation_t **dts_aggregations; /* aggregation array */
1129 1130 vmem_t *dts_aggid_arena; /* arena for aggregation IDs */
1130 1131 uint64_t dts_errors; /* total number of errors */
1131 1132 uint32_t dts_speculations_busy; /* number of spec. busy */
1132 1133 uint32_t dts_speculations_unavail; /* number of spec unavail */
1133 1134 uint32_t dts_stkstroverflows; /* stack string tab overflows */
1134 1135 uint32_t dts_dblerrors; /* errors in ERROR probes */
1135 1136 uint32_t dts_reserve; /* space reserved for END */
1136 1137 hrtime_t dts_laststatus; /* time of last status */
|
↓ open down ↓ |
200 lines elided |
↑ open up ↑ |
1137 1138 cyclic_id_t dts_cleaner; /* cleaning cyclic */
1138 1139 cyclic_id_t dts_deadman; /* deadman cyclic */
1139 1140 hrtime_t dts_alive; /* time last alive */
1140 1141 char dts_speculates; /* boolean: has speculations */
1141 1142 char dts_destructive; /* boolean: has dest. actions */
1142 1143 int dts_nformats; /* number of formats */
1143 1144 char **dts_formats; /* format string array */
1144 1145 dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
1145 1146 dtrace_cred_t dts_cred; /* credentials */
1146 1147 size_t dts_nretained; /* number of retained enabs */
1148 + int dts_getf; /* number of getf() calls */
1147 1149 };
1148 1150
1149 1151 struct dtrace_provider {
1150 1152 dtrace_pattr_t dtpv_attr; /* provider attributes */
1151 1153 dtrace_ppriv_t dtpv_priv; /* provider privileges */
1152 1154 dtrace_pops_t dtpv_pops; /* provider operations */
1153 1155 char *dtpv_name; /* provider name */
1154 1156 void *dtpv_arg; /* provider argument */
1155 1157 hrtime_t dtpv_defunct; /* when made defunct */
1156 1158 struct dtrace_provider *dtpv_next; /* next provider */
1157 1159 };
1158 1160
1159 1161 struct dtrace_meta {
1160 1162 dtrace_mops_t dtm_mops; /* meta provider operations */
1161 1163 char *dtm_name; /* meta provider name */
1162 1164 void *dtm_arg; /* meta provider user arg */
1163 1165 uint64_t dtm_count; /* no. of associated provs. */
1164 1166 };
1165 1167
1166 1168 /*
1167 1169 * DTrace Enablings
1168 1170 *
1169 1171 * A dtrace_enabling structure is used to track a collection of ECB
1170 1172 * descriptions -- before they have been turned into actual ECBs. This is
1171 1173 * created as a result of DOF processing, and is generally used to generate
1172 1174 * ECBs immediately thereafter. However, enablings are also generally
1173 1175 * retained should the probes they describe be created at a later time; as
1174 1176 * each new module or provider registers with the framework, the retained
1175 1177 * enablings are reevaluated, with any new match resulting in new ECBs. To
1176 1178 * prevent probes from being matched more than once, the enabling tracks the
1177 1179 * last probe generation matched, and only matches probes from subsequent
1178 1180 * generations.
1179 1181 */
1180 1182 typedef struct dtrace_enabling {
1181 1183 dtrace_ecbdesc_t **dten_desc; /* all ECB descriptions */
1182 1184 int dten_ndesc; /* number of ECB descriptions */
1183 1185 int dten_maxdesc; /* size of ECB array */
1184 1186 dtrace_vstate_t *dten_vstate; /* associated variable state */
1185 1187 dtrace_genid_t dten_probegen; /* matched probe generation */
1186 1188 dtrace_ecbdesc_t *dten_current; /* current ECB description */
1187 1189 int dten_error; /* current error value */
1188 1190 int dten_primed; /* boolean: set if primed */
1189 1191 struct dtrace_enabling *dten_prev; /* previous enabling */
1190 1192 struct dtrace_enabling *dten_next; /* next enabling */
1191 1193 } dtrace_enabling_t;
1192 1194
1193 1195 /*
1194 1196 * DTrace Anonymous Enablings
1195 1197 *
1196 1198 * Anonymous enablings are DTrace enablings that are not associated with a
1197 1199 * controlling process, but rather derive their enabling from DOF stored as
1198 1200 * properties in the dtrace.conf file. If there is an anonymous enabling, a
1199 1201 * DTrace consumer state and enabling are created on attach. The state may be
1200 1202 * subsequently grabbed by the first consumer specifying the "grabanon"
1201 1203 * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will
1202 1204 * refuse to unload.
1203 1205 */
1204 1206 typedef struct dtrace_anon {
1205 1207 dtrace_state_t *dta_state; /* DTrace consumer state */
1206 1208 dtrace_enabling_t *dta_enabling; /* pointer to enabling */
1207 1209 processorid_t dta_beganon; /* which CPU BEGIN ran on */
1208 1210 } dtrace_anon_t;
1209 1211
1210 1212 /*
1211 1213 * DTrace Error Debugging
1212 1214 */
1213 1215 #ifdef DEBUG
1214 1216 #define DTRACE_ERRDEBUG
1215 1217 #endif
1216 1218
1217 1219 #ifdef DTRACE_ERRDEBUG
1218 1220
1219 1221 typedef struct dtrace_errhash {
1220 1222 const char *dter_msg; /* error message */
1221 1223 int dter_count; /* number of times seen */
1222 1224 } dtrace_errhash_t;
1223 1225
1224 1226 #define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */
1225 1227
1226 1228 #endif /* DTRACE_ERRDEBUG */
1227 1229
1228 1230 /*
1229 1231 * DTrace Toxic Ranges
1230 1232 *
1231 1233 * DTrace supports safe loads from probe context; if the address turns out to
1232 1234 * be invalid, a bit will be set by the kernel indicating that DTrace
1233 1235 * encountered a memory error, and DTrace will propagate the error to the user
1234 1236 * accordingly. However, there may exist some regions of memory in which an
1235 1237 * arbitrary load can change system state, and from which it is impossible to
1236 1238 * recover from such a load after it has been attempted. Examples of this may
1237 1239 * include memory in which programmable I/O registers are mapped (for which a
1238 1240 * read may have some implications for the device) or (in the specific case of
1239 1241 * UltraSPARC-I and -II) the virtual address hole. The platform is required
1240 1242 * to make DTrace aware of these toxic ranges; DTrace will then check that
1241 1243 * target addresses are not in a toxic range before attempting to issue a
1242 1244 * safe load.
1243 1245 */
1244 1246 typedef struct dtrace_toxrange {
1245 1247 uintptr_t dtt_base; /* base of toxic range */
1246 1248 uintptr_t dtt_limit; /* limit of toxic range */
1247 1249 } dtrace_toxrange_t;
1248 1250
1249 1251 extern uint64_t dtrace_getarg(int, int);
1250 1252 extern greg_t dtrace_getfp(void);
1251 1253 extern int dtrace_getipl(void);
1252 1254 extern uintptr_t dtrace_caller(int);
1253 1255 extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
1254 1256 extern void *dtrace_casptr(void *, void *, void *);
1255 1257 extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1256 1258 extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1257 1259 extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1258 1260 extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t,
1259 1261 volatile uint16_t *);
1260 1262 extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
1261 1263 extern ulong_t dtrace_getreg(struct regs *, uint_t);
1262 1264 extern uint64_t dtrace_getvmreg(uint_t, volatile uint16_t *);
1263 1265 extern int dtrace_getstackdepth(int);
1264 1266 extern void dtrace_getupcstack(uint64_t *, int);
1265 1267 extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
1266 1268 extern int dtrace_getustackdepth(void);
1267 1269 extern uintptr_t dtrace_fulword(void *);
1268 1270 extern uint8_t dtrace_fuword8(void *);
1269 1271 extern uint16_t dtrace_fuword16(void *);
1270 1272 extern uint32_t dtrace_fuword32(void *);
1271 1273 extern uint64_t dtrace_fuword64(void *);
1272 1274 extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
1273 1275 int, uintptr_t);
1274 1276 extern int dtrace_assfail(const char *, const char *, int);
1275 1277 extern int dtrace_attached(void);
1276 1278 extern hrtime_t dtrace_gethrestime();
1277 1279
1278 1280 #ifdef __sparc
1279 1281 extern void dtrace_flush_windows(void);
1280 1282 extern void dtrace_flush_user_windows(void);
1281 1283 extern uint_t dtrace_getotherwin(void);
1282 1284 extern uint_t dtrace_getfprs(void);
1283 1285 #else
1284 1286 extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
1285 1287 extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
1286 1288 #endif
1287 1289
1288 1290 /*
1289 1291 * DTrace Assertions
1290 1292 *
1291 1293 * DTrace calls ASSERT from probe context. To assure that a failed ASSERT
1292 1294 * does not induce a markedly more catastrophic failure (e.g., one from which
1293 1295 * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
1294 1296 * may safely be called from probe context. This header file must thus be
1295 1297 * included by any DTrace component that calls ASSERT from probe context, and
1296 1298 * _only_ by those components. (The only exception to this is kernel
1297 1299 * debugging infrastructure at user-level that doesn't depend on calling
1298 1300 * ASSERT.)
1299 1301 */
1300 1302 #undef ASSERT
1301 1303 #ifdef DEBUG
1302 1304 #define ASSERT(EX) ((void)((EX) || \
1303 1305 dtrace_assfail(#EX, __FILE__, __LINE__)))
1304 1306 #else
1305 1307 #define ASSERT(X) ((void)0)
1306 1308 #endif
1307 1309
1308 1310 #ifdef __cplusplus
1309 1311 }
1310 1312 #endif
1311 1313
1312 1314 #endif /* _SYS_DTRACE_IMPL_H */
|
↓ open down ↓ |
156 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX