Print this page
7127 remove -Wno-missing-braces from Makefile.uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/dtrace/fasttrap.c
+++ new/usr/src/uts/common/dtrace/fasttrap.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
29 29 */
30 30
31 31 #include <sys/atomic.h>
32 32 #include <sys/errno.h>
33 33 #include <sys/stat.h>
34 34 #include <sys/modctl.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/systm.h>
37 37 #include <sys/ddi.h>
38 38 #include <sys/sunddi.h>
39 39 #include <sys/cpuvar.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/strsubr.h>
42 42 #include <sys/fasttrap.h>
43 43 #include <sys/fasttrap_impl.h>
44 44 #include <sys/fasttrap_isa.h>
45 45 #include <sys/dtrace.h>
46 46 #include <sys/dtrace_impl.h>
47 47 #include <sys/sysmacros.h>
48 48 #include <sys/proc.h>
49 49 #include <sys/priv.h>
50 50 #include <sys/policy.h>
51 51 #include <util/qsort.h>
52 52
53 53 /*
54 54 * User-Land Trap-Based Tracing
55 55 * ----------------------------
56 56 *
57 57 * The fasttrap provider allows DTrace consumers to instrument any user-level
58 58 * instruction to gather data; this includes probes with semantic
59 59 * signifigance like entry and return as well as simple offsets into the
60 60 * function. While the specific techniques used are very ISA specific, the
61 61 * methodology is generalizable to any architecture.
62 62 *
63 63 *
64 64 * The General Methodology
65 65 * -----------------------
66 66 *
67 67 * With the primary goal of tracing every user-land instruction and the
68 68 * limitation that we can't trust user space so don't want to rely on much
69 69 * information there, we begin by replacing the instructions we want to trace
70 70 * with trap instructions. Each instruction we overwrite is saved into a hash
71 71 * table keyed by process ID and pc address. When we enter the kernel due to
72 72 * this trap instruction, we need the effects of the replaced instruction to
73 73 * appear to have occurred before we proceed with the user thread's
74 74 * execution.
75 75 *
76 76 * Each user level thread is represented by a ulwp_t structure which is
77 77 * always easily accessible through a register. The most basic way to produce
78 78 * the effects of the instruction we replaced is to copy that instruction out
79 79 * to a bit of scratch space reserved in the user thread's ulwp_t structure
80 80 * (a sort of kernel-private thread local storage), set the PC to that
81 81 * scratch space and single step. When we reenter the kernel after single
82 82 * stepping the instruction we must then adjust the PC to point to what would
83 83 * normally be the next instruction. Of course, special care must be taken
84 84 * for branches and jumps, but these represent such a small fraction of any
85 85 * instruction set that writing the code to emulate these in the kernel is
86 86 * not too difficult.
87 87 *
88 88 * Return probes may require several tracepoints to trace every return site,
89 89 * and, conversely, each tracepoint may activate several probes (the entry
90 90 * and offset 0 probes, for example). To solve this muliplexing problem,
91 91 * tracepoints contain lists of probes to activate and probes contain lists
92 92 * of tracepoints to enable. If a probe is activated, it adds its ID to
93 93 * existing tracepoints or creates new ones as necessary.
94 94 *
95 95 * Most probes are activated _before_ the instruction is executed, but return
96 96 * probes are activated _after_ the effects of the last instruction of the
97 97 * function are visible. Return probes must be fired _after_ we have
98 98 * single-stepped the instruction whereas all other probes are fired
99 99 * beforehand.
100 100 *
101 101 *
102 102 * Lock Ordering
103 103 * -------------
104 104 *
105 105 * The lock ordering below -- both internally and with respect to the DTrace
106 106 * framework -- is a little tricky and bears some explanation. Each provider
107 107 * has a lock (ftp_mtx) that protects its members including reference counts
108 108 * for enabled probes (ftp_rcount), consumers actively creating probes
109 109 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
110 110 * from being freed. A provider is looked up by taking the bucket lock for the
111 111 * provider hash table, and is returned with its lock held. The provider lock
112 112 * may be taken in functions invoked by the DTrace framework, but may not be
113 113 * held while calling functions in the DTrace framework.
114 114 *
115 115 * To ensure consistency over multiple calls to the DTrace framework, the
116 116 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
117 117 * not be taken when holding the provider lock as that would create a cyclic
118 118 * lock ordering. In situations where one would naturally take the provider
119 119 * lock and then the creation lock, we instead up a reference count to prevent
120 120 * the provider from disappearing, drop the provider lock, and acquire the
121 121 * creation lock.
122 122 *
123 123 * Briefly:
124 124 * bucket lock before provider lock
125 125 * DTrace before provider lock
126 126 * creation lock before DTrace
127 127 * never hold the provider lock and creation lock simultaneously
128 128 */
129 129
130 130 static dev_info_t *fasttrap_devi;
131 131 static dtrace_meta_provider_id_t fasttrap_meta_id;
132 132
133 133 static timeout_id_t fasttrap_timeout;
134 134 static kmutex_t fasttrap_cleanup_mtx;
135 135 static uint_t fasttrap_cleanup_work;
136 136
137 137 /*
138 138 * Generation count on modifications to the global tracepoint lookup table.
139 139 */
140 140 static volatile uint64_t fasttrap_mod_gen;
141 141
142 142 /*
143 143 * When the fasttrap provider is loaded, fasttrap_max is set to either
144 144 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
145 145 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
146 146 * incremented by the number of tracepoints that may be associated with that
147 147 * probe; fasttrap_total is capped at fasttrap_max.
148 148 */
149 149 #define FASTTRAP_MAX_DEFAULT 250000
150 150 static uint32_t fasttrap_max;
151 151 static uint32_t fasttrap_total;
152 152
153 153
154 154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
155 155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
156 156 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
157 157
158 158 #define FASTTRAP_PID_NAME "pid"
159 159
160 160 fasttrap_hash_t fasttrap_tpoints;
161 161 static fasttrap_hash_t fasttrap_provs;
162 162 static fasttrap_hash_t fasttrap_procs;
163 163
164 164 static uint64_t fasttrap_pid_count; /* pid ref count */
165 165 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
166 166
167 167 #define FASTTRAP_ENABLE_FAIL 1
168 168 #define FASTTRAP_ENABLE_PARTIAL 2
169 169
170 170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
171 171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
172 172
173 173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
174 174 const dtrace_pattr_t *);
175 175 static void fasttrap_provider_retire(pid_t, const char *, int);
176 176 static void fasttrap_provider_free(fasttrap_provider_t *);
177 177
178 178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
179 179 static void fasttrap_proc_release(fasttrap_proc_t *);
180 180
181 181 #define FASTTRAP_PROVS_INDEX(pid, name) \
182 182 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
183 183
184 184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
185 185
186 186 static int
187 187 fasttrap_highbit(ulong_t i)
188 188 {
189 189 int h = 1;
190 190
191 191 if (i == 0)
192 192 return (0);
193 193 #ifdef _LP64
194 194 if (i & 0xffffffff00000000ul) {
195 195 h += 32; i >>= 32;
196 196 }
197 197 #endif
198 198 if (i & 0xffff0000) {
199 199 h += 16; i >>= 16;
200 200 }
201 201 if (i & 0xff00) {
202 202 h += 8; i >>= 8;
203 203 }
204 204 if (i & 0xf0) {
205 205 h += 4; i >>= 4;
206 206 }
207 207 if (i & 0xc) {
208 208 h += 2; i >>= 2;
209 209 }
210 210 if (i & 0x2) {
211 211 h += 1;
212 212 }
213 213 return (h);
214 214 }
215 215
216 216 static uint_t
217 217 fasttrap_hash_str(const char *p)
218 218 {
219 219 unsigned int g;
220 220 uint_t hval = 0;
221 221
222 222 while (*p) {
223 223 hval = (hval << 4) + *p++;
224 224 if ((g = (hval & 0xf0000000)) != 0)
225 225 hval ^= g >> 24;
226 226 hval &= ~g;
227 227 }
228 228 return (hval);
229 229 }
230 230
231 231 void
232 232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
233 233 {
234 234 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
235 235
236 236 sqp->sq_info.si_signo = SIGTRAP;
237 237 sqp->sq_info.si_code = TRAP_DTRACE;
238 238 sqp->sq_info.si_addr = (caddr_t)pc;
239 239
240 240 mutex_enter(&p->p_lock);
241 241 sigaddqa(p, t, sqp);
242 242 mutex_exit(&p->p_lock);
243 243
244 244 if (t != NULL)
245 245 aston(t);
246 246 }
247 247
248 248 /*
249 249 * This function ensures that no threads are actively using the memory
250 250 * associated with probes that were formerly live.
251 251 */
252 252 static void
253 253 fasttrap_mod_barrier(uint64_t gen)
254 254 {
255 255 int i;
256 256
257 257 if (gen < fasttrap_mod_gen)
258 258 return;
259 259
260 260 fasttrap_mod_gen++;
261 261
262 262 for (i = 0; i < NCPU; i++) {
263 263 mutex_enter(&cpu_core[i].cpuc_pid_lock);
264 264 mutex_exit(&cpu_core[i].cpuc_pid_lock);
265 265 }
266 266 }
267 267
268 268 /*
269 269 * This is the timeout's callback for cleaning up the providers and their
270 270 * probes.
271 271 */
272 272 /*ARGSUSED*/
273 273 static void
274 274 fasttrap_pid_cleanup_cb(void *data)
275 275 {
276 276 fasttrap_provider_t **fpp, *fp;
277 277 fasttrap_bucket_t *bucket;
278 278 dtrace_provider_id_t provid;
279 279 int i, later, rval;
280 280
281 281 static volatile int in = 0;
282 282 ASSERT(in == 0);
283 283 in = 1;
284 284
285 285 mutex_enter(&fasttrap_cleanup_mtx);
286 286 while (fasttrap_cleanup_work) {
287 287 fasttrap_cleanup_work = 0;
288 288 mutex_exit(&fasttrap_cleanup_mtx);
289 289
290 290 later = 0;
291 291
292 292 /*
293 293 * Iterate over all the providers trying to remove the marked
294 294 * ones. If a provider is marked but not retired, we just
295 295 * have to take a crack at removing it -- it's no big deal if
296 296 * we can't.
297 297 */
298 298 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
299 299 bucket = &fasttrap_provs.fth_table[i];
300 300 mutex_enter(&bucket->ftb_mtx);
301 301 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
302 302
303 303 while ((fp = *fpp) != NULL) {
304 304 if (!fp->ftp_marked) {
305 305 fpp = &fp->ftp_next;
306 306 continue;
307 307 }
308 308
309 309 mutex_enter(&fp->ftp_mtx);
310 310
311 311 /*
312 312 * If this provider has consumers actively
313 313 * creating probes (ftp_ccount) or is a USDT
314 314 * provider (ftp_mcount), we can't unregister
315 315 * or even condense.
316 316 */
317 317 if (fp->ftp_ccount != 0 ||
318 318 fp->ftp_mcount != 0) {
319 319 mutex_exit(&fp->ftp_mtx);
320 320 fp->ftp_marked = 0;
321 321 continue;
322 322 }
323 323
324 324 if (!fp->ftp_retired || fp->ftp_rcount != 0)
325 325 fp->ftp_marked = 0;
326 326
327 327 mutex_exit(&fp->ftp_mtx);
328 328
329 329 /*
330 330 * If we successfully unregister this
331 331 * provider we can remove it from the hash
332 332 * chain and free the memory. If our attempt
333 333 * to unregister fails and this is a retired
334 334 * provider, increment our flag to try again
335 335 * pretty soon. If we've consumed more than
336 336 * half of our total permitted number of
337 337 * probes call dtrace_condense() to try to
338 338 * clean out the unenabled probes.
339 339 */
340 340 provid = fp->ftp_provid;
341 341 if ((rval = dtrace_unregister(provid)) != 0) {
342 342 if (fasttrap_total > fasttrap_max / 2)
343 343 (void) dtrace_condense(provid);
344 344
345 345 if (rval == EAGAIN)
346 346 fp->ftp_marked = 1;
347 347
348 348 later += fp->ftp_marked;
349 349 fpp = &fp->ftp_next;
350 350 } else {
351 351 *fpp = fp->ftp_next;
352 352 fasttrap_provider_free(fp);
353 353 }
354 354 }
355 355 mutex_exit(&bucket->ftb_mtx);
356 356 }
357 357
358 358 mutex_enter(&fasttrap_cleanup_mtx);
359 359 }
360 360
361 361 ASSERT(fasttrap_timeout != 0);
362 362
363 363 /*
364 364 * If we were unable to remove a retired provider, try again after
365 365 * a second. This situation can occur in certain circumstances where
366 366 * providers cannot be unregistered even though they have no probes
367 367 * enabled because of an execution of dtrace -l or something similar.
368 368 * If the timeout has been disabled (set to 1 because we're trying
369 369 * to detach), we set fasttrap_cleanup_work to ensure that we'll
370 370 * get a chance to do that work if and when the timeout is reenabled
371 371 * (if detach fails).
372 372 */
373 373 if (later > 0) {
374 374 if (fasttrap_timeout != (timeout_id_t)1) {
375 375 fasttrap_timeout =
376 376 timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
377 377 }
378 378
379 379 fasttrap_cleanup_work = 1;
380 380 } else {
381 381 fasttrap_timeout = 0;
382 382 }
383 383
384 384 mutex_exit(&fasttrap_cleanup_mtx);
385 385 in = 0;
386 386 }
387 387
388 388 /*
389 389 * Activates the asynchronous cleanup mechanism.
390 390 */
391 391 static void
392 392 fasttrap_pid_cleanup(void)
393 393 {
394 394 mutex_enter(&fasttrap_cleanup_mtx);
395 395 fasttrap_cleanup_work = 1;
396 396 if (fasttrap_timeout == 0)
397 397 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
398 398 mutex_exit(&fasttrap_cleanup_mtx);
399 399 }
400 400
401 401 /*
402 402 * This is called from cfork() via dtrace_fasttrap_fork(). The child
403 403 * process's address space is (roughly) a copy of the parent process's so
404 404 * we have to remove all the instrumentation we had previously enabled in the
405 405 * parent.
406 406 */
407 407 static void
408 408 fasttrap_fork(proc_t *p, proc_t *cp)
409 409 {
410 410 pid_t ppid = p->p_pid;
411 411 int i;
412 412
413 413 ASSERT(curproc == p);
414 414 ASSERT(p->p_proc_flag & P_PR_LOCK);
415 415 ASSERT(p->p_dtrace_count > 0);
416 416 ASSERT(cp->p_dtrace_count == 0);
417 417
418 418 /*
419 419 * This would be simpler and faster if we maintained per-process
420 420 * hash tables of enabled tracepoints. It could, however, potentially
421 421 * slow down execution of a tracepoint since we'd need to go
422 422 * through two levels of indirection. In the future, we should
423 423 * consider either maintaining per-process ancillary lists of
424 424 * enabled tracepoints or hanging a pointer to a per-process hash
425 425 * table of enabled tracepoints off the proc structure.
426 426 */
427 427
428 428 /*
429 429 * We don't have to worry about the child process disappearing
430 430 * because we're in fork().
431 431 */
432 432 mutex_enter(&cp->p_lock);
433 433 sprlock_proc(cp);
434 434 mutex_exit(&cp->p_lock);
435 435
436 436 /*
437 437 * Iterate over every tracepoint looking for ones that belong to the
438 438 * parent process, and remove each from the child process.
439 439 */
440 440 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
441 441 fasttrap_tracepoint_t *tp;
442 442 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
443 443
444 444 mutex_enter(&bucket->ftb_mtx);
445 445 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
446 446 if (tp->ftt_pid == ppid &&
447 447 tp->ftt_proc->ftpc_acount != 0) {
448 448 int ret = fasttrap_tracepoint_remove(cp, tp);
449 449 ASSERT(ret == 0);
450 450
451 451 /*
452 452 * The count of active providers can only be
453 453 * decremented (i.e. to zero) during exec,
454 454 * exit, and removal of a meta provider so it
455 455 * should be impossible to drop the count
456 456 * mid-fork.
457 457 */
458 458 ASSERT(tp->ftt_proc->ftpc_acount != 0);
459 459 }
460 460 }
461 461 mutex_exit(&bucket->ftb_mtx);
462 462 }
463 463
464 464 mutex_enter(&cp->p_lock);
465 465 sprunlock(cp);
466 466 }
467 467
468 468 /*
469 469 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
470 470 * is set on the proc structure to indicate that there is a pid provider
471 471 * associated with this process.
472 472 */
473 473 static void
474 474 fasttrap_exec_exit(proc_t *p)
475 475 {
476 476 ASSERT(p == curproc);
477 477 ASSERT(MUTEX_HELD(&p->p_lock));
478 478
479 479 mutex_exit(&p->p_lock);
480 480
481 481 /*
482 482 * We clean up the pid provider for this process here; user-land
483 483 * static probes are handled by the meta-provider remove entry point.
484 484 */
485 485 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
486 486
487 487 mutex_enter(&p->p_lock);
488 488 }
489 489
490 490
491 491 /*ARGSUSED*/
492 492 static void
493 493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
494 494 {
495 495 /*
496 496 * There are no "default" pid probes.
497 497 */
498 498 }
499 499
500 500 static int
501 501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
502 502 {
503 503 fasttrap_tracepoint_t *tp, *new_tp = NULL;
504 504 fasttrap_bucket_t *bucket;
505 505 fasttrap_id_t *id;
506 506 pid_t pid;
507 507 uintptr_t pc;
508 508
509 509 ASSERT(index < probe->ftp_ntps);
510 510
511 511 pid = probe->ftp_pid;
512 512 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
513 513 id = &probe->ftp_tps[index].fit_id;
514 514
515 515 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
516 516
517 517 ASSERT(!(p->p_flag & SVFORK));
518 518
519 519 /*
520 520 * Before we make any modifications, make sure we've imposed a barrier
521 521 * on the generation in which this probe was last modified.
522 522 */
523 523 fasttrap_mod_barrier(probe->ftp_gen);
524 524
525 525 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
526 526
527 527 /*
528 528 * If the tracepoint has already been enabled, just add our id to the
529 529 * list of interested probes. This may be our second time through
530 530 * this path in which case we'll have constructed the tracepoint we'd
531 531 * like to install. If we can't find a match, and have an allocated
532 532 * tracepoint ready to go, enable that one now.
533 533 *
534 534 * A tracepoint whose process is defunct is also considered defunct.
535 535 */
536 536 again:
537 537 mutex_enter(&bucket->ftb_mtx);
538 538 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
539 539 /*
540 540 * Note that it's safe to access the active count on the
541 541 * associated proc structure because we know that at least one
542 542 * provider (this one) will still be around throughout this
543 543 * operation.
544 544 */
545 545 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
546 546 tp->ftt_proc->ftpc_acount == 0)
547 547 continue;
548 548
549 549 /*
550 550 * Now that we've found a matching tracepoint, it would be
551 551 * a decent idea to confirm that the tracepoint is still
552 552 * enabled and the trap instruction hasn't been overwritten.
553 553 * Since this is a little hairy, we'll punt for now.
554 554 */
555 555
556 556 /*
557 557 * This can't be the first interested probe. We don't have
558 558 * to worry about another thread being in the midst of
559 559 * deleting this tracepoint (which would be the only valid
560 560 * reason for a tracepoint to have no interested probes)
561 561 * since we're holding P_PR_LOCK for this process.
562 562 */
563 563 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
564 564
565 565 switch (id->fti_ptype) {
566 566 case DTFTP_ENTRY:
567 567 case DTFTP_OFFSETS:
568 568 case DTFTP_IS_ENABLED:
569 569 id->fti_next = tp->ftt_ids;
570 570 membar_producer();
571 571 tp->ftt_ids = id;
572 572 membar_producer();
573 573 break;
574 574
575 575 case DTFTP_RETURN:
576 576 case DTFTP_POST_OFFSETS:
577 577 id->fti_next = tp->ftt_retids;
578 578 membar_producer();
579 579 tp->ftt_retids = id;
580 580 membar_producer();
581 581 break;
582 582
583 583 default:
584 584 ASSERT(0);
585 585 }
586 586
587 587 mutex_exit(&bucket->ftb_mtx);
588 588
589 589 if (new_tp != NULL) {
590 590 new_tp->ftt_ids = NULL;
591 591 new_tp->ftt_retids = NULL;
592 592 }
593 593
594 594 return (0);
595 595 }
596 596
597 597 /*
598 598 * If we have a good tracepoint ready to go, install it now while
599 599 * we have the lock held and no one can screw with us.
600 600 */
601 601 if (new_tp != NULL) {
602 602 int rc = 0;
603 603
604 604 new_tp->ftt_next = bucket->ftb_data;
605 605 membar_producer();
606 606 bucket->ftb_data = new_tp;
607 607 membar_producer();
608 608 mutex_exit(&bucket->ftb_mtx);
609 609
610 610 /*
611 611 * Activate the tracepoint in the ISA-specific manner.
612 612 * If this fails, we need to report the failure, but
613 613 * indicate that this tracepoint must still be disabled
614 614 * by calling fasttrap_tracepoint_disable().
615 615 */
616 616 if (fasttrap_tracepoint_install(p, new_tp) != 0)
617 617 rc = FASTTRAP_ENABLE_PARTIAL;
618 618
619 619 /*
620 620 * Increment the count of the number of tracepoints active in
621 621 * the victim process.
622 622 */
623 623 ASSERT(p->p_proc_flag & P_PR_LOCK);
624 624 p->p_dtrace_count++;
625 625
626 626 return (rc);
627 627 }
628 628
629 629 mutex_exit(&bucket->ftb_mtx);
630 630
631 631 /*
632 632 * Initialize the tracepoint that's been preallocated with the probe.
633 633 */
634 634 new_tp = probe->ftp_tps[index].fit_tp;
635 635
636 636 ASSERT(new_tp->ftt_pid == pid);
637 637 ASSERT(new_tp->ftt_pc == pc);
638 638 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
639 639 ASSERT(new_tp->ftt_ids == NULL);
640 640 ASSERT(new_tp->ftt_retids == NULL);
641 641
642 642 switch (id->fti_ptype) {
643 643 case DTFTP_ENTRY:
644 644 case DTFTP_OFFSETS:
645 645 case DTFTP_IS_ENABLED:
646 646 id->fti_next = NULL;
647 647 new_tp->ftt_ids = id;
648 648 break;
649 649
650 650 case DTFTP_RETURN:
651 651 case DTFTP_POST_OFFSETS:
652 652 id->fti_next = NULL;
653 653 new_tp->ftt_retids = id;
654 654 break;
655 655
656 656 default:
657 657 ASSERT(0);
658 658 }
659 659
660 660 /*
661 661 * If the ISA-dependent initialization goes to plan, go back to the
662 662 * beginning and try to install this freshly made tracepoint.
663 663 */
664 664 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
665 665 goto again;
666 666
667 667 new_tp->ftt_ids = NULL;
668 668 new_tp->ftt_retids = NULL;
669 669
670 670 return (FASTTRAP_ENABLE_FAIL);
671 671 }
672 672
673 673 static void
674 674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
675 675 {
676 676 fasttrap_bucket_t *bucket;
677 677 fasttrap_provider_t *provider = probe->ftp_prov;
678 678 fasttrap_tracepoint_t **pp, *tp;
679 679 fasttrap_id_t *id, **idp;
680 680 pid_t pid;
681 681 uintptr_t pc;
682 682
683 683 ASSERT(index < probe->ftp_ntps);
684 684
685 685 pid = probe->ftp_pid;
686 686 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
687 687 id = &probe->ftp_tps[index].fit_id;
688 688
689 689 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
690 690
691 691 /*
692 692 * Find the tracepoint and make sure that our id is one of the
693 693 * ones registered with it.
694 694 */
695 695 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
696 696 mutex_enter(&bucket->ftb_mtx);
697 697 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
698 698 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
699 699 tp->ftt_proc == provider->ftp_proc)
700 700 break;
701 701 }
702 702
703 703 /*
704 704 * If we somehow lost this tracepoint, we're in a world of hurt.
705 705 */
706 706 ASSERT(tp != NULL);
707 707
708 708 switch (id->fti_ptype) {
709 709 case DTFTP_ENTRY:
710 710 case DTFTP_OFFSETS:
711 711 case DTFTP_IS_ENABLED:
712 712 ASSERT(tp->ftt_ids != NULL);
713 713 idp = &tp->ftt_ids;
714 714 break;
715 715
716 716 case DTFTP_RETURN:
717 717 case DTFTP_POST_OFFSETS:
718 718 ASSERT(tp->ftt_retids != NULL);
719 719 idp = &tp->ftt_retids;
720 720 break;
721 721
722 722 default:
723 723 ASSERT(0);
724 724 }
725 725
726 726 while ((*idp)->fti_probe != probe) {
727 727 idp = &(*idp)->fti_next;
728 728 ASSERT(*idp != NULL);
729 729 }
730 730
731 731 id = *idp;
732 732 *idp = id->fti_next;
733 733 membar_producer();
734 734
735 735 ASSERT(id->fti_probe == probe);
736 736
737 737 /*
738 738 * If there are other registered enablings of this tracepoint, we're
739 739 * all done, but if this was the last probe assocated with this
740 740 * this tracepoint, we need to remove and free it.
741 741 */
742 742 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
743 743
744 744 /*
745 745 * If the current probe's tracepoint is in use, swap it
746 746 * for an unused tracepoint.
747 747 */
748 748 if (tp == probe->ftp_tps[index].fit_tp) {
749 749 fasttrap_probe_t *tmp_probe;
750 750 fasttrap_tracepoint_t **tmp_tp;
751 751 uint_t tmp_index;
752 752
753 753 if (tp->ftt_ids != NULL) {
754 754 tmp_probe = tp->ftt_ids->fti_probe;
755 755 /* LINTED - alignment */
756 756 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
757 757 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
758 758 } else {
759 759 tmp_probe = tp->ftt_retids->fti_probe;
760 760 /* LINTED - alignment */
761 761 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
762 762 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
763 763 }
764 764
765 765 ASSERT(*tmp_tp != NULL);
766 766 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
767 767 ASSERT((*tmp_tp)->ftt_ids == NULL);
768 768 ASSERT((*tmp_tp)->ftt_retids == NULL);
769 769
770 770 probe->ftp_tps[index].fit_tp = *tmp_tp;
771 771 *tmp_tp = tp;
772 772 }
773 773
774 774 mutex_exit(&bucket->ftb_mtx);
775 775
776 776 /*
777 777 * Tag the modified probe with the generation in which it was
778 778 * changed.
779 779 */
780 780 probe->ftp_gen = fasttrap_mod_gen;
781 781 return;
782 782 }
783 783
784 784 mutex_exit(&bucket->ftb_mtx);
785 785
786 786 /*
787 787 * We can't safely remove the tracepoint from the set of active
788 788 * tracepoints until we've actually removed the fasttrap instruction
789 789 * from the process's text. We can, however, operate on this
790 790 * tracepoint secure in the knowledge that no other thread is going to
791 791 * be looking at it since we hold P_PR_LOCK on the process if it's
792 792 * live or we hold the provider lock on the process if it's dead and
793 793 * gone.
794 794 */
795 795
796 796 /*
797 797 * We only need to remove the actual instruction if we're looking
798 798 * at an existing process
799 799 */
800 800 if (p != NULL) {
801 801 /*
802 802 * If we fail to restore the instruction we need to kill
803 803 * this process since it's in a completely unrecoverable
804 804 * state.
805 805 */
806 806 if (fasttrap_tracepoint_remove(p, tp) != 0)
807 807 fasttrap_sigtrap(p, NULL, pc);
808 808
809 809 /*
810 810 * Decrement the count of the number of tracepoints active
811 811 * in the victim process.
812 812 */
813 813 ASSERT(p->p_proc_flag & P_PR_LOCK);
814 814 p->p_dtrace_count--;
815 815 }
816 816
817 817 /*
818 818 * Remove the probe from the hash table of active tracepoints.
819 819 */
820 820 mutex_enter(&bucket->ftb_mtx);
821 821 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
822 822 ASSERT(*pp != NULL);
823 823 while (*pp != tp) {
824 824 pp = &(*pp)->ftt_next;
825 825 ASSERT(*pp != NULL);
826 826 }
827 827
828 828 *pp = tp->ftt_next;
829 829 membar_producer();
830 830
831 831 mutex_exit(&bucket->ftb_mtx);
832 832
833 833 /*
834 834 * Tag the modified probe with the generation in which it was changed.
835 835 */
836 836 probe->ftp_gen = fasttrap_mod_gen;
837 837 }
838 838
839 839 static void
840 840 fasttrap_enable_callbacks(void)
841 841 {
842 842 /*
843 843 * We don't have to play the rw lock game here because we're
844 844 * providing something rather than taking something away --
845 845 * we can be sure that no threads have tried to follow this
846 846 * function pointer yet.
847 847 */
848 848 mutex_enter(&fasttrap_count_mtx);
849 849 if (fasttrap_pid_count == 0) {
850 850 ASSERT(dtrace_pid_probe_ptr == NULL);
851 851 ASSERT(dtrace_return_probe_ptr == NULL);
852 852 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
853 853 dtrace_return_probe_ptr = &fasttrap_return_probe;
854 854 }
855 855 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
856 856 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
857 857 fasttrap_pid_count++;
858 858 mutex_exit(&fasttrap_count_mtx);
859 859 }
860 860
861 861 static void
862 862 fasttrap_disable_callbacks(void)
863 863 {
864 864 ASSERT(MUTEX_HELD(&cpu_lock));
865 865
866 866 mutex_enter(&fasttrap_count_mtx);
867 867 ASSERT(fasttrap_pid_count > 0);
868 868 fasttrap_pid_count--;
869 869 if (fasttrap_pid_count == 0) {
870 870 cpu_t *cur, *cpu = CPU;
871 871
872 872 for (cur = cpu->cpu_next_onln; cur != cpu;
873 873 cur = cur->cpu_next_onln) {
874 874 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
875 875 }
876 876
877 877 dtrace_pid_probe_ptr = NULL;
878 878 dtrace_return_probe_ptr = NULL;
879 879
880 880 for (cur = cpu->cpu_next_onln; cur != cpu;
881 881 cur = cur->cpu_next_onln) {
882 882 rw_exit(&cur->cpu_ft_lock);
883 883 }
884 884 }
885 885 mutex_exit(&fasttrap_count_mtx);
886 886 }
887 887
888 888 /*ARGSUSED*/
889 889 static int
890 890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
891 891 {
892 892 fasttrap_probe_t *probe = parg;
893 893 proc_t *p;
894 894 int i, rc;
895 895
896 896 ASSERT(probe != NULL);
897 897 ASSERT(!probe->ftp_enabled);
898 898 ASSERT(id == probe->ftp_id);
899 899 ASSERT(MUTEX_HELD(&cpu_lock));
900 900
901 901 /*
902 902 * Increment the count of enabled probes on this probe's provider;
903 903 * the provider can't go away while the probe still exists. We
904 904 * must increment this even if we aren't able to properly enable
905 905 * this probe.
906 906 */
907 907 mutex_enter(&probe->ftp_prov->ftp_mtx);
908 908 probe->ftp_prov->ftp_rcount++;
909 909 mutex_exit(&probe->ftp_prov->ftp_mtx);
910 910
911 911 /*
912 912 * If this probe's provider is retired (meaning it was valid in a
913 913 * previously exec'ed incarnation of this address space), bail out. The
914 914 * provider can't go away while we're in this code path.
915 915 */
916 916 if (probe->ftp_prov->ftp_retired)
917 917 return (0);
918 918
919 919 /*
920 920 * If we can't find the process, it may be that we're in the context of
921 921 * a fork in which the traced process is being born and we're copying
922 922 * USDT probes. Otherwise, the process is gone so bail.
923 923 */
924 924 if ((p = sprlock(probe->ftp_pid)) == NULL) {
925 925 if ((curproc->p_flag & SFORKING) == 0)
926 926 return (0);
927 927
928 928 mutex_enter(&pidlock);
929 929 p = prfind(probe->ftp_pid);
930 930
931 931 if (p == NULL) {
932 932 /*
933 933 * So it's not that the target process is being born,
934 934 * it's that it isn't there at all (and we simply
935 935 * happen to be forking). Anyway, we know that the
936 936 * target is definitely gone, so bail out.
937 937 */
938 938 mutex_exit(&pidlock);
939 939 return (0);
940 940 }
941 941
942 942 /*
943 943 * Confirm that curproc is indeed forking the process in which
944 944 * we're trying to enable probes.
945 945 */
946 946 ASSERT(p->p_parent == curproc);
947 947 ASSERT(p->p_stat == SIDL);
948 948
949 949 mutex_enter(&p->p_lock);
950 950 mutex_exit(&pidlock);
951 951
952 952 sprlock_proc(p);
953 953 }
954 954
955 955 ASSERT(!(p->p_flag & SVFORK));
956 956 mutex_exit(&p->p_lock);
957 957
958 958 /*
959 959 * We have to enable the trap entry point before any user threads have
960 960 * the chance to execute the trap instruction we're about to place
961 961 * in their process's text.
962 962 */
963 963 fasttrap_enable_callbacks();
964 964
965 965 /*
966 966 * Enable all the tracepoints and add this probe's id to each
967 967 * tracepoint's list of active probes.
968 968 */
969 969 for (i = 0; i < probe->ftp_ntps; i++) {
970 970 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
971 971 /*
972 972 * If enabling the tracepoint failed completely,
973 973 * we don't have to disable it; if the failure
974 974 * was only partial we must disable it.
975 975 */
976 976 if (rc == FASTTRAP_ENABLE_FAIL)
977 977 i--;
978 978 else
979 979 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
980 980
981 981 /*
982 982 * Back up and pull out all the tracepoints we've
983 983 * created so far for this probe.
984 984 */
985 985 while (i >= 0) {
986 986 fasttrap_tracepoint_disable(p, probe, i);
987 987 i--;
988 988 }
989 989
990 990 mutex_enter(&p->p_lock);
991 991 sprunlock(p);
992 992
993 993 /*
994 994 * Since we're not actually enabling this probe,
995 995 * drop our reference on the trap table entry.
996 996 */
997 997 fasttrap_disable_callbacks();
998 998 return (0);
999 999 }
1000 1000 }
1001 1001
1002 1002 mutex_enter(&p->p_lock);
1003 1003 sprunlock(p);
1004 1004
1005 1005 probe->ftp_enabled = 1;
1006 1006 return (0);
1007 1007 }
1008 1008
1009 1009 /*ARGSUSED*/
1010 1010 static void
1011 1011 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1012 1012 {
1013 1013 fasttrap_probe_t *probe = parg;
1014 1014 fasttrap_provider_t *provider = probe->ftp_prov;
1015 1015 proc_t *p;
1016 1016 int i, whack = 0;
1017 1017
1018 1018 ASSERT(id == probe->ftp_id);
1019 1019
1020 1020 /*
1021 1021 * We won't be able to acquire a /proc-esque lock on the process
1022 1022 * iff the process is dead and gone. In this case, we rely on the
1023 1023 * provider lock as a point of mutual exclusion to prevent other
1024 1024 * DTrace consumers from disabling this probe.
1025 1025 */
1026 1026 if ((p = sprlock(probe->ftp_pid)) != NULL) {
1027 1027 ASSERT(!(p->p_flag & SVFORK));
1028 1028 mutex_exit(&p->p_lock);
1029 1029 }
1030 1030
1031 1031 mutex_enter(&provider->ftp_mtx);
1032 1032
1033 1033 /*
1034 1034 * Disable all the associated tracepoints (for fully enabled probes).
1035 1035 */
1036 1036 if (probe->ftp_enabled) {
1037 1037 for (i = 0; i < probe->ftp_ntps; i++) {
1038 1038 fasttrap_tracepoint_disable(p, probe, i);
1039 1039 }
1040 1040 }
1041 1041
1042 1042 ASSERT(provider->ftp_rcount > 0);
1043 1043 provider->ftp_rcount--;
1044 1044
1045 1045 if (p != NULL) {
1046 1046 /*
1047 1047 * Even though we may not be able to remove it entirely, we
1048 1048 * mark this retired provider to get a chance to remove some
1049 1049 * of the associated probes.
1050 1050 */
1051 1051 if (provider->ftp_retired && !provider->ftp_marked)
1052 1052 whack = provider->ftp_marked = 1;
1053 1053 mutex_exit(&provider->ftp_mtx);
1054 1054
1055 1055 mutex_enter(&p->p_lock);
1056 1056 sprunlock(p);
1057 1057 } else {
1058 1058 /*
1059 1059 * If the process is dead, we're just waiting for the
1060 1060 * last probe to be disabled to be able to free it.
1061 1061 */
1062 1062 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1063 1063 whack = provider->ftp_marked = 1;
1064 1064 mutex_exit(&provider->ftp_mtx);
1065 1065 }
1066 1066
1067 1067 if (whack)
1068 1068 fasttrap_pid_cleanup();
1069 1069
1070 1070 if (!probe->ftp_enabled)
1071 1071 return;
1072 1072
1073 1073 probe->ftp_enabled = 0;
1074 1074
1075 1075 ASSERT(MUTEX_HELD(&cpu_lock));
1076 1076 fasttrap_disable_callbacks();
1077 1077 }
1078 1078
1079 1079 /*ARGSUSED*/
1080 1080 static void
1081 1081 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1082 1082 dtrace_argdesc_t *desc)
1083 1083 {
1084 1084 fasttrap_probe_t *probe = parg;
1085 1085 char *str;
1086 1086 int i, ndx;
1087 1087
1088 1088 desc->dtargd_native[0] = '\0';
1089 1089 desc->dtargd_xlate[0] = '\0';
1090 1090
1091 1091 if (probe->ftp_prov->ftp_retired != 0 ||
1092 1092 desc->dtargd_ndx >= probe->ftp_nargs) {
1093 1093 desc->dtargd_ndx = DTRACE_ARGNONE;
1094 1094 return;
1095 1095 }
1096 1096
1097 1097 ndx = (probe->ftp_argmap != NULL) ?
1098 1098 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1099 1099
1100 1100 str = probe->ftp_ntypes;
1101 1101 for (i = 0; i < ndx; i++) {
1102 1102 str += strlen(str) + 1;
1103 1103 }
1104 1104
1105 1105 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1106 1106 (void) strcpy(desc->dtargd_native, str);
1107 1107
1108 1108 if (probe->ftp_xtypes == NULL)
1109 1109 return;
1110 1110
1111 1111 str = probe->ftp_xtypes;
1112 1112 for (i = 0; i < desc->dtargd_ndx; i++) {
1113 1113 str += strlen(str) + 1;
1114 1114 }
1115 1115
1116 1116 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1117 1117 (void) strcpy(desc->dtargd_xlate, str);
1118 1118 }
1119 1119
1120 1120 /*ARGSUSED*/
1121 1121 static void
1122 1122 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1123 1123 {
1124 1124 fasttrap_probe_t *probe = parg;
1125 1125 int i;
1126 1126 size_t size;
1127 1127
1128 1128 ASSERT(probe != NULL);
1129 1129 ASSERT(!probe->ftp_enabled);
1130 1130 ASSERT(fasttrap_total >= probe->ftp_ntps);
1131 1131
1132 1132 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1133 1133 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1134 1134
1135 1135 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1136 1136 fasttrap_mod_barrier(probe->ftp_gen);
1137 1137
1138 1138 for (i = 0; i < probe->ftp_ntps; i++) {
1139 1139 kmem_free(probe->ftp_tps[i].fit_tp,
1140 1140 sizeof (fasttrap_tracepoint_t));
1141 1141 }
1142 1142
1143 1143 kmem_free(probe, size);
1144 1144 }
1145 1145
1146 1146
1147 1147 static const dtrace_pattr_t pid_attr = {
1148 1148 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1149 1149 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1150 1150 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1151 1151 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1152 1152 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1153 1153 };
1154 1154
1155 1155 static dtrace_pops_t pid_pops = {
1156 1156 fasttrap_pid_provide,
1157 1157 NULL,
1158 1158 fasttrap_pid_enable,
1159 1159 fasttrap_pid_disable,
1160 1160 NULL,
1161 1161 NULL,
1162 1162 fasttrap_pid_getargdesc,
1163 1163 fasttrap_pid_getarg,
1164 1164 NULL,
1165 1165 fasttrap_pid_destroy
1166 1166 };
1167 1167
1168 1168 static dtrace_pops_t usdt_pops = {
1169 1169 fasttrap_pid_provide,
1170 1170 NULL,
1171 1171 fasttrap_pid_enable,
1172 1172 fasttrap_pid_disable,
1173 1173 NULL,
1174 1174 NULL,
1175 1175 fasttrap_pid_getargdesc,
1176 1176 fasttrap_usdt_getarg,
1177 1177 NULL,
1178 1178 fasttrap_pid_destroy
1179 1179 };
1180 1180
1181 1181 static fasttrap_proc_t *
1182 1182 fasttrap_proc_lookup(pid_t pid)
1183 1183 {
1184 1184 fasttrap_bucket_t *bucket;
1185 1185 fasttrap_proc_t *fprc, *new_fprc;
1186 1186
1187 1187 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1188 1188 mutex_enter(&bucket->ftb_mtx);
1189 1189
1190 1190 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1191 1191 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1192 1192 mutex_enter(&fprc->ftpc_mtx);
1193 1193 mutex_exit(&bucket->ftb_mtx);
1194 1194 fprc->ftpc_rcount++;
1195 1195 atomic_inc_64(&fprc->ftpc_acount);
1196 1196 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1197 1197 mutex_exit(&fprc->ftpc_mtx);
1198 1198
1199 1199 return (fprc);
1200 1200 }
1201 1201 }
1202 1202
1203 1203 /*
1204 1204 * Drop the bucket lock so we don't try to perform a sleeping
1205 1205 * allocation under it.
1206 1206 */
1207 1207 mutex_exit(&bucket->ftb_mtx);
1208 1208
1209 1209 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1210 1210 new_fprc->ftpc_pid = pid;
1211 1211 new_fprc->ftpc_rcount = 1;
1212 1212 new_fprc->ftpc_acount = 1;
1213 1213
1214 1214 mutex_enter(&bucket->ftb_mtx);
1215 1215
1216 1216 /*
1217 1217 * Take another lap through the list to make sure a proc hasn't
1218 1218 * been created for this pid while we weren't under the bucket lock.
1219 1219 */
1220 1220 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1221 1221 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1222 1222 mutex_enter(&fprc->ftpc_mtx);
1223 1223 mutex_exit(&bucket->ftb_mtx);
1224 1224 fprc->ftpc_rcount++;
1225 1225 atomic_inc_64(&fprc->ftpc_acount);
1226 1226 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1227 1227 mutex_exit(&fprc->ftpc_mtx);
1228 1228
1229 1229 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1230 1230
1231 1231 return (fprc);
1232 1232 }
1233 1233 }
1234 1234
1235 1235 new_fprc->ftpc_next = bucket->ftb_data;
1236 1236 bucket->ftb_data = new_fprc;
1237 1237
1238 1238 mutex_exit(&bucket->ftb_mtx);
1239 1239
1240 1240 return (new_fprc);
1241 1241 }
1242 1242
1243 1243 static void
1244 1244 fasttrap_proc_release(fasttrap_proc_t *proc)
1245 1245 {
1246 1246 fasttrap_bucket_t *bucket;
1247 1247 fasttrap_proc_t *fprc, **fprcp;
1248 1248 pid_t pid = proc->ftpc_pid;
1249 1249
1250 1250 mutex_enter(&proc->ftpc_mtx);
1251 1251
1252 1252 ASSERT(proc->ftpc_rcount != 0);
1253 1253 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1254 1254
1255 1255 if (--proc->ftpc_rcount != 0) {
1256 1256 mutex_exit(&proc->ftpc_mtx);
1257 1257 return;
1258 1258 }
1259 1259
1260 1260 mutex_exit(&proc->ftpc_mtx);
1261 1261
1262 1262 /*
1263 1263 * There should definitely be no live providers associated with this
1264 1264 * process at this point.
1265 1265 */
1266 1266 ASSERT(proc->ftpc_acount == 0);
1267 1267
1268 1268 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1269 1269 mutex_enter(&bucket->ftb_mtx);
1270 1270
1271 1271 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1272 1272 while ((fprc = *fprcp) != NULL) {
1273 1273 if (fprc == proc)
1274 1274 break;
1275 1275
1276 1276 fprcp = &fprc->ftpc_next;
1277 1277 }
1278 1278
1279 1279 /*
1280 1280 * Something strange has happened if we can't find the proc.
1281 1281 */
1282 1282 ASSERT(fprc != NULL);
1283 1283
1284 1284 *fprcp = fprc->ftpc_next;
1285 1285
1286 1286 mutex_exit(&bucket->ftb_mtx);
1287 1287
1288 1288 kmem_free(fprc, sizeof (fasttrap_proc_t));
1289 1289 }
1290 1290
1291 1291 /*
1292 1292 * Lookup a fasttrap-managed provider based on its name and associated pid.
1293 1293 * If the pattr argument is non-NULL, this function instantiates the provider
1294 1294 * if it doesn't exist otherwise it returns NULL. The provider is returned
1295 1295 * with its lock held.
1296 1296 */
1297 1297 static fasttrap_provider_t *
1298 1298 fasttrap_provider_lookup(pid_t pid, const char *name,
1299 1299 const dtrace_pattr_t *pattr)
1300 1300 {
1301 1301 fasttrap_provider_t *fp, *new_fp = NULL;
1302 1302 fasttrap_bucket_t *bucket;
1303 1303 char provname[DTRACE_PROVNAMELEN];
1304 1304 proc_t *p;
1305 1305 cred_t *cred;
1306 1306
1307 1307 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1308 1308 ASSERT(pattr != NULL);
1309 1309
1310 1310 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1311 1311 mutex_enter(&bucket->ftb_mtx);
1312 1312
1313 1313 /*
1314 1314 * Take a lap through the list and return the match if we find it.
1315 1315 */
1316 1316 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1317 1317 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1318 1318 !fp->ftp_retired) {
1319 1319 mutex_enter(&fp->ftp_mtx);
1320 1320 mutex_exit(&bucket->ftb_mtx);
1321 1321 return (fp);
1322 1322 }
1323 1323 }
1324 1324
1325 1325 /*
1326 1326 * Drop the bucket lock so we don't try to perform a sleeping
1327 1327 * allocation under it.
1328 1328 */
1329 1329 mutex_exit(&bucket->ftb_mtx);
1330 1330
1331 1331 /*
1332 1332 * Make sure the process exists, isn't a child created as the result
1333 1333 * of a vfork(2), and isn't a zombie (but may be in fork).
1334 1334 */
1335 1335 mutex_enter(&pidlock);
1336 1336 if ((p = prfind(pid)) == NULL) {
1337 1337 mutex_exit(&pidlock);
1338 1338 return (NULL);
1339 1339 }
1340 1340 mutex_enter(&p->p_lock);
1341 1341 mutex_exit(&pidlock);
1342 1342 if (p->p_flag & (SVFORK | SEXITING)) {
1343 1343 mutex_exit(&p->p_lock);
1344 1344 return (NULL);
1345 1345 }
1346 1346
1347 1347 /*
1348 1348 * Increment p_dtrace_probes so that the process knows to inform us
1349 1349 * when it exits or execs. fasttrap_provider_free() decrements this
1350 1350 * when we're done with this provider.
1351 1351 */
1352 1352 p->p_dtrace_probes++;
1353 1353
1354 1354 /*
1355 1355 * Grab the credentials for this process so we have
1356 1356 * something to pass to dtrace_register().
1357 1357 */
1358 1358 mutex_enter(&p->p_crlock);
1359 1359 crhold(p->p_cred);
1360 1360 cred = p->p_cred;
1361 1361 mutex_exit(&p->p_crlock);
1362 1362 mutex_exit(&p->p_lock);
1363 1363
1364 1364 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1365 1365 new_fp->ftp_pid = pid;
1366 1366 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1367 1367
1368 1368 ASSERT(new_fp->ftp_proc != NULL);
1369 1369
1370 1370 mutex_enter(&bucket->ftb_mtx);
1371 1371
1372 1372 /*
1373 1373 * Take another lap through the list to make sure a provider hasn't
1374 1374 * been created for this pid while we weren't under the bucket lock.
1375 1375 */
1376 1376 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1377 1377 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1378 1378 !fp->ftp_retired) {
1379 1379 mutex_enter(&fp->ftp_mtx);
1380 1380 mutex_exit(&bucket->ftb_mtx);
1381 1381 fasttrap_provider_free(new_fp);
1382 1382 crfree(cred);
1383 1383 return (fp);
1384 1384 }
1385 1385 }
1386 1386
1387 1387 (void) strcpy(new_fp->ftp_name, name);
1388 1388
1389 1389 /*
1390 1390 * Fail and return NULL if either the provider name is too long
1391 1391 * or we fail to register this new provider with the DTrace
1392 1392 * framework. Note that this is the only place we ever construct
1393 1393 * the full provider name -- we keep it in pieces in the provider
1394 1394 * structure.
1395 1395 */
1396 1396 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1397 1397 sizeof (provname) ||
1398 1398 dtrace_register(provname, pattr,
1399 1399 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1400 1400 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1401 1401 &new_fp->ftp_provid) != 0) {
1402 1402 mutex_exit(&bucket->ftb_mtx);
1403 1403 fasttrap_provider_free(new_fp);
1404 1404 crfree(cred);
1405 1405 return (NULL);
1406 1406 }
1407 1407
1408 1408 new_fp->ftp_next = bucket->ftb_data;
1409 1409 bucket->ftb_data = new_fp;
1410 1410
1411 1411 mutex_enter(&new_fp->ftp_mtx);
1412 1412 mutex_exit(&bucket->ftb_mtx);
1413 1413
1414 1414 crfree(cred);
1415 1415 return (new_fp);
1416 1416 }
1417 1417
1418 1418 static void
1419 1419 fasttrap_provider_free(fasttrap_provider_t *provider)
1420 1420 {
1421 1421 pid_t pid = provider->ftp_pid;
1422 1422 proc_t *p;
1423 1423
1424 1424 /*
1425 1425 * There need to be no associated enabled probes, no consumers
1426 1426 * creating probes, and no meta providers referencing this provider.
1427 1427 */
1428 1428 ASSERT(provider->ftp_rcount == 0);
1429 1429 ASSERT(provider->ftp_ccount == 0);
1430 1430 ASSERT(provider->ftp_mcount == 0);
1431 1431
1432 1432 /*
1433 1433 * If this provider hasn't been retired, we need to explicitly drop the
1434 1434 * count of active providers on the associated process structure.
1435 1435 */
1436 1436 if (!provider->ftp_retired) {
1437 1437 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1438 1438 ASSERT(provider->ftp_proc->ftpc_acount <
1439 1439 provider->ftp_proc->ftpc_rcount);
1440 1440 }
1441 1441
1442 1442 fasttrap_proc_release(provider->ftp_proc);
1443 1443
1444 1444 kmem_free(provider, sizeof (fasttrap_provider_t));
1445 1445
1446 1446 /*
1447 1447 * Decrement p_dtrace_probes on the process whose provider we're
1448 1448 * freeing. We don't have to worry about clobbering somone else's
1449 1449 * modifications to it because we have locked the bucket that
1450 1450 * corresponds to this process's hash chain in the provider hash
1451 1451 * table. Don't sweat it if we can't find the process.
1452 1452 */
1453 1453 mutex_enter(&pidlock);
1454 1454 if ((p = prfind(pid)) == NULL) {
1455 1455 mutex_exit(&pidlock);
1456 1456 return;
1457 1457 }
1458 1458
1459 1459 mutex_enter(&p->p_lock);
1460 1460 mutex_exit(&pidlock);
1461 1461
1462 1462 p->p_dtrace_probes--;
1463 1463 mutex_exit(&p->p_lock);
1464 1464 }
1465 1465
1466 1466 static void
1467 1467 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1468 1468 {
1469 1469 fasttrap_provider_t *fp;
1470 1470 fasttrap_bucket_t *bucket;
1471 1471 dtrace_provider_id_t provid;
1472 1472
1473 1473 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1474 1474
1475 1475 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1476 1476 mutex_enter(&bucket->ftb_mtx);
1477 1477
1478 1478 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1479 1479 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1480 1480 !fp->ftp_retired)
1481 1481 break;
1482 1482 }
1483 1483
1484 1484 if (fp == NULL) {
1485 1485 mutex_exit(&bucket->ftb_mtx);
1486 1486 return;
1487 1487 }
1488 1488
1489 1489 mutex_enter(&fp->ftp_mtx);
1490 1490 ASSERT(!mprov || fp->ftp_mcount > 0);
1491 1491 if (mprov && --fp->ftp_mcount != 0) {
1492 1492 mutex_exit(&fp->ftp_mtx);
1493 1493 mutex_exit(&bucket->ftb_mtx);
1494 1494 return;
1495 1495 }
1496 1496
1497 1497 /*
1498 1498 * Mark the provider to be removed in our post-processing step, mark it
1499 1499 * retired, and drop the active count on its proc. Marking it indicates
1500 1500 * that we should try to remove it; setting the retired flag indicates
1501 1501 * that we're done with this provider; dropping the active the proc
1502 1502 * releases our hold, and when this reaches zero (as it will during
1503 1503 * exit or exec) the proc and associated providers become defunct.
1504 1504 *
1505 1505 * We obviously need to take the bucket lock before the provider lock
1506 1506 * to perform the lookup, but we need to drop the provider lock
1507 1507 * before calling into the DTrace framework since we acquire the
1508 1508 * provider lock in callbacks invoked from the DTrace framework. The
1509 1509 * bucket lock therefore protects the integrity of the provider hash
1510 1510 * table.
1511 1511 */
1512 1512 atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1513 1513 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1514 1514
1515 1515 fp->ftp_retired = 1;
1516 1516 fp->ftp_marked = 1;
1517 1517 provid = fp->ftp_provid;
1518 1518 mutex_exit(&fp->ftp_mtx);
1519 1519
1520 1520 /*
1521 1521 * We don't have to worry about invalidating the same provider twice
1522 1522 * since fasttrap_provider_lookup() will ignore provider that have
1523 1523 * been marked as retired.
1524 1524 */
1525 1525 dtrace_invalidate(provid);
1526 1526
1527 1527 mutex_exit(&bucket->ftb_mtx);
1528 1528
1529 1529 fasttrap_pid_cleanup();
1530 1530 }
1531 1531
1532 1532 static int
1533 1533 fasttrap_uint32_cmp(const void *ap, const void *bp)
1534 1534 {
1535 1535 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1536 1536 }
1537 1537
1538 1538 static int
1539 1539 fasttrap_uint64_cmp(const void *ap, const void *bp)
1540 1540 {
1541 1541 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1542 1542 }
1543 1543
1544 1544 static int
1545 1545 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1546 1546 {
1547 1547 fasttrap_provider_t *provider;
1548 1548 fasttrap_probe_t *pp;
1549 1549 fasttrap_tracepoint_t *tp;
1550 1550 char *name;
1551 1551 int i, aframes, whack;
1552 1552
1553 1553 /*
1554 1554 * There needs to be at least one desired trace point.
1555 1555 */
1556 1556 if (pdata->ftps_noffs == 0)
1557 1557 return (EINVAL);
1558 1558
1559 1559 switch (pdata->ftps_type) {
1560 1560 case DTFTP_ENTRY:
1561 1561 name = "entry";
1562 1562 aframes = FASTTRAP_ENTRY_AFRAMES;
1563 1563 break;
1564 1564 case DTFTP_RETURN:
1565 1565 name = "return";
1566 1566 aframes = FASTTRAP_RETURN_AFRAMES;
1567 1567 break;
1568 1568 case DTFTP_OFFSETS:
1569 1569 name = NULL;
1570 1570 break;
1571 1571 default:
1572 1572 return (EINVAL);
1573 1573 }
1574 1574
1575 1575 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1576 1576 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1577 1577 return (ESRCH);
1578 1578
1579 1579 /*
1580 1580 * Increment this reference count to indicate that a consumer is
1581 1581 * actively adding a new probe associated with this provider. This
1582 1582 * prevents the provider from being deleted -- we'll need to check
1583 1583 * for pending deletions when we drop this reference count.
1584 1584 */
1585 1585 provider->ftp_ccount++;
1586 1586 mutex_exit(&provider->ftp_mtx);
1587 1587
1588 1588 /*
1589 1589 * Grab the creation lock to ensure consistency between calls to
1590 1590 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1591 1591 * other threads creating probes. We must drop the provider lock
1592 1592 * before taking this lock to avoid a three-way deadlock with the
1593 1593 * DTrace framework.
1594 1594 */
1595 1595 mutex_enter(&provider->ftp_cmtx);
1596 1596
1597 1597 if (name == NULL) {
1598 1598 for (i = 0; i < pdata->ftps_noffs; i++) {
1599 1599 char name_str[17];
1600 1600
1601 1601 (void) sprintf(name_str, "%llx",
1602 1602 (unsigned long long)pdata->ftps_offs[i]);
1603 1603
1604 1604 if (dtrace_probe_lookup(provider->ftp_provid,
1605 1605 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1606 1606 continue;
1607 1607
1608 1608 atomic_inc_32(&fasttrap_total);
1609 1609
1610 1610 if (fasttrap_total > fasttrap_max) {
1611 1611 atomic_dec_32(&fasttrap_total);
1612 1612 goto no_mem;
1613 1613 }
1614 1614
1615 1615 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1616 1616
1617 1617 pp->ftp_prov = provider;
1618 1618 pp->ftp_faddr = pdata->ftps_pc;
1619 1619 pp->ftp_fsize = pdata->ftps_size;
1620 1620 pp->ftp_pid = pdata->ftps_pid;
1621 1621 pp->ftp_ntps = 1;
1622 1622
1623 1623 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1624 1624 KM_SLEEP);
1625 1625
1626 1626 tp->ftt_proc = provider->ftp_proc;
1627 1627 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1628 1628 tp->ftt_pid = pdata->ftps_pid;
1629 1629
1630 1630 pp->ftp_tps[0].fit_tp = tp;
1631 1631 pp->ftp_tps[0].fit_id.fti_probe = pp;
1632 1632 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1633 1633
1634 1634 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1635 1635 pdata->ftps_mod, pdata->ftps_func, name_str,
1636 1636 FASTTRAP_OFFSET_AFRAMES, pp);
1637 1637 }
1638 1638
1639 1639 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1640 1640 pdata->ftps_func, name) == 0) {
1641 1641 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1642 1642
1643 1643 if (fasttrap_total > fasttrap_max) {
1644 1644 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1645 1645 goto no_mem;
1646 1646 }
1647 1647
1648 1648 /*
1649 1649 * Make sure all tracepoint program counter values are unique.
1650 1650 * We later assume that each probe has exactly one tracepoint
1651 1651 * for a given pc.
1652 1652 */
1653 1653 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1654 1654 sizeof (uint64_t), fasttrap_uint64_cmp);
1655 1655 for (i = 1; i < pdata->ftps_noffs; i++) {
1656 1656 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1657 1657 continue;
1658 1658
1659 1659 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1660 1660 goto no_mem;
1661 1661 }
1662 1662
1663 1663 ASSERT(pdata->ftps_noffs > 0);
1664 1664 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1665 1665 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1666 1666
1667 1667 pp->ftp_prov = provider;
1668 1668 pp->ftp_faddr = pdata->ftps_pc;
1669 1669 pp->ftp_fsize = pdata->ftps_size;
1670 1670 pp->ftp_pid = pdata->ftps_pid;
1671 1671 pp->ftp_ntps = pdata->ftps_noffs;
1672 1672
1673 1673 for (i = 0; i < pdata->ftps_noffs; i++) {
1674 1674 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1675 1675 KM_SLEEP);
1676 1676
1677 1677 tp->ftt_proc = provider->ftp_proc;
1678 1678 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1679 1679 tp->ftt_pid = pdata->ftps_pid;
1680 1680
1681 1681 pp->ftp_tps[i].fit_tp = tp;
1682 1682 pp->ftp_tps[i].fit_id.fti_probe = pp;
1683 1683 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1684 1684 }
1685 1685
1686 1686 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1687 1687 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1688 1688 }
1689 1689
1690 1690 mutex_exit(&provider->ftp_cmtx);
1691 1691
1692 1692 /*
1693 1693 * We know that the provider is still valid since we incremented the
1694 1694 * creation reference count. If someone tried to clean up this provider
1695 1695 * while we were using it (e.g. because the process called exec(2) or
1696 1696 * exit(2)), take note of that and try to clean it up now.
1697 1697 */
1698 1698 mutex_enter(&provider->ftp_mtx);
1699 1699 provider->ftp_ccount--;
1700 1700 whack = provider->ftp_retired;
1701 1701 mutex_exit(&provider->ftp_mtx);
1702 1702
1703 1703 if (whack)
1704 1704 fasttrap_pid_cleanup();
1705 1705
1706 1706 return (0);
1707 1707
1708 1708 no_mem:
1709 1709 /*
1710 1710 * If we've exhausted the allowable resources, we'll try to remove
1711 1711 * this provider to free some up. This is to cover the case where
1712 1712 * the user has accidentally created many more probes than was
1713 1713 * intended (e.g. pid123:::).
1714 1714 */
1715 1715 mutex_exit(&provider->ftp_cmtx);
1716 1716 mutex_enter(&provider->ftp_mtx);
1717 1717 provider->ftp_ccount--;
1718 1718 provider->ftp_marked = 1;
1719 1719 mutex_exit(&provider->ftp_mtx);
1720 1720
1721 1721 fasttrap_pid_cleanup();
1722 1722
1723 1723 return (ENOMEM);
1724 1724 }
1725 1725
1726 1726 /*ARGSUSED*/
1727 1727 static void *
1728 1728 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1729 1729 {
1730 1730 fasttrap_provider_t *provider;
1731 1731
1732 1732 /*
1733 1733 * A 32-bit unsigned integer (like a pid for example) can be
1734 1734 * expressed in 10 or fewer decimal digits. Make sure that we'll
1735 1735 * have enough space for the provider name.
1736 1736 */
1737 1737 if (strlen(dhpv->dthpv_provname) + 10 >=
1738 1738 sizeof (provider->ftp_name)) {
1739 1739 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1740 1740 "name too long to accomodate pid", dhpv->dthpv_provname);
1741 1741 return (NULL);
1742 1742 }
1743 1743
1744 1744 /*
1745 1745 * Don't let folks spoof the true pid provider.
1746 1746 */
1747 1747 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1748 1748 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1749 1749 "%s is an invalid name", dhpv->dthpv_provname,
1750 1750 FASTTRAP_PID_NAME);
1751 1751 return (NULL);
1752 1752 }
1753 1753
1754 1754 /*
1755 1755 * The highest stability class that fasttrap supports is ISA; cap
1756 1756 * the stability of the new provider accordingly.
1757 1757 */
1758 1758 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1759 1759 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1760 1760 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1761 1761 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1762 1762 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1763 1763 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1764 1764 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1765 1765 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1766 1766 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1767 1767 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1768 1768
1769 1769 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1770 1770 &dhpv->dthpv_pattr)) == NULL) {
1771 1771 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1772 1772 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1773 1773 return (NULL);
1774 1774 }
1775 1775
1776 1776 /*
1777 1777 * Up the meta provider count so this provider isn't removed until
1778 1778 * the meta provider has been told to remove it.
1779 1779 */
1780 1780 provider->ftp_mcount++;
1781 1781
1782 1782 mutex_exit(&provider->ftp_mtx);
1783 1783
1784 1784 return (provider);
1785 1785 }
1786 1786
1787 1787 /*
1788 1788 * We know a few things about our context here: we know that the probe being
1789 1789 * created doesn't already exist (DTrace won't load DOF at the same address
1790 1790 * twice, even if explicitly told to do so) and we know that we are
1791 1791 * single-threaded with respect to the meta provider machinery. Knowing that
1792 1792 * this is a new probe and that there is no way for us to race with another
1793 1793 * operation on this provider allows us an important optimization: we need not
1794 1794 * lookup a probe before adding it. Saving this lookup is important because
1795 1795 * this code is in the fork path for processes with USDT probes, and lookups
1796 1796 * here are potentially very expensive because of long hash conflicts on
1797 1797 * module, function and name (DTrace doesn't hash on provider name).
1798 1798 */
1799 1799 /*ARGSUSED*/
1800 1800 static void
1801 1801 fasttrap_meta_create_probe(void *arg, void *parg,
1802 1802 dtrace_helper_probedesc_t *dhpb)
1803 1803 {
1804 1804 fasttrap_provider_t *provider = parg;
1805 1805 fasttrap_probe_t *pp;
1806 1806 fasttrap_tracepoint_t *tp;
1807 1807 int i, j;
1808 1808 uint32_t ntps;
1809 1809
1810 1810 /*
1811 1811 * Since the meta provider count is non-zero we don't have to worry
1812 1812 * about this provider disappearing.
1813 1813 */
1814 1814 ASSERT(provider->ftp_mcount > 0);
1815 1815
1816 1816 /*
1817 1817 * The offsets must be unique.
1818 1818 */
1819 1819 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1820 1820 fasttrap_uint32_cmp);
1821 1821 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1822 1822 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1823 1823 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1824 1824 return;
1825 1825 }
1826 1826
1827 1827 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1828 1828 fasttrap_uint32_cmp);
1829 1829 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1830 1830 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1831 1831 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1832 1832 return;
1833 1833 }
1834 1834
1835 1835 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1836 1836 ASSERT(ntps > 0);
1837 1837
1838 1838 atomic_add_32(&fasttrap_total, ntps);
1839 1839
1840 1840 if (fasttrap_total > fasttrap_max) {
1841 1841 atomic_add_32(&fasttrap_total, -ntps);
1842 1842 return;
1843 1843 }
1844 1844
1845 1845 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1846 1846
1847 1847 pp->ftp_prov = provider;
1848 1848 pp->ftp_pid = provider->ftp_pid;
1849 1849 pp->ftp_ntps = ntps;
1850 1850 pp->ftp_nargs = dhpb->dthpb_xargc;
1851 1851 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1852 1852 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1853 1853
1854 1854 /*
1855 1855 * First create a tracepoint for each actual point of interest.
1856 1856 */
1857 1857 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1858 1858 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1859 1859
1860 1860 tp->ftt_proc = provider->ftp_proc;
1861 1861 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1862 1862 tp->ftt_pid = provider->ftp_pid;
1863 1863
1864 1864 pp->ftp_tps[i].fit_tp = tp;
1865 1865 pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 1866 #ifdef __sparc
1867 1867 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1868 1868 #else
1869 1869 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1870 1870 #endif
1871 1871 }
1872 1872
1873 1873 /*
1874 1874 * Then create a tracepoint for each is-enabled point.
1875 1875 */
1876 1876 for (j = 0; i < ntps; i++, j++) {
1877 1877 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1878 1878
1879 1879 tp->ftt_proc = provider->ftp_proc;
1880 1880 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1881 1881 tp->ftt_pid = provider->ftp_pid;
1882 1882
1883 1883 pp->ftp_tps[i].fit_tp = tp;
1884 1884 pp->ftp_tps[i].fit_id.fti_probe = pp;
1885 1885 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1886 1886 }
1887 1887
1888 1888 /*
1889 1889 * If the arguments are shuffled around we set the argument remapping
1890 1890 * table. Later, when the probe fires, we only remap the arguments
1891 1891 * if the table is non-NULL.
1892 1892 */
1893 1893 for (i = 0; i < dhpb->dthpb_xargc; i++) {
1894 1894 if (dhpb->dthpb_args[i] != i) {
1895 1895 pp->ftp_argmap = dhpb->dthpb_args;
1896 1896 break;
1897 1897 }
1898 1898 }
1899 1899
1900 1900 /*
1901 1901 * The probe is fully constructed -- register it with DTrace.
1902 1902 */
1903 1903 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1904 1904 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1905 1905 }
1906 1906
1907 1907 /*ARGSUSED*/
1908 1908 static void
1909 1909 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1910 1910 {
1911 1911 /*
1912 1912 * Clean up the USDT provider. There may be active consumers of the
1913 1913 * provider busy adding probes, no damage will actually befall the
1914 1914 * provider until that count has dropped to zero. This just puts
1915 1915 * the provider on death row.
1916 1916 */
1917 1917 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1918 1918 }
1919 1919
1920 1920 static dtrace_mops_t fasttrap_mops = {
1921 1921 fasttrap_meta_create_probe,
1922 1922 fasttrap_meta_provide,
1923 1923 fasttrap_meta_remove
1924 1924 };
1925 1925
1926 1926 /*ARGSUSED*/
1927 1927 static int
1928 1928 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1929 1929 {
1930 1930 return (0);
1931 1931 }
1932 1932
1933 1933 /*ARGSUSED*/
1934 1934 static int
1935 1935 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1936 1936 {
1937 1937 if (!dtrace_attached())
1938 1938 return (EAGAIN);
1939 1939
1940 1940 if (cmd == FASTTRAPIOC_MAKEPROBE) {
1941 1941 fasttrap_probe_spec_t *uprobe = (void *)arg;
1942 1942 fasttrap_probe_spec_t *probe;
1943 1943 uint64_t noffs;
1944 1944 size_t size;
1945 1945 int ret, err;
1946 1946
1947 1947 if (copyin(&uprobe->ftps_noffs, &noffs,
1948 1948 sizeof (uprobe->ftps_noffs)))
1949 1949 return (EFAULT);
1950 1950
1951 1951 /*
1952 1952 * Probes must have at least one tracepoint.
1953 1953 */
1954 1954 if (noffs == 0)
1955 1955 return (EINVAL);
1956 1956
1957 1957 size = sizeof (fasttrap_probe_spec_t) +
1958 1958 sizeof (probe->ftps_offs[0]) * (noffs - 1);
1959 1959
1960 1960 if (size > 1024 * 1024)
1961 1961 return (ENOMEM);
1962 1962
1963 1963 probe = kmem_alloc(size, KM_SLEEP);
1964 1964
1965 1965 if (copyin(uprobe, probe, size) != 0 ||
1966 1966 probe->ftps_noffs != noffs) {
1967 1967 kmem_free(probe, size);
1968 1968 return (EFAULT);
1969 1969 }
1970 1970
1971 1971 /*
1972 1972 * Verify that the function and module strings contain no
1973 1973 * funny characters.
1974 1974 */
1975 1975 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1976 1976 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977 1977 ret = EINVAL;
1978 1978 goto err;
1979 1979 }
1980 1980
1981 1981 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1982 1982 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1983 1983 ret = EINVAL;
1984 1984 goto err;
1985 1985 }
1986 1986
1987 1987 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1988 1988 proc_t *p;
1989 1989 pid_t pid = probe->ftps_pid;
1990 1990
1991 1991 mutex_enter(&pidlock);
1992 1992 /*
1993 1993 * Report an error if the process doesn't exist
1994 1994 * or is actively being birthed.
1995 1995 */
1996 1996 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1997 1997 mutex_exit(&pidlock);
1998 1998 return (ESRCH);
1999 1999 }
2000 2000 mutex_enter(&p->p_lock);
2001 2001 mutex_exit(&pidlock);
2002 2002
2003 2003 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2004 2004 VREAD | VWRITE)) != 0) {
2005 2005 mutex_exit(&p->p_lock);
2006 2006 return (ret);
2007 2007 }
2008 2008
2009 2009 mutex_exit(&p->p_lock);
2010 2010 }
2011 2011
2012 2012 ret = fasttrap_add_probe(probe);
2013 2013 err:
2014 2014 kmem_free(probe, size);
2015 2015
2016 2016 return (ret);
2017 2017
2018 2018 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2019 2019 fasttrap_instr_query_t instr;
2020 2020 fasttrap_tracepoint_t *tp;
2021 2021 uint_t index;
2022 2022 int ret;
2023 2023
2024 2024 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2025 2025 return (EFAULT);
2026 2026
2027 2027 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2028 2028 proc_t *p;
2029 2029 pid_t pid = instr.ftiq_pid;
2030 2030
2031 2031 mutex_enter(&pidlock);
2032 2032 /*
2033 2033 * Report an error if the process doesn't exist
2034 2034 * or is actively being birthed.
2035 2035 */
2036 2036 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2037 2037 mutex_exit(&pidlock);
2038 2038 return (ESRCH);
2039 2039 }
2040 2040 mutex_enter(&p->p_lock);
2041 2041 mutex_exit(&pidlock);
2042 2042
2043 2043 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2044 2044 VREAD)) != 0) {
2045 2045 mutex_exit(&p->p_lock);
2046 2046 return (ret);
2047 2047 }
2048 2048
2049 2049 mutex_exit(&p->p_lock);
2050 2050 }
2051 2051
2052 2052 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2053 2053
2054 2054 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2055 2055 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2056 2056 while (tp != NULL) {
2057 2057 if (instr.ftiq_pid == tp->ftt_pid &&
2058 2058 instr.ftiq_pc == tp->ftt_pc &&
2059 2059 tp->ftt_proc->ftpc_acount != 0)
2060 2060 break;
2061 2061
2062 2062 tp = tp->ftt_next;
2063 2063 }
2064 2064
2065 2065 if (tp == NULL) {
2066 2066 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067 2067 return (ENOENT);
2068 2068 }
2069 2069
2070 2070 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2071 2071 sizeof (instr.ftiq_instr));
2072 2072 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2073 2073
2074 2074 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2075 2075 return (EFAULT);
2076 2076
2077 2077 return (0);
2078 2078 }
2079 2079
2080 2080 return (EINVAL);
2081 2081 }
2082 2082
2083 2083 static struct cb_ops fasttrap_cb_ops = {
2084 2084 fasttrap_open, /* open */
2085 2085 nodev, /* close */
2086 2086 nulldev, /* strategy */
2087 2087 nulldev, /* print */
2088 2088 nodev, /* dump */
2089 2089 nodev, /* read */
2090 2090 nodev, /* write */
2091 2091 fasttrap_ioctl, /* ioctl */
2092 2092 nodev, /* devmap */
2093 2093 nodev, /* mmap */
2094 2094 nodev, /* segmap */
2095 2095 nochpoll, /* poll */
2096 2096 ddi_prop_op, /* cb_prop_op */
2097 2097 0, /* streamtab */
2098 2098 D_NEW | D_MP /* Driver compatibility flag */
2099 2099 };
2100 2100
2101 2101 /*ARGSUSED*/
2102 2102 static int
2103 2103 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2104 2104 {
2105 2105 int error;
2106 2106
2107 2107 switch (infocmd) {
2108 2108 case DDI_INFO_DEVT2DEVINFO:
2109 2109 *result = (void *)fasttrap_devi;
2110 2110 error = DDI_SUCCESS;
2111 2111 break;
2112 2112 case DDI_INFO_DEVT2INSTANCE:
2113 2113 *result = (void *)0;
2114 2114 error = DDI_SUCCESS;
2115 2115 break;
2116 2116 default:
2117 2117 error = DDI_FAILURE;
2118 2118 }
2119 2119 return (error);
2120 2120 }
2121 2121
2122 2122 static int
2123 2123 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2124 2124 {
2125 2125 ulong_t nent;
2126 2126
2127 2127 switch (cmd) {
2128 2128 case DDI_ATTACH:
2129 2129 break;
2130 2130 case DDI_RESUME:
2131 2131 return (DDI_SUCCESS);
2132 2132 default:
2133 2133 return (DDI_FAILURE);
2134 2134 }
2135 2135
2136 2136 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2137 2137 DDI_PSEUDO, NULL) == DDI_FAILURE) {
2138 2138 ddi_remove_minor_node(devi, NULL);
2139 2139 return (DDI_FAILURE);
2140 2140 }
2141 2141
2142 2142 ddi_report_dev(devi);
2143 2143 fasttrap_devi = devi;
2144 2144
2145 2145 /*
2146 2146 * Install our hooks into fork(2), exec(2), and exit(2).
2147 2147 */
2148 2148 dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2149 2149 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2150 2150 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2151 2151
2152 2152 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2153 2153 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2154 2154 fasttrap_total = 0;
2155 2155
2156 2156 /*
2157 2157 * Conjure up the tracepoints hashtable...
2158 2158 */
2159 2159 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2160 2160 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2161 2161
2162 2162 if (nent == 0 || nent > 0x1000000)
2163 2163 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2164 2164
2165 2165 if (ISP2(nent))
2166 2166 fasttrap_tpoints.fth_nent = nent;
2167 2167 else
2168 2168 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2169 2169 ASSERT(fasttrap_tpoints.fth_nent > 0);
2170 2170 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2171 2171 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2172 2172 sizeof (fasttrap_bucket_t), KM_SLEEP);
2173 2173
2174 2174 /*
2175 2175 * ... and the providers hash table...
2176 2176 */
2177 2177 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2178 2178 if (ISP2(nent))
2179 2179 fasttrap_provs.fth_nent = nent;
2180 2180 else
2181 2181 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2182 2182 ASSERT(fasttrap_provs.fth_nent > 0);
2183 2183 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2184 2184 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2185 2185 sizeof (fasttrap_bucket_t), KM_SLEEP);
2186 2186
2187 2187 /*
2188 2188 * ... and the procs hash table.
2189 2189 */
2190 2190 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2191 2191 if (ISP2(nent))
2192 2192 fasttrap_procs.fth_nent = nent;
2193 2193 else
2194 2194 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2195 2195 ASSERT(fasttrap_procs.fth_nent > 0);
2196 2196 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2197 2197 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2198 2198 sizeof (fasttrap_bucket_t), KM_SLEEP);
2199 2199
2200 2200 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2201 2201 &fasttrap_meta_id);
2202 2202
2203 2203 return (DDI_SUCCESS);
2204 2204 }
2205 2205
2206 2206 static int
2207 2207 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2208 2208 {
2209 2209 int i, fail = 0;
2210 2210 timeout_id_t tmp;
2211 2211
2212 2212 switch (cmd) {
2213 2213 case DDI_DETACH:
2214 2214 break;
2215 2215 case DDI_SUSPEND:
2216 2216 return (DDI_SUCCESS);
2217 2217 default:
2218 2218 return (DDI_FAILURE);
2219 2219 }
2220 2220
2221 2221 /*
2222 2222 * Unregister the meta-provider to make sure no new fasttrap-
2223 2223 * managed providers come along while we're trying to close up
2224 2224 * shop. If we fail to detach, we'll need to re-register as a
2225 2225 * meta-provider. We can fail to unregister as a meta-provider
2226 2226 * if providers we manage still exist.
2227 2227 */
2228 2228 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2229 2229 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2230 2230 return (DDI_FAILURE);
2231 2231
2232 2232 /*
2233 2233 * Prevent any new timeouts from running by setting fasttrap_timeout
2234 2234 * to a non-zero value, and wait for the current timeout to complete.
2235 2235 */
2236 2236 mutex_enter(&fasttrap_cleanup_mtx);
2237 2237 fasttrap_cleanup_work = 0;
2238 2238
2239 2239 while (fasttrap_timeout != (timeout_id_t)1) {
2240 2240 tmp = fasttrap_timeout;
2241 2241 fasttrap_timeout = (timeout_id_t)1;
2242 2242
2243 2243 if (tmp != 0) {
2244 2244 mutex_exit(&fasttrap_cleanup_mtx);
2245 2245 (void) untimeout(tmp);
2246 2246 mutex_enter(&fasttrap_cleanup_mtx);
2247 2247 }
2248 2248 }
2249 2249
2250 2250 fasttrap_cleanup_work = 0;
2251 2251 mutex_exit(&fasttrap_cleanup_mtx);
2252 2252
2253 2253 /*
2254 2254 * Iterate over all of our providers. If there's still a process
2255 2255 * that corresponds to that pid, fail to detach.
2256 2256 */
2257 2257 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2258 2258 fasttrap_provider_t **fpp, *fp;
2259 2259 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2260 2260
2261 2261 mutex_enter(&bucket->ftb_mtx);
2262 2262 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2263 2263 while ((fp = *fpp) != NULL) {
2264 2264 /*
2265 2265 * Acquire and release the lock as a simple way of
2266 2266 * waiting for any other consumer to finish with
2267 2267 * this provider. A thread must first acquire the
2268 2268 * bucket lock so there's no chance of another thread
2269 2269 * blocking on the provider's lock.
2270 2270 */
2271 2271 mutex_enter(&fp->ftp_mtx);
2272 2272 mutex_exit(&fp->ftp_mtx);
2273 2273
2274 2274 if (dtrace_unregister(fp->ftp_provid) != 0) {
2275 2275 fail = 1;
2276 2276 fpp = &fp->ftp_next;
2277 2277 } else {
2278 2278 *fpp = fp->ftp_next;
2279 2279 fasttrap_provider_free(fp);
2280 2280 }
2281 2281 }
2282 2282
2283 2283 mutex_exit(&bucket->ftb_mtx);
2284 2284 }
2285 2285
2286 2286 if (fail) {
2287 2287 uint_t work;
2288 2288 /*
2289 2289 * If we're failing to detach, we need to unblock timeouts
2290 2290 * and start a new timeout if any work has accumulated while
2291 2291 * we've been unsuccessfully trying to detach.
2292 2292 */
2293 2293 mutex_enter(&fasttrap_cleanup_mtx);
2294 2294 fasttrap_timeout = 0;
2295 2295 work = fasttrap_cleanup_work;
2296 2296 mutex_exit(&fasttrap_cleanup_mtx);
2297 2297
2298 2298 if (work)
2299 2299 fasttrap_pid_cleanup();
2300 2300
2301 2301 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2302 2302 &fasttrap_meta_id);
2303 2303
2304 2304 return (DDI_FAILURE);
2305 2305 }
2306 2306
2307 2307 #ifdef DEBUG
2308 2308 mutex_enter(&fasttrap_count_mtx);
2309 2309 ASSERT(fasttrap_pid_count == 0);
2310 2310 mutex_exit(&fasttrap_count_mtx);
2311 2311 #endif
2312 2312
2313 2313 kmem_free(fasttrap_tpoints.fth_table,
2314 2314 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2315 2315 fasttrap_tpoints.fth_nent = 0;
2316 2316
2317 2317 kmem_free(fasttrap_provs.fth_table,
2318 2318 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2319 2319 fasttrap_provs.fth_nent = 0;
2320 2320
2321 2321 kmem_free(fasttrap_procs.fth_table,
2322 2322 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2323 2323 fasttrap_procs.fth_nent = 0;
2324 2324
2325 2325 /*
2326 2326 * We know there are no tracepoints in any process anywhere in
2327 2327 * the system so there is no process which has its p_dtrace_count
2328 2328 * greater than zero, therefore we know that no thread can actively
2329 2329 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2330 2330 * and fasttrap_exec() and fasttrap_exit().
2331 2331 */
2332 2332 ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2333 2333 dtrace_fasttrap_fork_ptr = NULL;
2334 2334
2335 2335 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2336 2336 dtrace_fasttrap_exec_ptr = NULL;
2337 2337
2338 2338 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2339 2339 dtrace_fasttrap_exit_ptr = NULL;
2340 2340
2341 2341 ddi_remove_minor_node(devi, NULL);
2342 2342
2343 2343 return (DDI_SUCCESS);
2344 2344 }
2345 2345
2346 2346 static struct dev_ops fasttrap_ops = {
2347 2347 DEVO_REV, /* devo_rev */
2348 2348 0, /* refcnt */
2349 2349 fasttrap_info, /* get_dev_info */
2350 2350 nulldev, /* identify */
2351 2351 nulldev, /* probe */
2352 2352 fasttrap_attach, /* attach */
2353 2353 fasttrap_detach, /* detach */
2354 2354 nodev, /* reset */
2355 2355 &fasttrap_cb_ops, /* driver operations */
2356 2356 NULL, /* bus operations */
2357 2357 nodev, /* dev power */
2358 2358 ddi_quiesce_not_needed, /* quiesce */
2359 2359 };
2360 2360
2361 2361 /*
|
↓ open down ↓ |
2361 lines elided |
↑ open up ↑ |
2362 2362 * Module linkage information for the kernel.
2363 2363 */
2364 2364 static struct modldrv modldrv = {
2365 2365 &mod_driverops, /* module type (this is a pseudo driver) */
2366 2366 "Fasttrap Tracing", /* name of module */
2367 2367 &fasttrap_ops, /* driver ops */
2368 2368 };
2369 2369
2370 2370 static struct modlinkage modlinkage = {
2371 2371 MODREV_1,
2372 - (void *)&modldrv,
2373 - NULL
2372 + { (void *)&modldrv, NULL }
2374 2373 };
2375 2374
2376 2375 int
2377 2376 _init(void)
2378 2377 {
2379 2378 return (mod_install(&modlinkage));
2380 2379 }
2381 2380
2382 2381 int
2383 2382 _info(struct modinfo *modinfop)
2384 2383 {
2385 2384 return (mod_info(&modlinkage, modinfop));
2386 2385 }
2387 2386
2388 2387 int
2389 2388 _fini(void)
2390 2389 {
2391 2390 return (mod_remove(&modlinkage));
2392 2391 }
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX