1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
29 */
30
31 #include <sys/atomic.h>
32 #include <sys/errno.h>
33 #include <sys/stat.h>
34 #include <sys/modctl.h>
35 #include <sys/conf.h>
36 #include <sys/systm.h>
37 #include <sys/ddi.h>
38 #include <sys/sunddi.h>
39 #include <sys/cpuvar.h>
40 #include <sys/kmem.h>
41 #include <sys/strsubr.h>
42 #include <sys/fasttrap.h>
43 #include <sys/fasttrap_impl.h>
44 #include <sys/fasttrap_isa.h>
45 #include <sys/dtrace.h>
46 #include <sys/dtrace_impl.h>
47 #include <sys/sysmacros.h>
48 #include <sys/proc.h>
49 #include <sys/priv.h>
50 #include <sys/policy.h>
51 #include <util/qsort.h>
52
53 /*
54 * User-Land Trap-Based Tracing
55 * ----------------------------
56 *
57 * The fasttrap provider allows DTrace consumers to instrument any user-level
58 * instruction to gather data; this includes probes with semantic
59 * signifigance like entry and return as well as simple offsets into the
60 * function. While the specific techniques used are very ISA specific, the
61 * methodology is generalizable to any architecture.
62 *
63 *
64 * The General Methodology
65 * -----------------------
66 *
67 * With the primary goal of tracing every user-land instruction and the
68 * limitation that we can't trust user space so don't want to rely on much
69 * information there, we begin by replacing the instructions we want to trace
70 * with trap instructions. Each instruction we overwrite is saved into a hash
71 * table keyed by process ID and pc address. When we enter the kernel due to
72 * this trap instruction, we need the effects of the replaced instruction to
73 * appear to have occurred before we proceed with the user thread's
74 * execution.
75 *
76 * Each user level thread is represented by a ulwp_t structure which is
77 * always easily accessible through a register. The most basic way to produce
78 * the effects of the instruction we replaced is to copy that instruction out
79 * to a bit of scratch space reserved in the user thread's ulwp_t structure
80 * (a sort of kernel-private thread local storage), set the PC to that
81 * scratch space and single step. When we reenter the kernel after single
82 * stepping the instruction we must then adjust the PC to point to what would
83 * normally be the next instruction. Of course, special care must be taken
84 * for branches and jumps, but these represent such a small fraction of any
85 * instruction set that writing the code to emulate these in the kernel is
86 * not too difficult.
87 *
88 * Return probes may require several tracepoints to trace every return site,
89 * and, conversely, each tracepoint may activate several probes (the entry
90 * and offset 0 probes, for example). To solve this muliplexing problem,
91 * tracepoints contain lists of probes to activate and probes contain lists
92 * of tracepoints to enable. If a probe is activated, it adds its ID to
93 * existing tracepoints or creates new ones as necessary.
94 *
95 * Most probes are activated _before_ the instruction is executed, but return
96 * probes are activated _after_ the effects of the last instruction of the
97 * function are visible. Return probes must be fired _after_ we have
98 * single-stepped the instruction whereas all other probes are fired
99 * beforehand.
100 *
101 *
102 * Lock Ordering
103 * -------------
104 *
105 * The lock ordering below -- both internally and with respect to the DTrace
106 * framework -- is a little tricky and bears some explanation. Each provider
107 * has a lock (ftp_mtx) that protects its members including reference counts
108 * for enabled probes (ftp_rcount), consumers actively creating probes
109 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
110 * from being freed. A provider is looked up by taking the bucket lock for the
111 * provider hash table, and is returned with its lock held. The provider lock
112 * may be taken in functions invoked by the DTrace framework, but may not be
113 * held while calling functions in the DTrace framework.
114 *
115 * To ensure consistency over multiple calls to the DTrace framework, the
116 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
117 * not be taken when holding the provider lock as that would create a cyclic
118 * lock ordering. In situations where one would naturally take the provider
119 * lock and then the creation lock, we instead up a reference count to prevent
120 * the provider from disappearing, drop the provider lock, and acquire the
121 * creation lock.
122 *
123 * Briefly:
124 * bucket lock before provider lock
125 * DTrace before provider lock
126 * creation lock before DTrace
127 * never hold the provider lock and creation lock simultaneously
128 */
129
130 static dev_info_t *fasttrap_devi;
131 static dtrace_meta_provider_id_t fasttrap_meta_id;
132
133 static timeout_id_t fasttrap_timeout;
134 static kmutex_t fasttrap_cleanup_mtx;
135 static uint_t fasttrap_cleanup_work;
136
137 /*
138 * Generation count on modifications to the global tracepoint lookup table.
139 */
140 static volatile uint64_t fasttrap_mod_gen;
141
142 /*
143 * When the fasttrap provider is loaded, fasttrap_max is set to either
144 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
145 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
146 * incremented by the number of tracepoints that may be associated with that
147 * probe; fasttrap_total is capped at fasttrap_max.
148 */
149 #define FASTTRAP_MAX_DEFAULT 250000
150 static uint32_t fasttrap_max;
151 static uint32_t fasttrap_total;
152
153
154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
156 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
157
158 #define FASTTRAP_PID_NAME "pid"
159
160 fasttrap_hash_t fasttrap_tpoints;
161 static fasttrap_hash_t fasttrap_provs;
162 static fasttrap_hash_t fasttrap_procs;
163
164 static uint64_t fasttrap_pid_count; /* pid ref count */
165 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
166
167 #define FASTTRAP_ENABLE_FAIL 1
168 #define FASTTRAP_ENABLE_PARTIAL 2
169
170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
172
173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
174 const dtrace_pattr_t *);
175 static void fasttrap_provider_retire(pid_t, const char *, int);
176 static void fasttrap_provider_free(fasttrap_provider_t *);
177
178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
179 static void fasttrap_proc_release(fasttrap_proc_t *);
180
181 #define FASTTRAP_PROVS_INDEX(pid, name) \
182 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
183
184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
185
186 static int
187 fasttrap_highbit(ulong_t i)
188 {
189 int h = 1;
190
191 if (i == 0)
192 return (0);
193 #ifdef _LP64
194 if (i & 0xffffffff00000000ul) {
195 h += 32; i >>= 32;
196 }
197 #endif
198 if (i & 0xffff0000) {
199 h += 16; i >>= 16;
200 }
201 if (i & 0xff00) {
202 h += 8; i >>= 8;
203 }
204 if (i & 0xf0) {
205 h += 4; i >>= 4;
206 }
207 if (i & 0xc) {
208 h += 2; i >>= 2;
209 }
210 if (i & 0x2) {
211 h += 1;
212 }
213 return (h);
214 }
215
216 static uint_t
217 fasttrap_hash_str(const char *p)
218 {
219 unsigned int g;
220 uint_t hval = 0;
221
222 while (*p) {
223 hval = (hval << 4) + *p++;
224 if ((g = (hval & 0xf0000000)) != 0)
225 hval ^= g >> 24;
226 hval &= ~g;
227 }
228 return (hval);
229 }
230
231 void
232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
233 {
234 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
235
236 sqp->sq_info.si_signo = SIGTRAP;
237 sqp->sq_info.si_code = TRAP_DTRACE;
238 sqp->sq_info.si_addr = (caddr_t)pc;
239
240 mutex_enter(&p->p_lock);
241 sigaddqa(p, t, sqp);
242 mutex_exit(&p->p_lock);
243
244 if (t != NULL)
245 aston(t);
246 }
247
248 /*
249 * This function ensures that no threads are actively using the memory
250 * associated with probes that were formerly live.
251 */
252 static void
253 fasttrap_mod_barrier(uint64_t gen)
254 {
255 int i;
256
257 if (gen < fasttrap_mod_gen)
258 return;
259
260 fasttrap_mod_gen++;
261
262 for (i = 0; i < NCPU; i++) {
263 mutex_enter(&cpu_core[i].cpuc_pid_lock);
264 mutex_exit(&cpu_core[i].cpuc_pid_lock);
265 }
266 }
267
268 /*
269 * This is the timeout's callback for cleaning up the providers and their
270 * probes.
271 */
272 /*ARGSUSED*/
273 static void
274 fasttrap_pid_cleanup_cb(void *data)
275 {
276 fasttrap_provider_t **fpp, *fp;
277 fasttrap_bucket_t *bucket;
278 dtrace_provider_id_t provid;
279 int i, later, rval;
280
281 static volatile int in = 0;
282 ASSERT(in == 0);
283 in = 1;
284
285 mutex_enter(&fasttrap_cleanup_mtx);
286 while (fasttrap_cleanup_work) {
287 fasttrap_cleanup_work = 0;
288 mutex_exit(&fasttrap_cleanup_mtx);
289
290 later = 0;
291
292 /*
293 * Iterate over all the providers trying to remove the marked
294 * ones. If a provider is marked but not retired, we just
295 * have to take a crack at removing it -- it's no big deal if
296 * we can't.
297 */
298 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
299 bucket = &fasttrap_provs.fth_table[i];
300 mutex_enter(&bucket->ftb_mtx);
301 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
302
303 while ((fp = *fpp) != NULL) {
304 if (!fp->ftp_marked) {
305 fpp = &fp->ftp_next;
306 continue;
307 }
308
309 mutex_enter(&fp->ftp_mtx);
310
311 /*
312 * If this provider has consumers actively
313 * creating probes (ftp_ccount) or is a USDT
314 * provider (ftp_mcount), we can't unregister
315 * or even condense.
316 */
317 if (fp->ftp_ccount != 0 ||
318 fp->ftp_mcount != 0) {
319 mutex_exit(&fp->ftp_mtx);
320 fp->ftp_marked = 0;
321 continue;
322 }
323
324 if (!fp->ftp_retired || fp->ftp_rcount != 0)
325 fp->ftp_marked = 0;
326
327 mutex_exit(&fp->ftp_mtx);
328
329 /*
330 * If we successfully unregister this
331 * provider we can remove it from the hash
332 * chain and free the memory. If our attempt
333 * to unregister fails and this is a retired
334 * provider, increment our flag to try again
335 * pretty soon. If we've consumed more than
336 * half of our total permitted number of
337 * probes call dtrace_condense() to try to
338 * clean out the unenabled probes.
339 */
340 provid = fp->ftp_provid;
341 if ((rval = dtrace_unregister(provid)) != 0) {
342 if (fasttrap_total > fasttrap_max / 2)
343 (void) dtrace_condense(provid);
344
345 if (rval == EAGAIN)
346 fp->ftp_marked = 1;
347
348 later += fp->ftp_marked;
349 fpp = &fp->ftp_next;
350 } else {
351 *fpp = fp->ftp_next;
352 fasttrap_provider_free(fp);
353 }
354 }
355 mutex_exit(&bucket->ftb_mtx);
356 }
357
358 mutex_enter(&fasttrap_cleanup_mtx);
359 }
360
361 ASSERT(fasttrap_timeout != 0);
362
363 /*
364 * If we were unable to remove a retired provider, try again after
365 * a second. This situation can occur in certain circumstances where
366 * providers cannot be unregistered even though they have no probes
367 * enabled because of an execution of dtrace -l or something similar.
368 * If the timeout has been disabled (set to 1 because we're trying
369 * to detach), we set fasttrap_cleanup_work to ensure that we'll
370 * get a chance to do that work if and when the timeout is reenabled
371 * (if detach fails).
372 */
373 if (later > 0) {
374 if (fasttrap_timeout != (timeout_id_t)1) {
375 fasttrap_timeout =
376 timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
377 }
378
379 fasttrap_cleanup_work = 1;
380 } else {
381 fasttrap_timeout = 0;
382 }
383
384 mutex_exit(&fasttrap_cleanup_mtx);
385 in = 0;
386 }
387
388 /*
389 * Activates the asynchronous cleanup mechanism.
390 */
391 static void
392 fasttrap_pid_cleanup(void)
393 {
394 mutex_enter(&fasttrap_cleanup_mtx);
395 fasttrap_cleanup_work = 1;
396 if (fasttrap_timeout == 0)
397 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
398 mutex_exit(&fasttrap_cleanup_mtx);
399 }
400
401 /*
402 * This is called from cfork() via dtrace_fasttrap_fork(). The child
403 * process's address space is (roughly) a copy of the parent process's so
404 * we have to remove all the instrumentation we had previously enabled in the
405 * parent.
406 */
407 static void
408 fasttrap_fork(proc_t *p, proc_t *cp)
409 {
410 pid_t ppid = p->p_pid;
411 int i;
412
413 ASSERT(curproc == p);
414 ASSERT(p->p_proc_flag & P_PR_LOCK);
415 ASSERT(p->p_dtrace_count > 0);
416 ASSERT(cp->p_dtrace_count == 0);
417
418 /*
419 * This would be simpler and faster if we maintained per-process
420 * hash tables of enabled tracepoints. It could, however, potentially
421 * slow down execution of a tracepoint since we'd need to go
422 * through two levels of indirection. In the future, we should
423 * consider either maintaining per-process ancillary lists of
424 * enabled tracepoints or hanging a pointer to a per-process hash
425 * table of enabled tracepoints off the proc structure.
426 */
427
428 /*
429 * We don't have to worry about the child process disappearing
430 * because we're in fork().
431 */
432 mutex_enter(&cp->p_lock);
433 sprlock_proc(cp);
434 mutex_exit(&cp->p_lock);
435
436 /*
437 * Iterate over every tracepoint looking for ones that belong to the
438 * parent process, and remove each from the child process.
439 */
440 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
441 fasttrap_tracepoint_t *tp;
442 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
443
444 mutex_enter(&bucket->ftb_mtx);
445 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
446 if (tp->ftt_pid == ppid &&
447 tp->ftt_proc->ftpc_acount != 0) {
448 int ret = fasttrap_tracepoint_remove(cp, tp);
449 ASSERT(ret == 0);
450
451 /*
452 * The count of active providers can only be
453 * decremented (i.e. to zero) during exec,
454 * exit, and removal of a meta provider so it
455 * should be impossible to drop the count
456 * mid-fork.
457 */
458 ASSERT(tp->ftt_proc->ftpc_acount != 0);
459 }
460 }
461 mutex_exit(&bucket->ftb_mtx);
462 }
463
464 mutex_enter(&cp->p_lock);
465 sprunlock(cp);
466 }
467
468 /*
469 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
470 * is set on the proc structure to indicate that there is a pid provider
471 * associated with this process.
472 */
473 static void
474 fasttrap_exec_exit(proc_t *p)
475 {
476 ASSERT(p == curproc);
477 ASSERT(MUTEX_HELD(&p->p_lock));
478
479 mutex_exit(&p->p_lock);
480
481 /*
482 * We clean up the pid provider for this process here; user-land
483 * static probes are handled by the meta-provider remove entry point.
484 */
485 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
486
487 mutex_enter(&p->p_lock);
488 }
489
490
491 /*ARGSUSED*/
492 static void
493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
494 {
495 /*
496 * There are no "default" pid probes.
497 */
498 }
499
500 static int
501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
502 {
503 fasttrap_tracepoint_t *tp, *new_tp = NULL;
504 fasttrap_bucket_t *bucket;
505 fasttrap_id_t *id;
506 pid_t pid;
507 uintptr_t pc;
508
509 ASSERT(index < probe->ftp_ntps);
510
511 pid = probe->ftp_pid;
512 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
513 id = &probe->ftp_tps[index].fit_id;
514
515 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
516
517 ASSERT(!(p->p_flag & SVFORK));
518
519 /*
520 * Before we make any modifications, make sure we've imposed a barrier
521 * on the generation in which this probe was last modified.
522 */
523 fasttrap_mod_barrier(probe->ftp_gen);
524
525 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
526
527 /*
528 * If the tracepoint has already been enabled, just add our id to the
529 * list of interested probes. This may be our second time through
530 * this path in which case we'll have constructed the tracepoint we'd
531 * like to install. If we can't find a match, and have an allocated
532 * tracepoint ready to go, enable that one now.
533 *
534 * A tracepoint whose process is defunct is also considered defunct.
535 */
536 again:
537 mutex_enter(&bucket->ftb_mtx);
538 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
539 /*
540 * Note that it's safe to access the active count on the
541 * associated proc structure because we know that at least one
542 * provider (this one) will still be around throughout this
543 * operation.
544 */
545 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
546 tp->ftt_proc->ftpc_acount == 0)
547 continue;
548
549 /*
550 * Now that we've found a matching tracepoint, it would be
551 * a decent idea to confirm that the tracepoint is still
552 * enabled and the trap instruction hasn't been overwritten.
553 * Since this is a little hairy, we'll punt for now.
554 */
555
556 /*
557 * This can't be the first interested probe. We don't have
558 * to worry about another thread being in the midst of
559 * deleting this tracepoint (which would be the only valid
560 * reason for a tracepoint to have no interested probes)
561 * since we're holding P_PR_LOCK for this process.
562 */
563 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
564
565 switch (id->fti_ptype) {
566 case DTFTP_ENTRY:
567 case DTFTP_OFFSETS:
568 case DTFTP_IS_ENABLED:
569 id->fti_next = tp->ftt_ids;
570 membar_producer();
571 tp->ftt_ids = id;
572 membar_producer();
573 break;
574
575 case DTFTP_RETURN:
576 case DTFTP_POST_OFFSETS:
577 id->fti_next = tp->ftt_retids;
578 membar_producer();
579 tp->ftt_retids = id;
580 membar_producer();
581 break;
582
583 default:
584 ASSERT(0);
585 }
586
587 mutex_exit(&bucket->ftb_mtx);
588
589 if (new_tp != NULL) {
590 new_tp->ftt_ids = NULL;
591 new_tp->ftt_retids = NULL;
592 }
593
594 return (0);
595 }
596
597 /*
598 * If we have a good tracepoint ready to go, install it now while
599 * we have the lock held and no one can screw with us.
600 */
601 if (new_tp != NULL) {
602 int rc = 0;
603
604 new_tp->ftt_next = bucket->ftb_data;
605 membar_producer();
606 bucket->ftb_data = new_tp;
607 membar_producer();
608 mutex_exit(&bucket->ftb_mtx);
609
610 /*
611 * Activate the tracepoint in the ISA-specific manner.
612 * If this fails, we need to report the failure, but
613 * indicate that this tracepoint must still be disabled
614 * by calling fasttrap_tracepoint_disable().
615 */
616 if (fasttrap_tracepoint_install(p, new_tp) != 0)
617 rc = FASTTRAP_ENABLE_PARTIAL;
618
619 /*
620 * Increment the count of the number of tracepoints active in
621 * the victim process.
622 */
623 ASSERT(p->p_proc_flag & P_PR_LOCK);
624 p->p_dtrace_count++;
625
626 return (rc);
627 }
628
629 mutex_exit(&bucket->ftb_mtx);
630
631 /*
632 * Initialize the tracepoint that's been preallocated with the probe.
633 */
634 new_tp = probe->ftp_tps[index].fit_tp;
635
636 ASSERT(new_tp->ftt_pid == pid);
637 ASSERT(new_tp->ftt_pc == pc);
638 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
639 ASSERT(new_tp->ftt_ids == NULL);
640 ASSERT(new_tp->ftt_retids == NULL);
641
642 switch (id->fti_ptype) {
643 case DTFTP_ENTRY:
644 case DTFTP_OFFSETS:
645 case DTFTP_IS_ENABLED:
646 id->fti_next = NULL;
647 new_tp->ftt_ids = id;
648 break;
649
650 case DTFTP_RETURN:
651 case DTFTP_POST_OFFSETS:
652 id->fti_next = NULL;
653 new_tp->ftt_retids = id;
654 break;
655
656 default:
657 ASSERT(0);
658 }
659
660 /*
661 * If the ISA-dependent initialization goes to plan, go back to the
662 * beginning and try to install this freshly made tracepoint.
663 */
664 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
665 goto again;
666
667 new_tp->ftt_ids = NULL;
668 new_tp->ftt_retids = NULL;
669
670 return (FASTTRAP_ENABLE_FAIL);
671 }
672
673 static void
674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
675 {
676 fasttrap_bucket_t *bucket;
677 fasttrap_provider_t *provider = probe->ftp_prov;
678 fasttrap_tracepoint_t **pp, *tp;
679 fasttrap_id_t *id, **idp;
680 pid_t pid;
681 uintptr_t pc;
682
683 ASSERT(index < probe->ftp_ntps);
684
685 pid = probe->ftp_pid;
686 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
687 id = &probe->ftp_tps[index].fit_id;
688
689 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
690
691 /*
692 * Find the tracepoint and make sure that our id is one of the
693 * ones registered with it.
694 */
695 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
696 mutex_enter(&bucket->ftb_mtx);
697 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
698 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
699 tp->ftt_proc == provider->ftp_proc)
700 break;
701 }
702
703 /*
704 * If we somehow lost this tracepoint, we're in a world of hurt.
705 */
706 ASSERT(tp != NULL);
707
708 switch (id->fti_ptype) {
709 case DTFTP_ENTRY:
710 case DTFTP_OFFSETS:
711 case DTFTP_IS_ENABLED:
712 ASSERT(tp->ftt_ids != NULL);
713 idp = &tp->ftt_ids;
714 break;
715
716 case DTFTP_RETURN:
717 case DTFTP_POST_OFFSETS:
718 ASSERT(tp->ftt_retids != NULL);
719 idp = &tp->ftt_retids;
720 break;
721
722 default:
723 ASSERT(0);
724 }
725
726 while ((*idp)->fti_probe != probe) {
727 idp = &(*idp)->fti_next;
728 ASSERT(*idp != NULL);
729 }
730
731 id = *idp;
732 *idp = id->fti_next;
733 membar_producer();
734
735 ASSERT(id->fti_probe == probe);
736
737 /*
738 * If there are other registered enablings of this tracepoint, we're
739 * all done, but if this was the last probe assocated with this
740 * this tracepoint, we need to remove and free it.
741 */
742 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
743
744 /*
745 * If the current probe's tracepoint is in use, swap it
746 * for an unused tracepoint.
747 */
748 if (tp == probe->ftp_tps[index].fit_tp) {
749 fasttrap_probe_t *tmp_probe;
750 fasttrap_tracepoint_t **tmp_tp;
751 uint_t tmp_index;
752
753 if (tp->ftt_ids != NULL) {
754 tmp_probe = tp->ftt_ids->fti_probe;
755 /* LINTED - alignment */
756 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
757 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
758 } else {
759 tmp_probe = tp->ftt_retids->fti_probe;
760 /* LINTED - alignment */
761 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
762 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
763 }
764
765 ASSERT(*tmp_tp != NULL);
766 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
767 ASSERT((*tmp_tp)->ftt_ids == NULL);
768 ASSERT((*tmp_tp)->ftt_retids == NULL);
769
770 probe->ftp_tps[index].fit_tp = *tmp_tp;
771 *tmp_tp = tp;
772 }
773
774 mutex_exit(&bucket->ftb_mtx);
775
776 /*
777 * Tag the modified probe with the generation in which it was
778 * changed.
779 */
780 probe->ftp_gen = fasttrap_mod_gen;
781 return;
782 }
783
784 mutex_exit(&bucket->ftb_mtx);
785
786 /*
787 * We can't safely remove the tracepoint from the set of active
788 * tracepoints until we've actually removed the fasttrap instruction
789 * from the process's text. We can, however, operate on this
790 * tracepoint secure in the knowledge that no other thread is going to
791 * be looking at it since we hold P_PR_LOCK on the process if it's
792 * live or we hold the provider lock on the process if it's dead and
793 * gone.
794 */
795
796 /*
797 * We only need to remove the actual instruction if we're looking
798 * at an existing process
799 */
800 if (p != NULL) {
801 /*
802 * If we fail to restore the instruction we need to kill
803 * this process since it's in a completely unrecoverable
804 * state.
805 */
806 if (fasttrap_tracepoint_remove(p, tp) != 0)
807 fasttrap_sigtrap(p, NULL, pc);
808
809 /*
810 * Decrement the count of the number of tracepoints active
811 * in the victim process.
812 */
813 ASSERT(p->p_proc_flag & P_PR_LOCK);
814 p->p_dtrace_count--;
815 }
816
817 /*
818 * Remove the probe from the hash table of active tracepoints.
819 */
820 mutex_enter(&bucket->ftb_mtx);
821 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
822 ASSERT(*pp != NULL);
823 while (*pp != tp) {
824 pp = &(*pp)->ftt_next;
825 ASSERT(*pp != NULL);
826 }
827
828 *pp = tp->ftt_next;
829 membar_producer();
830
831 mutex_exit(&bucket->ftb_mtx);
832
833 /*
834 * Tag the modified probe with the generation in which it was changed.
835 */
836 probe->ftp_gen = fasttrap_mod_gen;
837 }
838
839 static void
840 fasttrap_enable_callbacks(void)
841 {
842 /*
843 * We don't have to play the rw lock game here because we're
844 * providing something rather than taking something away --
845 * we can be sure that no threads have tried to follow this
846 * function pointer yet.
847 */
848 mutex_enter(&fasttrap_count_mtx);
849 if (fasttrap_pid_count == 0) {
850 ASSERT(dtrace_pid_probe_ptr == NULL);
851 ASSERT(dtrace_return_probe_ptr == NULL);
852 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
853 dtrace_return_probe_ptr = &fasttrap_return_probe;
854 }
855 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
856 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
857 fasttrap_pid_count++;
858 mutex_exit(&fasttrap_count_mtx);
859 }
860
861 static void
862 fasttrap_disable_callbacks(void)
863 {
864 ASSERT(MUTEX_HELD(&cpu_lock));
865
866 mutex_enter(&fasttrap_count_mtx);
867 ASSERT(fasttrap_pid_count > 0);
868 fasttrap_pid_count--;
869 if (fasttrap_pid_count == 0) {
870 cpu_t *cur, *cpu = CPU;
871
872 for (cur = cpu->cpu_next_onln; cur != cpu;
873 cur = cur->cpu_next_onln) {
874 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
875 }
876
877 dtrace_pid_probe_ptr = NULL;
878 dtrace_return_probe_ptr = NULL;
879
880 for (cur = cpu->cpu_next_onln; cur != cpu;
881 cur = cur->cpu_next_onln) {
882 rw_exit(&cur->cpu_ft_lock);
883 }
884 }
885 mutex_exit(&fasttrap_count_mtx);
886 }
887
888 /*ARGSUSED*/
889 static int
890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
891 {
892 fasttrap_probe_t *probe = parg;
893 proc_t *p;
894 int i, rc;
895
896 ASSERT(probe != NULL);
897 ASSERT(!probe->ftp_enabled);
898 ASSERT(id == probe->ftp_id);
899 ASSERT(MUTEX_HELD(&cpu_lock));
900
901 /*
902 * Increment the count of enabled probes on this probe's provider;
903 * the provider can't go away while the probe still exists. We
904 * must increment this even if we aren't able to properly enable
905 * this probe.
906 */
907 mutex_enter(&probe->ftp_prov->ftp_mtx);
908 probe->ftp_prov->ftp_rcount++;
909 mutex_exit(&probe->ftp_prov->ftp_mtx);
910
911 /*
912 * If this probe's provider is retired (meaning it was valid in a
913 * previously exec'ed incarnation of this address space), bail out. The
914 * provider can't go away while we're in this code path.
915 */
916 if (probe->ftp_prov->ftp_retired)
917 return (0);
918
919 /*
920 * If we can't find the process, it may be that we're in the context of
921 * a fork in which the traced process is being born and we're copying
922 * USDT probes. Otherwise, the process is gone so bail.
923 */
924 if ((p = sprlock(probe->ftp_pid)) == NULL) {
925 if ((curproc->p_flag & SFORKING) == 0)
926 return (0);
927
928 mutex_enter(&pidlock);
929 p = prfind(probe->ftp_pid);
930
931 if (p == NULL) {
932 /*
933 * So it's not that the target process is being born,
934 * it's that it isn't there at all (and we simply
935 * happen to be forking). Anyway, we know that the
936 * target is definitely gone, so bail out.
937 */
938 mutex_exit(&pidlock);
939 return (0);
940 }
941
942 /*
943 * Confirm that curproc is indeed forking the process in which
944 * we're trying to enable probes.
945 */
946 ASSERT(p->p_parent == curproc);
947 ASSERT(p->p_stat == SIDL);
948
949 mutex_enter(&p->p_lock);
950 mutex_exit(&pidlock);
951
952 sprlock_proc(p);
953 }
954
955 ASSERT(!(p->p_flag & SVFORK));
956 mutex_exit(&p->p_lock);
957
958 /*
959 * We have to enable the trap entry point before any user threads have
960 * the chance to execute the trap instruction we're about to place
961 * in their process's text.
962 */
963 fasttrap_enable_callbacks();
964
965 /*
966 * Enable all the tracepoints and add this probe's id to each
967 * tracepoint's list of active probes.
968 */
969 for (i = 0; i < probe->ftp_ntps; i++) {
970 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
971 /*
972 * If enabling the tracepoint failed completely,
973 * we don't have to disable it; if the failure
974 * was only partial we must disable it.
975 */
976 if (rc == FASTTRAP_ENABLE_FAIL)
977 i--;
978 else
979 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
980
981 /*
982 * Back up and pull out all the tracepoints we've
983 * created so far for this probe.
984 */
985 while (i >= 0) {
986 fasttrap_tracepoint_disable(p, probe, i);
987 i--;
988 }
989
990 mutex_enter(&p->p_lock);
991 sprunlock(p);
992
993 /*
994 * Since we're not actually enabling this probe,
995 * drop our reference on the trap table entry.
996 */
997 fasttrap_disable_callbacks();
998 return (0);
999 }
1000 }
1001
1002 mutex_enter(&p->p_lock);
1003 sprunlock(p);
1004
1005 probe->ftp_enabled = 1;
1006 return (0);
1007 }
1008
1009 /*ARGSUSED*/
1010 static void
1011 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1012 {
1013 fasttrap_probe_t *probe = parg;
1014 fasttrap_provider_t *provider = probe->ftp_prov;
1015 proc_t *p;
1016 int i, whack = 0;
1017
1018 ASSERT(id == probe->ftp_id);
1019
1020 /*
1021 * We won't be able to acquire a /proc-esque lock on the process
1022 * iff the process is dead and gone. In this case, we rely on the
1023 * provider lock as a point of mutual exclusion to prevent other
1024 * DTrace consumers from disabling this probe.
1025 */
1026 if ((p = sprlock(probe->ftp_pid)) != NULL) {
1027 ASSERT(!(p->p_flag & SVFORK));
1028 mutex_exit(&p->p_lock);
1029 }
1030
1031 mutex_enter(&provider->ftp_mtx);
1032
1033 /*
1034 * Disable all the associated tracepoints (for fully enabled probes).
1035 */
1036 if (probe->ftp_enabled) {
1037 for (i = 0; i < probe->ftp_ntps; i++) {
1038 fasttrap_tracepoint_disable(p, probe, i);
1039 }
1040 }
1041
1042 ASSERT(provider->ftp_rcount > 0);
1043 provider->ftp_rcount--;
1044
1045 if (p != NULL) {
1046 /*
1047 * Even though we may not be able to remove it entirely, we
1048 * mark this retired provider to get a chance to remove some
1049 * of the associated probes.
1050 */
1051 if (provider->ftp_retired && !provider->ftp_marked)
1052 whack = provider->ftp_marked = 1;
1053 mutex_exit(&provider->ftp_mtx);
1054
1055 mutex_enter(&p->p_lock);
1056 sprunlock(p);
1057 } else {
1058 /*
1059 * If the process is dead, we're just waiting for the
1060 * last probe to be disabled to be able to free it.
1061 */
1062 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1063 whack = provider->ftp_marked = 1;
1064 mutex_exit(&provider->ftp_mtx);
1065 }
1066
1067 if (whack)
1068 fasttrap_pid_cleanup();
1069
1070 if (!probe->ftp_enabled)
1071 return;
1072
1073 probe->ftp_enabled = 0;
1074
1075 ASSERT(MUTEX_HELD(&cpu_lock));
1076 fasttrap_disable_callbacks();
1077 }
1078
1079 /*ARGSUSED*/
1080 static void
1081 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1082 dtrace_argdesc_t *desc)
1083 {
1084 fasttrap_probe_t *probe = parg;
1085 char *str;
1086 int i, ndx;
1087
1088 desc->dtargd_native[0] = '\0';
1089 desc->dtargd_xlate[0] = '\0';
1090
1091 if (probe->ftp_prov->ftp_retired != 0 ||
1092 desc->dtargd_ndx >= probe->ftp_nargs) {
1093 desc->dtargd_ndx = DTRACE_ARGNONE;
1094 return;
1095 }
1096
1097 ndx = (probe->ftp_argmap != NULL) ?
1098 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1099
1100 str = probe->ftp_ntypes;
1101 for (i = 0; i < ndx; i++) {
1102 str += strlen(str) + 1;
1103 }
1104
1105 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1106 (void) strcpy(desc->dtargd_native, str);
1107
1108 if (probe->ftp_xtypes == NULL)
1109 return;
1110
1111 str = probe->ftp_xtypes;
1112 for (i = 0; i < desc->dtargd_ndx; i++) {
1113 str += strlen(str) + 1;
1114 }
1115
1116 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1117 (void) strcpy(desc->dtargd_xlate, str);
1118 }
1119
1120 /*ARGSUSED*/
1121 static void
1122 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1123 {
1124 fasttrap_probe_t *probe = parg;
1125 int i;
1126 size_t size;
1127
1128 ASSERT(probe != NULL);
1129 ASSERT(!probe->ftp_enabled);
1130 ASSERT(fasttrap_total >= probe->ftp_ntps);
1131
1132 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1133 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1134
1135 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1136 fasttrap_mod_barrier(probe->ftp_gen);
1137
1138 for (i = 0; i < probe->ftp_ntps; i++) {
1139 kmem_free(probe->ftp_tps[i].fit_tp,
1140 sizeof (fasttrap_tracepoint_t));
1141 }
1142
1143 kmem_free(probe, size);
1144 }
1145
1146
1147 static const dtrace_pattr_t pid_attr = {
1148 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1149 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1150 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1151 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1152 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1153 };
1154
1155 static dtrace_pops_t pid_pops = {
1156 fasttrap_pid_provide,
1157 NULL,
1158 fasttrap_pid_enable,
1159 fasttrap_pid_disable,
1160 NULL,
1161 NULL,
1162 fasttrap_pid_getargdesc,
1163 fasttrap_pid_getarg,
1164 NULL,
1165 fasttrap_pid_destroy
1166 };
1167
1168 static dtrace_pops_t usdt_pops = {
1169 fasttrap_pid_provide,
1170 NULL,
1171 fasttrap_pid_enable,
1172 fasttrap_pid_disable,
1173 NULL,
1174 NULL,
1175 fasttrap_pid_getargdesc,
1176 fasttrap_usdt_getarg,
1177 NULL,
1178 fasttrap_pid_destroy
1179 };
1180
1181 static fasttrap_proc_t *
1182 fasttrap_proc_lookup(pid_t pid)
1183 {
1184 fasttrap_bucket_t *bucket;
1185 fasttrap_proc_t *fprc, *new_fprc;
1186
1187 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1188 mutex_enter(&bucket->ftb_mtx);
1189
1190 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1191 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1192 mutex_enter(&fprc->ftpc_mtx);
1193 mutex_exit(&bucket->ftb_mtx);
1194 fprc->ftpc_rcount++;
1195 atomic_inc_64(&fprc->ftpc_acount);
1196 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1197 mutex_exit(&fprc->ftpc_mtx);
1198
1199 return (fprc);
1200 }
1201 }
1202
1203 /*
1204 * Drop the bucket lock so we don't try to perform a sleeping
1205 * allocation under it.
1206 */
1207 mutex_exit(&bucket->ftb_mtx);
1208
1209 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1210 new_fprc->ftpc_pid = pid;
1211 new_fprc->ftpc_rcount = 1;
1212 new_fprc->ftpc_acount = 1;
1213
1214 mutex_enter(&bucket->ftb_mtx);
1215
1216 /*
1217 * Take another lap through the list to make sure a proc hasn't
1218 * been created for this pid while we weren't under the bucket lock.
1219 */
1220 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1221 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1222 mutex_enter(&fprc->ftpc_mtx);
1223 mutex_exit(&bucket->ftb_mtx);
1224 fprc->ftpc_rcount++;
1225 atomic_inc_64(&fprc->ftpc_acount);
1226 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1227 mutex_exit(&fprc->ftpc_mtx);
1228
1229 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1230
1231 return (fprc);
1232 }
1233 }
1234
1235 new_fprc->ftpc_next = bucket->ftb_data;
1236 bucket->ftb_data = new_fprc;
1237
1238 mutex_exit(&bucket->ftb_mtx);
1239
1240 return (new_fprc);
1241 }
1242
1243 static void
1244 fasttrap_proc_release(fasttrap_proc_t *proc)
1245 {
1246 fasttrap_bucket_t *bucket;
1247 fasttrap_proc_t *fprc, **fprcp;
1248 pid_t pid = proc->ftpc_pid;
1249
1250 mutex_enter(&proc->ftpc_mtx);
1251
1252 ASSERT(proc->ftpc_rcount != 0);
1253 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1254
1255 if (--proc->ftpc_rcount != 0) {
1256 mutex_exit(&proc->ftpc_mtx);
1257 return;
1258 }
1259
1260 mutex_exit(&proc->ftpc_mtx);
1261
1262 /*
1263 * There should definitely be no live providers associated with this
1264 * process at this point.
1265 */
1266 ASSERT(proc->ftpc_acount == 0);
1267
1268 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1269 mutex_enter(&bucket->ftb_mtx);
1270
1271 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1272 while ((fprc = *fprcp) != NULL) {
1273 if (fprc == proc)
1274 break;
1275
1276 fprcp = &fprc->ftpc_next;
1277 }
1278
1279 /*
1280 * Something strange has happened if we can't find the proc.
1281 */
1282 ASSERT(fprc != NULL);
1283
1284 *fprcp = fprc->ftpc_next;
1285
1286 mutex_exit(&bucket->ftb_mtx);
1287
1288 kmem_free(fprc, sizeof (fasttrap_proc_t));
1289 }
1290
1291 /*
1292 * Lookup a fasttrap-managed provider based on its name and associated pid.
1293 * If the pattr argument is non-NULL, this function instantiates the provider
1294 * if it doesn't exist otherwise it returns NULL. The provider is returned
1295 * with its lock held.
1296 */
1297 static fasttrap_provider_t *
1298 fasttrap_provider_lookup(pid_t pid, const char *name,
1299 const dtrace_pattr_t *pattr)
1300 {
1301 fasttrap_provider_t *fp, *new_fp = NULL;
1302 fasttrap_bucket_t *bucket;
1303 char provname[DTRACE_PROVNAMELEN];
1304 proc_t *p;
1305 cred_t *cred;
1306
1307 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1308 ASSERT(pattr != NULL);
1309
1310 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1311 mutex_enter(&bucket->ftb_mtx);
1312
1313 /*
1314 * Take a lap through the list and return the match if we find it.
1315 */
1316 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1317 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1318 !fp->ftp_retired) {
1319 mutex_enter(&fp->ftp_mtx);
1320 mutex_exit(&bucket->ftb_mtx);
1321 return (fp);
1322 }
1323 }
1324
1325 /*
1326 * Drop the bucket lock so we don't try to perform a sleeping
1327 * allocation under it.
1328 */
1329 mutex_exit(&bucket->ftb_mtx);
1330
1331 /*
1332 * Make sure the process exists, isn't a child created as the result
1333 * of a vfork(2), and isn't a zombie (but may be in fork).
1334 */
1335 mutex_enter(&pidlock);
1336 if ((p = prfind(pid)) == NULL) {
1337 mutex_exit(&pidlock);
1338 return (NULL);
1339 }
1340 mutex_enter(&p->p_lock);
1341 mutex_exit(&pidlock);
1342 if (p->p_flag & (SVFORK | SEXITING)) {
1343 mutex_exit(&p->p_lock);
1344 return (NULL);
1345 }
1346
1347 /*
1348 * Increment p_dtrace_probes so that the process knows to inform us
1349 * when it exits or execs. fasttrap_provider_free() decrements this
1350 * when we're done with this provider.
1351 */
1352 p->p_dtrace_probes++;
1353
1354 /*
1355 * Grab the credentials for this process so we have
1356 * something to pass to dtrace_register().
1357 */
1358 mutex_enter(&p->p_crlock);
1359 crhold(p->p_cred);
1360 cred = p->p_cred;
1361 mutex_exit(&p->p_crlock);
1362 mutex_exit(&p->p_lock);
1363
1364 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1365 new_fp->ftp_pid = pid;
1366 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1367
1368 ASSERT(new_fp->ftp_proc != NULL);
1369
1370 mutex_enter(&bucket->ftb_mtx);
1371
1372 /*
1373 * Take another lap through the list to make sure a provider hasn't
1374 * been created for this pid while we weren't under the bucket lock.
1375 */
1376 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1377 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1378 !fp->ftp_retired) {
1379 mutex_enter(&fp->ftp_mtx);
1380 mutex_exit(&bucket->ftb_mtx);
1381 fasttrap_provider_free(new_fp);
1382 crfree(cred);
1383 return (fp);
1384 }
1385 }
1386
1387 (void) strcpy(new_fp->ftp_name, name);
1388
1389 /*
1390 * Fail and return NULL if either the provider name is too long
1391 * or we fail to register this new provider with the DTrace
1392 * framework. Note that this is the only place we ever construct
1393 * the full provider name -- we keep it in pieces in the provider
1394 * structure.
1395 */
1396 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1397 sizeof (provname) ||
1398 dtrace_register(provname, pattr,
1399 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1400 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1401 &new_fp->ftp_provid) != 0) {
1402 mutex_exit(&bucket->ftb_mtx);
1403 fasttrap_provider_free(new_fp);
1404 crfree(cred);
1405 return (NULL);
1406 }
1407
1408 new_fp->ftp_next = bucket->ftb_data;
1409 bucket->ftb_data = new_fp;
1410
1411 mutex_enter(&new_fp->ftp_mtx);
1412 mutex_exit(&bucket->ftb_mtx);
1413
1414 crfree(cred);
1415 return (new_fp);
1416 }
1417
1418 static void
1419 fasttrap_provider_free(fasttrap_provider_t *provider)
1420 {
1421 pid_t pid = provider->ftp_pid;
1422 proc_t *p;
1423
1424 /*
1425 * There need to be no associated enabled probes, no consumers
1426 * creating probes, and no meta providers referencing this provider.
1427 */
1428 ASSERT(provider->ftp_rcount == 0);
1429 ASSERT(provider->ftp_ccount == 0);
1430 ASSERT(provider->ftp_mcount == 0);
1431
1432 /*
1433 * If this provider hasn't been retired, we need to explicitly drop the
1434 * count of active providers on the associated process structure.
1435 */
1436 if (!provider->ftp_retired) {
1437 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1438 ASSERT(provider->ftp_proc->ftpc_acount <
1439 provider->ftp_proc->ftpc_rcount);
1440 }
1441
1442 fasttrap_proc_release(provider->ftp_proc);
1443
1444 kmem_free(provider, sizeof (fasttrap_provider_t));
1445
1446 /*
1447 * Decrement p_dtrace_probes on the process whose provider we're
1448 * freeing. We don't have to worry about clobbering somone else's
1449 * modifications to it because we have locked the bucket that
1450 * corresponds to this process's hash chain in the provider hash
1451 * table. Don't sweat it if we can't find the process.
1452 */
1453 mutex_enter(&pidlock);
1454 if ((p = prfind(pid)) == NULL) {
1455 mutex_exit(&pidlock);
1456 return;
1457 }
1458
1459 mutex_enter(&p->p_lock);
1460 mutex_exit(&pidlock);
1461
1462 p->p_dtrace_probes--;
1463 mutex_exit(&p->p_lock);
1464 }
1465
1466 static void
1467 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1468 {
1469 fasttrap_provider_t *fp;
1470 fasttrap_bucket_t *bucket;
1471 dtrace_provider_id_t provid;
1472
1473 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1474
1475 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1476 mutex_enter(&bucket->ftb_mtx);
1477
1478 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1479 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1480 !fp->ftp_retired)
1481 break;
1482 }
1483
1484 if (fp == NULL) {
1485 mutex_exit(&bucket->ftb_mtx);
1486 return;
1487 }
1488
1489 mutex_enter(&fp->ftp_mtx);
1490 ASSERT(!mprov || fp->ftp_mcount > 0);
1491 if (mprov && --fp->ftp_mcount != 0) {
1492 mutex_exit(&fp->ftp_mtx);
1493 mutex_exit(&bucket->ftb_mtx);
1494 return;
1495 }
1496
1497 /*
1498 * Mark the provider to be removed in our post-processing step, mark it
1499 * retired, and drop the active count on its proc. Marking it indicates
1500 * that we should try to remove it; setting the retired flag indicates
1501 * that we're done with this provider; dropping the active the proc
1502 * releases our hold, and when this reaches zero (as it will during
1503 * exit or exec) the proc and associated providers become defunct.
1504 *
1505 * We obviously need to take the bucket lock before the provider lock
1506 * to perform the lookup, but we need to drop the provider lock
1507 * before calling into the DTrace framework since we acquire the
1508 * provider lock in callbacks invoked from the DTrace framework. The
1509 * bucket lock therefore protects the integrity of the provider hash
1510 * table.
1511 */
1512 atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1513 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1514
1515 fp->ftp_retired = 1;
1516 fp->ftp_marked = 1;
1517 provid = fp->ftp_provid;
1518 mutex_exit(&fp->ftp_mtx);
1519
1520 /*
1521 * We don't have to worry about invalidating the same provider twice
1522 * since fasttrap_provider_lookup() will ignore provider that have
1523 * been marked as retired.
1524 */
1525 dtrace_invalidate(provid);
1526
1527 mutex_exit(&bucket->ftb_mtx);
1528
1529 fasttrap_pid_cleanup();
1530 }
1531
1532 static int
1533 fasttrap_uint32_cmp(const void *ap, const void *bp)
1534 {
1535 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1536 }
1537
1538 static int
1539 fasttrap_uint64_cmp(const void *ap, const void *bp)
1540 {
1541 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1542 }
1543
1544 static int
1545 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1546 {
1547 fasttrap_provider_t *provider;
1548 fasttrap_probe_t *pp;
1549 fasttrap_tracepoint_t *tp;
1550 char *name;
1551 int i, aframes, whack;
1552
1553 /*
1554 * There needs to be at least one desired trace point.
1555 */
1556 if (pdata->ftps_noffs == 0)
1557 return (EINVAL);
1558
1559 switch (pdata->ftps_type) {
1560 case DTFTP_ENTRY:
1561 name = "entry";
1562 aframes = FASTTRAP_ENTRY_AFRAMES;
1563 break;
1564 case DTFTP_RETURN:
1565 name = "return";
1566 aframes = FASTTRAP_RETURN_AFRAMES;
1567 break;
1568 case DTFTP_OFFSETS:
1569 name = NULL;
1570 break;
1571 default:
1572 return (EINVAL);
1573 }
1574
1575 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1576 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1577 return (ESRCH);
1578
1579 /*
1580 * Increment this reference count to indicate that a consumer is
1581 * actively adding a new probe associated with this provider. This
1582 * prevents the provider from being deleted -- we'll need to check
1583 * for pending deletions when we drop this reference count.
1584 */
1585 provider->ftp_ccount++;
1586 mutex_exit(&provider->ftp_mtx);
1587
1588 /*
1589 * Grab the creation lock to ensure consistency between calls to
1590 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1591 * other threads creating probes. We must drop the provider lock
1592 * before taking this lock to avoid a three-way deadlock with the
1593 * DTrace framework.
1594 */
1595 mutex_enter(&provider->ftp_cmtx);
1596
1597 if (name == NULL) {
1598 for (i = 0; i < pdata->ftps_noffs; i++) {
1599 char name_str[17];
1600
1601 (void) sprintf(name_str, "%llx",
1602 (unsigned long long)pdata->ftps_offs[i]);
1603
1604 if (dtrace_probe_lookup(provider->ftp_provid,
1605 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1606 continue;
1607
1608 atomic_inc_32(&fasttrap_total);
1609
1610 if (fasttrap_total > fasttrap_max) {
1611 atomic_dec_32(&fasttrap_total);
1612 goto no_mem;
1613 }
1614
1615 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1616
1617 pp->ftp_prov = provider;
1618 pp->ftp_faddr = pdata->ftps_pc;
1619 pp->ftp_fsize = pdata->ftps_size;
1620 pp->ftp_pid = pdata->ftps_pid;
1621 pp->ftp_ntps = 1;
1622
1623 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1624 KM_SLEEP);
1625
1626 tp->ftt_proc = provider->ftp_proc;
1627 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1628 tp->ftt_pid = pdata->ftps_pid;
1629
1630 pp->ftp_tps[0].fit_tp = tp;
1631 pp->ftp_tps[0].fit_id.fti_probe = pp;
1632 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1633
1634 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1635 pdata->ftps_mod, pdata->ftps_func, name_str,
1636 FASTTRAP_OFFSET_AFRAMES, pp);
1637 }
1638
1639 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1640 pdata->ftps_func, name) == 0) {
1641 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1642
1643 if (fasttrap_total > fasttrap_max) {
1644 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1645 goto no_mem;
1646 }
1647
1648 /*
1649 * Make sure all tracepoint program counter values are unique.
1650 * We later assume that each probe has exactly one tracepoint
1651 * for a given pc.
1652 */
1653 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1654 sizeof (uint64_t), fasttrap_uint64_cmp);
1655 for (i = 1; i < pdata->ftps_noffs; i++) {
1656 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1657 continue;
1658
1659 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1660 goto no_mem;
1661 }
1662
1663 ASSERT(pdata->ftps_noffs > 0);
1664 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1665 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1666
1667 pp->ftp_prov = provider;
1668 pp->ftp_faddr = pdata->ftps_pc;
1669 pp->ftp_fsize = pdata->ftps_size;
1670 pp->ftp_pid = pdata->ftps_pid;
1671 pp->ftp_ntps = pdata->ftps_noffs;
1672
1673 for (i = 0; i < pdata->ftps_noffs; i++) {
1674 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1675 KM_SLEEP);
1676
1677 tp->ftt_proc = provider->ftp_proc;
1678 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1679 tp->ftt_pid = pdata->ftps_pid;
1680
1681 pp->ftp_tps[i].fit_tp = tp;
1682 pp->ftp_tps[i].fit_id.fti_probe = pp;
1683 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1684 }
1685
1686 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1687 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1688 }
1689
1690 mutex_exit(&provider->ftp_cmtx);
1691
1692 /*
1693 * We know that the provider is still valid since we incremented the
1694 * creation reference count. If someone tried to clean up this provider
1695 * while we were using it (e.g. because the process called exec(2) or
1696 * exit(2)), take note of that and try to clean it up now.
1697 */
1698 mutex_enter(&provider->ftp_mtx);
1699 provider->ftp_ccount--;
1700 whack = provider->ftp_retired;
1701 mutex_exit(&provider->ftp_mtx);
1702
1703 if (whack)
1704 fasttrap_pid_cleanup();
1705
1706 return (0);
1707
1708 no_mem:
1709 /*
1710 * If we've exhausted the allowable resources, we'll try to remove
1711 * this provider to free some up. This is to cover the case where
1712 * the user has accidentally created many more probes than was
1713 * intended (e.g. pid123:::).
1714 */
1715 mutex_exit(&provider->ftp_cmtx);
1716 mutex_enter(&provider->ftp_mtx);
1717 provider->ftp_ccount--;
1718 provider->ftp_marked = 1;
1719 mutex_exit(&provider->ftp_mtx);
1720
1721 fasttrap_pid_cleanup();
1722
1723 return (ENOMEM);
1724 }
1725
1726 /*ARGSUSED*/
1727 static void *
1728 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1729 {
1730 fasttrap_provider_t *provider;
1731
1732 /*
1733 * A 32-bit unsigned integer (like a pid for example) can be
1734 * expressed in 10 or fewer decimal digits. Make sure that we'll
1735 * have enough space for the provider name.
1736 */
1737 if (strlen(dhpv->dthpv_provname) + 10 >=
1738 sizeof (provider->ftp_name)) {
1739 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1740 "name too long to accomodate pid", dhpv->dthpv_provname);
1741 return (NULL);
1742 }
1743
1744 /*
1745 * Don't let folks spoof the true pid provider.
1746 */
1747 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1748 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1749 "%s is an invalid name", dhpv->dthpv_provname,
1750 FASTTRAP_PID_NAME);
1751 return (NULL);
1752 }
1753
1754 /*
1755 * The highest stability class that fasttrap supports is ISA; cap
1756 * the stability of the new provider accordingly.
1757 */
1758 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1759 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1760 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1761 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1762 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1763 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1764 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1765 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1766 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1767 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1768
1769 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1770 &dhpv->dthpv_pattr)) == NULL) {
1771 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1772 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1773 return (NULL);
1774 }
1775
1776 /*
1777 * Up the meta provider count so this provider isn't removed until
1778 * the meta provider has been told to remove it.
1779 */
1780 provider->ftp_mcount++;
1781
1782 mutex_exit(&provider->ftp_mtx);
1783
1784 return (provider);
1785 }
1786
1787 /*
1788 * We know a few things about our context here: we know that the probe being
1789 * created doesn't already exist (DTrace won't load DOF at the same address
1790 * twice, even if explicitly told to do so) and we know that we are
1791 * single-threaded with respect to the meta provider machinery. Knowing that
1792 * this is a new probe and that there is no way for us to race with another
1793 * operation on this provider allows us an important optimization: we need not
1794 * lookup a probe before adding it. Saving this lookup is important because
1795 * this code is in the fork path for processes with USDT probes, and lookups
1796 * here are potentially very expensive because of long hash conflicts on
1797 * module, function and name (DTrace doesn't hash on provider name).
1798 */
1799 /*ARGSUSED*/
1800 static void
1801 fasttrap_meta_create_probe(void *arg, void *parg,
1802 dtrace_helper_probedesc_t *dhpb)
1803 {
1804 fasttrap_provider_t *provider = parg;
1805 fasttrap_probe_t *pp;
1806 fasttrap_tracepoint_t *tp;
1807 int i, j;
1808 uint32_t ntps;
1809
1810 /*
1811 * Since the meta provider count is non-zero we don't have to worry
1812 * about this provider disappearing.
1813 */
1814 ASSERT(provider->ftp_mcount > 0);
1815
1816 /*
1817 * The offsets must be unique.
1818 */
1819 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1820 fasttrap_uint32_cmp);
1821 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1822 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1823 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1824 return;
1825 }
1826
1827 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1828 fasttrap_uint32_cmp);
1829 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1830 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1831 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1832 return;
1833 }
1834
1835 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1836 ASSERT(ntps > 0);
1837
1838 atomic_add_32(&fasttrap_total, ntps);
1839
1840 if (fasttrap_total > fasttrap_max) {
1841 atomic_add_32(&fasttrap_total, -ntps);
1842 return;
1843 }
1844
1845 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1846
1847 pp->ftp_prov = provider;
1848 pp->ftp_pid = provider->ftp_pid;
1849 pp->ftp_ntps = ntps;
1850 pp->ftp_nargs = dhpb->dthpb_xargc;
1851 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1852 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1853
1854 /*
1855 * First create a tracepoint for each actual point of interest.
1856 */
1857 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1858 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1859
1860 tp->ftt_proc = provider->ftp_proc;
1861 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1862 tp->ftt_pid = provider->ftp_pid;
1863
1864 pp->ftp_tps[i].fit_tp = tp;
1865 pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 #ifdef __sparc
1867 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1868 #else
1869 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1870 #endif
1871 }
1872
1873 /*
1874 * Then create a tracepoint for each is-enabled point.
1875 */
1876 for (j = 0; i < ntps; i++, j++) {
1877 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1878
1879 tp->ftt_proc = provider->ftp_proc;
1880 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1881 tp->ftt_pid = provider->ftp_pid;
1882
1883 pp->ftp_tps[i].fit_tp = tp;
1884 pp->ftp_tps[i].fit_id.fti_probe = pp;
1885 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1886 }
1887
1888 /*
1889 * If the arguments are shuffled around we set the argument remapping
1890 * table. Later, when the probe fires, we only remap the arguments
1891 * if the table is non-NULL.
1892 */
1893 for (i = 0; i < dhpb->dthpb_xargc; i++) {
1894 if (dhpb->dthpb_args[i] != i) {
1895 pp->ftp_argmap = dhpb->dthpb_args;
1896 break;
1897 }
1898 }
1899
1900 /*
1901 * The probe is fully constructed -- register it with DTrace.
1902 */
1903 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1904 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1905 }
1906
1907 /*ARGSUSED*/
1908 static void
1909 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1910 {
1911 /*
1912 * Clean up the USDT provider. There may be active consumers of the
1913 * provider busy adding probes, no damage will actually befall the
1914 * provider until that count has dropped to zero. This just puts
1915 * the provider on death row.
1916 */
1917 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1918 }
1919
1920 static dtrace_mops_t fasttrap_mops = {
1921 fasttrap_meta_create_probe,
1922 fasttrap_meta_provide,
1923 fasttrap_meta_remove
1924 };
1925
1926 /*ARGSUSED*/
1927 static int
1928 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1929 {
1930 return (0);
1931 }
1932
1933 /*ARGSUSED*/
1934 static int
1935 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1936 {
1937 if (!dtrace_attached())
1938 return (EAGAIN);
1939
1940 if (cmd == FASTTRAPIOC_MAKEPROBE) {
1941 fasttrap_probe_spec_t *uprobe = (void *)arg;
1942 fasttrap_probe_spec_t *probe;
1943 uint64_t noffs;
1944 size_t size;
1945 int ret, err;
1946
1947 if (copyin(&uprobe->ftps_noffs, &noffs,
1948 sizeof (uprobe->ftps_noffs)))
1949 return (EFAULT);
1950
1951 /*
1952 * Probes must have at least one tracepoint.
1953 */
1954 if (noffs == 0)
1955 return (EINVAL);
1956
1957 size = sizeof (fasttrap_probe_spec_t) +
1958 sizeof (probe->ftps_offs[0]) * (noffs - 1);
1959
1960 if (size > 1024 * 1024)
1961 return (ENOMEM);
1962
1963 probe = kmem_alloc(size, KM_SLEEP);
1964
1965 if (copyin(uprobe, probe, size) != 0 ||
1966 probe->ftps_noffs != noffs) {
1967 kmem_free(probe, size);
1968 return (EFAULT);
1969 }
1970
1971 /*
1972 * Verify that the function and module strings contain no
1973 * funny characters.
1974 */
1975 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1976 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977 ret = EINVAL;
1978 goto err;
1979 }
1980
1981 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1982 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1983 ret = EINVAL;
1984 goto err;
1985 }
1986
1987 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1988 proc_t *p;
1989 pid_t pid = probe->ftps_pid;
1990
1991 mutex_enter(&pidlock);
1992 /*
1993 * Report an error if the process doesn't exist
1994 * or is actively being birthed.
1995 */
1996 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1997 mutex_exit(&pidlock);
1998 return (ESRCH);
1999 }
2000 mutex_enter(&p->p_lock);
2001 mutex_exit(&pidlock);
2002
2003 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2004 VREAD | VWRITE)) != 0) {
2005 mutex_exit(&p->p_lock);
2006 return (ret);
2007 }
2008
2009 mutex_exit(&p->p_lock);
2010 }
2011
2012 ret = fasttrap_add_probe(probe);
2013 err:
2014 kmem_free(probe, size);
2015
2016 return (ret);
2017
2018 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2019 fasttrap_instr_query_t instr;
2020 fasttrap_tracepoint_t *tp;
2021 uint_t index;
2022 int ret;
2023
2024 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2025 return (EFAULT);
2026
2027 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2028 proc_t *p;
2029 pid_t pid = instr.ftiq_pid;
2030
2031 mutex_enter(&pidlock);
2032 /*
2033 * Report an error if the process doesn't exist
2034 * or is actively being birthed.
2035 */
2036 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2037 mutex_exit(&pidlock);
2038 return (ESRCH);
2039 }
2040 mutex_enter(&p->p_lock);
2041 mutex_exit(&pidlock);
2042
2043 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2044 VREAD)) != 0) {
2045 mutex_exit(&p->p_lock);
2046 return (ret);
2047 }
2048
2049 mutex_exit(&p->p_lock);
2050 }
2051
2052 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2053
2054 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2055 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2056 while (tp != NULL) {
2057 if (instr.ftiq_pid == tp->ftt_pid &&
2058 instr.ftiq_pc == tp->ftt_pc &&
2059 tp->ftt_proc->ftpc_acount != 0)
2060 break;
2061
2062 tp = tp->ftt_next;
2063 }
2064
2065 if (tp == NULL) {
2066 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067 return (ENOENT);
2068 }
2069
2070 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2071 sizeof (instr.ftiq_instr));
2072 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2073
2074 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2075 return (EFAULT);
2076
2077 return (0);
2078 }
2079
2080 return (EINVAL);
2081 }
2082
2083 static struct cb_ops fasttrap_cb_ops = {
2084 fasttrap_open, /* open */
2085 nodev, /* close */
2086 nulldev, /* strategy */
2087 nulldev, /* print */
2088 nodev, /* dump */
2089 nodev, /* read */
2090 nodev, /* write */
2091 fasttrap_ioctl, /* ioctl */
2092 nodev, /* devmap */
2093 nodev, /* mmap */
2094 nodev, /* segmap */
2095 nochpoll, /* poll */
2096 ddi_prop_op, /* cb_prop_op */
2097 0, /* streamtab */
2098 D_NEW | D_MP /* Driver compatibility flag */
2099 };
2100
2101 /*ARGSUSED*/
2102 static int
2103 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2104 {
2105 int error;
2106
2107 switch (infocmd) {
2108 case DDI_INFO_DEVT2DEVINFO:
2109 *result = (void *)fasttrap_devi;
2110 error = DDI_SUCCESS;
2111 break;
2112 case DDI_INFO_DEVT2INSTANCE:
2113 *result = (void *)0;
2114 error = DDI_SUCCESS;
2115 break;
2116 default:
2117 error = DDI_FAILURE;
2118 }
2119 return (error);
2120 }
2121
2122 static int
2123 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2124 {
2125 ulong_t nent;
2126
2127 switch (cmd) {
2128 case DDI_ATTACH:
2129 break;
2130 case DDI_RESUME:
2131 return (DDI_SUCCESS);
2132 default:
2133 return (DDI_FAILURE);
2134 }
2135
2136 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2137 DDI_PSEUDO, NULL) == DDI_FAILURE) {
2138 ddi_remove_minor_node(devi, NULL);
2139 return (DDI_FAILURE);
2140 }
2141
2142 ddi_report_dev(devi);
2143 fasttrap_devi = devi;
2144
2145 /*
2146 * Install our hooks into fork(2), exec(2), and exit(2).
2147 */
2148 dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2149 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2150 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2151
2152 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2153 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2154 fasttrap_total = 0;
2155
2156 /*
2157 * Conjure up the tracepoints hashtable...
2158 */
2159 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2160 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2161
2162 if (nent == 0 || nent > 0x1000000)
2163 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2164
2165 if (ISP2(nent))
2166 fasttrap_tpoints.fth_nent = nent;
2167 else
2168 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2169 ASSERT(fasttrap_tpoints.fth_nent > 0);
2170 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2171 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2172 sizeof (fasttrap_bucket_t), KM_SLEEP);
2173
2174 /*
2175 * ... and the providers hash table...
2176 */
2177 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2178 if (ISP2(nent))
2179 fasttrap_provs.fth_nent = nent;
2180 else
2181 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2182 ASSERT(fasttrap_provs.fth_nent > 0);
2183 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2184 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2185 sizeof (fasttrap_bucket_t), KM_SLEEP);
2186
2187 /*
2188 * ... and the procs hash table.
2189 */
2190 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2191 if (ISP2(nent))
2192 fasttrap_procs.fth_nent = nent;
2193 else
2194 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2195 ASSERT(fasttrap_procs.fth_nent > 0);
2196 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2197 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2198 sizeof (fasttrap_bucket_t), KM_SLEEP);
2199
2200 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2201 &fasttrap_meta_id);
2202
2203 return (DDI_SUCCESS);
2204 }
2205
2206 static int
2207 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2208 {
2209 int i, fail = 0;
2210 timeout_id_t tmp;
2211
2212 switch (cmd) {
2213 case DDI_DETACH:
2214 break;
2215 case DDI_SUSPEND:
2216 return (DDI_SUCCESS);
2217 default:
2218 return (DDI_FAILURE);
2219 }
2220
2221 /*
2222 * Unregister the meta-provider to make sure no new fasttrap-
2223 * managed providers come along while we're trying to close up
2224 * shop. If we fail to detach, we'll need to re-register as a
2225 * meta-provider. We can fail to unregister as a meta-provider
2226 * if providers we manage still exist.
2227 */
2228 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2229 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2230 return (DDI_FAILURE);
2231
2232 /*
2233 * Prevent any new timeouts from running by setting fasttrap_timeout
2234 * to a non-zero value, and wait for the current timeout to complete.
2235 */
2236 mutex_enter(&fasttrap_cleanup_mtx);
2237 fasttrap_cleanup_work = 0;
2238
2239 while (fasttrap_timeout != (timeout_id_t)1) {
2240 tmp = fasttrap_timeout;
2241 fasttrap_timeout = (timeout_id_t)1;
2242
2243 if (tmp != 0) {
2244 mutex_exit(&fasttrap_cleanup_mtx);
2245 (void) untimeout(tmp);
2246 mutex_enter(&fasttrap_cleanup_mtx);
2247 }
2248 }
2249
2250 fasttrap_cleanup_work = 0;
2251 mutex_exit(&fasttrap_cleanup_mtx);
2252
2253 /*
2254 * Iterate over all of our providers. If there's still a process
2255 * that corresponds to that pid, fail to detach.
2256 */
2257 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2258 fasttrap_provider_t **fpp, *fp;
2259 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2260
2261 mutex_enter(&bucket->ftb_mtx);
2262 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2263 while ((fp = *fpp) != NULL) {
2264 /*
2265 * Acquire and release the lock as a simple way of
2266 * waiting for any other consumer to finish with
2267 * this provider. A thread must first acquire the
2268 * bucket lock so there's no chance of another thread
2269 * blocking on the provider's lock.
2270 */
2271 mutex_enter(&fp->ftp_mtx);
2272 mutex_exit(&fp->ftp_mtx);
2273
2274 if (dtrace_unregister(fp->ftp_provid) != 0) {
2275 fail = 1;
2276 fpp = &fp->ftp_next;
2277 } else {
2278 *fpp = fp->ftp_next;
2279 fasttrap_provider_free(fp);
2280 }
2281 }
2282
2283 mutex_exit(&bucket->ftb_mtx);
2284 }
2285
2286 if (fail) {
2287 uint_t work;
2288 /*
2289 * If we're failing to detach, we need to unblock timeouts
2290 * and start a new timeout if any work has accumulated while
2291 * we've been unsuccessfully trying to detach.
2292 */
2293 mutex_enter(&fasttrap_cleanup_mtx);
2294 fasttrap_timeout = 0;
2295 work = fasttrap_cleanup_work;
2296 mutex_exit(&fasttrap_cleanup_mtx);
2297
2298 if (work)
2299 fasttrap_pid_cleanup();
2300
2301 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2302 &fasttrap_meta_id);
2303
2304 return (DDI_FAILURE);
2305 }
2306
2307 #ifdef DEBUG
2308 mutex_enter(&fasttrap_count_mtx);
2309 ASSERT(fasttrap_pid_count == 0);
2310 mutex_exit(&fasttrap_count_mtx);
2311 #endif
2312
2313 kmem_free(fasttrap_tpoints.fth_table,
2314 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2315 fasttrap_tpoints.fth_nent = 0;
2316
2317 kmem_free(fasttrap_provs.fth_table,
2318 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2319 fasttrap_provs.fth_nent = 0;
2320
2321 kmem_free(fasttrap_procs.fth_table,
2322 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2323 fasttrap_procs.fth_nent = 0;
2324
2325 /*
2326 * We know there are no tracepoints in any process anywhere in
2327 * the system so there is no process which has its p_dtrace_count
2328 * greater than zero, therefore we know that no thread can actively
2329 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2330 * and fasttrap_exec() and fasttrap_exit().
2331 */
2332 ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2333 dtrace_fasttrap_fork_ptr = NULL;
2334
2335 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2336 dtrace_fasttrap_exec_ptr = NULL;
2337
2338 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2339 dtrace_fasttrap_exit_ptr = NULL;
2340
2341 ddi_remove_minor_node(devi, NULL);
2342
2343 return (DDI_SUCCESS);
2344 }
2345
2346 static struct dev_ops fasttrap_ops = {
2347 DEVO_REV, /* devo_rev */
2348 0, /* refcnt */
2349 fasttrap_info, /* get_dev_info */
2350 nulldev, /* identify */
2351 nulldev, /* probe */
2352 fasttrap_attach, /* attach */
2353 fasttrap_detach, /* detach */
2354 nodev, /* reset */
2355 &fasttrap_cb_ops, /* driver operations */
2356 NULL, /* bus operations */
2357 nodev, /* dev power */
2358 ddi_quiesce_not_needed, /* quiesce */
2359 };
2360
2361 /*
2362 * Module linkage information for the kernel.
2363 */
2364 static struct modldrv modldrv = {
2365 &mod_driverops, /* module type (this is a pseudo driver) */
2366 "Fasttrap Tracing", /* name of module */
2367 &fasttrap_ops, /* driver ops */
2368 };
2369
2370 static struct modlinkage modlinkage = {
2371 MODREV_1,
2372 (void *)&modldrv,
2373 NULL
2374 };
2375
2376 int
2377 _init(void)
2378 {
2379 return (mod_install(&modlinkage));
2380 }
2381
2382 int
2383 _info(struct modinfo *modinfop)
2384 {
2385 return (mod_info(&modlinkage, modinfop));
2386 }
2387
2388 int
2389 _fini(void)
2390 {
2391 return (mod_remove(&modlinkage));
2392 }