1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * Copyright (c) 2015, Joyent, Inc. All rights reserved.
  29  */
  30 
  31 #include <sys/atomic.h>
  32 #include <sys/errno.h>
  33 #include <sys/stat.h>
  34 #include <sys/modctl.h>
  35 #include <sys/conf.h>
  36 #include <sys/systm.h>
  37 #include <sys/ddi.h>
  38 #include <sys/sunddi.h>
  39 #include <sys/cpuvar.h>
  40 #include <sys/kmem.h>
  41 #include <sys/strsubr.h>
  42 #include <sys/fasttrap.h>
  43 #include <sys/fasttrap_impl.h>
  44 #include <sys/fasttrap_isa.h>
  45 #include <sys/dtrace.h>
  46 #include <sys/dtrace_impl.h>
  47 #include <sys/sysmacros.h>
  48 #include <sys/proc.h>
  49 #include <sys/priv.h>
  50 #include <sys/policy.h>
  51 #include <util/qsort.h>
  52 
  53 /*
  54  * User-Land Trap-Based Tracing
  55  * ----------------------------
  56  *
  57  * The fasttrap provider allows DTrace consumers to instrument any user-level
  58  * instruction to gather data; this includes probes with semantic
  59  * signifigance like entry and return as well as simple offsets into the
  60  * function. While the specific techniques used are very ISA specific, the
  61  * methodology is generalizable to any architecture.
  62  *
  63  *
  64  * The General Methodology
  65  * -----------------------
  66  *
  67  * With the primary goal of tracing every user-land instruction and the
  68  * limitation that we can't trust user space so don't want to rely on much
  69  * information there, we begin by replacing the instructions we want to trace
  70  * with trap instructions. Each instruction we overwrite is saved into a hash
  71  * table keyed by process ID and pc address. When we enter the kernel due to
  72  * this trap instruction, we need the effects of the replaced instruction to
  73  * appear to have occurred before we proceed with the user thread's
  74  * execution.
  75  *
  76  * Each user level thread is represented by a ulwp_t structure which is
  77  * always easily accessible through a register. The most basic way to produce
  78  * the effects of the instruction we replaced is to copy that instruction out
  79  * to a bit of scratch space reserved in the user thread's ulwp_t structure
  80  * (a sort of kernel-private thread local storage), set the PC to that
  81  * scratch space and single step. When we reenter the kernel after single
  82  * stepping the instruction we must then adjust the PC to point to what would
  83  * normally be the next instruction. Of course, special care must be taken
  84  * for branches and jumps, but these represent such a small fraction of any
  85  * instruction set that writing the code to emulate these in the kernel is
  86  * not too difficult.
  87  *
  88  * Return probes may require several tracepoints to trace every return site,
  89  * and, conversely, each tracepoint may activate several probes (the entry
  90  * and offset 0 probes, for example). To solve this muliplexing problem,
  91  * tracepoints contain lists of probes to activate and probes contain lists
  92  * of tracepoints to enable. If a probe is activated, it adds its ID to
  93  * existing tracepoints or creates new ones as necessary.
  94  *
  95  * Most probes are activated _before_ the instruction is executed, but return
  96  * probes are activated _after_ the effects of the last instruction of the
  97  * function are visible. Return probes must be fired _after_ we have
  98  * single-stepped the instruction whereas all other probes are fired
  99  * beforehand.
 100  *
 101  *
 102  * Lock Ordering
 103  * -------------
 104  *
 105  * The lock ordering below -- both internally and with respect to the DTrace
 106  * framework -- is a little tricky and bears some explanation. Each provider
 107  * has a lock (ftp_mtx) that protects its members including reference counts
 108  * for enabled probes (ftp_rcount), consumers actively creating probes
 109  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
 110  * from being freed. A provider is looked up by taking the bucket lock for the
 111  * provider hash table, and is returned with its lock held. The provider lock
 112  * may be taken in functions invoked by the DTrace framework, but may not be
 113  * held while calling functions in the DTrace framework.
 114  *
 115  * To ensure consistency over multiple calls to the DTrace framework, the
 116  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
 117  * not be taken when holding the provider lock as that would create a cyclic
 118  * lock ordering. In situations where one would naturally take the provider
 119  * lock and then the creation lock, we instead up a reference count to prevent
 120  * the provider from disappearing, drop the provider lock, and acquire the
 121  * creation lock.
 122  *
 123  * Briefly:
 124  *      bucket lock before provider lock
 125  *      DTrace before provider lock
 126  *      creation lock before DTrace
 127  *      never hold the provider lock and creation lock simultaneously
 128  */
 129 
 130 static dev_info_t *fasttrap_devi;
 131 static dtrace_meta_provider_id_t fasttrap_meta_id;
 132 
 133 static timeout_id_t fasttrap_timeout;
 134 static kmutex_t fasttrap_cleanup_mtx;
 135 static uint_t fasttrap_cleanup_work;
 136 
 137 /*
 138  * Generation count on modifications to the global tracepoint lookup table.
 139  */
 140 static volatile uint64_t fasttrap_mod_gen;
 141 
 142 /*
 143  * When the fasttrap provider is loaded, fasttrap_max is set to either
 144  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
 145  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
 146  * incremented by the number of tracepoints that may be associated with that
 147  * probe; fasttrap_total is capped at fasttrap_max.
 148  */
 149 #define FASTTRAP_MAX_DEFAULT            250000
 150 static uint32_t fasttrap_max;
 151 static uint32_t fasttrap_total;
 152 
 153 
 154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE   0x4000
 155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
 156 #define FASTTRAP_PROCS_DEFAULT_SIZE     0x100
 157 
 158 #define FASTTRAP_PID_NAME               "pid"
 159 
 160 fasttrap_hash_t                 fasttrap_tpoints;
 161 static fasttrap_hash_t          fasttrap_provs;
 162 static fasttrap_hash_t          fasttrap_procs;
 163 
 164 static uint64_t                 fasttrap_pid_count;     /* pid ref count */
 165 static kmutex_t                 fasttrap_count_mtx;     /* lock on ref count */
 166 
 167 #define FASTTRAP_ENABLE_FAIL    1
 168 #define FASTTRAP_ENABLE_PARTIAL 2
 169 
 170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
 171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
 172 
 173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
 174     const dtrace_pattr_t *);
 175 static void fasttrap_provider_retire(pid_t, const char *, int);
 176 static void fasttrap_provider_free(fasttrap_provider_t *);
 177 
 178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
 179 static void fasttrap_proc_release(fasttrap_proc_t *);
 180 
 181 #define FASTTRAP_PROVS_INDEX(pid, name) \
 182         ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
 183 
 184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
 185 
 186 static int
 187 fasttrap_highbit(ulong_t i)
 188 {
 189         int h = 1;
 190 
 191         if (i == 0)
 192                 return (0);
 193 #ifdef _LP64
 194         if (i & 0xffffffff00000000ul) {
 195                 h += 32; i >>= 32;
 196         }
 197 #endif
 198         if (i & 0xffff0000) {
 199                 h += 16; i >>= 16;
 200         }
 201         if (i & 0xff00) {
 202                 h += 8; i >>= 8;
 203         }
 204         if (i & 0xf0) {
 205                 h += 4; i >>= 4;
 206         }
 207         if (i & 0xc) {
 208                 h += 2; i >>= 2;
 209         }
 210         if (i & 0x2) {
 211                 h += 1;
 212         }
 213         return (h);
 214 }
 215 
 216 static uint_t
 217 fasttrap_hash_str(const char *p)
 218 {
 219         unsigned int g;
 220         uint_t hval = 0;
 221 
 222         while (*p) {
 223                 hval = (hval << 4) + *p++;
 224                 if ((g = (hval & 0xf0000000)) != 0)
 225                         hval ^= g >> 24;
 226                 hval &= ~g;
 227         }
 228         return (hval);
 229 }
 230 
 231 void
 232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
 233 {
 234         sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
 235 
 236         sqp->sq_info.si_signo = SIGTRAP;
 237         sqp->sq_info.si_code = TRAP_DTRACE;
 238         sqp->sq_info.si_addr = (caddr_t)pc;
 239 
 240         mutex_enter(&p->p_lock);
 241         sigaddqa(p, t, sqp);
 242         mutex_exit(&p->p_lock);
 243 
 244         if (t != NULL)
 245                 aston(t);
 246 }
 247 
 248 /*
 249  * This function ensures that no threads are actively using the memory
 250  * associated with probes that were formerly live.
 251  */
 252 static void
 253 fasttrap_mod_barrier(uint64_t gen)
 254 {
 255         int i;
 256 
 257         if (gen < fasttrap_mod_gen)
 258                 return;
 259 
 260         fasttrap_mod_gen++;
 261 
 262         for (i = 0; i < NCPU; i++) {
 263                 mutex_enter(&cpu_core[i].cpuc_pid_lock);
 264                 mutex_exit(&cpu_core[i].cpuc_pid_lock);
 265         }
 266 }
 267 
 268 /*
 269  * This is the timeout's callback for cleaning up the providers and their
 270  * probes.
 271  */
 272 /*ARGSUSED*/
 273 static void
 274 fasttrap_pid_cleanup_cb(void *data)
 275 {
 276         fasttrap_provider_t **fpp, *fp;
 277         fasttrap_bucket_t *bucket;
 278         dtrace_provider_id_t provid;
 279         int i, later, rval;
 280 
 281         static volatile int in = 0;
 282         ASSERT(in == 0);
 283         in = 1;
 284 
 285         mutex_enter(&fasttrap_cleanup_mtx);
 286         while (fasttrap_cleanup_work) {
 287                 fasttrap_cleanup_work = 0;
 288                 mutex_exit(&fasttrap_cleanup_mtx);
 289 
 290                 later = 0;
 291 
 292                 /*
 293                  * Iterate over all the providers trying to remove the marked
 294                  * ones. If a provider is marked but not retired, we just
 295                  * have to take a crack at removing it -- it's no big deal if
 296                  * we can't.
 297                  */
 298                 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
 299                         bucket = &fasttrap_provs.fth_table[i];
 300                         mutex_enter(&bucket->ftb_mtx);
 301                         fpp = (fasttrap_provider_t **)&bucket->ftb_data;
 302 
 303                         while ((fp = *fpp) != NULL) {
 304                                 if (!fp->ftp_marked) {
 305                                         fpp = &fp->ftp_next;
 306                                         continue;
 307                                 }
 308 
 309                                 mutex_enter(&fp->ftp_mtx);
 310 
 311                                 /*
 312                                  * If this provider has consumers actively
 313                                  * creating probes (ftp_ccount) or is a USDT
 314                                  * provider (ftp_mcount), we can't unregister
 315                                  * or even condense.
 316                                  */
 317                                 if (fp->ftp_ccount != 0 ||
 318                                     fp->ftp_mcount != 0) {
 319                                         mutex_exit(&fp->ftp_mtx);
 320                                         fp->ftp_marked = 0;
 321                                         continue;
 322                                 }
 323 
 324                                 if (!fp->ftp_retired || fp->ftp_rcount != 0)
 325                                         fp->ftp_marked = 0;
 326 
 327                                 mutex_exit(&fp->ftp_mtx);
 328 
 329                                 /*
 330                                  * If we successfully unregister this
 331                                  * provider we can remove it from the hash
 332                                  * chain and free the memory. If our attempt
 333                                  * to unregister fails and this is a retired
 334                                  * provider, increment our flag to try again
 335                                  * pretty soon. If we've consumed more than
 336                                  * half of our total permitted number of
 337                                  * probes call dtrace_condense() to try to
 338                                  * clean out the unenabled probes.
 339                                  */
 340                                 provid = fp->ftp_provid;
 341                                 if ((rval = dtrace_unregister(provid)) != 0) {
 342                                         if (fasttrap_total > fasttrap_max / 2)
 343                                                 (void) dtrace_condense(provid);
 344 
 345                                         if (rval == EAGAIN)
 346                                                 fp->ftp_marked = 1;
 347 
 348                                         later += fp->ftp_marked;
 349                                         fpp = &fp->ftp_next;
 350                                 } else {
 351                                         *fpp = fp->ftp_next;
 352                                         fasttrap_provider_free(fp);
 353                                 }
 354                         }
 355                         mutex_exit(&bucket->ftb_mtx);
 356                 }
 357 
 358                 mutex_enter(&fasttrap_cleanup_mtx);
 359         }
 360 
 361         ASSERT(fasttrap_timeout != 0);
 362 
 363         /*
 364          * If we were unable to remove a retired provider, try again after
 365          * a second. This situation can occur in certain circumstances where
 366          * providers cannot be unregistered even though they have no probes
 367          * enabled because of an execution of dtrace -l or something similar.
 368          * If the timeout has been disabled (set to 1 because we're trying
 369          * to detach), we set fasttrap_cleanup_work to ensure that we'll
 370          * get a chance to do that work if and when the timeout is reenabled
 371          * (if detach fails).
 372          */
 373         if (later > 0) {
 374                 if (fasttrap_timeout != (timeout_id_t)1) {
 375                         fasttrap_timeout =
 376                             timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
 377                 }
 378 
 379                 fasttrap_cleanup_work = 1;
 380         } else {
 381                 fasttrap_timeout = 0;
 382         }
 383 
 384         mutex_exit(&fasttrap_cleanup_mtx);
 385         in = 0;
 386 }
 387 
 388 /*
 389  * Activates the asynchronous cleanup mechanism.
 390  */
 391 static void
 392 fasttrap_pid_cleanup(void)
 393 {
 394         mutex_enter(&fasttrap_cleanup_mtx);
 395         fasttrap_cleanup_work = 1;
 396         if (fasttrap_timeout == 0)
 397                 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
 398         mutex_exit(&fasttrap_cleanup_mtx);
 399 }
 400 
 401 /*
 402  * This is called from cfork() via dtrace_fasttrap_fork(). The child
 403  * process's address space is (roughly) a copy of the parent process's so
 404  * we have to remove all the instrumentation we had previously enabled in the
 405  * parent.
 406  */
 407 static void
 408 fasttrap_fork(proc_t *p, proc_t *cp)
 409 {
 410         pid_t ppid = p->p_pid;
 411         int i;
 412 
 413         ASSERT(curproc == p);
 414         ASSERT(p->p_proc_flag & P_PR_LOCK);
 415         ASSERT(p->p_dtrace_count > 0);
 416         ASSERT(cp->p_dtrace_count == 0);
 417 
 418         /*
 419          * This would be simpler and faster if we maintained per-process
 420          * hash tables of enabled tracepoints. It could, however, potentially
 421          * slow down execution of a tracepoint since we'd need to go
 422          * through two levels of indirection. In the future, we should
 423          * consider either maintaining per-process ancillary lists of
 424          * enabled tracepoints or hanging a pointer to a per-process hash
 425          * table of enabled tracepoints off the proc structure.
 426          */
 427 
 428         /*
 429          * We don't have to worry about the child process disappearing
 430          * because we're in fork().
 431          */
 432         mutex_enter(&cp->p_lock);
 433         sprlock_proc(cp);
 434         mutex_exit(&cp->p_lock);
 435 
 436         /*
 437          * Iterate over every tracepoint looking for ones that belong to the
 438          * parent process, and remove each from the child process.
 439          */
 440         for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
 441                 fasttrap_tracepoint_t *tp;
 442                 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
 443 
 444                 mutex_enter(&bucket->ftb_mtx);
 445                 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 446                         if (tp->ftt_pid == ppid &&
 447                             tp->ftt_proc->ftpc_acount != 0) {
 448                                 int ret = fasttrap_tracepoint_remove(cp, tp);
 449                                 ASSERT(ret == 0);
 450 
 451                                 /*
 452                                  * The count of active providers can only be
 453                                  * decremented (i.e. to zero) during exec,
 454                                  * exit, and removal of a meta provider so it
 455                                  * should be impossible to drop the count
 456                                  * mid-fork.
 457                                  */
 458                                 ASSERT(tp->ftt_proc->ftpc_acount != 0);
 459                         }
 460                 }
 461                 mutex_exit(&bucket->ftb_mtx);
 462         }
 463 
 464         mutex_enter(&cp->p_lock);
 465         sprunlock(cp);
 466 }
 467 
 468 /*
 469  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
 470  * is set on the proc structure to indicate that there is a pid provider
 471  * associated with this process.
 472  */
 473 static void
 474 fasttrap_exec_exit(proc_t *p)
 475 {
 476         ASSERT(p == curproc);
 477         ASSERT(MUTEX_HELD(&p->p_lock));
 478 
 479         mutex_exit(&p->p_lock);
 480 
 481         /*
 482          * We clean up the pid provider for this process here; user-land
 483          * static probes are handled by the meta-provider remove entry point.
 484          */
 485         fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
 486 
 487         mutex_enter(&p->p_lock);
 488 }
 489 
 490 
 491 /*ARGSUSED*/
 492 static void
 493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
 494 {
 495         /*
 496          * There are no "default" pid probes.
 497          */
 498 }
 499 
 500 static int
 501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
 502 {
 503         fasttrap_tracepoint_t *tp, *new_tp = NULL;
 504         fasttrap_bucket_t *bucket;
 505         fasttrap_id_t *id;
 506         pid_t pid;
 507         uintptr_t pc;
 508 
 509         ASSERT(index < probe->ftp_ntps);
 510 
 511         pid = probe->ftp_pid;
 512         pc = probe->ftp_tps[index].fit_tp->ftt_pc;
 513         id = &probe->ftp_tps[index].fit_id;
 514 
 515         ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
 516 
 517         ASSERT(!(p->p_flag & SVFORK));
 518 
 519         /*
 520          * Before we make any modifications, make sure we've imposed a barrier
 521          * on the generation in which this probe was last modified.
 522          */
 523         fasttrap_mod_barrier(probe->ftp_gen);
 524 
 525         bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 526 
 527         /*
 528          * If the tracepoint has already been enabled, just add our id to the
 529          * list of interested probes. This may be our second time through
 530          * this path in which case we'll have constructed the tracepoint we'd
 531          * like to install. If we can't find a match, and have an allocated
 532          * tracepoint ready to go, enable that one now.
 533          *
 534          * A tracepoint whose process is defunct is also considered defunct.
 535          */
 536 again:
 537         mutex_enter(&bucket->ftb_mtx);
 538         for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 539                 /*
 540                  * Note that it's safe to access the active count on the
 541                  * associated proc structure because we know that at least one
 542                  * provider (this one) will still be around throughout this
 543                  * operation.
 544                  */
 545                 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
 546                     tp->ftt_proc->ftpc_acount == 0)
 547                         continue;
 548 
 549                 /*
 550                  * Now that we've found a matching tracepoint, it would be
 551                  * a decent idea to confirm that the tracepoint is still
 552                  * enabled and the trap instruction hasn't been overwritten.
 553                  * Since this is a little hairy, we'll punt for now.
 554                  */
 555 
 556                 /*
 557                  * This can't be the first interested probe. We don't have
 558                  * to worry about another thread being in the midst of
 559                  * deleting this tracepoint (which would be the only valid
 560                  * reason for a tracepoint to have no interested probes)
 561                  * since we're holding P_PR_LOCK for this process.
 562                  */
 563                 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
 564 
 565                 switch (id->fti_ptype) {
 566                 case DTFTP_ENTRY:
 567                 case DTFTP_OFFSETS:
 568                 case DTFTP_IS_ENABLED:
 569                         id->fti_next = tp->ftt_ids;
 570                         membar_producer();
 571                         tp->ftt_ids = id;
 572                         membar_producer();
 573                         break;
 574 
 575                 case DTFTP_RETURN:
 576                 case DTFTP_POST_OFFSETS:
 577                         id->fti_next = tp->ftt_retids;
 578                         membar_producer();
 579                         tp->ftt_retids = id;
 580                         membar_producer();
 581                         break;
 582 
 583                 default:
 584                         ASSERT(0);
 585                 }
 586 
 587                 mutex_exit(&bucket->ftb_mtx);
 588 
 589                 if (new_tp != NULL) {
 590                         new_tp->ftt_ids = NULL;
 591                         new_tp->ftt_retids = NULL;
 592                 }
 593 
 594                 return (0);
 595         }
 596 
 597         /*
 598          * If we have a good tracepoint ready to go, install it now while
 599          * we have the lock held and no one can screw with us.
 600          */
 601         if (new_tp != NULL) {
 602                 int rc = 0;
 603 
 604                 new_tp->ftt_next = bucket->ftb_data;
 605                 membar_producer();
 606                 bucket->ftb_data = new_tp;
 607                 membar_producer();
 608                 mutex_exit(&bucket->ftb_mtx);
 609 
 610                 /*
 611                  * Activate the tracepoint in the ISA-specific manner.
 612                  * If this fails, we need to report the failure, but
 613                  * indicate that this tracepoint must still be disabled
 614                  * by calling fasttrap_tracepoint_disable().
 615                  */
 616                 if (fasttrap_tracepoint_install(p, new_tp) != 0)
 617                         rc = FASTTRAP_ENABLE_PARTIAL;
 618 
 619                 /*
 620                  * Increment the count of the number of tracepoints active in
 621                  * the victim process.
 622                  */
 623                 ASSERT(p->p_proc_flag & P_PR_LOCK);
 624                 p->p_dtrace_count++;
 625 
 626                 return (rc);
 627         }
 628 
 629         mutex_exit(&bucket->ftb_mtx);
 630 
 631         /*
 632          * Initialize the tracepoint that's been preallocated with the probe.
 633          */
 634         new_tp = probe->ftp_tps[index].fit_tp;
 635 
 636         ASSERT(new_tp->ftt_pid == pid);
 637         ASSERT(new_tp->ftt_pc == pc);
 638         ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
 639         ASSERT(new_tp->ftt_ids == NULL);
 640         ASSERT(new_tp->ftt_retids == NULL);
 641 
 642         switch (id->fti_ptype) {
 643         case DTFTP_ENTRY:
 644         case DTFTP_OFFSETS:
 645         case DTFTP_IS_ENABLED:
 646                 id->fti_next = NULL;
 647                 new_tp->ftt_ids = id;
 648                 break;
 649 
 650         case DTFTP_RETURN:
 651         case DTFTP_POST_OFFSETS:
 652                 id->fti_next = NULL;
 653                 new_tp->ftt_retids = id;
 654                 break;
 655 
 656         default:
 657                 ASSERT(0);
 658         }
 659 
 660         /*
 661          * If the ISA-dependent initialization goes to plan, go back to the
 662          * beginning and try to install this freshly made tracepoint.
 663          */
 664         if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
 665                 goto again;
 666 
 667         new_tp->ftt_ids = NULL;
 668         new_tp->ftt_retids = NULL;
 669 
 670         return (FASTTRAP_ENABLE_FAIL);
 671 }
 672 
 673 static void
 674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
 675 {
 676         fasttrap_bucket_t *bucket;
 677         fasttrap_provider_t *provider = probe->ftp_prov;
 678         fasttrap_tracepoint_t **pp, *tp;
 679         fasttrap_id_t *id, **idp;
 680         pid_t pid;
 681         uintptr_t pc;
 682 
 683         ASSERT(index < probe->ftp_ntps);
 684 
 685         pid = probe->ftp_pid;
 686         pc = probe->ftp_tps[index].fit_tp->ftt_pc;
 687         id = &probe->ftp_tps[index].fit_id;
 688 
 689         ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
 690 
 691         /*
 692          * Find the tracepoint and make sure that our id is one of the
 693          * ones registered with it.
 694          */
 695         bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 696         mutex_enter(&bucket->ftb_mtx);
 697         for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
 698                 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
 699                     tp->ftt_proc == provider->ftp_proc)
 700                         break;
 701         }
 702 
 703         /*
 704          * If we somehow lost this tracepoint, we're in a world of hurt.
 705          */
 706         ASSERT(tp != NULL);
 707 
 708         switch (id->fti_ptype) {
 709         case DTFTP_ENTRY:
 710         case DTFTP_OFFSETS:
 711         case DTFTP_IS_ENABLED:
 712                 ASSERT(tp->ftt_ids != NULL);
 713                 idp = &tp->ftt_ids;
 714                 break;
 715 
 716         case DTFTP_RETURN:
 717         case DTFTP_POST_OFFSETS:
 718                 ASSERT(tp->ftt_retids != NULL);
 719                 idp = &tp->ftt_retids;
 720                 break;
 721 
 722         default:
 723                 ASSERT(0);
 724         }
 725 
 726         while ((*idp)->fti_probe != probe) {
 727                 idp = &(*idp)->fti_next;
 728                 ASSERT(*idp != NULL);
 729         }
 730 
 731         id = *idp;
 732         *idp = id->fti_next;
 733         membar_producer();
 734 
 735         ASSERT(id->fti_probe == probe);
 736 
 737         /*
 738          * If there are other registered enablings of this tracepoint, we're
 739          * all done, but if this was the last probe assocated with this
 740          * this tracepoint, we need to remove and free it.
 741          */
 742         if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
 743 
 744                 /*
 745                  * If the current probe's tracepoint is in use, swap it
 746                  * for an unused tracepoint.
 747                  */
 748                 if (tp == probe->ftp_tps[index].fit_tp) {
 749                         fasttrap_probe_t *tmp_probe;
 750                         fasttrap_tracepoint_t **tmp_tp;
 751                         uint_t tmp_index;
 752 
 753                         if (tp->ftt_ids != NULL) {
 754                                 tmp_probe = tp->ftt_ids->fti_probe;
 755                                 /* LINTED - alignment */
 756                                 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
 757                                 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
 758                         } else {
 759                                 tmp_probe = tp->ftt_retids->fti_probe;
 760                                 /* LINTED - alignment */
 761                                 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
 762                                 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
 763                         }
 764 
 765                         ASSERT(*tmp_tp != NULL);
 766                         ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
 767                         ASSERT((*tmp_tp)->ftt_ids == NULL);
 768                         ASSERT((*tmp_tp)->ftt_retids == NULL);
 769 
 770                         probe->ftp_tps[index].fit_tp = *tmp_tp;
 771                         *tmp_tp = tp;
 772                 }
 773 
 774                 mutex_exit(&bucket->ftb_mtx);
 775 
 776                 /*
 777                  * Tag the modified probe with the generation in which it was
 778                  * changed.
 779                  */
 780                 probe->ftp_gen = fasttrap_mod_gen;
 781                 return;
 782         }
 783 
 784         mutex_exit(&bucket->ftb_mtx);
 785 
 786         /*
 787          * We can't safely remove the tracepoint from the set of active
 788          * tracepoints until we've actually removed the fasttrap instruction
 789          * from the process's text. We can, however, operate on this
 790          * tracepoint secure in the knowledge that no other thread is going to
 791          * be looking at it since we hold P_PR_LOCK on the process if it's
 792          * live or we hold the provider lock on the process if it's dead and
 793          * gone.
 794          */
 795 
 796         /*
 797          * We only need to remove the actual instruction if we're looking
 798          * at an existing process
 799          */
 800         if (p != NULL) {
 801                 /*
 802                  * If we fail to restore the instruction we need to kill
 803                  * this process since it's in a completely unrecoverable
 804                  * state.
 805                  */
 806                 if (fasttrap_tracepoint_remove(p, tp) != 0)
 807                         fasttrap_sigtrap(p, NULL, pc);
 808 
 809                 /*
 810                  * Decrement the count of the number of tracepoints active
 811                  * in the victim process.
 812                  */
 813                 ASSERT(p->p_proc_flag & P_PR_LOCK);
 814                 p->p_dtrace_count--;
 815         }
 816 
 817         /*
 818          * Remove the probe from the hash table of active tracepoints.
 819          */
 820         mutex_enter(&bucket->ftb_mtx);
 821         pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
 822         ASSERT(*pp != NULL);
 823         while (*pp != tp) {
 824                 pp = &(*pp)->ftt_next;
 825                 ASSERT(*pp != NULL);
 826         }
 827 
 828         *pp = tp->ftt_next;
 829         membar_producer();
 830 
 831         mutex_exit(&bucket->ftb_mtx);
 832 
 833         /*
 834          * Tag the modified probe with the generation in which it was changed.
 835          */
 836         probe->ftp_gen = fasttrap_mod_gen;
 837 }
 838 
 839 static void
 840 fasttrap_enable_callbacks(void)
 841 {
 842         /*
 843          * We don't have to play the rw lock game here because we're
 844          * providing something rather than taking something away --
 845          * we can be sure that no threads have tried to follow this
 846          * function pointer yet.
 847          */
 848         mutex_enter(&fasttrap_count_mtx);
 849         if (fasttrap_pid_count == 0) {
 850                 ASSERT(dtrace_pid_probe_ptr == NULL);
 851                 ASSERT(dtrace_return_probe_ptr == NULL);
 852                 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
 853                 dtrace_return_probe_ptr = &fasttrap_return_probe;
 854         }
 855         ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
 856         ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
 857         fasttrap_pid_count++;
 858         mutex_exit(&fasttrap_count_mtx);
 859 }
 860 
 861 static void
 862 fasttrap_disable_callbacks(void)
 863 {
 864         ASSERT(MUTEX_HELD(&cpu_lock));
 865 
 866         mutex_enter(&fasttrap_count_mtx);
 867         ASSERT(fasttrap_pid_count > 0);
 868         fasttrap_pid_count--;
 869         if (fasttrap_pid_count == 0) {
 870                 cpu_t *cur, *cpu = CPU;
 871 
 872                 for (cur = cpu->cpu_next_onln; cur != cpu;
 873                     cur = cur->cpu_next_onln) {
 874                         rw_enter(&cur->cpu_ft_lock, RW_WRITER);
 875                 }
 876 
 877                 dtrace_pid_probe_ptr = NULL;
 878                 dtrace_return_probe_ptr = NULL;
 879 
 880                 for (cur = cpu->cpu_next_onln; cur != cpu;
 881                     cur = cur->cpu_next_onln) {
 882                         rw_exit(&cur->cpu_ft_lock);
 883                 }
 884         }
 885         mutex_exit(&fasttrap_count_mtx);
 886 }
 887 
 888 /*ARGSUSED*/
 889 static int
 890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 891 {
 892         fasttrap_probe_t *probe = parg;
 893         proc_t *p;
 894         int i, rc;
 895 
 896         ASSERT(probe != NULL);
 897         ASSERT(!probe->ftp_enabled);
 898         ASSERT(id == probe->ftp_id);
 899         ASSERT(MUTEX_HELD(&cpu_lock));
 900 
 901         /*
 902          * Increment the count of enabled probes on this probe's provider;
 903          * the provider can't go away while the probe still exists. We
 904          * must increment this even if we aren't able to properly enable
 905          * this probe.
 906          */
 907         mutex_enter(&probe->ftp_prov->ftp_mtx);
 908         probe->ftp_prov->ftp_rcount++;
 909         mutex_exit(&probe->ftp_prov->ftp_mtx);
 910 
 911         /*
 912          * If this probe's provider is retired (meaning it was valid in a
 913          * previously exec'ed incarnation of this address space), bail out. The
 914          * provider can't go away while we're in this code path.
 915          */
 916         if (probe->ftp_prov->ftp_retired)
 917                 return (0);
 918 
 919         /*
 920          * If we can't find the process, it may be that we're in the context of
 921          * a fork in which the traced process is being born and we're copying
 922          * USDT probes. Otherwise, the process is gone so bail.
 923          */
 924         if ((p = sprlock(probe->ftp_pid)) == NULL) {
 925                 if ((curproc->p_flag & SFORKING) == 0)
 926                         return (0);
 927 
 928                 mutex_enter(&pidlock);
 929                 p = prfind(probe->ftp_pid);
 930 
 931                 if (p == NULL) {
 932                         /*
 933                          * So it's not that the target process is being born,
 934                          * it's that it isn't there at all (and we simply
 935                          * happen to be forking).  Anyway, we know that the
 936                          * target is definitely gone, so bail out.
 937                          */
 938                         mutex_exit(&pidlock);
 939                         return (0);
 940                 }
 941 
 942                 /*
 943                  * Confirm that curproc is indeed forking the process in which
 944                  * we're trying to enable probes.
 945                  */
 946                 ASSERT(p->p_parent == curproc);
 947                 ASSERT(p->p_stat == SIDL);
 948 
 949                 mutex_enter(&p->p_lock);
 950                 mutex_exit(&pidlock);
 951 
 952                 sprlock_proc(p);
 953         }
 954 
 955         ASSERT(!(p->p_flag & SVFORK));
 956         mutex_exit(&p->p_lock);
 957 
 958         /*
 959          * We have to enable the trap entry point before any user threads have
 960          * the chance to execute the trap instruction we're about to place
 961          * in their process's text.
 962          */
 963         fasttrap_enable_callbacks();
 964 
 965         /*
 966          * Enable all the tracepoints and add this probe's id to each
 967          * tracepoint's list of active probes.
 968          */
 969         for (i = 0; i < probe->ftp_ntps; i++) {
 970                 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
 971                         /*
 972                          * If enabling the tracepoint failed completely,
 973                          * we don't have to disable it; if the failure
 974                          * was only partial we must disable it.
 975                          */
 976                         if (rc == FASTTRAP_ENABLE_FAIL)
 977                                 i--;
 978                         else
 979                                 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
 980 
 981                         /*
 982                          * Back up and pull out all the tracepoints we've
 983                          * created so far for this probe.
 984                          */
 985                         while (i >= 0) {
 986                                 fasttrap_tracepoint_disable(p, probe, i);
 987                                 i--;
 988                         }
 989 
 990                         mutex_enter(&p->p_lock);
 991                         sprunlock(p);
 992 
 993                         /*
 994                          * Since we're not actually enabling this probe,
 995                          * drop our reference on the trap table entry.
 996                          */
 997                         fasttrap_disable_callbacks();
 998                         return (0);
 999                 }
1000         }
1001 
1002         mutex_enter(&p->p_lock);
1003         sprunlock(p);
1004 
1005         probe->ftp_enabled = 1;
1006         return (0);
1007 }
1008 
1009 /*ARGSUSED*/
1010 static void
1011 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1012 {
1013         fasttrap_probe_t *probe = parg;
1014         fasttrap_provider_t *provider = probe->ftp_prov;
1015         proc_t *p;
1016         int i, whack = 0;
1017 
1018         ASSERT(id == probe->ftp_id);
1019 
1020         /*
1021          * We won't be able to acquire a /proc-esque lock on the process
1022          * iff the process is dead and gone. In this case, we rely on the
1023          * provider lock as a point of mutual exclusion to prevent other
1024          * DTrace consumers from disabling this probe.
1025          */
1026         if ((p = sprlock(probe->ftp_pid)) != NULL) {
1027                 ASSERT(!(p->p_flag & SVFORK));
1028                 mutex_exit(&p->p_lock);
1029         }
1030 
1031         mutex_enter(&provider->ftp_mtx);
1032 
1033         /*
1034          * Disable all the associated tracepoints (for fully enabled probes).
1035          */
1036         if (probe->ftp_enabled) {
1037                 for (i = 0; i < probe->ftp_ntps; i++) {
1038                         fasttrap_tracepoint_disable(p, probe, i);
1039                 }
1040         }
1041 
1042         ASSERT(provider->ftp_rcount > 0);
1043         provider->ftp_rcount--;
1044 
1045         if (p != NULL) {
1046                 /*
1047                  * Even though we may not be able to remove it entirely, we
1048                  * mark this retired provider to get a chance to remove some
1049                  * of the associated probes.
1050                  */
1051                 if (provider->ftp_retired && !provider->ftp_marked)
1052                         whack = provider->ftp_marked = 1;
1053                 mutex_exit(&provider->ftp_mtx);
1054 
1055                 mutex_enter(&p->p_lock);
1056                 sprunlock(p);
1057         } else {
1058                 /*
1059                  * If the process is dead, we're just waiting for the
1060                  * last probe to be disabled to be able to free it.
1061                  */
1062                 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1063                         whack = provider->ftp_marked = 1;
1064                 mutex_exit(&provider->ftp_mtx);
1065         }
1066 
1067         if (whack)
1068                 fasttrap_pid_cleanup();
1069 
1070         if (!probe->ftp_enabled)
1071                 return;
1072 
1073         probe->ftp_enabled = 0;
1074 
1075         ASSERT(MUTEX_HELD(&cpu_lock));
1076         fasttrap_disable_callbacks();
1077 }
1078 
1079 /*ARGSUSED*/
1080 static void
1081 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1082     dtrace_argdesc_t *desc)
1083 {
1084         fasttrap_probe_t *probe = parg;
1085         char *str;
1086         int i, ndx;
1087 
1088         desc->dtargd_native[0] = '\0';
1089         desc->dtargd_xlate[0] = '\0';
1090 
1091         if (probe->ftp_prov->ftp_retired != 0 ||
1092             desc->dtargd_ndx >= probe->ftp_nargs) {
1093                 desc->dtargd_ndx = DTRACE_ARGNONE;
1094                 return;
1095         }
1096 
1097         ndx = (probe->ftp_argmap != NULL) ?
1098             probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1099 
1100         str = probe->ftp_ntypes;
1101         for (i = 0; i < ndx; i++) {
1102                 str += strlen(str) + 1;
1103         }
1104 
1105         ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1106         (void) strcpy(desc->dtargd_native, str);
1107 
1108         if (probe->ftp_xtypes == NULL)
1109                 return;
1110 
1111         str = probe->ftp_xtypes;
1112         for (i = 0; i < desc->dtargd_ndx; i++) {
1113                 str += strlen(str) + 1;
1114         }
1115 
1116         ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1117         (void) strcpy(desc->dtargd_xlate, str);
1118 }
1119 
1120 /*ARGSUSED*/
1121 static void
1122 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1123 {
1124         fasttrap_probe_t *probe = parg;
1125         int i;
1126         size_t size;
1127 
1128         ASSERT(probe != NULL);
1129         ASSERT(!probe->ftp_enabled);
1130         ASSERT(fasttrap_total >= probe->ftp_ntps);
1131 
1132         atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1133         size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1134 
1135         if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1136                 fasttrap_mod_barrier(probe->ftp_gen);
1137 
1138         for (i = 0; i < probe->ftp_ntps; i++) {
1139                 kmem_free(probe->ftp_tps[i].fit_tp,
1140                     sizeof (fasttrap_tracepoint_t));
1141         }
1142 
1143         kmem_free(probe, size);
1144 }
1145 
1146 
1147 static const dtrace_pattr_t pid_attr = {
1148 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1149 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1150 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1151 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1152 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1153 };
1154 
1155 static dtrace_pops_t pid_pops = {
1156         fasttrap_pid_provide,
1157         NULL,
1158         fasttrap_pid_enable,
1159         fasttrap_pid_disable,
1160         NULL,
1161         NULL,
1162         fasttrap_pid_getargdesc,
1163         fasttrap_pid_getarg,
1164         NULL,
1165         fasttrap_pid_destroy
1166 };
1167 
1168 static dtrace_pops_t usdt_pops = {
1169         fasttrap_pid_provide,
1170         NULL,
1171         fasttrap_pid_enable,
1172         fasttrap_pid_disable,
1173         NULL,
1174         NULL,
1175         fasttrap_pid_getargdesc,
1176         fasttrap_usdt_getarg,
1177         NULL,
1178         fasttrap_pid_destroy
1179 };
1180 
1181 static fasttrap_proc_t *
1182 fasttrap_proc_lookup(pid_t pid)
1183 {
1184         fasttrap_bucket_t *bucket;
1185         fasttrap_proc_t *fprc, *new_fprc;
1186 
1187         bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1188         mutex_enter(&bucket->ftb_mtx);
1189 
1190         for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1191                 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1192                         mutex_enter(&fprc->ftpc_mtx);
1193                         mutex_exit(&bucket->ftb_mtx);
1194                         fprc->ftpc_rcount++;
1195                         atomic_inc_64(&fprc->ftpc_acount);
1196                         ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1197                         mutex_exit(&fprc->ftpc_mtx);
1198 
1199                         return (fprc);
1200                 }
1201         }
1202 
1203         /*
1204          * Drop the bucket lock so we don't try to perform a sleeping
1205          * allocation under it.
1206          */
1207         mutex_exit(&bucket->ftb_mtx);
1208 
1209         new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1210         new_fprc->ftpc_pid = pid;
1211         new_fprc->ftpc_rcount = 1;
1212         new_fprc->ftpc_acount = 1;
1213 
1214         mutex_enter(&bucket->ftb_mtx);
1215 
1216         /*
1217          * Take another lap through the list to make sure a proc hasn't
1218          * been created for this pid while we weren't under the bucket lock.
1219          */
1220         for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1221                 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1222                         mutex_enter(&fprc->ftpc_mtx);
1223                         mutex_exit(&bucket->ftb_mtx);
1224                         fprc->ftpc_rcount++;
1225                         atomic_inc_64(&fprc->ftpc_acount);
1226                         ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1227                         mutex_exit(&fprc->ftpc_mtx);
1228 
1229                         kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1230 
1231                         return (fprc);
1232                 }
1233         }
1234 
1235         new_fprc->ftpc_next = bucket->ftb_data;
1236         bucket->ftb_data = new_fprc;
1237 
1238         mutex_exit(&bucket->ftb_mtx);
1239 
1240         return (new_fprc);
1241 }
1242 
1243 static void
1244 fasttrap_proc_release(fasttrap_proc_t *proc)
1245 {
1246         fasttrap_bucket_t *bucket;
1247         fasttrap_proc_t *fprc, **fprcp;
1248         pid_t pid = proc->ftpc_pid;
1249 
1250         mutex_enter(&proc->ftpc_mtx);
1251 
1252         ASSERT(proc->ftpc_rcount != 0);
1253         ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1254 
1255         if (--proc->ftpc_rcount != 0) {
1256                 mutex_exit(&proc->ftpc_mtx);
1257                 return;
1258         }
1259 
1260         mutex_exit(&proc->ftpc_mtx);
1261 
1262         /*
1263          * There should definitely be no live providers associated with this
1264          * process at this point.
1265          */
1266         ASSERT(proc->ftpc_acount == 0);
1267 
1268         bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1269         mutex_enter(&bucket->ftb_mtx);
1270 
1271         fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1272         while ((fprc = *fprcp) != NULL) {
1273                 if (fprc == proc)
1274                         break;
1275 
1276                 fprcp = &fprc->ftpc_next;
1277         }
1278 
1279         /*
1280          * Something strange has happened if we can't find the proc.
1281          */
1282         ASSERT(fprc != NULL);
1283 
1284         *fprcp = fprc->ftpc_next;
1285 
1286         mutex_exit(&bucket->ftb_mtx);
1287 
1288         kmem_free(fprc, sizeof (fasttrap_proc_t));
1289 }
1290 
1291 /*
1292  * Lookup a fasttrap-managed provider based on its name and associated pid.
1293  * If the pattr argument is non-NULL, this function instantiates the provider
1294  * if it doesn't exist otherwise it returns NULL. The provider is returned
1295  * with its lock held.
1296  */
1297 static fasttrap_provider_t *
1298 fasttrap_provider_lookup(pid_t pid, const char *name,
1299     const dtrace_pattr_t *pattr)
1300 {
1301         fasttrap_provider_t *fp, *new_fp = NULL;
1302         fasttrap_bucket_t *bucket;
1303         char provname[DTRACE_PROVNAMELEN];
1304         proc_t *p;
1305         cred_t *cred;
1306 
1307         ASSERT(strlen(name) < sizeof (fp->ftp_name));
1308         ASSERT(pattr != NULL);
1309 
1310         bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1311         mutex_enter(&bucket->ftb_mtx);
1312 
1313         /*
1314          * Take a lap through the list and return the match if we find it.
1315          */
1316         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1317                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1318                     !fp->ftp_retired) {
1319                         mutex_enter(&fp->ftp_mtx);
1320                         mutex_exit(&bucket->ftb_mtx);
1321                         return (fp);
1322                 }
1323         }
1324 
1325         /*
1326          * Drop the bucket lock so we don't try to perform a sleeping
1327          * allocation under it.
1328          */
1329         mutex_exit(&bucket->ftb_mtx);
1330 
1331         /*
1332          * Make sure the process exists, isn't a child created as the result
1333          * of a vfork(2), and isn't a zombie (but may be in fork).
1334          */
1335         mutex_enter(&pidlock);
1336         if ((p = prfind(pid)) == NULL) {
1337                 mutex_exit(&pidlock);
1338                 return (NULL);
1339         }
1340         mutex_enter(&p->p_lock);
1341         mutex_exit(&pidlock);
1342         if (p->p_flag & (SVFORK | SEXITING)) {
1343                 mutex_exit(&p->p_lock);
1344                 return (NULL);
1345         }
1346 
1347         /*
1348          * Increment p_dtrace_probes so that the process knows to inform us
1349          * when it exits or execs. fasttrap_provider_free() decrements this
1350          * when we're done with this provider.
1351          */
1352         p->p_dtrace_probes++;
1353 
1354         /*
1355          * Grab the credentials for this process so we have
1356          * something to pass to dtrace_register().
1357          */
1358         mutex_enter(&p->p_crlock);
1359         crhold(p->p_cred);
1360         cred = p->p_cred;
1361         mutex_exit(&p->p_crlock);
1362         mutex_exit(&p->p_lock);
1363 
1364         new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1365         new_fp->ftp_pid = pid;
1366         new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1367 
1368         ASSERT(new_fp->ftp_proc != NULL);
1369 
1370         mutex_enter(&bucket->ftb_mtx);
1371 
1372         /*
1373          * Take another lap through the list to make sure a provider hasn't
1374          * been created for this pid while we weren't under the bucket lock.
1375          */
1376         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1377                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1378                     !fp->ftp_retired) {
1379                         mutex_enter(&fp->ftp_mtx);
1380                         mutex_exit(&bucket->ftb_mtx);
1381                         fasttrap_provider_free(new_fp);
1382                         crfree(cred);
1383                         return (fp);
1384                 }
1385         }
1386 
1387         (void) strcpy(new_fp->ftp_name, name);
1388 
1389         /*
1390          * Fail and return NULL if either the provider name is too long
1391          * or we fail to register this new provider with the DTrace
1392          * framework. Note that this is the only place we ever construct
1393          * the full provider name -- we keep it in pieces in the provider
1394          * structure.
1395          */
1396         if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1397             sizeof (provname) ||
1398             dtrace_register(provname, pattr,
1399             DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1400             pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1401             &new_fp->ftp_provid) != 0) {
1402                 mutex_exit(&bucket->ftb_mtx);
1403                 fasttrap_provider_free(new_fp);
1404                 crfree(cred);
1405                 return (NULL);
1406         }
1407 
1408         new_fp->ftp_next = bucket->ftb_data;
1409         bucket->ftb_data = new_fp;
1410 
1411         mutex_enter(&new_fp->ftp_mtx);
1412         mutex_exit(&bucket->ftb_mtx);
1413 
1414         crfree(cred);
1415         return (new_fp);
1416 }
1417 
1418 static void
1419 fasttrap_provider_free(fasttrap_provider_t *provider)
1420 {
1421         pid_t pid = provider->ftp_pid;
1422         proc_t *p;
1423 
1424         /*
1425          * There need to be no associated enabled probes, no consumers
1426          * creating probes, and no meta providers referencing this provider.
1427          */
1428         ASSERT(provider->ftp_rcount == 0);
1429         ASSERT(provider->ftp_ccount == 0);
1430         ASSERT(provider->ftp_mcount == 0);
1431 
1432         /*
1433          * If this provider hasn't been retired, we need to explicitly drop the
1434          * count of active providers on the associated process structure.
1435          */
1436         if (!provider->ftp_retired) {
1437                 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1438                 ASSERT(provider->ftp_proc->ftpc_acount <
1439                     provider->ftp_proc->ftpc_rcount);
1440         }
1441 
1442         fasttrap_proc_release(provider->ftp_proc);
1443 
1444         kmem_free(provider, sizeof (fasttrap_provider_t));
1445 
1446         /*
1447          * Decrement p_dtrace_probes on the process whose provider we're
1448          * freeing. We don't have to worry about clobbering somone else's
1449          * modifications to it because we have locked the bucket that
1450          * corresponds to this process's hash chain in the provider hash
1451          * table. Don't sweat it if we can't find the process.
1452          */
1453         mutex_enter(&pidlock);
1454         if ((p = prfind(pid)) == NULL) {
1455                 mutex_exit(&pidlock);
1456                 return;
1457         }
1458 
1459         mutex_enter(&p->p_lock);
1460         mutex_exit(&pidlock);
1461 
1462         p->p_dtrace_probes--;
1463         mutex_exit(&p->p_lock);
1464 }
1465 
1466 static void
1467 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1468 {
1469         fasttrap_provider_t *fp;
1470         fasttrap_bucket_t *bucket;
1471         dtrace_provider_id_t provid;
1472 
1473         ASSERT(strlen(name) < sizeof (fp->ftp_name));
1474 
1475         bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1476         mutex_enter(&bucket->ftb_mtx);
1477 
1478         for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1479                 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1480                     !fp->ftp_retired)
1481                         break;
1482         }
1483 
1484         if (fp == NULL) {
1485                 mutex_exit(&bucket->ftb_mtx);
1486                 return;
1487         }
1488 
1489         mutex_enter(&fp->ftp_mtx);
1490         ASSERT(!mprov || fp->ftp_mcount > 0);
1491         if (mprov && --fp->ftp_mcount != 0)  {
1492                 mutex_exit(&fp->ftp_mtx);
1493                 mutex_exit(&bucket->ftb_mtx);
1494                 return;
1495         }
1496 
1497         /*
1498          * Mark the provider to be removed in our post-processing step, mark it
1499          * retired, and drop the active count on its proc. Marking it indicates
1500          * that we should try to remove it; setting the retired flag indicates
1501          * that we're done with this provider; dropping the active the proc
1502          * releases our hold, and when this reaches zero (as it will during
1503          * exit or exec) the proc and associated providers become defunct.
1504          *
1505          * We obviously need to take the bucket lock before the provider lock
1506          * to perform the lookup, but we need to drop the provider lock
1507          * before calling into the DTrace framework since we acquire the
1508          * provider lock in callbacks invoked from the DTrace framework. The
1509          * bucket lock therefore protects the integrity of the provider hash
1510          * table.
1511          */
1512         atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1513         ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1514 
1515         fp->ftp_retired = 1;
1516         fp->ftp_marked = 1;
1517         provid = fp->ftp_provid;
1518         mutex_exit(&fp->ftp_mtx);
1519 
1520         /*
1521          * We don't have to worry about invalidating the same provider twice
1522          * since fasttrap_provider_lookup() will ignore provider that have
1523          * been marked as retired.
1524          */
1525         dtrace_invalidate(provid);
1526 
1527         mutex_exit(&bucket->ftb_mtx);
1528 
1529         fasttrap_pid_cleanup();
1530 }
1531 
1532 static int
1533 fasttrap_uint32_cmp(const void *ap, const void *bp)
1534 {
1535         return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1536 }
1537 
1538 static int
1539 fasttrap_uint64_cmp(const void *ap, const void *bp)
1540 {
1541         return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1542 }
1543 
1544 static int
1545 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1546 {
1547         fasttrap_provider_t *provider;
1548         fasttrap_probe_t *pp;
1549         fasttrap_tracepoint_t *tp;
1550         char *name;
1551         int i, aframes, whack;
1552 
1553         /*
1554          * There needs to be at least one desired trace point.
1555          */
1556         if (pdata->ftps_noffs == 0)
1557                 return (EINVAL);
1558 
1559         switch (pdata->ftps_type) {
1560         case DTFTP_ENTRY:
1561                 name = "entry";
1562                 aframes = FASTTRAP_ENTRY_AFRAMES;
1563                 break;
1564         case DTFTP_RETURN:
1565                 name = "return";
1566                 aframes = FASTTRAP_RETURN_AFRAMES;
1567                 break;
1568         case DTFTP_OFFSETS:
1569                 name = NULL;
1570                 break;
1571         default:
1572                 return (EINVAL);
1573         }
1574 
1575         if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1576             FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1577                 return (ESRCH);
1578 
1579         /*
1580          * Increment this reference count to indicate that a consumer is
1581          * actively adding a new probe associated with this provider. This
1582          * prevents the provider from being deleted -- we'll need to check
1583          * for pending deletions when we drop this reference count.
1584          */
1585         provider->ftp_ccount++;
1586         mutex_exit(&provider->ftp_mtx);
1587 
1588         /*
1589          * Grab the creation lock to ensure consistency between calls to
1590          * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1591          * other threads creating probes. We must drop the provider lock
1592          * before taking this lock to avoid a three-way deadlock with the
1593          * DTrace framework.
1594          */
1595         mutex_enter(&provider->ftp_cmtx);
1596 
1597         if (name == NULL) {
1598                 for (i = 0; i < pdata->ftps_noffs; i++) {
1599                         char name_str[17];
1600 
1601                         (void) sprintf(name_str, "%llx",
1602                             (unsigned long long)pdata->ftps_offs[i]);
1603 
1604                         if (dtrace_probe_lookup(provider->ftp_provid,
1605                             pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1606                                 continue;
1607 
1608                         atomic_inc_32(&fasttrap_total);
1609 
1610                         if (fasttrap_total > fasttrap_max) {
1611                                 atomic_dec_32(&fasttrap_total);
1612                                 goto no_mem;
1613                         }
1614 
1615                         pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1616 
1617                         pp->ftp_prov = provider;
1618                         pp->ftp_faddr = pdata->ftps_pc;
1619                         pp->ftp_fsize = pdata->ftps_size;
1620                         pp->ftp_pid = pdata->ftps_pid;
1621                         pp->ftp_ntps = 1;
1622 
1623                         tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1624                             KM_SLEEP);
1625 
1626                         tp->ftt_proc = provider->ftp_proc;
1627                         tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1628                         tp->ftt_pid = pdata->ftps_pid;
1629 
1630                         pp->ftp_tps[0].fit_tp = tp;
1631                         pp->ftp_tps[0].fit_id.fti_probe = pp;
1632                         pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1633 
1634                         pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1635                             pdata->ftps_mod, pdata->ftps_func, name_str,
1636                             FASTTRAP_OFFSET_AFRAMES, pp);
1637                 }
1638 
1639         } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1640             pdata->ftps_func, name) == 0) {
1641                 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1642 
1643                 if (fasttrap_total > fasttrap_max) {
1644                         atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1645                         goto no_mem;
1646                 }
1647 
1648                 /*
1649                  * Make sure all tracepoint program counter values are unique.
1650                  * We later assume that each probe has exactly one tracepoint
1651                  * for a given pc.
1652                  */
1653                 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1654                     sizeof (uint64_t), fasttrap_uint64_cmp);
1655                 for (i = 1; i < pdata->ftps_noffs; i++) {
1656                         if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1657                                 continue;
1658 
1659                         atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1660                         goto no_mem;
1661                 }
1662 
1663                 ASSERT(pdata->ftps_noffs > 0);
1664                 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1665                     ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1666 
1667                 pp->ftp_prov = provider;
1668                 pp->ftp_faddr = pdata->ftps_pc;
1669                 pp->ftp_fsize = pdata->ftps_size;
1670                 pp->ftp_pid = pdata->ftps_pid;
1671                 pp->ftp_ntps = pdata->ftps_noffs;
1672 
1673                 for (i = 0; i < pdata->ftps_noffs; i++) {
1674                         tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1675                             KM_SLEEP);
1676 
1677                         tp->ftt_proc = provider->ftp_proc;
1678                         tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1679                         tp->ftt_pid = pdata->ftps_pid;
1680 
1681                         pp->ftp_tps[i].fit_tp = tp;
1682                         pp->ftp_tps[i].fit_id.fti_probe = pp;
1683                         pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1684                 }
1685 
1686                 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1687                     pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1688         }
1689 
1690         mutex_exit(&provider->ftp_cmtx);
1691 
1692         /*
1693          * We know that the provider is still valid since we incremented the
1694          * creation reference count. If someone tried to clean up this provider
1695          * while we were using it (e.g. because the process called exec(2) or
1696          * exit(2)), take note of that and try to clean it up now.
1697          */
1698         mutex_enter(&provider->ftp_mtx);
1699         provider->ftp_ccount--;
1700         whack = provider->ftp_retired;
1701         mutex_exit(&provider->ftp_mtx);
1702 
1703         if (whack)
1704                 fasttrap_pid_cleanup();
1705 
1706         return (0);
1707 
1708 no_mem:
1709         /*
1710          * If we've exhausted the allowable resources, we'll try to remove
1711          * this provider to free some up. This is to cover the case where
1712          * the user has accidentally created many more probes than was
1713          * intended (e.g. pid123:::).
1714          */
1715         mutex_exit(&provider->ftp_cmtx);
1716         mutex_enter(&provider->ftp_mtx);
1717         provider->ftp_ccount--;
1718         provider->ftp_marked = 1;
1719         mutex_exit(&provider->ftp_mtx);
1720 
1721         fasttrap_pid_cleanup();
1722 
1723         return (ENOMEM);
1724 }
1725 
1726 /*ARGSUSED*/
1727 static void *
1728 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1729 {
1730         fasttrap_provider_t *provider;
1731 
1732         /*
1733          * A 32-bit unsigned integer (like a pid for example) can be
1734          * expressed in 10 or fewer decimal digits. Make sure that we'll
1735          * have enough space for the provider name.
1736          */
1737         if (strlen(dhpv->dthpv_provname) + 10 >=
1738             sizeof (provider->ftp_name)) {
1739                 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1740                     "name too long to accomodate pid", dhpv->dthpv_provname);
1741                 return (NULL);
1742         }
1743 
1744         /*
1745          * Don't let folks spoof the true pid provider.
1746          */
1747         if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1748                 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1749                     "%s is an invalid name", dhpv->dthpv_provname,
1750                     FASTTRAP_PID_NAME);
1751                 return (NULL);
1752         }
1753 
1754         /*
1755          * The highest stability class that fasttrap supports is ISA; cap
1756          * the stability of the new provider accordingly.
1757          */
1758         if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1759                 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1760         if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1761                 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1762         if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1763                 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1764         if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1765                 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1766         if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1767                 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1768 
1769         if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1770             &dhpv->dthpv_pattr)) == NULL) {
1771                 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1772                     "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1773                 return (NULL);
1774         }
1775 
1776         /*
1777          * Up the meta provider count so this provider isn't removed until
1778          * the meta provider has been told to remove it.
1779          */
1780         provider->ftp_mcount++;
1781 
1782         mutex_exit(&provider->ftp_mtx);
1783 
1784         return (provider);
1785 }
1786 
1787 /*
1788  * We know a few things about our context here:  we know that the probe being
1789  * created doesn't already exist (DTrace won't load DOF at the same address
1790  * twice, even if explicitly told to do so) and we know that we are
1791  * single-threaded with respect to the meta provider machinery. Knowing that
1792  * this is a new probe and that there is no way for us to race with another
1793  * operation on this provider allows us an important optimization: we need not
1794  * lookup a probe before adding it.  Saving this lookup is important because
1795  * this code is in the fork path for processes with USDT probes, and lookups
1796  * here are potentially very expensive because of long hash conflicts on
1797  * module, function and name (DTrace doesn't hash on provider name).
1798  */
1799 /*ARGSUSED*/
1800 static void
1801 fasttrap_meta_create_probe(void *arg, void *parg,
1802     dtrace_helper_probedesc_t *dhpb)
1803 {
1804         fasttrap_provider_t *provider = parg;
1805         fasttrap_probe_t *pp;
1806         fasttrap_tracepoint_t *tp;
1807         int i, j;
1808         uint32_t ntps;
1809 
1810         /*
1811          * Since the meta provider count is non-zero we don't have to worry
1812          * about this provider disappearing.
1813          */
1814         ASSERT(provider->ftp_mcount > 0);
1815 
1816         /*
1817          * The offsets must be unique.
1818          */
1819         qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1820             fasttrap_uint32_cmp);
1821         for (i = 1; i < dhpb->dthpb_noffs; i++) {
1822                 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1823                     dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1824                         return;
1825         }
1826 
1827         qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1828             fasttrap_uint32_cmp);
1829         for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1830                 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1831                     dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1832                         return;
1833         }
1834 
1835         ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1836         ASSERT(ntps > 0);
1837 
1838         atomic_add_32(&fasttrap_total, ntps);
1839 
1840         if (fasttrap_total > fasttrap_max) {
1841                 atomic_add_32(&fasttrap_total, -ntps);
1842                 return;
1843         }
1844 
1845         pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1846 
1847         pp->ftp_prov = provider;
1848         pp->ftp_pid = provider->ftp_pid;
1849         pp->ftp_ntps = ntps;
1850         pp->ftp_nargs = dhpb->dthpb_xargc;
1851         pp->ftp_xtypes = dhpb->dthpb_xtypes;
1852         pp->ftp_ntypes = dhpb->dthpb_ntypes;
1853 
1854         /*
1855          * First create a tracepoint for each actual point of interest.
1856          */
1857         for (i = 0; i < dhpb->dthpb_noffs; i++) {
1858                 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1859 
1860                 tp->ftt_proc = provider->ftp_proc;
1861                 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1862                 tp->ftt_pid = provider->ftp_pid;
1863 
1864                 pp->ftp_tps[i].fit_tp = tp;
1865                 pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 #ifdef __sparc
1867                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1868 #else
1869                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1870 #endif
1871         }
1872 
1873         /*
1874          * Then create a tracepoint for each is-enabled point.
1875          */
1876         for (j = 0; i < ntps; i++, j++) {
1877                 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1878 
1879                 tp->ftt_proc = provider->ftp_proc;
1880                 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1881                 tp->ftt_pid = provider->ftp_pid;
1882 
1883                 pp->ftp_tps[i].fit_tp = tp;
1884                 pp->ftp_tps[i].fit_id.fti_probe = pp;
1885                 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1886         }
1887 
1888         /*
1889          * If the arguments are shuffled around we set the argument remapping
1890          * table. Later, when the probe fires, we only remap the arguments
1891          * if the table is non-NULL.
1892          */
1893         for (i = 0; i < dhpb->dthpb_xargc; i++) {
1894                 if (dhpb->dthpb_args[i] != i) {
1895                         pp->ftp_argmap = dhpb->dthpb_args;
1896                         break;
1897                 }
1898         }
1899 
1900         /*
1901          * The probe is fully constructed -- register it with DTrace.
1902          */
1903         pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1904             dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1905 }
1906 
1907 /*ARGSUSED*/
1908 static void
1909 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1910 {
1911         /*
1912          * Clean up the USDT provider. There may be active consumers of the
1913          * provider busy adding probes, no damage will actually befall the
1914          * provider until that count has dropped to zero. This just puts
1915          * the provider on death row.
1916          */
1917         fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1918 }
1919 
1920 static dtrace_mops_t fasttrap_mops = {
1921         fasttrap_meta_create_probe,
1922         fasttrap_meta_provide,
1923         fasttrap_meta_remove
1924 };
1925 
1926 /*ARGSUSED*/
1927 static int
1928 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1929 {
1930         return (0);
1931 }
1932 
1933 /*ARGSUSED*/
1934 static int
1935 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1936 {
1937         if (!dtrace_attached())
1938                 return (EAGAIN);
1939 
1940         if (cmd == FASTTRAPIOC_MAKEPROBE) {
1941                 fasttrap_probe_spec_t *uprobe = (void *)arg;
1942                 fasttrap_probe_spec_t *probe;
1943                 uint64_t noffs;
1944                 size_t size;
1945                 int ret, err;
1946 
1947                 if (copyin(&uprobe->ftps_noffs, &noffs,
1948                     sizeof (uprobe->ftps_noffs)))
1949                         return (EFAULT);
1950 
1951                 /*
1952                  * Probes must have at least one tracepoint.
1953                  */
1954                 if (noffs == 0)
1955                         return (EINVAL);
1956 
1957                 size = sizeof (fasttrap_probe_spec_t) +
1958                     sizeof (probe->ftps_offs[0]) * (noffs - 1);
1959 
1960                 if (size > 1024 * 1024)
1961                         return (ENOMEM);
1962 
1963                 probe = kmem_alloc(size, KM_SLEEP);
1964 
1965                 if (copyin(uprobe, probe, size) != 0 ||
1966                     probe->ftps_noffs != noffs) {
1967                         kmem_free(probe, size);
1968                         return (EFAULT);
1969                 }
1970 
1971                 /*
1972                  * Verify that the function and module strings contain no
1973                  * funny characters.
1974                  */
1975                 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1976                     NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977                         ret = EINVAL;
1978                         goto err;
1979                 }
1980 
1981                 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1982                     NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1983                         ret = EINVAL;
1984                         goto err;
1985                 }
1986 
1987                 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1988                         proc_t *p;
1989                         pid_t pid = probe->ftps_pid;
1990 
1991                         mutex_enter(&pidlock);
1992                         /*
1993                          * Report an error if the process doesn't exist
1994                          * or is actively being birthed.
1995                          */
1996                         if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1997                                 mutex_exit(&pidlock);
1998                                 return (ESRCH);
1999                         }
2000                         mutex_enter(&p->p_lock);
2001                         mutex_exit(&pidlock);
2002 
2003                         if ((ret = priv_proc_cred_perm(cr, p, NULL,
2004                             VREAD | VWRITE)) != 0) {
2005                                 mutex_exit(&p->p_lock);
2006                                 return (ret);
2007                         }
2008 
2009                         mutex_exit(&p->p_lock);
2010                 }
2011 
2012                 ret = fasttrap_add_probe(probe);
2013 err:
2014                 kmem_free(probe, size);
2015 
2016                 return (ret);
2017 
2018         } else if (cmd == FASTTRAPIOC_GETINSTR) {
2019                 fasttrap_instr_query_t instr;
2020                 fasttrap_tracepoint_t *tp;
2021                 uint_t index;
2022                 int ret;
2023 
2024                 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2025                         return (EFAULT);
2026 
2027                 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2028                         proc_t *p;
2029                         pid_t pid = instr.ftiq_pid;
2030 
2031                         mutex_enter(&pidlock);
2032                         /*
2033                          * Report an error if the process doesn't exist
2034                          * or is actively being birthed.
2035                          */
2036                         if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2037                                 mutex_exit(&pidlock);
2038                                 return (ESRCH);
2039                         }
2040                         mutex_enter(&p->p_lock);
2041                         mutex_exit(&pidlock);
2042 
2043                         if ((ret = priv_proc_cred_perm(cr, p, NULL,
2044                             VREAD)) != 0) {
2045                                 mutex_exit(&p->p_lock);
2046                                 return (ret);
2047                         }
2048 
2049                         mutex_exit(&p->p_lock);
2050                 }
2051 
2052                 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2053 
2054                 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2055                 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2056                 while (tp != NULL) {
2057                         if (instr.ftiq_pid == tp->ftt_pid &&
2058                             instr.ftiq_pc == tp->ftt_pc &&
2059                             tp->ftt_proc->ftpc_acount != 0)
2060                                 break;
2061 
2062                         tp = tp->ftt_next;
2063                 }
2064 
2065                 if (tp == NULL) {
2066                         mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067                         return (ENOENT);
2068                 }
2069 
2070                 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2071                     sizeof (instr.ftiq_instr));
2072                 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2073 
2074                 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2075                         return (EFAULT);
2076 
2077                 return (0);
2078         }
2079 
2080         return (EINVAL);
2081 }
2082 
2083 static struct cb_ops fasttrap_cb_ops = {
2084         fasttrap_open,          /* open */
2085         nodev,                  /* close */
2086         nulldev,                /* strategy */
2087         nulldev,                /* print */
2088         nodev,                  /* dump */
2089         nodev,                  /* read */
2090         nodev,                  /* write */
2091         fasttrap_ioctl,         /* ioctl */
2092         nodev,                  /* devmap */
2093         nodev,                  /* mmap */
2094         nodev,                  /* segmap */
2095         nochpoll,               /* poll */
2096         ddi_prop_op,            /* cb_prop_op */
2097         0,                      /* streamtab  */
2098         D_NEW | D_MP            /* Driver compatibility flag */
2099 };
2100 
2101 /*ARGSUSED*/
2102 static int
2103 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2104 {
2105         int error;
2106 
2107         switch (infocmd) {
2108         case DDI_INFO_DEVT2DEVINFO:
2109                 *result = (void *)fasttrap_devi;
2110                 error = DDI_SUCCESS;
2111                 break;
2112         case DDI_INFO_DEVT2INSTANCE:
2113                 *result = (void *)0;
2114                 error = DDI_SUCCESS;
2115                 break;
2116         default:
2117                 error = DDI_FAILURE;
2118         }
2119         return (error);
2120 }
2121 
2122 static int
2123 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2124 {
2125         ulong_t nent;
2126 
2127         switch (cmd) {
2128         case DDI_ATTACH:
2129                 break;
2130         case DDI_RESUME:
2131                 return (DDI_SUCCESS);
2132         default:
2133                 return (DDI_FAILURE);
2134         }
2135 
2136         if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2137             DDI_PSEUDO, NULL) == DDI_FAILURE) {
2138                 ddi_remove_minor_node(devi, NULL);
2139                 return (DDI_FAILURE);
2140         }
2141 
2142         ddi_report_dev(devi);
2143         fasttrap_devi = devi;
2144 
2145         /*
2146          * Install our hooks into fork(2), exec(2), and exit(2).
2147          */
2148         dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2149         dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2150         dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2151 
2152         fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2153             "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2154         fasttrap_total = 0;
2155 
2156         /*
2157          * Conjure up the tracepoints hashtable...
2158          */
2159         nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2160             "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2161 
2162         if (nent == 0 || nent > 0x1000000)
2163                 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2164 
2165         if (ISP2(nent))
2166                 fasttrap_tpoints.fth_nent = nent;
2167         else
2168                 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2169         ASSERT(fasttrap_tpoints.fth_nent > 0);
2170         fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2171         fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2172             sizeof (fasttrap_bucket_t), KM_SLEEP);
2173 
2174         /*
2175          * ... and the providers hash table...
2176          */
2177         nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2178         if (ISP2(nent))
2179                 fasttrap_provs.fth_nent = nent;
2180         else
2181                 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2182         ASSERT(fasttrap_provs.fth_nent > 0);
2183         fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2184         fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2185             sizeof (fasttrap_bucket_t), KM_SLEEP);
2186 
2187         /*
2188          * ... and the procs hash table.
2189          */
2190         nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2191         if (ISP2(nent))
2192                 fasttrap_procs.fth_nent = nent;
2193         else
2194                 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2195         ASSERT(fasttrap_procs.fth_nent > 0);
2196         fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2197         fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2198             sizeof (fasttrap_bucket_t), KM_SLEEP);
2199 
2200         (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2201             &fasttrap_meta_id);
2202 
2203         return (DDI_SUCCESS);
2204 }
2205 
2206 static int
2207 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2208 {
2209         int i, fail = 0;
2210         timeout_id_t tmp;
2211 
2212         switch (cmd) {
2213         case DDI_DETACH:
2214                 break;
2215         case DDI_SUSPEND:
2216                 return (DDI_SUCCESS);
2217         default:
2218                 return (DDI_FAILURE);
2219         }
2220 
2221         /*
2222          * Unregister the meta-provider to make sure no new fasttrap-
2223          * managed providers come along while we're trying to close up
2224          * shop. If we fail to detach, we'll need to re-register as a
2225          * meta-provider. We can fail to unregister as a meta-provider
2226          * if providers we manage still exist.
2227          */
2228         if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2229             dtrace_meta_unregister(fasttrap_meta_id) != 0)
2230                 return (DDI_FAILURE);
2231 
2232         /*
2233          * Prevent any new timeouts from running by setting fasttrap_timeout
2234          * to a non-zero value, and wait for the current timeout to complete.
2235          */
2236         mutex_enter(&fasttrap_cleanup_mtx);
2237         fasttrap_cleanup_work = 0;
2238 
2239         while (fasttrap_timeout != (timeout_id_t)1) {
2240                 tmp = fasttrap_timeout;
2241                 fasttrap_timeout = (timeout_id_t)1;
2242 
2243                 if (tmp != 0) {
2244                         mutex_exit(&fasttrap_cleanup_mtx);
2245                         (void) untimeout(tmp);
2246                         mutex_enter(&fasttrap_cleanup_mtx);
2247                 }
2248         }
2249 
2250         fasttrap_cleanup_work = 0;
2251         mutex_exit(&fasttrap_cleanup_mtx);
2252 
2253         /*
2254          * Iterate over all of our providers. If there's still a process
2255          * that corresponds to that pid, fail to detach.
2256          */
2257         for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2258                 fasttrap_provider_t **fpp, *fp;
2259                 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2260 
2261                 mutex_enter(&bucket->ftb_mtx);
2262                 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2263                 while ((fp = *fpp) != NULL) {
2264                         /*
2265                          * Acquire and release the lock as a simple way of
2266                          * waiting for any other consumer to finish with
2267                          * this provider. A thread must first acquire the
2268                          * bucket lock so there's no chance of another thread
2269                          * blocking on the provider's lock.
2270                          */
2271                         mutex_enter(&fp->ftp_mtx);
2272                         mutex_exit(&fp->ftp_mtx);
2273 
2274                         if (dtrace_unregister(fp->ftp_provid) != 0) {
2275                                 fail = 1;
2276                                 fpp = &fp->ftp_next;
2277                         } else {
2278                                 *fpp = fp->ftp_next;
2279                                 fasttrap_provider_free(fp);
2280                         }
2281                 }
2282 
2283                 mutex_exit(&bucket->ftb_mtx);
2284         }
2285 
2286         if (fail) {
2287                 uint_t work;
2288                 /*
2289                  * If we're failing to detach, we need to unblock timeouts
2290                  * and start a new timeout if any work has accumulated while
2291                  * we've been unsuccessfully trying to detach.
2292                  */
2293                 mutex_enter(&fasttrap_cleanup_mtx);
2294                 fasttrap_timeout = 0;
2295                 work = fasttrap_cleanup_work;
2296                 mutex_exit(&fasttrap_cleanup_mtx);
2297 
2298                 if (work)
2299                         fasttrap_pid_cleanup();
2300 
2301                 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2302                     &fasttrap_meta_id);
2303 
2304                 return (DDI_FAILURE);
2305         }
2306 
2307 #ifdef DEBUG
2308         mutex_enter(&fasttrap_count_mtx);
2309         ASSERT(fasttrap_pid_count == 0);
2310         mutex_exit(&fasttrap_count_mtx);
2311 #endif
2312 
2313         kmem_free(fasttrap_tpoints.fth_table,
2314             fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2315         fasttrap_tpoints.fth_nent = 0;
2316 
2317         kmem_free(fasttrap_provs.fth_table,
2318             fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2319         fasttrap_provs.fth_nent = 0;
2320 
2321         kmem_free(fasttrap_procs.fth_table,
2322             fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2323         fasttrap_procs.fth_nent = 0;
2324 
2325         /*
2326          * We know there are no tracepoints in any process anywhere in
2327          * the system so there is no process which has its p_dtrace_count
2328          * greater than zero, therefore we know that no thread can actively
2329          * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2330          * and fasttrap_exec() and fasttrap_exit().
2331          */
2332         ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2333         dtrace_fasttrap_fork_ptr = NULL;
2334 
2335         ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2336         dtrace_fasttrap_exec_ptr = NULL;
2337 
2338         ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2339         dtrace_fasttrap_exit_ptr = NULL;
2340 
2341         ddi_remove_minor_node(devi, NULL);
2342 
2343         return (DDI_SUCCESS);
2344 }
2345 
2346 static struct dev_ops fasttrap_ops = {
2347         DEVO_REV,               /* devo_rev */
2348         0,                      /* refcnt */
2349         fasttrap_info,          /* get_dev_info */
2350         nulldev,                /* identify */
2351         nulldev,                /* probe */
2352         fasttrap_attach,        /* attach */
2353         fasttrap_detach,        /* detach */
2354         nodev,                  /* reset */
2355         &fasttrap_cb_ops,   /* driver operations */
2356         NULL,                   /* bus operations */
2357         nodev,                  /* dev power */
2358         ddi_quiesce_not_needed,         /* quiesce */
2359 };
2360 
2361 /*
2362  * Module linkage information for the kernel.
2363  */
2364 static struct modldrv modldrv = {
2365         &mod_driverops,             /* module type (this is a pseudo driver) */
2366         "Fasttrap Tracing",     /* name of module */
2367         &fasttrap_ops,              /* driver ops */
2368 };
2369 
2370 static struct modlinkage modlinkage = {
2371         MODREV_1,
2372         { (void *)&modldrv, NULL }
2373 };
2374 
2375 int
2376 _init(void)
2377 {
2378         return (mod_install(&modlinkage));
2379 }
2380 
2381 int
2382 _info(struct modinfo *modinfop)
2383 {
2384         return (mod_info(&modlinkage, modinfop));
2385 }
2386 
2387 int
2388 _fini(void)
2389 {
2390         return (mod_remove(&modlinkage));
2391 }