1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 
  27 #include <sys/errno.h>
  28 #include <sys/stat.h>
  29 #include <sys/modctl.h>
  30 #include <sys/conf.h>
  31 #include <sys/systm.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/cpuvar.h>
  35 #include <sys/kmem.h>
  36 #include <sys/strsubr.h>
  37 #include <sys/dtrace.h>
  38 #include <sys/kobj.h>
  39 #include <sys/modctl.h>
  40 #include <sys/atomic.h>
  41 #include <vm/seg_kmem.h>
  42 #include <sys/stack.h>
  43 #include <sys/ctf_api.h>
  44 #include <sys/sysmacros.h>
  45 
  46 static dev_info_t               *fbt_devi;
  47 static dtrace_provider_id_t     fbt_id;
  48 static uintptr_t                fbt_trampoline;
  49 static caddr_t                  fbt_trampoline_window;
  50 static size_t                   fbt_trampoline_size;
  51 static int                      fbt_verbose = 0;
  52 
  53 /*
  54  * Various interesting bean counters.
  55  */
  56 static int                      fbt_entry;
  57 static int                      fbt_ret;
  58 static int                      fbt_retl;
  59 static int                      fbt_retl_jmptab;
  60 static int                      fbt_retl_twoinstr;
  61 static int                      fbt_retl_tailcall;
  62 static int                      fbt_retl_tailjmpl;
  63 static int                      fbt_leaf_functions;
  64 
  65 extern char                     stubs_base[];
  66 extern char                     stubs_end[];
  67 
  68 #define FBT_REG_G0              0
  69 #define FBT_REG_G1              1
  70 #define FBT_REG_O0              8
  71 #define FBT_REG_O1              9
  72 #define FBT_REG_O2              10
  73 #define FBT_REG_O3              11
  74 #define FBT_REG_O4              12
  75 #define FBT_REG_O5              13
  76 #define FBT_REG_O6              14
  77 #define FBT_REG_O7              15
  78 #define FBT_REG_I0              24
  79 #define FBT_REG_I1              25
  80 #define FBT_REG_I2              26
  81 #define FBT_REG_I3              27
  82 #define FBT_REG_I4              28
  83 #define FBT_REG_I7              31
  84 #define FBT_REG_L0              16
  85 #define FBT_REG_L1              17
  86 #define FBT_REG_L2              18
  87 #define FBT_REG_L3              19
  88 #define FBT_REG_PC              5
  89 
  90 #define FBT_REG_ISGLOBAL(r)     ((r) < 8)
  91 #define FBT_REG_ISOUTPUT(r)     ((r) >= 8 && (r) < 16)
  92 #define FBT_REG_ISLOCAL(r)      ((r) >= 16 && (r) < 24)
  93 #define FBT_REG_ISVOLATILE(r)   \
  94         ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
  95 #define FBT_REG_NLOCALS         8
  96 
  97 #define FBT_REG_MARKLOCAL(locals, r)    \
  98         if (FBT_REG_ISLOCAL(r)) \
  99                 (locals)[(r) - FBT_REG_L0] = 1;
 100 
 101 #define FBT_REG_INITLOCALS(local, locals)       \
 102         for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++)  \
 103                 (locals)[(local)] = 0; \
 104         (local) = FBT_REG_L0
 105 
 106 #define FBT_REG_ALLOCLOCAL(local, locals)       \
 107         while ((locals)[(local) - FBT_REG_L0]) \
 108                 (local)++; \
 109         (locals)[(local) - FBT_REG_L0] = 1;
 110 
 111 #define FBT_OP_MASK             0xc0000000
 112 #define FBT_OP_SHIFT            30
 113 #define FBT_OP(val)             ((val) & FBT_FMT1_MASK)
 114 
 115 #define FBT_SIMM13_MASK         0x1fff
 116 #define FBT_SIMM13_MAX          ((int32_t)0xfff)
 117 #define FBT_IMM22_MASK          0x3fffff
 118 #define FBT_IMM22_SHIFT         10
 119 #define FBT_IMM10_MASK          0x3ff
 120 
 121 #define FBT_DISP30_MASK         0x3fffffff
 122 #define FBT_DISP30(from, to)    \
 123         (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)
 124 
 125 #define FBT_DISP22_MASK         0x3fffff
 126 #define FBT_DISP22(from, to)    \
 127         (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)
 128 
 129 #define FBT_DISP19_MASK         0x7ffff
 130 #define FBT_DISP19(from, to)    \
 131         (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)
 132 
 133 #define FBT_DISP16_HISHIFT      20
 134 #define FBT_DISP16_HIMASK       (0x3 << FBT_DISP16_HISHIFT)
 135 #define FBT_DISP16_LOMASK       (0x3fff)
 136 #define FBT_DISP16_MASK         (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
 137 #define FBT_DISP16(val) \
 138         ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))
 139 
 140 #define FBT_DISP14_MASK         0x3fff
 141 #define FBT_DISP14(from, to)    \
 142         (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)
 143 
 144 #define FBT_OP0                 (((uint32_t)0) << FBT_OP_SHIFT)
 145 #define FBT_OP1                 (((uint32_t)1) << FBT_OP_SHIFT)
 146 #define FBT_OP2                 (((uint32_t)2) << FBT_OP_SHIFT)
 147 #define FBT_ILLTRAP             0
 148 
 149 #define FBT_ANNUL_SHIFT         29
 150 #define FBT_ANNUL               (1 << FBT_ANNUL_SHIFT)
 151 
 152 #define FBT_FMT3_OP3_SHIFT      19
 153 #define FBT_FMT3_OP_MASK        0xc1f80000
 154 #define FBT_FMT3_OP(val)        ((val) & FBT_FMT3_OP_MASK)
 155 
 156 #define FBT_FMT3_RD_SHIFT       25
 157 #define FBT_FMT3_RD_MASK        (0x1f << FBT_FMT3_RD_SHIFT)
 158 #define FBT_FMT3_RD(val)        \
 159         (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)
 160 
 161 #define FBT_FMT3_RS1_SHIFT      14
 162 #define FBT_FMT3_RS1_MASK       (0x1f << FBT_FMT3_RS1_SHIFT)
 163 #define FBT_FMT3_RS1(val)       \
 164         (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
 165 #define FBT_FMT3_RS1_SET(val, rs1) \
 166         (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)
 167 
 168 #define FBT_FMT3_RS2_SHIFT      0
 169 #define FBT_FMT3_RS2_MASK       (0x1f << FBT_FMT3_RS2_SHIFT)
 170 #define FBT_FMT3_RS2(val)       \
 171         (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
 172 #define FBT_FMT3_RS2_SET(val, rs2) \
 173         (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)
 174 
 175 #define FBT_FMT3_IMM_SHIFT      13
 176 #define FBT_FMT3_IMM            (1 << FBT_FMT3_IMM_SHIFT)
 177 #define FBT_FMT3_SIMM13_MASK    FBT_SIMM13_MASK
 178 
 179 #define FBT_FMT3_ISIMM(val)     ((val) & FBT_FMT3_IMM)
 180 #define FBT_FMT3_SIMM13(val)    ((val) & FBT_FMT3_SIMM13_MASK)
 181 
 182 #define FBT_FMT2_OP2_SHIFT      22
 183 #define FBT_FMT2_OP2_MASK       (0x7 << FBT_FMT2_OP2_SHIFT)
 184 #define FBT_FMT2_RD_SHIFT       25
 185 
 186 #define FBT_FMT1_OP(val)        ((val) & FBT_OP_MASK)
 187 #define FBT_FMT1_DISP30(val)    ((val) & FBT_DISP30_MASK)
 188 
 189 #define FBT_FMT2_OP2_BPCC       (0x01 << FBT_FMT2_OP2_SHIFT)
 190 #define FBT_FMT2_OP2_BCC        (0x02 << FBT_FMT2_OP2_SHIFT)
 191 #define FBT_FMT2_OP2_BPR        (0x03 << FBT_FMT2_OP2_SHIFT)
 192 #define FBT_FMT2_OP2_SETHI      (0x04 << FBT_FMT2_OP2_SHIFT)
 193 
 194 #define FBT_FMT2_COND_SHIFT     25
 195 #define FBT_FMT2_COND_BA        (0x8 << FBT_FMT2_COND_SHIFT)
 196 #define FBT_FMT2_COND_BL        (0x3 << FBT_FMT2_COND_SHIFT)
 197 #define FBT_FMT2_COND_BGE       (0xb << FBT_FMT2_COND_SHIFT)
 198 
 199 #define FBT_OP_RESTORE          (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
 200 #define FBT_OP_SAVE             (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
 201 #define FBT_OP_JMPL             (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
 202 #define FBT_OP_RETURN           (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
 203 #define FBT_OP_CALL             FBT_OP1
 204 #define FBT_OP_SETHI            (FBT_OP0 | FBT_FMT2_OP2_SETHI)
 205 #define FBT_OP_ADD              (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
 206 #define FBT_OP_OR               (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
 207 #define FBT_OP_SUB              (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
 208 #define FBT_OP_CC               (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
 209 #define FBT_OP_BA               (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
 210 #define FBT_OP_BL               (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
 211 #define FBT_OP_BGE              (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
 212 #define FBT_OP_BAPCC            (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
 213 #define FBT_OP_RD               (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))
 214 
 215 #define FBT_ORLO(rs, val, rd) \
 216         (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
 217         ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))
 218 
 219 #define FBT_ORSIMM13(rs, val, rd) \
 220         (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
 221         ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
 222 
 223 #define FBT_ADDSIMM13(rs, val, rd) \
 224         (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
 225         ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
 226 
 227 #define FBT_ADD(rs1, rs2, rd) \
 228         (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
 229         ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
 230 
 231 #define FBT_CMP(rs1, rs2) \
 232         (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
 233         ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))
 234 
 235 #define FBT_MOV(rs, rd) \
 236         (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
 237         ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
 238 
 239 #define FBT_SETHI(val, reg)     \
 240         (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
 241         ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))
 242 
 243 #define FBT_CALL(orig, dest)    (FBT_OP_CALL | FBT_DISP30(orig, dest))
 244 
 245 #define FBT_RET \
 246         (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
 247         (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))
 248 
 249 #define FBT_SAVEIMM(rd, val, rs1)       \
 250         (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
 251         ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
 252 
 253 #define FBT_RESTORE(rd, rs1, rs2)       \
 254         (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
 255         ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))
 256 
 257 #define FBT_RETURN(rs1, val)            \
 258         (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
 259         FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
 260 
 261 #define FBT_BA(orig, dest)      (FBT_OP_BA | FBT_DISP22(orig, dest))
 262 #define FBT_BAA(orig, dest)     (FBT_BA(orig, dest) | FBT_ANNUL)
 263 #define FBT_BL(orig, dest)      (FBT_OP_BL | FBT_DISP22(orig, dest))
 264 #define FBT_BGE(orig, dest)     (FBT_OP_BGE | FBT_DISP22(orig, dest))
 265 #define FBT_BDEST(va, instr)    ((uintptr_t)(va) + \
 266         (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
 267 #define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \
 268         (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
 269 #define FBT_BPRDEST(va, instr)  ((uintptr_t)(va) + \
 270         (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))
 271 
 272 /*
 273  * We're only going to treat a save as safe if (a) both rs1 and rd are
 274  * %sp and (b) if the instruction has a simm, the value isn't 0.
 275  */
 276 #define FBT_IS_SAVE(instr)      \
 277         (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
 278         FBT_FMT3_RD(instr) == FBT_REG_O6 && \
 279         FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
 280         !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))
 281 
 282 #define FBT_IS_BA(instr)        (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
 283 #define FBT_IS_BAPCC(instr)     (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)
 284 
 285 #define FBT_IS_RDPC(instr)      ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
 286         (FBT_FMT3_RD(instr) == FBT_REG_PC))
 287 
 288 #define FBT_IS_PCRELATIVE(instr)        \
 289         ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
 290         ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
 291         ((instr) & FBT_OP_MASK) == FBT_OP1 || \
 292         FBT_IS_RDPC(instr))
 293 
 294 #define FBT_IS_CTI(instr)       \
 295         ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
 296         ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
 297         ((instr) & FBT_OP_MASK) == FBT_OP1 || \
 298         (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
 299         (FBT_FMT3_OP(instr) == FBT_OP_RETURN))
 300 
 301 #define FBT_PROBENAME_ENTRY     "entry"
 302 #define FBT_PROBENAME_RETURN    "return"
 303 #define FBT_ESTIMATE_ID         (UINT32_MAX)
 304 #define FBT_COUNTER(id, count)  if ((id) != FBT_ESTIMATE_ID) (count)++
 305 
 306 #define FBT_ENTENT_MAXSIZE      (16 * sizeof (uint32_t))
 307 #define FBT_RETENT_MAXSIZE      (11 * sizeof (uint32_t))
 308 #define FBT_RETLENT_MAXSIZE     (23 * sizeof (uint32_t))
 309 #define FBT_ENT_MAXSIZE         \
 310         MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)
 311 
 312 typedef struct fbt_probe {
 313         char            *fbtp_name;
 314         dtrace_id_t     fbtp_id;
 315         uintptr_t       fbtp_addr;
 316         struct modctl   *fbtp_ctl;
 317         int             fbtp_loadcnt;
 318         int             fbtp_symndx;
 319         int             fbtp_primary;
 320         int             fbtp_return;
 321         uint32_t        *fbtp_patchpoint;
 322         uint32_t        fbtp_patchval;
 323         uint32_t        fbtp_savedval;
 324         struct fbt_probe *fbtp_next;
 325 } fbt_probe_t;
 326 
 327 typedef struct fbt_trampoline {
 328         uintptr_t       fbtt_va;
 329         uintptr_t       fbtt_limit;
 330         uintptr_t       fbtt_next;
 331 } fbt_trampoline_t;
 332 
 333 static caddr_t
 334 fbt_trampoline_map(uintptr_t tramp, size_t size)
 335 {
 336         uintptr_t offs;
 337         page_t **ppl;
 338 
 339         ASSERT(fbt_trampoline_window == NULL);
 340         ASSERT(fbt_trampoline_size == 0);
 341         ASSERT(fbt_trampoline == NULL);
 342 
 343         size += tramp & PAGEOFFSET;
 344         fbt_trampoline = tramp & PAGEMASK;
 345         fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
 346         fbt_trampoline_window =
 347             vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);
 348 
 349         (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
 350             fbt_trampoline_size, S_WRITE);
 351 
 352         for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
 353                 hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
 354                     hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
 355                     PROT_READ | PROT_WRITE,
 356                     HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
 357         }
 358 
 359         as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
 360             S_WRITE);
 361 
 362         return (fbt_trampoline_window + (tramp & PAGEOFFSET));
 363 }
 364 
 365 static void
 366 fbt_trampoline_unmap()
 367 {
 368         ASSERT(fbt_trampoline_window != NULL);
 369         ASSERT(fbt_trampoline_size != 0);
 370         ASSERT(fbt_trampoline != NULL);
 371 
 372         membar_enter();
 373         sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
 374         sync_icache(fbt_trampoline_window, fbt_trampoline_size);
 375 
 376         hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
 377             HAT_UNLOAD_UNLOCK);
 378 
 379         vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);
 380 
 381         fbt_trampoline_window = NULL;
 382         fbt_trampoline = NULL;
 383         fbt_trampoline_size = 0;
 384 }
 385 
 386 static uintptr_t
 387 fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
 388     int nargs)
 389 {
 390         uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
 391         uint32_t first = *instr;
 392         uintptr_t va = tramp->fbtt_va;
 393         uintptr_t base = tramp->fbtt_next;
 394 
 395         if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
 396                 /*
 397                  * There isn't sufficient room for this entry; return failure.
 398                  */
 399                 return (0);
 400         }
 401 
 402         FBT_COUNTER(id, fbt_entry);
 403 
 404         if (FBT_IS_SAVE(first)) {
 405                 *tinstr++ = first;
 406         } else {
 407                 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
 408         }
 409 
 410         if (id > (uint32_t)FBT_SIMM13_MAX) {
 411                 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
 412                 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
 413         } else {
 414                 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
 415         }
 416 
 417         if (nargs >= 1)
 418                 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);
 419 
 420         if (nargs >= 2)
 421                 *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);
 422 
 423         if (nargs >= 3)
 424                 *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);
 425 
 426         if (nargs >= 4)
 427                 *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);
 428 
 429         if (nargs >= 5)
 430                 *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);
 431 
 432         if (FBT_IS_SAVE(first)) {
 433                 uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);
 434 
 435                 *tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
 436                 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
 437                 tinstr++;
 438                 *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
 439         } else {
 440                 uintptr_t slot = *--tinstr;
 441                 uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
 442                 uint32_t delay = first;
 443 
 444                 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
 445                 tinstr++;
 446                 *tinstr++ = slot;
 447                 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
 448 
 449                 if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
 450                         /*
 451                          * This is a special case:  we are instrumenting a
 452                          * a non-annulled branch-always (or variant).  We'll
 453                          * return directly to the destination of the branch,
 454                          * copying the instruction in the delay slot here,
 455                          * and then executing it in the slot of a ba.
 456                          */
 457                         if (FBT_IS_BA(first)) {
 458                                 ret = FBT_BDEST(instr, *instr);
 459                         } else {
 460                                 ret = FBT_BPCCDEST(instr, *instr);
 461                         }
 462 
 463                         delay = *(instr + 1);
 464                 }
 465 
 466                 if ((first & FBT_OP_MASK) != FBT_OP0 ||
 467                     (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
 468                         *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
 469                         tinstr++;
 470                         *tinstr++ = delay;
 471                 } else {
 472                         /*
 473                          * If this is a branch-on-register, we have a little
 474                          * more work to do:  because the displacement is only
 475                          * sixteen bits, we're going to thunk the branch into
 476                          * the trampoline, and then ba,a to the appropriate
 477                          * destination in the branch targets.  That is, we're
 478                          * constructing this sequence in the trampoline:
 479                          *
 480                          *              br[cc]  %[rs], 1f
 481                          *              <delay-instruction>
 482                          *              ba,a    <not-taken-destination>
 483                          *      1:      ba,a    <taken-destination>
 484                          *
 485                          */
 486                         uintptr_t targ = FBT_BPRDEST(instr, first);
 487 
 488                         *tinstr = first & ~(FBT_DISP16_MASK);
 489                         *tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
 490                         tinstr++;
 491                         *tinstr++ = *(instr + 1);
 492                         *tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
 493                             ret + sizeof (uint32_t));
 494                         tinstr++;
 495                         *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
 496                         tinstr++;
 497                 }
 498         }
 499 
 500         tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
 501         tramp->fbtt_next = (uintptr_t)tinstr;
 502 
 503         return (1);
 504 }
 505 
 506 /*
 507  * We are patching control-transfer/restore couplets.  There are three
 508  * variants of couplet:
 509  *
 510  * (a)  return          rs1 + imm
 511  *      delay
 512  *
 513  * (b)  jmpl            rs1 + (rs2 | offset), rd
 514  *      restore         rs1, rs2 | imm, rd
 515  *
 516  * (c)  call            displacement
 517  *      restore         rs1, rs2 | imm, rd
 518  *
 519  * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
 520  * or delay is a DCTI, we fail.  If rd from the jmpl in (b) is something other
 521  * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
 522  * through a register), we fail.
 523  *
 524  * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
 525  * potentially outputs and/or globals.  Because these registers cannot be
 526  * relied upon across the call to dtrace_probe(), we move rs1 into an unused
 527  * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
 528  * to be:
 529  *
 530  *      restore         ls0, ls1, rd
 531  *
 532  * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
 533  * If the jmpl uses outputs or globals, we restructure it to be:
 534  *
 535  *      jmpl            ls2 + (ls3 | offset), (%g0 | %o7)
 536  *
 537  */
 538 /*ARGSUSED*/
 539 static int
 540 fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
 541 {
 542         int rd;
 543 
 544         if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
 545                 uint32_t delay = *(instr + 1);
 546 
 547                 if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
 548                         /*
 549                          * It's unclear if we should warn about this or not.
 550                          * We really wouldn't expect the compiler to generate
 551                          * return instructions with something other than %i7
 552                          * as rs1 and 8 as the simm13 -- it would just be
 553                          * mean-spirited.  That said, such a construct isn't
 554                          * necessarily incorrect.  Sill, we err on the side of
 555                          * caution and warn about it...
 556                          */
 557                         cmn_err(CE_NOTE, "cannot instrument return of %s at "
 558                             "%p: non-canonical return instruction", name,
 559                             (void *)instr);
 560                         return (0);
 561                 }
 562 
 563                 if (FBT_IS_CTI(delay)) {
 564                         /*
 565                          * This is even weirder -- a DCTI coupled with a
 566                          * return instruction.  Similar constructs are used to
 567                          * return from utraps, but these typically have the
 568                          * return in the slot -- and we wouldn't expect to see
 569                          * it in the kernel regardless.  At any rate, we don't
 570                          * want to try to instrument this construct, whatever
 571                          * it may be.
 572                          */
 573                         cmn_err(CE_NOTE, "cannot instrument return of %s at "
 574                             "%p: CTI in delay slot of return instruction",
 575                             name, (void *)instr);
 576                         return (0);
 577                 }
 578 
 579                 if (FBT_IS_PCRELATIVE(delay)) {
 580                         /*
 581                          * This is also very weird, but might be correct code
 582                          * if the function is (for example) returning the
 583                          * address of the delay instruction of the return as
 584                          * its return value (e.g. "rd %pc, %o0" in the slot).
 585                          * Perhaps correct, but still too weird to not warn
 586                          * about it...
 587                          */
 588                         cmn_err(CE_NOTE, "cannot instrument return of %s at "
 589                             "%p: PC-relative instruction in delay slot of "
 590                             "return instruction", name, (void *)instr);
 591                         return (0);
 592                 }
 593 
 594                 return (1);
 595         }
 596 
 597         if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
 598                 return (0);
 599 
 600         if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
 601                 return (1);
 602 
 603         if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
 604                 return (0);
 605 
 606         rd = FBT_FMT3_RD(*instr);
 607 
 608         if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
 609                 return (1);
 610 
 611         /*
 612          * We have encountered a jmpl that is storing the calling %pc in
 613          * some register besides %i7, %o7 or %g0.  This is strange; emit
 614          * a warning and fail.
 615          */
 616         cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
 617             "jmpl destination register", name, (void *)instr);
 618         return (0);
 619 }
 620 
 621 static int
 622 fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
 623 {
 624         if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
 625             (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
 626             FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
 627                 /*
 628                  * If this is a call (or a jmpl that links into %o7), we can
 629                  * patch it iff the next instruction uses %o7 as a destination
 630                  * register.  Because there is an ABI responsibility to
 631                  * restore %o7 to the value before the call/jmpl, we don't
 632                  * particularly care how this routine is managing to restore
 633                  * it (mov, add, ld or divx for all we care).  If it doesn't
 634                  * seem to be restoring it at all, however, we'll refuse
 635                  * to patch it.
 636                  */
 637                 uint32_t delay = *(instr + 1);
 638                 uint32_t op, rd;
 639 
 640                 op = FBT_FMT1_OP(delay);
 641                 rd = FBT_FMT3_RD(delay);
 642 
 643                 if (op != FBT_OP2 || rd != FBT_REG_O7) {
 644                         /*
 645                          * This is odd.  Before we assume that we're looking
 646                          * at something bizarre (and warn accordingly), we'll
 647                          * check to see if it's obviously a jump table entry.
 648                          */
 649                         if (*instr < (uintptr_t)instr &&
 650                             *instr >= (uintptr_t)instr - offset)
 651                                 return (0);
 652 
 653                         cmn_err(CE_NOTE, "cannot instrument return of %s at "
 654                             "%p: leaf jmpl/call delay isn't restoring %%o7",
 655                             name, (void *)instr);
 656                         return (0);
 657                 }
 658 
 659                 return (1);
 660         }
 661 
 662         if (offset == sizeof (uint32_t)) {
 663                 /*
 664                  * If this is the second instruction in the function, we're
 665                  * going to allow it to be patched if the first instruction
 666                  * is a patchable return-from-leaf instruction.
 667                  */
 668                 if (fbt_canpatch_retl(instr - 1, 0, name))
 669                         return (1);
 670         }
 671 
 672         if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
 673                 return (0);
 674 
 675         if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
 676                 return (0);
 677 
 678         return (1);
 679 }
 680 
 681 /*ARGSUSED*/
 682 static uint32_t
 683 fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
 684     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
 685 {
 686         uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
 687         uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
 688         uintptr_t va = tramp->fbtt_va;
 689         uintptr_t base = tramp->fbtt_next;
 690         uint32_t locals[FBT_REG_NLOCALS], local;
 691 
 692         if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
 693                 /*
 694                  * There isn't sufficient room for this entry; return failure.
 695                  */
 696                 return (FBT_ILLTRAP);
 697         }
 698 
 699         FBT_COUNTER(id, fbt_ret);
 700 
 701         if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
 702                 /*
 703                  * To handle the case of the return instruction, we'll emit a
 704                  * restore, followed by the instruction in the slot (which
 705                  * we'll transplant here), and then another save.  While it
 706                  * may seem intellectually unsatisfying to emit the additional
 707                  * restore/save couplet, one can take solace in the fact that
 708                  * we don't do this if the instruction in the return delay
 709                  * slot is a nop -- which it is nearly 90% of the time with
 710                  * gcc.  (And besides, this couplet can't induce unnecessary
 711                  * spill/fill traps; rewriting the delay instruction to be
 712                  * in terms of the current window hardly seems worth the
 713                  * trouble -- let alone the risk.)
 714                  */
 715                 uint32_t delay = *(instr + 1);
 716                 ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));
 717 
 718                 cti = FBT_RET;
 719                 restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
 720 
 721                 if (delay != FBT_SETHI(0, FBT_REG_G0)) {
 722                         *tinstr++ = restore;
 723                         *tinstr++ = delay;
 724                         *tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
 725                             -SA(MINFRAME), FBT_REG_O6);
 726                 }
 727         }
 728 
 729         FBT_REG_INITLOCALS(local, locals);
 730 
 731         /*
 732          * Mark the locals used in the jmpl.
 733          */
 734         if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
 735                 uint32_t rs1 = FBT_FMT3_RS1(cti);
 736                 FBT_REG_MARKLOCAL(locals, rs1);
 737 
 738                 if (!FBT_FMT3_ISIMM(cti)) {
 739                         uint32_t rs2 = FBT_FMT3_RS2(cti);
 740                         FBT_REG_MARKLOCAL(locals, rs2);
 741                 }
 742         }
 743 
 744         /*
 745          * And mark the locals used in the restore.
 746          */
 747         rs1 = FBT_FMT3_RS1(restore);
 748         FBT_REG_MARKLOCAL(locals, rs1);
 749 
 750         if (!FBT_FMT3_ISIMM(restore)) {
 751                 uint32_t rs2 = FBT_FMT3_RS2(restore);
 752                 FBT_REG_MARKLOCAL(locals, rs2);
 753         }
 754 
 755         if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
 756                 uint32_t rs1 = FBT_FMT3_RS1(cti);
 757 
 758                 if (FBT_REG_ISVOLATILE(rs1)) {
 759                         FBT_REG_ALLOCLOCAL(local, locals);
 760                         FBT_FMT3_RS1_SET(cti, local);
 761                         *tinstr++ = FBT_MOV(rs1, local);
 762                 }
 763 
 764                 if (!FBT_FMT3_ISIMM(cti)) {
 765                         uint32_t rs2 = FBT_FMT3_RS2(cti);
 766 
 767                         if (FBT_REG_ISVOLATILE(rs2)) {
 768                                 FBT_REG_ALLOCLOCAL(local, locals);
 769                                 FBT_FMT3_RS2_SET(cti, local);
 770                                 *tinstr++ = FBT_MOV(rs2, local);
 771                         }
 772                 }
 773         }
 774 
 775         rs1 = FBT_FMT3_RS1(restore);
 776 
 777         if (FBT_REG_ISVOLATILE(rs1)) {
 778                 FBT_REG_ALLOCLOCAL(local, locals);
 779                 FBT_FMT3_RS1_SET(restore, local);
 780                 *tinstr++ = FBT_MOV(rs1, local);
 781         }
 782 
 783         if (!FBT_FMT3_ISIMM(restore)) {
 784                 uint32_t rs2 = FBT_FMT3_RS2(restore);
 785 
 786                 if (FBT_REG_ISVOLATILE(rs2)) {
 787                         FBT_REG_ALLOCLOCAL(local, locals);
 788                         FBT_FMT3_RS2_SET(restore, local);
 789                         *tinstr++ = FBT_MOV(rs2, local);
 790                 }
 791         }
 792 
 793         if (id > (uint32_t)FBT_SIMM13_MAX) {
 794                 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
 795                 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
 796         } else {
 797                 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
 798         }
 799 
 800         if (offset > (uint32_t)FBT_SIMM13_MAX) {
 801                 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
 802                 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
 803         } else {
 804                 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
 805         }
 806 
 807         *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
 808         tinstr++;
 809 
 810         if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
 811                 /*
 812                  * If the destination register of the restore is %o0, we
 813                  * need to perform the implied calculation to derive the
 814                  * return value.
 815                  */
 816                 uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
 817                 add &= ~FBT_FMT3_RD_MASK;
 818                 *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
 819         } else {
 820                 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
 821         }
 822 
 823         /*
 824          * If the control transfer instruction is %pc-relative (i.e. a
 825          * call), we need to reset it appropriately.
 826          */
 827         if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
 828                 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
 829                 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
 830                 tinstr++;
 831         } else {
 832                 *tinstr++ = cti;
 833         }
 834 
 835         *tinstr++ = restore;
 836         tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
 837         tramp->fbtt_next = (uintptr_t)tinstr;
 838 
 839         return (FBT_BAA(instr, va));
 840 }
 841 
 842 static uint32_t
 843 fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
 844     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
 845 {
 846         uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
 847         uintptr_t va = tramp->fbtt_va;
 848         uintptr_t base = tramp->fbtt_next;
 849         uint32_t cti = *instr, dest;
 850         int annul = 0;
 851 
 852         FBT_COUNTER(id, fbt_retl);
 853 
 854         if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
 855                 /*
 856                  * There isn't sufficient room for this entry; return failure.
 857                  */
 858                 return (FBT_ILLTRAP);
 859         }
 860 
 861         if (offset == sizeof (uint32_t) &&
 862             fbt_canpatch_retl(instr - 1, 0, name)) {
 863                 *tinstr++ = *instr;
 864                 annul = 1;
 865                 FBT_COUNTER(id, fbt_retl_twoinstr);
 866         } else {
 867                 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
 868                     FBT_FMT3_RD(cti) != FBT_REG_O7 &&
 869                     FBT_FMT3_RS1(cti) != FBT_REG_O7) {
 870                         annul = 1;
 871                         *tinstr++ = *(instr + 1);
 872                 }
 873         }
 874 
 875         *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
 876 
 877         if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
 878                 uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;
 879 
 880                 /*
 881                  * If we have a jmpl and it's in terms of output registers, we
 882                  * need to rewrite it to be in terms of the corresponding input
 883                  * registers.  If it's in terms of the globals, we'll rewrite
 884                  * it to be in terms of locals.
 885                  */
 886                 rs1 = FBT_FMT3_RS1(cti);
 887 
 888                 if (FBT_REG_ISOUTPUT(rs1))
 889                         rs1 += o2i;
 890 
 891                 if (FBT_REG_ISGLOBAL(rs1)) {
 892                         *tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
 893                         rs1 = FBT_REG_L0;
 894                 }
 895 
 896                 FBT_FMT3_RS1_SET(cti, rs1);
 897 
 898                 if (!FBT_FMT3_ISIMM(cti)) {
 899                         rs2 = FBT_FMT3_RS2(cti);
 900 
 901                         if (FBT_REG_ISOUTPUT(rs2))
 902                                 rs2 += o2i;
 903 
 904                         if (FBT_REG_ISGLOBAL(rs2)) {
 905                                 *tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
 906                                 rs2 = FBT_REG_L1;
 907                         }
 908 
 909                         FBT_FMT3_RS2_SET(cti, rs2);
 910                 }
 911 
 912                 /*
 913                  * Now we need to check the rd and source register for the jmpl;
 914                  * If neither rd nor the source register is %o7, then we might
 915                  * have a jmp that is actually part of a jump table.  We need
 916                  * to generate the code to compare it to the base and limit of
 917                  * the function.
 918                  */
 919                 if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
 920                         uintptr_t base = (uintptr_t)funcbase;
 921                         uintptr_t limit = (uintptr_t)funclim;
 922 
 923                         FBT_COUNTER(id, fbt_retl_jmptab);
 924 
 925                         if (FBT_FMT3_ISIMM(cti)) {
 926                                 *tinstr++ = FBT_ADDSIMM13(rs1,
 927                                     FBT_FMT3_SIMM13(cti), FBT_REG_L2);
 928                         } else {
 929                                 *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
 930                         }
 931 
 932                         *tinstr++ = FBT_SETHI(base, FBT_REG_L3);
 933                         *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
 934                         *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
 935                         *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
 936                         *tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
 937                         *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
 938                         *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
 939                         *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
 940                         *tinstr++ = FBT_SETHI(0, FBT_REG_G0);
 941                         *tinstr++ = cti;
 942                         *tinstr++ = FBT_RESTORE(FBT_REG_G0,
 943                             FBT_REG_G0, FBT_REG_G0);
 944                 }
 945         }
 946 
 947         if (id > (uint32_t)FBT_SIMM13_MAX) {
 948                 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
 949                 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
 950         } else {
 951                 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
 952         }
 953 
 954         if (offset > (uint32_t)FBT_SIMM13_MAX) {
 955                 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
 956                 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
 957         } else {
 958                 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
 959         }
 960 
 961         *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
 962         tinstr++;
 963         *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
 964 
 965         /*
 966          * If the control transfer instruction is %pc-relative (i.e. a
 967          * call), we need to reset it appropriately.
 968          */
 969         if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
 970                 FBT_COUNTER(id, fbt_retl_tailcall);
 971                 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
 972                 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
 973                 tinstr++;
 974                 annul = 1;
 975         } else {
 976                 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
 977                         *tinstr++ = cti;
 978 
 979                         if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
 980                                 FBT_COUNTER(id, fbt_retl_tailjmpl);
 981                                 annul = 1;
 982                         }
 983                 } else {
 984                         *tinstr++ = FBT_RET;
 985                 }
 986         }
 987 
 988         *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
 989 
 990         tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
 991         tramp->fbtt_next = (uintptr_t)tinstr;
 992 
 993         return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
 994 }
 995 
 996 /*ARGSUSED*/
 997 static void
 998 fbt_provide_module(void *arg, struct modctl *ctl)
 999 {
1000         struct module *mp = ctl->mod_mp;
1001         char *modname = ctl->mod_modname;
1002         char *str = mp->strings;
1003         int nsyms = mp->nsyms;
1004         Shdr *symhdr = mp->symhdr;
1005         size_t symsize;
1006         char *name;
1007         int i;
1008         fbt_probe_t *fbt, *retfbt;
1009         fbt_trampoline_t tramp;
1010         uintptr_t offset;
1011         int primary = 0;
1012         ctf_file_t *fp = NULL;
1013         int error;
1014         int estimate = 1;
1015         uint32_t faketramp[50];
1016         size_t fbt_size = 0;
1017 
1018         /*
1019          * Employees of dtrace and their families are ineligible.  Void
1020          * where prohibited.
1021          */
1022         if (strcmp(modname, "dtrace") == 0)
1023                 return;
1024 
1025         if (ctl->mod_requisites != NULL) {
1026                 struct modctl_list *list;
1027 
1028                 list = (struct modctl_list *)ctl->mod_requisites;
1029 
1030                 for (; list != NULL; list = list->modl_next) {
1031                         if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
1032                                 return;
1033                 }
1034         }
1035 
1036         /*
1037          * KMDB is ineligible for instrumentation -- it may execute in
1038          * any context, including probe context.
1039          */
1040         if (strcmp(modname, "kmdbmod") == 0)
1041                 return;
1042 
1043         if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) {
1044                 /*
1045                  * If this module doesn't (yet) have its string or symbol
1046                  * table allocated, clear out.
1047                  */
1048                 return;
1049         }
1050 
1051         symsize = symhdr->sh_entsize;
1052 
1053         if (mp->fbt_nentries) {
1054                 /*
1055                  * This module has some FBT entries allocated; we're afraid
1056                  * to screw with it.
1057                  */
1058                 return;
1059         }
1060 
1061         if (mp->fbt_tab != NULL)
1062                 estimate = 0;
1063 
1064         /*
1065          * This is a hack for unix/genunix/krtld.
1066          */
1067         primary = vmem_contains(heap_arena, (void *)ctl,
1068             sizeof (struct modctl)) == 0;
1069         kobj_textwin_alloc(mp);
1070 
1071         /*
1072          * Open the CTF data for the module.  We'll use this to determine the
1073          * functions that can be instrumented.  Note that this call can fail,
1074          * in which case we'll use heuristics to determine the functions that
1075          * can be instrumented.  (But in particular, leaf functions will not be
1076          * instrumented.)
1077          */
1078         fp = ctf_modopen(mp, &error);
1079 
1080 forreal:
1081         if (!estimate) {
1082                 tramp.fbtt_next =
1083                     (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
1084                     mp->fbt_size);
1085                 tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
1086                 tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
1087         }
1088 
1089         for (i = 1; i < nsyms; i++) {
1090                 ctf_funcinfo_t f;
1091                 uint32_t *instr, *base, *limit;
1092                 Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
1093                 int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
1094                 int (*canpatch)(uint32_t *, int, const char *);
1095                 uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
1096                     uint32_t, fbt_trampoline_t *, const char *);
1097 
1098                 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
1099                         continue;
1100 
1101                 /*
1102                  * Weak symbols are not candidates.  This could be made to
1103                  * work (where weak functions and their underlying function
1104                  * appear as two disjoint probes), but it's not simple.
1105                  */
1106                 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
1107                         continue;
1108 
1109                 name = str + sym->st_name;
1110 
1111                 if (strstr(name, "dtrace_") == name &&
1112                     strstr(name, "dtrace_safe_") != name) {
1113                         /*
1114                          * Anything beginning with "dtrace_" may be called
1115                          * from probe context unless it explitly indicates
1116                          * that it won't be called from probe context by
1117                          * using the prefix "dtrace_safe_".
1118                          */
1119                         continue;
1120                 }
1121 
1122                 if (strstr(name, "kdi_") == name ||
1123                     strstr(name, "_kdi_") != NULL) {
1124                         /*
1125                          * Any function name beginning with "kdi_" or
1126                          * containing the string "_kdi_" is a part of the
1127                          * kernel debugger interface and may be called in
1128                          * arbitrary context -- including probe context.
1129                          */
1130                         continue;
1131                 }
1132 
1133                 if (strstr(name, "__relocatable") != NULL) {
1134                         /*
1135                          * Anything with the string "__relocatable" anywhere
1136                          * in the function name is considered to be a function
1137                          * that may be manually relocated before execution.
1138                          * Because FBT uses a PC-relative technique for
1139                          * instrumentation, these functions cannot safely
1140                          * be instrumented by us.
1141                          */
1142                         continue;
1143                 }
1144 
1145                 if (strstr(name, "ip_ocsum") == name) {
1146                         /*
1147                          * The ip_ocsum_* family of routines are all ABI
1148                          * violators.  (They expect incoming arguments in the
1149                          * globals!)  Break the ABI?  No soup for you!
1150                          */
1151                         continue;
1152                 }
1153 
1154                 /*
1155                  * We want to scan the function for one (and only one) save.
1156                  * Any more indicates that something fancy is going on.
1157                  */
1158                 base = (uint32_t *)sym->st_value;
1159                 limit = (uint32_t *)(sym->st_value + sym->st_size);
1160 
1161                 /*
1162                  * We don't want to interpose on the module stubs.
1163                  */
1164                 if (base >= (uint32_t *)stubs_base &&
1165                     base <= (uint32_t *)stubs_end)
1166                         continue;
1167 
1168                 /*
1169                  * We can't safely trace a zero-length function...
1170                  */
1171                 if (base == limit)
1172                         continue;
1173 
1174                 /*
1175                  * Due to 4524008, _init and _fini may have a bloated st_size.
1176                  * While this bug was fixed quite some time ago, old drivers
1177                  * may be lurking.  We need to develop a better solution to
1178                  * this problem, such that correct _init and _fini functions
1179                  * (the vast majority) may be correctly traced.  One solution
1180                  * may be to scan through the entire symbol table to see if
1181                  * any symbol overlaps with _init.  If none does, set a bit in
1182                  * the module structure that this module has correct _init and
1183                  * _fini sizes.  This will cause some pain the first time a
1184                  * module is scanned, but at least it would be O(N) instead of
1185                  * O(N log N)...
1186                  */
1187                 if (strcmp(name, "_init") == 0)
1188                         continue;
1189 
1190                 if (strcmp(name, "_fini") == 0)
1191                         continue;
1192 
1193                 instr = base;
1194 
1195                 /*
1196                  * While we try hard to only trace safe functions (that is,
1197                  * functions at TL=0), one unsafe function manages to otherwise
1198                  * appear safe:  prom_trap().  We could discover prom_trap()
1199                  * if we added an additional rule:  in order to trace a
1200                  * function, we must either (a) discover a restore or (b)
1201                  * determine that the function does not have any unlinked
1202                  * control transfers to another function (i.e., the function
1203                  * never returns).  Unfortunately, as of this writing, one
1204                  * legitimate function (resume_from_zombie()) transfers
1205                  * control to a different function (_resume_from_idle())
1206                  * without executing a restore.  Barring a rule to figure out
1207                  * that resume_from_zombie() is safe while prom_trap() is not,
1208                  * we resort to hard-coding prom_trap() here.
1209                  */
1210                 if (strcmp(name, "prom_trap") == 0)
1211                         continue;
1212 
1213                 if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
1214                         nargs = f.ctc_argc;
1215                         have_ctf = 1;
1216                 } else {
1217                         nargs = 32;
1218                 }
1219 
1220                 /*
1221                  * If the first instruction of the function is a branch and
1222                  * it's not a branch-always-not-annulled, we're going to refuse
1223                  * to patch it.
1224                  */
1225                 if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
1226                     (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
1227                     (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
1228                         if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
1229                                 if (have_ctf) {
1230                                         cmn_err(CE_NOTE, "cannot instrument %s:"
1231                                             " begins with non-ba, "
1232                                             "non-br CTI", name);
1233                                 }
1234                                 continue;
1235                         }
1236                 }
1237 
1238                 while (!FBT_IS_SAVE(*instr)) {
1239                         /*
1240                          * Before we assume that this is a leaf routine, check
1241                          * forward in the basic block for a save.
1242                          */
1243                         int op = *instr & FBT_OP_MASK;
1244                         int op2 = *instr & FBT_FMT2_OP2_MASK;
1245 
1246                         if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
1247                                 /*
1248                                  * This is a CTI.  If we see a subsequent
1249                                  * save, we will refuse to process this
1250                                  * routine unless both of the following are
1251                                  * true:
1252                                  *
1253                                  *  (a) The branch is not annulled
1254                                  *
1255                                  *  (b) The subsequent save is in the delay
1256                                  *      slot of the branch
1257                                  */
1258                                 if ((*instr & FBT_ANNUL) ||
1259                                     !FBT_IS_SAVE(*(instr + 1))) {
1260                                         cti = 1;
1261                                 } else {
1262                                         instr++;
1263                                         break;
1264                                 }
1265                         }
1266 
1267                         if (op == FBT_OP1)
1268                                 cti = 1;
1269 
1270                         if (++instr == limit)
1271                                 break;
1272                 }
1273 
1274                 if (instr < limit && cti) {
1275                         /*
1276                          * If we found a CTI before the save, we need to not
1277                          * do anything.  But if we have CTF information, this
1278                          * is weird enough that it merits a message.
1279                          */
1280                         if (!have_ctf)
1281                                 continue;
1282 
1283                         cmn_err(CE_NOTE, "cannot instrument %s: "
1284                             "save not in first basic block", name);
1285                         continue;
1286                 }
1287 
1288                 if (instr == limit) {
1289                         if (!have_ctf)
1290                                 continue;
1291                         is_leaf = 1;
1292 
1293                         if (!estimate)
1294                                 fbt_leaf_functions++;
1295 
1296                         canpatch = fbt_canpatch_retl;
1297                         patch = fbt_patch_retl;
1298                 } else {
1299                         canpatch = fbt_canpatch_return;
1300                         patch = fbt_patch_return;
1301                 }
1302 
1303                 if (!have_ctf && !is_leaf) {
1304                         /*
1305                          * Before we assume that this isn't something tricky,
1306                          * look for other saves.  If we find them, there are
1307                          * multiple entry points here (or something), and we'll
1308                          * leave it alone.
1309                          */
1310                         while (++instr < limit) {
1311                                 if (FBT_IS_SAVE(*instr))
1312                                         break;
1313                         }
1314 
1315                         if (instr != limit)
1316                                 continue;
1317                 }
1318 
1319                 instr = base;
1320 
1321                 if (FBT_IS_CTI(*instr)) {
1322                         /*
1323                          * If we have a CTI, we want to be sure that we don't
1324                          * have a CTI or a PC-relative instruction in the
1325                          * delay slot -- we want to be able to thunk the
1326                          * instruction into the trampoline without worrying
1327                          * about either DCTIs or relocations.  It would be
1328                          * very odd for the compiler to generate this kind of
1329                          * code, so we warn about it if we have CTF
1330                          * information.
1331                          */
1332                         if (FBT_IS_CTI(*(instr + 1))) {
1333                                 if (!have_ctf)
1334                                         continue;
1335 
1336                                 cmn_err(CE_NOTE, "cannot instrument %s: "
1337                                     "CTI in delay slot of first instruction",
1338                                     name);
1339                                 continue;
1340                         }
1341 
1342                         if (FBT_IS_PCRELATIVE(*(instr + 1))) {
1343                                 if (!have_ctf)
1344                                         continue;
1345 
1346                                 cmn_err(CE_NOTE, "cannot instrument %s: "
1347                                     "PC-relative instruction in delay slot of"
1348                                     " first instruction", name);
1349                                 continue;
1350                         }
1351                 }
1352 
1353                 if (estimate) {
1354                         tramp.fbtt_next = (uintptr_t)faketramp;
1355                         tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1356                         (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
1357                             &tramp, nargs);
1358                         fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1359                 } else {
1360                         fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1361                         fbt->fbtp_name = name;
1362                         fbt->fbtp_ctl = ctl;
1363                         fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1364                             name, FBT_PROBENAME_ENTRY, 1, fbt);
1365                         fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);
1366 
1367                         if (!fbt_patch_entry(instr, fbt->fbtp_id,
1368                             &tramp, nargs)) {
1369                                 cmn_err(CE_WARN, "unexpectedly short FBT table "
1370                                     "in module %s (sym %d of %d)", modname,
1371                                     i, nsyms);
1372                                 break;
1373                         }
1374 
1375                         fbt->fbtp_patchpoint =
1376                             (uint32_t *)((uintptr_t)mp->textwin +
1377                             ((uintptr_t)instr - (uintptr_t)mp->text));
1378                         fbt->fbtp_savedval = *instr;
1379 
1380                         fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1381                         fbt->fbtp_primary = primary;
1382                         fbt->fbtp_symndx = i;
1383                         mp->fbt_nentries++;
1384                 }
1385 
1386                 retfbt = NULL;
1387 again:
1388                 if (++instr == limit)
1389                         continue;
1390 
1391                 offset = (uintptr_t)instr - (uintptr_t)base;
1392 
1393                 if (!(*canpatch)(instr, offset, name))
1394                         goto again;
1395 
1396                 if (estimate) {
1397                         tramp.fbtt_next = (uintptr_t)faketramp;
1398                         tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1399                         (void) (*patch)(instr, base, limit,
1400                             offset, FBT_ESTIMATE_ID, &tramp, name);
1401                         fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1402 
1403                         goto again;
1404                 }
1405 
1406                 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1407                 fbt->fbtp_name = name;
1408                 fbt->fbtp_ctl = ctl;
1409 
1410                 if (retfbt == NULL) {
1411                         fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1412                             name, FBT_PROBENAME_RETURN, 1, fbt);
1413                 } else {
1414                         retfbt->fbtp_next = fbt;
1415                         fbt->fbtp_id = retfbt->fbtp_id;
1416                 }
1417 
1418                 fbt->fbtp_return = 1;
1419                 retfbt = fbt;
1420 
1421                 if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
1422                     fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
1423                         cmn_err(CE_WARN, "unexpectedly short FBT table "
1424                             "in module %s (sym %d of %d)", modname, i, nsyms);
1425                         break;
1426                 }
1427 
1428                 fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
1429                     ((uintptr_t)instr - (uintptr_t)mp->text));
1430                 fbt->fbtp_savedval = *instr;
1431                 fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1432                 fbt->fbtp_primary = primary;
1433                 fbt->fbtp_symndx = i;
1434                 mp->fbt_nentries++;
1435 
1436                 goto again;
1437         }
1438 
1439         if (estimate) {
1440                 /*
1441                  * Slosh on another entry's worth...
1442                  */
1443                 fbt_size += FBT_ENT_MAXSIZE;
1444                 mp->fbt_size = fbt_size;
1445                 mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);
1446 
1447                 if (mp->fbt_tab == NULL) {
1448                         cmn_err(CE_WARN, "couldn't allocate FBT table "
1449                             "for module %s", modname);
1450                 } else {
1451                         estimate = 0;
1452                         goto forreal;
1453                 }
1454         } else {
1455                 fbt_trampoline_unmap();
1456         }
1457 
1458 error:
1459         if (fp != NULL)
1460                 ctf_close(fp);
1461 }
1462 
1463 /*ARGSUSED*/
1464 static void
1465 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
1466 {
1467         fbt_probe_t *fbt = parg, *next;
1468         struct modctl *ctl = fbt->fbtp_ctl;
1469 
1470         do {
1471                 if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
1472                         if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
1473                             ctl->mod_loaded) || fbt->fbtp_primary) {
1474                                 ((struct module *)
1475                                     (ctl->mod_mp))->fbt_nentries--;
1476                         }
1477                 }
1478 
1479                 next = fbt->fbtp_next;
1480                 kmem_free(fbt, sizeof (fbt_probe_t));
1481                 fbt = next;
1482         } while (fbt != NULL);
1483 }
1484 
1485 /*ARGSUSED*/
1486 static int
1487 fbt_enable(void *arg, dtrace_id_t id, void *parg)
1488 {
1489         fbt_probe_t *fbt = parg, *f;
1490         struct modctl *ctl = fbt->fbtp_ctl;
1491 
1492         ctl->mod_nenabled++;
1493 
1494         for (f = fbt; f != NULL; f = f->fbtp_next) {
1495                 if (f->fbtp_patchpoint == NULL) {
1496                         /*
1497                          * Due to a shortened FBT table, this entry was never
1498                          * completed; refuse to enable it.
1499                          */
1500                         if (fbt_verbose) {
1501                                 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1502                                     "(short FBT table in %s)",
1503                                     fbt->fbtp_name, ctl->mod_modname);
1504                         }
1505 
1506                         return (0);
1507                 }
1508         }
1509 
1510         /*
1511          * If this module has disappeared since we discovered its probes,
1512          * refuse to enable it.
1513          */
1514         if (!fbt->fbtp_primary && !ctl->mod_loaded) {
1515                 if (fbt_verbose) {
1516                         cmn_err(CE_NOTE, "fbt is failing for probe %s "
1517                             "(module %s unloaded)",
1518                             fbt->fbtp_name, ctl->mod_modname);
1519                 }
1520 
1521                 return (0);
1522         }
1523 
1524         /*
1525          * Now check that our modctl has the expected load count.  If it
1526          * doesn't, this module must have been unloaded and reloaded -- and
1527          * we're not going to touch it.
1528          */
1529         if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
1530                 if (fbt_verbose) {
1531                         cmn_err(CE_NOTE, "fbt is failing for probe %s "
1532                             "(module %s reloaded)",
1533                             fbt->fbtp_name, ctl->mod_modname);
1534                 }
1535 
1536                 return (0);
1537         }
1538 
1539         for (; fbt != NULL; fbt = fbt->fbtp_next)
1540                 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1541 
1542         return (0);
1543 }
1544 
1545 /*ARGSUSED*/
1546 static void
1547 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1548 {
1549         fbt_probe_t *fbt = parg, *f;
1550         struct modctl *ctl = fbt->fbtp_ctl;
1551 
1552         ASSERT(ctl->mod_nenabled > 0);
1553         ctl->mod_nenabled--;
1554 
1555         for (f = fbt; f != NULL; f = f->fbtp_next) {
1556                 if (f->fbtp_patchpoint == NULL)
1557                         return;
1558         }
1559 
1560         if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
1561             (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1562                 return;
1563 
1564         for (; fbt != NULL; fbt = fbt->fbtp_next)
1565                 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1566 }
1567 
1568 /*ARGSUSED*/
1569 static void
1570 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1571 {
1572         fbt_probe_t *fbt = parg;
1573         struct modctl *ctl = fbt->fbtp_ctl;
1574 
1575         if (!fbt->fbtp_primary && !ctl->mod_loaded)
1576                 return;
1577 
1578         if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1579                 return;
1580 
1581         ASSERT(ctl->mod_nenabled > 0);
1582 
1583         for (; fbt != NULL; fbt = fbt->fbtp_next)
1584                 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1585 }
1586 
1587 /*ARGSUSED*/
1588 static void
1589 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1590 {
1591         fbt_probe_t *fbt = parg;
1592         struct modctl *ctl = fbt->fbtp_ctl;
1593 
1594         if (!fbt->fbtp_primary && !ctl->mod_loaded)
1595                 return;
1596 
1597         if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1598                 return;
1599 
1600         ASSERT(ctl->mod_nenabled > 0);
1601 
1602         for (; fbt != NULL; fbt = fbt->fbtp_next)
1603                 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1604 }
1605 
1606 /*ARGSUSED*/
1607 static void
1608 fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
1609 {
1610         fbt_probe_t *fbt = parg;
1611         struct modctl *ctl = fbt->fbtp_ctl;
1612         struct module *mp = ctl->mod_mp;
1613         ctf_file_t *fp = NULL, *pfp;
1614         ctf_funcinfo_t f;
1615         int error;
1616         ctf_id_t argv[32], type;
1617         int argc = sizeof (argv) / sizeof (ctf_id_t);
1618         const char *parent;
1619 
1620         if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1621                 goto err;
1622 
1623         if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
1624                 (void) strcpy(desc->dtargd_native, "int");
1625                 return;
1626         }
1627 
1628         if ((fp = ctf_modopen(mp, &error)) == NULL) {
1629                 /*
1630                  * We have no CTF information for this module -- and therefore
1631                  * no args[] information.
1632                  */
1633                 goto err;
1634         }
1635 
1636         /*
1637          * If we have a parent container, we must manually import it.
1638          */
1639         if ((parent = ctf_parent_name(fp)) != NULL) {
1640                 struct modctl *mp = &modules;
1641                 struct modctl *mod = NULL;
1642 
1643                 /*
1644                  * We must iterate over all modules to find the module that
1645                  * is our parent.
1646                  */
1647                 do {
1648                         if (strcmp(mp->mod_modname, parent) == 0) {
1649                                 mod = mp;
1650                                 break;
1651                         }
1652                 } while ((mp = mp->mod_next) != &modules);
1653 
1654                 if (mod == NULL)
1655                         goto err;
1656 
1657                 if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
1658                         goto err;
1659 
1660                 if (ctf_import(fp, pfp) != 0) {
1661                         ctf_close(pfp);
1662                         goto err;
1663                 }
1664 
1665                 ctf_close(pfp);
1666         }
1667 
1668         if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
1669                 goto err;
1670 
1671         if (fbt->fbtp_return) {
1672                 if (desc->dtargd_ndx > 1)
1673                         goto err;
1674 
1675                 ASSERT(desc->dtargd_ndx == 1);
1676                 type = f.ctc_return;
1677         } else {
1678                 if (desc->dtargd_ndx + 1 > f.ctc_argc)
1679                         goto err;
1680 
1681                 if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
1682                         goto err;
1683 
1684                 type = argv[desc->dtargd_ndx];
1685         }
1686 
1687         if (ctf_type_name(fp, type, desc->dtargd_native,
1688             DTRACE_ARGTYPELEN) != NULL) {
1689                 ctf_close(fp);
1690                 return;
1691         }
1692 err:
1693         if (fp != NULL)
1694                 ctf_close(fp);
1695 
1696         desc->dtargd_ndx = DTRACE_ARGNONE;
1697 }
1698 
1699 static dtrace_pattr_t fbt_attr = {
1700 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1701 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1702 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1703 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1704 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
1705 };
1706 
1707 static dtrace_pops_t fbt_pops = {
1708         NULL,
1709         fbt_provide_module,
1710         fbt_enable,
1711         fbt_disable,
1712         fbt_suspend,
1713         fbt_resume,
1714         fbt_getargdesc,
1715         NULL,
1716         NULL,
1717         fbt_destroy
1718 };
1719 
1720 static int
1721 fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1722 {
1723         switch (cmd) {
1724         case DDI_ATTACH:
1725                 break;
1726         case DDI_RESUME:
1727                 return (DDI_SUCCESS);
1728         default:
1729                 return (DDI_FAILURE);
1730         }
1731 
1732         if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
1733             DDI_PSEUDO, NULL) == DDI_FAILURE ||
1734             dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
1735             &fbt_pops, NULL, &fbt_id) != 0) {
1736                 ddi_remove_minor_node(devi, NULL);
1737                 return (DDI_FAILURE);
1738         }
1739 
1740         ddi_report_dev(devi);
1741         fbt_devi = devi;
1742         return (DDI_SUCCESS);
1743 }
1744 
1745 static int
1746 fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
1747 {
1748         switch (cmd) {
1749         case DDI_DETACH:
1750                 break;
1751         case DDI_SUSPEND:
1752                 return (DDI_SUCCESS);
1753         default:
1754                 return (DDI_FAILURE);
1755         }
1756 
1757         if (dtrace_unregister(fbt_id) != 0)
1758                 return (DDI_FAILURE);
1759 
1760         ddi_remove_minor_node(devi, NULL);
1761         return (DDI_SUCCESS);
1762 }
1763 
1764 /*ARGSUSED*/
1765 static int
1766 fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1767 {
1768         int error;
1769 
1770         switch (infocmd) {
1771         case DDI_INFO_DEVT2DEVINFO:
1772                 *result = (void *)fbt_devi;
1773                 error = DDI_SUCCESS;
1774                 break;
1775         case DDI_INFO_DEVT2INSTANCE:
1776                 *result = (void *)0;
1777                 error = DDI_SUCCESS;
1778                 break;
1779         default:
1780                 error = DDI_FAILURE;
1781         }
1782         return (error);
1783 }
1784 
1785 /*ARGSUSED*/
1786 static int
1787 fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1788 {
1789         return (0);
1790 }
1791 
1792 static struct cb_ops fbt_cb_ops = {
1793         fbt_open,               /* open */
1794         nodev,                  /* close */
1795         nulldev,                /* strategy */
1796         nulldev,                /* print */
1797         nodev,                  /* dump */
1798         nodev,                  /* read */
1799         nodev,                  /* write */
1800         nodev,                  /* ioctl */
1801         nodev,                  /* devmap */
1802         nodev,                  /* mmap */
1803         nodev,                  /* segmap */
1804         nochpoll,               /* poll */
1805         ddi_prop_op,            /* cb_prop_op */
1806         0,                      /* streamtab  */
1807         D_NEW | D_MP            /* Driver compatibility flag */
1808 };
1809 
1810 static struct dev_ops fbt_ops = {
1811         DEVO_REV,               /* devo_rev */
1812         0,                      /* refcnt */
1813         fbt_info,               /* get_dev_info */
1814         nulldev,                /* identify */
1815         nulldev,                /* probe */
1816         fbt_attach,             /* attach */
1817         fbt_detach,             /* detach */
1818         nodev,                  /* reset */
1819         &fbt_cb_ops,                /* driver operations */
1820         NULL,                   /* bus operations */
1821         nodev,                  /* dev power */
1822         ddi_quiesce_not_needed,         /* quiesce */
1823 };
1824 
1825 /*
1826  * Module linkage information for the kernel.
1827  */
1828 static struct modldrv modldrv = {
1829         &mod_driverops,             /* module type (this is a pseudo driver) */
1830         "Function Boundary Tracing",    /* name of module */
1831         &fbt_ops,           /* driver ops */
1832 };
1833 
1834 static struct modlinkage modlinkage = {
1835         MODREV_1,
1836         (void *)&modldrv,
1837         NULL
1838 };
1839 
1840 int
1841 _init(void)
1842 {
1843         return (mod_install(&modlinkage));
1844 }
1845 
1846 int
1847 _info(struct modinfo *modinfop)
1848 {
1849         return (mod_info(&modlinkage, modinfop));
1850 }
1851 
1852 int
1853 _fini(void)
1854 {
1855         return (mod_remove(&modlinkage));
1856 }