Print this page
1730 DTrace should ignore type information from modules with cth_parlabel mismatches
Reviewed by: Keith Wesolowski <keith.wesolowski@joyent.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/sparc/dtrace/fbt.c
+++ new/usr/src/uts/sparc/dtrace/fbt.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26
27 27 #include <sys/errno.h>
28 28 #include <sys/stat.h>
29 29 #include <sys/modctl.h>
30 30 #include <sys/conf.h>
31 31 #include <sys/systm.h>
32 32 #include <sys/ddi.h>
33 33 #include <sys/sunddi.h>
34 34 #include <sys/cpuvar.h>
35 35 #include <sys/kmem.h>
36 36 #include <sys/strsubr.h>
37 37 #include <sys/dtrace.h>
38 38 #include <sys/kobj.h>
39 39 #include <sys/modctl.h>
40 40 #include <sys/atomic.h>
41 41 #include <vm/seg_kmem.h>
42 42 #include <sys/stack.h>
43 43 #include <sys/ctf_api.h>
44 44 #include <sys/sysmacros.h>
45 45
46 46 static dev_info_t *fbt_devi;
47 47 static dtrace_provider_id_t fbt_id;
48 48 static uintptr_t fbt_trampoline;
49 49 static caddr_t fbt_trampoline_window;
50 50 static size_t fbt_trampoline_size;
51 51 static int fbt_verbose = 0;
52 52
53 53 /*
54 54 * Various interesting bean counters.
55 55 */
56 56 static int fbt_entry;
57 57 static int fbt_ret;
58 58 static int fbt_retl;
59 59 static int fbt_retl_jmptab;
60 60 static int fbt_retl_twoinstr;
61 61 static int fbt_retl_tailcall;
62 62 static int fbt_retl_tailjmpl;
63 63 static int fbt_leaf_functions;
64 64
65 65 extern char stubs_base[];
66 66 extern char stubs_end[];
67 67
68 68 #define FBT_REG_G0 0
69 69 #define FBT_REG_G1 1
70 70 #define FBT_REG_O0 8
71 71 #define FBT_REG_O1 9
72 72 #define FBT_REG_O2 10
73 73 #define FBT_REG_O3 11
74 74 #define FBT_REG_O4 12
75 75 #define FBT_REG_O5 13
76 76 #define FBT_REG_O6 14
77 77 #define FBT_REG_O7 15
78 78 #define FBT_REG_I0 24
79 79 #define FBT_REG_I1 25
80 80 #define FBT_REG_I2 26
81 81 #define FBT_REG_I3 27
82 82 #define FBT_REG_I4 28
83 83 #define FBT_REG_I7 31
84 84 #define FBT_REG_L0 16
85 85 #define FBT_REG_L1 17
86 86 #define FBT_REG_L2 18
87 87 #define FBT_REG_L3 19
88 88 #define FBT_REG_PC 5
89 89
90 90 #define FBT_REG_ISGLOBAL(r) ((r) < 8)
91 91 #define FBT_REG_ISOUTPUT(r) ((r) >= 8 && (r) < 16)
92 92 #define FBT_REG_ISLOCAL(r) ((r) >= 16 && (r) < 24)
93 93 #define FBT_REG_ISVOLATILE(r) \
94 94 ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
95 95 #define FBT_REG_NLOCALS 8
96 96
97 97 #define FBT_REG_MARKLOCAL(locals, r) \
98 98 if (FBT_REG_ISLOCAL(r)) \
99 99 (locals)[(r) - FBT_REG_L0] = 1;
100 100
101 101 #define FBT_REG_INITLOCALS(local, locals) \
102 102 for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++) \
103 103 (locals)[(local)] = 0; \
104 104 (local) = FBT_REG_L0
105 105
106 106 #define FBT_REG_ALLOCLOCAL(local, locals) \
107 107 while ((locals)[(local) - FBT_REG_L0]) \
108 108 (local)++; \
109 109 (locals)[(local) - FBT_REG_L0] = 1;
110 110
111 111 #define FBT_OP_MASK 0xc0000000
112 112 #define FBT_OP_SHIFT 30
113 113 #define FBT_OP(val) ((val) & FBT_FMT1_MASK)
114 114
115 115 #define FBT_SIMM13_MASK 0x1fff
116 116 #define FBT_SIMM13_MAX ((int32_t)0xfff)
117 117 #define FBT_IMM22_MASK 0x3fffff
118 118 #define FBT_IMM22_SHIFT 10
119 119 #define FBT_IMM10_MASK 0x3ff
120 120
121 121 #define FBT_DISP30_MASK 0x3fffffff
122 122 #define FBT_DISP30(from, to) \
123 123 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)
124 124
125 125 #define FBT_DISP22_MASK 0x3fffff
126 126 #define FBT_DISP22(from, to) \
127 127 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)
128 128
129 129 #define FBT_DISP19_MASK 0x7ffff
130 130 #define FBT_DISP19(from, to) \
131 131 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)
132 132
133 133 #define FBT_DISP16_HISHIFT 20
134 134 #define FBT_DISP16_HIMASK (0x3 << FBT_DISP16_HISHIFT)
135 135 #define FBT_DISP16_LOMASK (0x3fff)
136 136 #define FBT_DISP16_MASK (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
137 137 #define FBT_DISP16(val) \
138 138 ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))
139 139
140 140 #define FBT_DISP14_MASK 0x3fff
141 141 #define FBT_DISP14(from, to) \
142 142 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)
143 143
144 144 #define FBT_OP0 (((uint32_t)0) << FBT_OP_SHIFT)
145 145 #define FBT_OP1 (((uint32_t)1) << FBT_OP_SHIFT)
146 146 #define FBT_OP2 (((uint32_t)2) << FBT_OP_SHIFT)
147 147 #define FBT_ILLTRAP 0
148 148
149 149 #define FBT_ANNUL_SHIFT 29
150 150 #define FBT_ANNUL (1 << FBT_ANNUL_SHIFT)
151 151
152 152 #define FBT_FMT3_OP3_SHIFT 19
153 153 #define FBT_FMT3_OP_MASK 0xc1f80000
154 154 #define FBT_FMT3_OP(val) ((val) & FBT_FMT3_OP_MASK)
155 155
156 156 #define FBT_FMT3_RD_SHIFT 25
157 157 #define FBT_FMT3_RD_MASK (0x1f << FBT_FMT3_RD_SHIFT)
158 158 #define FBT_FMT3_RD(val) \
159 159 (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)
160 160
161 161 #define FBT_FMT3_RS1_SHIFT 14
162 162 #define FBT_FMT3_RS1_MASK (0x1f << FBT_FMT3_RS1_SHIFT)
163 163 #define FBT_FMT3_RS1(val) \
164 164 (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
165 165 #define FBT_FMT3_RS1_SET(val, rs1) \
166 166 (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)
167 167
168 168 #define FBT_FMT3_RS2_SHIFT 0
169 169 #define FBT_FMT3_RS2_MASK (0x1f << FBT_FMT3_RS2_SHIFT)
170 170 #define FBT_FMT3_RS2(val) \
171 171 (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
172 172 #define FBT_FMT3_RS2_SET(val, rs2) \
173 173 (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)
174 174
175 175 #define FBT_FMT3_IMM_SHIFT 13
176 176 #define FBT_FMT3_IMM (1 << FBT_FMT3_IMM_SHIFT)
177 177 #define FBT_FMT3_SIMM13_MASK FBT_SIMM13_MASK
178 178
179 179 #define FBT_FMT3_ISIMM(val) ((val) & FBT_FMT3_IMM)
180 180 #define FBT_FMT3_SIMM13(val) ((val) & FBT_FMT3_SIMM13_MASK)
181 181
182 182 #define FBT_FMT2_OP2_SHIFT 22
183 183 #define FBT_FMT2_OP2_MASK (0x7 << FBT_FMT2_OP2_SHIFT)
184 184 #define FBT_FMT2_RD_SHIFT 25
185 185
186 186 #define FBT_FMT1_OP(val) ((val) & FBT_OP_MASK)
187 187 #define FBT_FMT1_DISP30(val) ((val) & FBT_DISP30_MASK)
188 188
189 189 #define FBT_FMT2_OP2_BPCC (0x01 << FBT_FMT2_OP2_SHIFT)
190 190 #define FBT_FMT2_OP2_BCC (0x02 << FBT_FMT2_OP2_SHIFT)
191 191 #define FBT_FMT2_OP2_BPR (0x03 << FBT_FMT2_OP2_SHIFT)
192 192 #define FBT_FMT2_OP2_SETHI (0x04 << FBT_FMT2_OP2_SHIFT)
193 193
194 194 #define FBT_FMT2_COND_SHIFT 25
195 195 #define FBT_FMT2_COND_BA (0x8 << FBT_FMT2_COND_SHIFT)
196 196 #define FBT_FMT2_COND_BL (0x3 << FBT_FMT2_COND_SHIFT)
197 197 #define FBT_FMT2_COND_BGE (0xb << FBT_FMT2_COND_SHIFT)
198 198
199 199 #define FBT_OP_RESTORE (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
200 200 #define FBT_OP_SAVE (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
201 201 #define FBT_OP_JMPL (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
202 202 #define FBT_OP_RETURN (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
203 203 #define FBT_OP_CALL FBT_OP1
204 204 #define FBT_OP_SETHI (FBT_OP0 | FBT_FMT2_OP2_SETHI)
205 205 #define FBT_OP_ADD (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
206 206 #define FBT_OP_OR (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
207 207 #define FBT_OP_SUB (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
208 208 #define FBT_OP_CC (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
209 209 #define FBT_OP_BA (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
210 210 #define FBT_OP_BL (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
211 211 #define FBT_OP_BGE (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
212 212 #define FBT_OP_BAPCC (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
213 213 #define FBT_OP_RD (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))
214 214
215 215 #define FBT_ORLO(rs, val, rd) \
216 216 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
217 217 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))
218 218
219 219 #define FBT_ORSIMM13(rs, val, rd) \
220 220 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
221 221 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
222 222
223 223 #define FBT_ADDSIMM13(rs, val, rd) \
224 224 (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
225 225 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
226 226
227 227 #define FBT_ADD(rs1, rs2, rd) \
228 228 (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
229 229 ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
230 230
231 231 #define FBT_CMP(rs1, rs2) \
232 232 (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
233 233 ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))
234 234
235 235 #define FBT_MOV(rs, rd) \
236 236 (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
237 237 ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
238 238
239 239 #define FBT_SETHI(val, reg) \
240 240 (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
241 241 ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))
242 242
243 243 #define FBT_CALL(orig, dest) (FBT_OP_CALL | FBT_DISP30(orig, dest))
244 244
245 245 #define FBT_RET \
246 246 (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
247 247 (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))
248 248
249 249 #define FBT_SAVEIMM(rd, val, rs1) \
250 250 (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
251 251 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
252 252
253 253 #define FBT_RESTORE(rd, rs1, rs2) \
254 254 (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
255 255 ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))
256 256
257 257 #define FBT_RETURN(rs1, val) \
258 258 (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
259 259 FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
260 260
261 261 #define FBT_BA(orig, dest) (FBT_OP_BA | FBT_DISP22(orig, dest))
262 262 #define FBT_BAA(orig, dest) (FBT_BA(orig, dest) | FBT_ANNUL)
263 263 #define FBT_BL(orig, dest) (FBT_OP_BL | FBT_DISP22(orig, dest))
264 264 #define FBT_BGE(orig, dest) (FBT_OP_BGE | FBT_DISP22(orig, dest))
265 265 #define FBT_BDEST(va, instr) ((uintptr_t)(va) + \
266 266 (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
267 267 #define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \
268 268 (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
269 269 #define FBT_BPRDEST(va, instr) ((uintptr_t)(va) + \
270 270 (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))
271 271
272 272 /*
273 273 * We're only going to treat a save as safe if (a) both rs1 and rd are
274 274 * %sp and (b) if the instruction has a simm, the value isn't 0.
275 275 */
276 276 #define FBT_IS_SAVE(instr) \
277 277 (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
278 278 FBT_FMT3_RD(instr) == FBT_REG_O6 && \
279 279 FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
280 280 !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))
281 281
282 282 #define FBT_IS_BA(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
283 283 #define FBT_IS_BAPCC(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)
284 284
285 285 #define FBT_IS_RDPC(instr) ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
286 286 (FBT_FMT3_RD(instr) == FBT_REG_PC))
287 287
288 288 #define FBT_IS_PCRELATIVE(instr) \
289 289 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
290 290 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
291 291 ((instr) & FBT_OP_MASK) == FBT_OP1 || \
292 292 FBT_IS_RDPC(instr))
293 293
294 294 #define FBT_IS_CTI(instr) \
295 295 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
296 296 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
297 297 ((instr) & FBT_OP_MASK) == FBT_OP1 || \
298 298 (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
299 299 (FBT_FMT3_OP(instr) == FBT_OP_RETURN))
300 300
301 301 #define FBT_PROBENAME_ENTRY "entry"
302 302 #define FBT_PROBENAME_RETURN "return"
303 303 #define FBT_ESTIMATE_ID (UINT32_MAX)
304 304 #define FBT_COUNTER(id, count) if ((id) != FBT_ESTIMATE_ID) (count)++
305 305
306 306 #define FBT_ENTENT_MAXSIZE (16 * sizeof (uint32_t))
307 307 #define FBT_RETENT_MAXSIZE (11 * sizeof (uint32_t))
308 308 #define FBT_RETLENT_MAXSIZE (23 * sizeof (uint32_t))
309 309 #define FBT_ENT_MAXSIZE \
310 310 MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)
311 311
312 312 typedef struct fbt_probe {
313 313 char *fbtp_name;
314 314 dtrace_id_t fbtp_id;
315 315 uintptr_t fbtp_addr;
316 316 struct modctl *fbtp_ctl;
317 317 int fbtp_loadcnt;
318 318 int fbtp_symndx;
319 319 int fbtp_primary;
320 320 int fbtp_return;
321 321 uint32_t *fbtp_patchpoint;
322 322 uint32_t fbtp_patchval;
323 323 uint32_t fbtp_savedval;
324 324 struct fbt_probe *fbtp_next;
325 325 } fbt_probe_t;
326 326
327 327 typedef struct fbt_trampoline {
328 328 uintptr_t fbtt_va;
329 329 uintptr_t fbtt_limit;
330 330 uintptr_t fbtt_next;
331 331 } fbt_trampoline_t;
332 332
333 333 static caddr_t
334 334 fbt_trampoline_map(uintptr_t tramp, size_t size)
335 335 {
336 336 uintptr_t offs;
337 337 page_t **ppl;
338 338
339 339 ASSERT(fbt_trampoline_window == NULL);
340 340 ASSERT(fbt_trampoline_size == 0);
341 341 ASSERT(fbt_trampoline == NULL);
342 342
343 343 size += tramp & PAGEOFFSET;
344 344 fbt_trampoline = tramp & PAGEMASK;
345 345 fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
346 346 fbt_trampoline_window =
347 347 vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);
348 348
349 349 (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
350 350 fbt_trampoline_size, S_WRITE);
351 351
352 352 for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
353 353 hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
354 354 hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
355 355 PROT_READ | PROT_WRITE,
356 356 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
357 357 }
358 358
359 359 as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
360 360 S_WRITE);
361 361
362 362 return (fbt_trampoline_window + (tramp & PAGEOFFSET));
363 363 }
364 364
365 365 static void
366 366 fbt_trampoline_unmap()
367 367 {
368 368 ASSERT(fbt_trampoline_window != NULL);
369 369 ASSERT(fbt_trampoline_size != 0);
370 370 ASSERT(fbt_trampoline != NULL);
371 371
372 372 membar_enter();
373 373 sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
374 374 sync_icache(fbt_trampoline_window, fbt_trampoline_size);
375 375
376 376 hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
377 377 HAT_UNLOAD_UNLOCK);
378 378
379 379 vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);
380 380
381 381 fbt_trampoline_window = NULL;
382 382 fbt_trampoline = NULL;
383 383 fbt_trampoline_size = 0;
384 384 }
385 385
386 386 static uintptr_t
387 387 fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
388 388 int nargs)
389 389 {
390 390 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
391 391 uint32_t first = *instr;
392 392 uintptr_t va = tramp->fbtt_va;
393 393 uintptr_t base = tramp->fbtt_next;
394 394
395 395 if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
396 396 /*
397 397 * There isn't sufficient room for this entry; return failure.
398 398 */
399 399 return (0);
400 400 }
401 401
402 402 FBT_COUNTER(id, fbt_entry);
403 403
404 404 if (FBT_IS_SAVE(first)) {
405 405 *tinstr++ = first;
406 406 } else {
407 407 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
408 408 }
409 409
410 410 if (id > (uint32_t)FBT_SIMM13_MAX) {
411 411 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
412 412 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
413 413 } else {
414 414 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
415 415 }
416 416
417 417 if (nargs >= 1)
418 418 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);
419 419
420 420 if (nargs >= 2)
421 421 *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);
422 422
423 423 if (nargs >= 3)
424 424 *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);
425 425
426 426 if (nargs >= 4)
427 427 *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);
428 428
429 429 if (nargs >= 5)
430 430 *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);
431 431
432 432 if (FBT_IS_SAVE(first)) {
433 433 uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);
434 434
435 435 *tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
436 436 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
437 437 tinstr++;
438 438 *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
439 439 } else {
440 440 uintptr_t slot = *--tinstr;
441 441 uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
442 442 uint32_t delay = first;
443 443
444 444 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
445 445 tinstr++;
446 446 *tinstr++ = slot;
447 447 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
448 448
449 449 if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
450 450 /*
451 451 * This is a special case: we are instrumenting a
452 452 * a non-annulled branch-always (or variant). We'll
453 453 * return directly to the destination of the branch,
454 454 * copying the instruction in the delay slot here,
455 455 * and then executing it in the slot of a ba.
456 456 */
457 457 if (FBT_IS_BA(first)) {
458 458 ret = FBT_BDEST(instr, *instr);
459 459 } else {
460 460 ret = FBT_BPCCDEST(instr, *instr);
461 461 }
462 462
463 463 delay = *(instr + 1);
464 464 }
465 465
466 466 if ((first & FBT_OP_MASK) != FBT_OP0 ||
467 467 (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
468 468 *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
469 469 tinstr++;
470 470 *tinstr++ = delay;
471 471 } else {
472 472 /*
473 473 * If this is a branch-on-register, we have a little
474 474 * more work to do: because the displacement is only
475 475 * sixteen bits, we're going to thunk the branch into
476 476 * the trampoline, and then ba,a to the appropriate
477 477 * destination in the branch targets. That is, we're
478 478 * constructing this sequence in the trampoline:
479 479 *
480 480 * br[cc] %[rs], 1f
481 481 * <delay-instruction>
482 482 * ba,a <not-taken-destination>
483 483 * 1: ba,a <taken-destination>
484 484 *
485 485 */
486 486 uintptr_t targ = FBT_BPRDEST(instr, first);
487 487
488 488 *tinstr = first & ~(FBT_DISP16_MASK);
489 489 *tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
490 490 tinstr++;
491 491 *tinstr++ = *(instr + 1);
492 492 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
493 493 ret + sizeof (uint32_t));
494 494 tinstr++;
495 495 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
496 496 tinstr++;
497 497 }
498 498 }
499 499
500 500 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
501 501 tramp->fbtt_next = (uintptr_t)tinstr;
502 502
503 503 return (1);
504 504 }
505 505
506 506 /*
507 507 * We are patching control-transfer/restore couplets. There are three
508 508 * variants of couplet:
509 509 *
510 510 * (a) return rs1 + imm
511 511 * delay
512 512 *
513 513 * (b) jmpl rs1 + (rs2 | offset), rd
514 514 * restore rs1, rs2 | imm, rd
515 515 *
516 516 * (c) call displacement
517 517 * restore rs1, rs2 | imm, rd
518 518 *
519 519 * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
520 520 * or delay is a DCTI, we fail. If rd from the jmpl in (b) is something other
521 521 * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
522 522 * through a register), we fail.
523 523 *
524 524 * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
525 525 * potentially outputs and/or globals. Because these registers cannot be
526 526 * relied upon across the call to dtrace_probe(), we move rs1 into an unused
527 527 * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
528 528 * to be:
529 529 *
530 530 * restore ls0, ls1, rd
531 531 *
532 532 * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
533 533 * If the jmpl uses outputs or globals, we restructure it to be:
534 534 *
535 535 * jmpl ls2 + (ls3 | offset), (%g0 | %o7)
536 536 *
537 537 */
538 538 /*ARGSUSED*/
539 539 static int
540 540 fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
541 541 {
542 542 int rd;
543 543
544 544 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
545 545 uint32_t delay = *(instr + 1);
546 546
547 547 if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
548 548 /*
549 549 * It's unclear if we should warn about this or not.
550 550 * We really wouldn't expect the compiler to generate
551 551 * return instructions with something other than %i7
552 552 * as rs1 and 8 as the simm13 -- it would just be
553 553 * mean-spirited. That said, such a construct isn't
554 554 * necessarily incorrect. Sill, we err on the side of
555 555 * caution and warn about it...
556 556 */
557 557 cmn_err(CE_NOTE, "cannot instrument return of %s at "
558 558 "%p: non-canonical return instruction", name,
559 559 (void *)instr);
560 560 return (0);
561 561 }
562 562
563 563 if (FBT_IS_CTI(delay)) {
564 564 /*
565 565 * This is even weirder -- a DCTI coupled with a
566 566 * return instruction. Similar constructs are used to
567 567 * return from utraps, but these typically have the
568 568 * return in the slot -- and we wouldn't expect to see
569 569 * it in the kernel regardless. At any rate, we don't
570 570 * want to try to instrument this construct, whatever
571 571 * it may be.
572 572 */
573 573 cmn_err(CE_NOTE, "cannot instrument return of %s at "
574 574 "%p: CTI in delay slot of return instruction",
575 575 name, (void *)instr);
576 576 return (0);
577 577 }
578 578
579 579 if (FBT_IS_PCRELATIVE(delay)) {
580 580 /*
581 581 * This is also very weird, but might be correct code
582 582 * if the function is (for example) returning the
583 583 * address of the delay instruction of the return as
584 584 * its return value (e.g. "rd %pc, %o0" in the slot).
585 585 * Perhaps correct, but still too weird to not warn
586 586 * about it...
587 587 */
588 588 cmn_err(CE_NOTE, "cannot instrument return of %s at "
589 589 "%p: PC-relative instruction in delay slot of "
590 590 "return instruction", name, (void *)instr);
591 591 return (0);
592 592 }
593 593
594 594 return (1);
595 595 }
596 596
597 597 if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
598 598 return (0);
599 599
600 600 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
601 601 return (1);
602 602
603 603 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
604 604 return (0);
605 605
606 606 rd = FBT_FMT3_RD(*instr);
607 607
608 608 if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
609 609 return (1);
610 610
611 611 /*
612 612 * We have encountered a jmpl that is storing the calling %pc in
613 613 * some register besides %i7, %o7 or %g0. This is strange; emit
614 614 * a warning and fail.
615 615 */
616 616 cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
617 617 "jmpl destination register", name, (void *)instr);
618 618 return (0);
619 619 }
620 620
621 621 static int
622 622 fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
623 623 {
624 624 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
625 625 (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
626 626 FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
627 627 /*
628 628 * If this is a call (or a jmpl that links into %o7), we can
629 629 * patch it iff the next instruction uses %o7 as a destination
630 630 * register. Because there is an ABI responsibility to
631 631 * restore %o7 to the value before the call/jmpl, we don't
632 632 * particularly care how this routine is managing to restore
633 633 * it (mov, add, ld or divx for all we care). If it doesn't
634 634 * seem to be restoring it at all, however, we'll refuse
635 635 * to patch it.
636 636 */
637 637 uint32_t delay = *(instr + 1);
638 638 uint32_t op, rd;
639 639
640 640 op = FBT_FMT1_OP(delay);
641 641 rd = FBT_FMT3_RD(delay);
642 642
643 643 if (op != FBT_OP2 || rd != FBT_REG_O7) {
644 644 /*
645 645 * This is odd. Before we assume that we're looking
646 646 * at something bizarre (and warn accordingly), we'll
647 647 * check to see if it's obviously a jump table entry.
648 648 */
649 649 if (*instr < (uintptr_t)instr &&
650 650 *instr >= (uintptr_t)instr - offset)
651 651 return (0);
652 652
653 653 cmn_err(CE_NOTE, "cannot instrument return of %s at "
654 654 "%p: leaf jmpl/call delay isn't restoring %%o7",
655 655 name, (void *)instr);
656 656 return (0);
657 657 }
658 658
659 659 return (1);
660 660 }
661 661
662 662 if (offset == sizeof (uint32_t)) {
663 663 /*
664 664 * If this is the second instruction in the function, we're
665 665 * going to allow it to be patched if the first instruction
666 666 * is a patchable return-from-leaf instruction.
667 667 */
668 668 if (fbt_canpatch_retl(instr - 1, 0, name))
669 669 return (1);
670 670 }
671 671
672 672 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
673 673 return (0);
674 674
675 675 if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
676 676 return (0);
677 677
678 678 return (1);
679 679 }
680 680
681 681 /*ARGSUSED*/
682 682 static uint32_t
683 683 fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
684 684 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
685 685 {
686 686 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
687 687 uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
688 688 uintptr_t va = tramp->fbtt_va;
689 689 uintptr_t base = tramp->fbtt_next;
690 690 uint32_t locals[FBT_REG_NLOCALS], local;
691 691
692 692 if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
693 693 /*
694 694 * There isn't sufficient room for this entry; return failure.
695 695 */
696 696 return (FBT_ILLTRAP);
697 697 }
698 698
699 699 FBT_COUNTER(id, fbt_ret);
700 700
701 701 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
702 702 /*
703 703 * To handle the case of the return instruction, we'll emit a
704 704 * restore, followed by the instruction in the slot (which
705 705 * we'll transplant here), and then another save. While it
706 706 * may seem intellectually unsatisfying to emit the additional
707 707 * restore/save couplet, one can take solace in the fact that
708 708 * we don't do this if the instruction in the return delay
709 709 * slot is a nop -- which it is nearly 90% of the time with
710 710 * gcc. (And besides, this couplet can't induce unnecessary
711 711 * spill/fill traps; rewriting the delay instruction to be
712 712 * in terms of the current window hardly seems worth the
713 713 * trouble -- let alone the risk.)
714 714 */
715 715 uint32_t delay = *(instr + 1);
716 716 ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));
717 717
718 718 cti = FBT_RET;
719 719 restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
720 720
721 721 if (delay != FBT_SETHI(0, FBT_REG_G0)) {
722 722 *tinstr++ = restore;
723 723 *tinstr++ = delay;
724 724 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
725 725 -SA(MINFRAME), FBT_REG_O6);
726 726 }
727 727 }
728 728
729 729 FBT_REG_INITLOCALS(local, locals);
730 730
731 731 /*
732 732 * Mark the locals used in the jmpl.
733 733 */
734 734 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
735 735 uint32_t rs1 = FBT_FMT3_RS1(cti);
736 736 FBT_REG_MARKLOCAL(locals, rs1);
737 737
738 738 if (!FBT_FMT3_ISIMM(cti)) {
739 739 uint32_t rs2 = FBT_FMT3_RS2(cti);
740 740 FBT_REG_MARKLOCAL(locals, rs2);
741 741 }
742 742 }
743 743
744 744 /*
745 745 * And mark the locals used in the restore.
746 746 */
747 747 rs1 = FBT_FMT3_RS1(restore);
748 748 FBT_REG_MARKLOCAL(locals, rs1);
749 749
750 750 if (!FBT_FMT3_ISIMM(restore)) {
751 751 uint32_t rs2 = FBT_FMT3_RS2(restore);
752 752 FBT_REG_MARKLOCAL(locals, rs2);
753 753 }
754 754
755 755 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
756 756 uint32_t rs1 = FBT_FMT3_RS1(cti);
757 757
758 758 if (FBT_REG_ISVOLATILE(rs1)) {
759 759 FBT_REG_ALLOCLOCAL(local, locals);
760 760 FBT_FMT3_RS1_SET(cti, local);
761 761 *tinstr++ = FBT_MOV(rs1, local);
762 762 }
763 763
764 764 if (!FBT_FMT3_ISIMM(cti)) {
765 765 uint32_t rs2 = FBT_FMT3_RS2(cti);
766 766
767 767 if (FBT_REG_ISVOLATILE(rs2)) {
768 768 FBT_REG_ALLOCLOCAL(local, locals);
769 769 FBT_FMT3_RS2_SET(cti, local);
770 770 *tinstr++ = FBT_MOV(rs2, local);
771 771 }
772 772 }
773 773 }
774 774
775 775 rs1 = FBT_FMT3_RS1(restore);
776 776
777 777 if (FBT_REG_ISVOLATILE(rs1)) {
778 778 FBT_REG_ALLOCLOCAL(local, locals);
779 779 FBT_FMT3_RS1_SET(restore, local);
780 780 *tinstr++ = FBT_MOV(rs1, local);
781 781 }
782 782
783 783 if (!FBT_FMT3_ISIMM(restore)) {
784 784 uint32_t rs2 = FBT_FMT3_RS2(restore);
785 785
786 786 if (FBT_REG_ISVOLATILE(rs2)) {
787 787 FBT_REG_ALLOCLOCAL(local, locals);
788 788 FBT_FMT3_RS2_SET(restore, local);
789 789 *tinstr++ = FBT_MOV(rs2, local);
790 790 }
791 791 }
792 792
793 793 if (id > (uint32_t)FBT_SIMM13_MAX) {
794 794 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
795 795 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
796 796 } else {
797 797 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
798 798 }
799 799
800 800 if (offset > (uint32_t)FBT_SIMM13_MAX) {
801 801 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
802 802 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
803 803 } else {
804 804 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
805 805 }
806 806
807 807 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
808 808 tinstr++;
809 809
810 810 if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
811 811 /*
812 812 * If the destination register of the restore is %o0, we
813 813 * need to perform the implied calculation to derive the
814 814 * return value.
815 815 */
816 816 uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
817 817 add &= ~FBT_FMT3_RD_MASK;
818 818 *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
819 819 } else {
820 820 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
821 821 }
822 822
823 823 /*
824 824 * If the control transfer instruction is %pc-relative (i.e. a
825 825 * call), we need to reset it appropriately.
826 826 */
827 827 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
828 828 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
829 829 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
830 830 tinstr++;
831 831 } else {
832 832 *tinstr++ = cti;
833 833 }
834 834
835 835 *tinstr++ = restore;
836 836 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
837 837 tramp->fbtt_next = (uintptr_t)tinstr;
838 838
839 839 return (FBT_BAA(instr, va));
840 840 }
841 841
842 842 static uint32_t
843 843 fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
844 844 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
845 845 {
846 846 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
847 847 uintptr_t va = tramp->fbtt_va;
848 848 uintptr_t base = tramp->fbtt_next;
849 849 uint32_t cti = *instr, dest;
850 850 int annul = 0;
851 851
852 852 FBT_COUNTER(id, fbt_retl);
853 853
854 854 if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
855 855 /*
856 856 * There isn't sufficient room for this entry; return failure.
857 857 */
858 858 return (FBT_ILLTRAP);
859 859 }
860 860
861 861 if (offset == sizeof (uint32_t) &&
862 862 fbt_canpatch_retl(instr - 1, 0, name)) {
863 863 *tinstr++ = *instr;
864 864 annul = 1;
865 865 FBT_COUNTER(id, fbt_retl_twoinstr);
866 866 } else {
867 867 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
868 868 FBT_FMT3_RD(cti) != FBT_REG_O7 &&
869 869 FBT_FMT3_RS1(cti) != FBT_REG_O7) {
870 870 annul = 1;
871 871 *tinstr++ = *(instr + 1);
872 872 }
873 873 }
874 874
875 875 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
876 876
877 877 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
878 878 uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;
879 879
880 880 /*
881 881 * If we have a jmpl and it's in terms of output registers, we
882 882 * need to rewrite it to be in terms of the corresponding input
883 883 * registers. If it's in terms of the globals, we'll rewrite
884 884 * it to be in terms of locals.
885 885 */
886 886 rs1 = FBT_FMT3_RS1(cti);
887 887
888 888 if (FBT_REG_ISOUTPUT(rs1))
889 889 rs1 += o2i;
890 890
891 891 if (FBT_REG_ISGLOBAL(rs1)) {
892 892 *tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
893 893 rs1 = FBT_REG_L0;
894 894 }
895 895
896 896 FBT_FMT3_RS1_SET(cti, rs1);
897 897
898 898 if (!FBT_FMT3_ISIMM(cti)) {
899 899 rs2 = FBT_FMT3_RS2(cti);
900 900
901 901 if (FBT_REG_ISOUTPUT(rs2))
902 902 rs2 += o2i;
903 903
904 904 if (FBT_REG_ISGLOBAL(rs2)) {
905 905 *tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
906 906 rs2 = FBT_REG_L1;
907 907 }
908 908
909 909 FBT_FMT3_RS2_SET(cti, rs2);
910 910 }
911 911
912 912 /*
913 913 * Now we need to check the rd and source register for the jmpl;
914 914 * If neither rd nor the source register is %o7, then we might
915 915 * have a jmp that is actually part of a jump table. We need
916 916 * to generate the code to compare it to the base and limit of
917 917 * the function.
918 918 */
919 919 if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
920 920 uintptr_t base = (uintptr_t)funcbase;
921 921 uintptr_t limit = (uintptr_t)funclim;
922 922
923 923 FBT_COUNTER(id, fbt_retl_jmptab);
924 924
925 925 if (FBT_FMT3_ISIMM(cti)) {
926 926 *tinstr++ = FBT_ADDSIMM13(rs1,
927 927 FBT_FMT3_SIMM13(cti), FBT_REG_L2);
928 928 } else {
929 929 *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
930 930 }
931 931
932 932 *tinstr++ = FBT_SETHI(base, FBT_REG_L3);
933 933 *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
934 934 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
935 935 *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
936 936 *tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
937 937 *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
938 938 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
939 939 *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
940 940 *tinstr++ = FBT_SETHI(0, FBT_REG_G0);
941 941 *tinstr++ = cti;
942 942 *tinstr++ = FBT_RESTORE(FBT_REG_G0,
943 943 FBT_REG_G0, FBT_REG_G0);
944 944 }
945 945 }
946 946
947 947 if (id > (uint32_t)FBT_SIMM13_MAX) {
948 948 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
949 949 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
950 950 } else {
951 951 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
952 952 }
953 953
954 954 if (offset > (uint32_t)FBT_SIMM13_MAX) {
955 955 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
956 956 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
957 957 } else {
958 958 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
959 959 }
960 960
961 961 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
962 962 tinstr++;
963 963 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
964 964
965 965 /*
966 966 * If the control transfer instruction is %pc-relative (i.e. a
967 967 * call), we need to reset it appropriately.
968 968 */
969 969 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
970 970 FBT_COUNTER(id, fbt_retl_tailcall);
971 971 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
972 972 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
973 973 tinstr++;
974 974 annul = 1;
975 975 } else {
976 976 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
977 977 *tinstr++ = cti;
978 978
979 979 if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
980 980 FBT_COUNTER(id, fbt_retl_tailjmpl);
981 981 annul = 1;
982 982 }
983 983 } else {
984 984 *tinstr++ = FBT_RET;
985 985 }
986 986 }
987 987
988 988 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
989 989
990 990 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
991 991 tramp->fbtt_next = (uintptr_t)tinstr;
992 992
993 993 return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
994 994 }
995 995
996 996 /*ARGSUSED*/
997 997 static void
998 998 fbt_provide_module(void *arg, struct modctl *ctl)
999 999 {
1000 1000 struct module *mp = ctl->mod_mp;
1001 1001 char *modname = ctl->mod_modname;
1002 1002 char *str = mp->strings;
1003 1003 int nsyms = mp->nsyms;
1004 1004 Shdr *symhdr = mp->symhdr;
1005 1005 size_t symsize;
1006 1006 char *name;
1007 1007 int i;
1008 1008 fbt_probe_t *fbt, *retfbt;
1009 1009 fbt_trampoline_t tramp;
1010 1010 uintptr_t offset;
1011 1011 int primary = 0;
1012 1012 ctf_file_t *fp = NULL;
1013 1013 int error;
1014 1014 int estimate = 1;
1015 1015 uint32_t faketramp[50];
1016 1016 size_t fbt_size = 0;
1017 1017
1018 1018 /*
1019 1019 * Employees of dtrace and their families are ineligible. Void
1020 1020 * where prohibited.
1021 1021 */
1022 1022 if (strcmp(modname, "dtrace") == 0)
1023 1023 return;
1024 1024
1025 1025 if (ctl->mod_requisites != NULL) {
1026 1026 struct modctl_list *list;
1027 1027
1028 1028 list = (struct modctl_list *)ctl->mod_requisites;
1029 1029
1030 1030 for (; list != NULL; list = list->modl_next) {
1031 1031 if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
1032 1032 return;
1033 1033 }
1034 1034 }
1035 1035
1036 1036 /*
1037 1037 * KMDB is ineligible for instrumentation -- it may execute in
1038 1038 * any context, including probe context.
1039 1039 */
1040 1040 if (strcmp(modname, "kmdbmod") == 0)
1041 1041 return;
1042 1042
1043 1043 if (str == NULL || symhdr == NULL || symhdr->sh_addr == NULL) {
1044 1044 /*
1045 1045 * If this module doesn't (yet) have its string or symbol
1046 1046 * table allocated, clear out.
1047 1047 */
1048 1048 return;
1049 1049 }
1050 1050
1051 1051 symsize = symhdr->sh_entsize;
1052 1052
1053 1053 if (mp->fbt_nentries) {
1054 1054 /*
1055 1055 * This module has some FBT entries allocated; we're afraid
1056 1056 * to screw with it.
1057 1057 */
1058 1058 return;
1059 1059 }
1060 1060
1061 1061 if (mp->fbt_tab != NULL)
1062 1062 estimate = 0;
1063 1063
1064 1064 /*
1065 1065 * This is a hack for unix/genunix/krtld.
1066 1066 */
1067 1067 primary = vmem_contains(heap_arena, (void *)ctl,
1068 1068 sizeof (struct modctl)) == 0;
1069 1069 kobj_textwin_alloc(mp);
1070 1070
1071 1071 /*
1072 1072 * Open the CTF data for the module. We'll use this to determine the
1073 1073 * functions that can be instrumented. Note that this call can fail,
1074 1074 * in which case we'll use heuristics to determine the functions that
1075 1075 * can be instrumented. (But in particular, leaf functions will not be
1076 1076 * instrumented.)
1077 1077 */
1078 1078 fp = ctf_modopen(mp, &error);
1079 1079
1080 1080 forreal:
1081 1081 if (!estimate) {
1082 1082 tramp.fbtt_next =
1083 1083 (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
1084 1084 mp->fbt_size);
1085 1085 tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
1086 1086 tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
1087 1087 }
1088 1088
1089 1089 for (i = 1; i < nsyms; i++) {
1090 1090 ctf_funcinfo_t f;
1091 1091 uint32_t *instr, *base, *limit;
1092 1092 Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
1093 1093 int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
1094 1094 int (*canpatch)(uint32_t *, int, const char *);
1095 1095 uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
1096 1096 uint32_t, fbt_trampoline_t *, const char *);
1097 1097
1098 1098 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
1099 1099 continue;
1100 1100
1101 1101 /*
1102 1102 * Weak symbols are not candidates. This could be made to
1103 1103 * work (where weak functions and their underlying function
1104 1104 * appear as two disjoint probes), but it's not simple.
1105 1105 */
1106 1106 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
1107 1107 continue;
1108 1108
1109 1109 name = str + sym->st_name;
1110 1110
1111 1111 if (strstr(name, "dtrace_") == name &&
1112 1112 strstr(name, "dtrace_safe_") != name) {
1113 1113 /*
1114 1114 * Anything beginning with "dtrace_" may be called
1115 1115 * from probe context unless it explitly indicates
1116 1116 * that it won't be called from probe context by
1117 1117 * using the prefix "dtrace_safe_".
1118 1118 */
1119 1119 continue;
1120 1120 }
1121 1121
1122 1122 if (strstr(name, "kdi_") == name ||
1123 1123 strstr(name, "_kdi_") != NULL) {
1124 1124 /*
1125 1125 * Any function name beginning with "kdi_" or
1126 1126 * containing the string "_kdi_" is a part of the
1127 1127 * kernel debugger interface and may be called in
1128 1128 * arbitrary context -- including probe context.
1129 1129 */
1130 1130 continue;
1131 1131 }
1132 1132
1133 1133 if (strstr(name, "__relocatable") != NULL) {
1134 1134 /*
1135 1135 * Anything with the string "__relocatable" anywhere
1136 1136 * in the function name is considered to be a function
1137 1137 * that may be manually relocated before execution.
1138 1138 * Because FBT uses a PC-relative technique for
1139 1139 * instrumentation, these functions cannot safely
1140 1140 * be instrumented by us.
1141 1141 */
1142 1142 continue;
1143 1143 }
1144 1144
1145 1145 if (strstr(name, "ip_ocsum") == name) {
1146 1146 /*
1147 1147 * The ip_ocsum_* family of routines are all ABI
1148 1148 * violators. (They expect incoming arguments in the
1149 1149 * globals!) Break the ABI? No soup for you!
1150 1150 */
1151 1151 continue;
1152 1152 }
1153 1153
1154 1154 /*
1155 1155 * We want to scan the function for one (and only one) save.
1156 1156 * Any more indicates that something fancy is going on.
1157 1157 */
1158 1158 base = (uint32_t *)sym->st_value;
1159 1159 limit = (uint32_t *)(sym->st_value + sym->st_size);
1160 1160
1161 1161 /*
1162 1162 * We don't want to interpose on the module stubs.
1163 1163 */
1164 1164 if (base >= (uint32_t *)stubs_base &&
1165 1165 base <= (uint32_t *)stubs_end)
1166 1166 continue;
1167 1167
1168 1168 /*
1169 1169 * We can't safely trace a zero-length function...
1170 1170 */
1171 1171 if (base == limit)
1172 1172 continue;
1173 1173
1174 1174 /*
1175 1175 * Due to 4524008, _init and _fini may have a bloated st_size.
1176 1176 * While this bug was fixed quite some time ago, old drivers
1177 1177 * may be lurking. We need to develop a better solution to
1178 1178 * this problem, such that correct _init and _fini functions
1179 1179 * (the vast majority) may be correctly traced. One solution
1180 1180 * may be to scan through the entire symbol table to see if
1181 1181 * any symbol overlaps with _init. If none does, set a bit in
1182 1182 * the module structure that this module has correct _init and
1183 1183 * _fini sizes. This will cause some pain the first time a
1184 1184 * module is scanned, but at least it would be O(N) instead of
1185 1185 * O(N log N)...
1186 1186 */
1187 1187 if (strcmp(name, "_init") == 0)
1188 1188 continue;
1189 1189
1190 1190 if (strcmp(name, "_fini") == 0)
1191 1191 continue;
1192 1192
1193 1193 instr = base;
1194 1194
1195 1195 /*
1196 1196 * While we try hard to only trace safe functions (that is,
1197 1197 * functions at TL=0), one unsafe function manages to otherwise
1198 1198 * appear safe: prom_trap(). We could discover prom_trap()
1199 1199 * if we added an additional rule: in order to trace a
1200 1200 * function, we must either (a) discover a restore or (b)
1201 1201 * determine that the function does not have any unlinked
1202 1202 * control transfers to another function (i.e., the function
1203 1203 * never returns). Unfortunately, as of this writing, one
1204 1204 * legitimate function (resume_from_zombie()) transfers
1205 1205 * control to a different function (_resume_from_idle())
1206 1206 * without executing a restore. Barring a rule to figure out
1207 1207 * that resume_from_zombie() is safe while prom_trap() is not,
1208 1208 * we resort to hard-coding prom_trap() here.
1209 1209 */
1210 1210 if (strcmp(name, "prom_trap") == 0)
1211 1211 continue;
1212 1212
1213 1213 if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
1214 1214 nargs = f.ctc_argc;
1215 1215 have_ctf = 1;
1216 1216 } else {
1217 1217 nargs = 32;
1218 1218 }
1219 1219
1220 1220 /*
1221 1221 * If the first instruction of the function is a branch and
1222 1222 * it's not a branch-always-not-annulled, we're going to refuse
1223 1223 * to patch it.
1224 1224 */
1225 1225 if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
1226 1226 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
1227 1227 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
1228 1228 if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
1229 1229 if (have_ctf) {
1230 1230 cmn_err(CE_NOTE, "cannot instrument %s:"
1231 1231 " begins with non-ba, "
1232 1232 "non-br CTI", name);
1233 1233 }
1234 1234 continue;
1235 1235 }
1236 1236 }
1237 1237
1238 1238 while (!FBT_IS_SAVE(*instr)) {
1239 1239 /*
1240 1240 * Before we assume that this is a leaf routine, check
1241 1241 * forward in the basic block for a save.
1242 1242 */
1243 1243 int op = *instr & FBT_OP_MASK;
1244 1244 int op2 = *instr & FBT_FMT2_OP2_MASK;
1245 1245
1246 1246 if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
1247 1247 /*
1248 1248 * This is a CTI. If we see a subsequent
1249 1249 * save, we will refuse to process this
1250 1250 * routine unless both of the following are
1251 1251 * true:
1252 1252 *
1253 1253 * (a) The branch is not annulled
1254 1254 *
1255 1255 * (b) The subsequent save is in the delay
1256 1256 * slot of the branch
1257 1257 */
1258 1258 if ((*instr & FBT_ANNUL) ||
1259 1259 !FBT_IS_SAVE(*(instr + 1))) {
1260 1260 cti = 1;
1261 1261 } else {
1262 1262 instr++;
1263 1263 break;
1264 1264 }
1265 1265 }
1266 1266
1267 1267 if (op == FBT_OP1)
1268 1268 cti = 1;
1269 1269
1270 1270 if (++instr == limit)
1271 1271 break;
1272 1272 }
1273 1273
1274 1274 if (instr < limit && cti) {
1275 1275 /*
1276 1276 * If we found a CTI before the save, we need to not
1277 1277 * do anything. But if we have CTF information, this
1278 1278 * is weird enough that it merits a message.
1279 1279 */
1280 1280 if (!have_ctf)
1281 1281 continue;
1282 1282
1283 1283 cmn_err(CE_NOTE, "cannot instrument %s: "
1284 1284 "save not in first basic block", name);
1285 1285 continue;
1286 1286 }
1287 1287
1288 1288 if (instr == limit) {
1289 1289 if (!have_ctf)
1290 1290 continue;
1291 1291 is_leaf = 1;
1292 1292
1293 1293 if (!estimate)
1294 1294 fbt_leaf_functions++;
1295 1295
1296 1296 canpatch = fbt_canpatch_retl;
1297 1297 patch = fbt_patch_retl;
1298 1298 } else {
1299 1299 canpatch = fbt_canpatch_return;
1300 1300 patch = fbt_patch_return;
1301 1301 }
1302 1302
1303 1303 if (!have_ctf && !is_leaf) {
1304 1304 /*
1305 1305 * Before we assume that this isn't something tricky,
1306 1306 * look for other saves. If we find them, there are
1307 1307 * multiple entry points here (or something), and we'll
1308 1308 * leave it alone.
1309 1309 */
1310 1310 while (++instr < limit) {
1311 1311 if (FBT_IS_SAVE(*instr))
1312 1312 break;
1313 1313 }
1314 1314
1315 1315 if (instr != limit)
1316 1316 continue;
1317 1317 }
1318 1318
1319 1319 instr = base;
1320 1320
1321 1321 if (FBT_IS_CTI(*instr)) {
1322 1322 /*
1323 1323 * If we have a CTI, we want to be sure that we don't
1324 1324 * have a CTI or a PC-relative instruction in the
1325 1325 * delay slot -- we want to be able to thunk the
1326 1326 * instruction into the trampoline without worrying
1327 1327 * about either DCTIs or relocations. It would be
1328 1328 * very odd for the compiler to generate this kind of
1329 1329 * code, so we warn about it if we have CTF
1330 1330 * information.
1331 1331 */
1332 1332 if (FBT_IS_CTI(*(instr + 1))) {
1333 1333 if (!have_ctf)
1334 1334 continue;
1335 1335
1336 1336 cmn_err(CE_NOTE, "cannot instrument %s: "
1337 1337 "CTI in delay slot of first instruction",
1338 1338 name);
1339 1339 continue;
1340 1340 }
1341 1341
1342 1342 if (FBT_IS_PCRELATIVE(*(instr + 1))) {
1343 1343 if (!have_ctf)
1344 1344 continue;
1345 1345
1346 1346 cmn_err(CE_NOTE, "cannot instrument %s: "
1347 1347 "PC-relative instruction in delay slot of"
1348 1348 " first instruction", name);
1349 1349 continue;
1350 1350 }
1351 1351 }
1352 1352
1353 1353 if (estimate) {
1354 1354 tramp.fbtt_next = (uintptr_t)faketramp;
1355 1355 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1356 1356 (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
1357 1357 &tramp, nargs);
1358 1358 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1359 1359 } else {
1360 1360 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1361 1361 fbt->fbtp_name = name;
1362 1362 fbt->fbtp_ctl = ctl;
1363 1363 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1364 1364 name, FBT_PROBENAME_ENTRY, 1, fbt);
1365 1365 fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);
1366 1366
1367 1367 if (!fbt_patch_entry(instr, fbt->fbtp_id,
1368 1368 &tramp, nargs)) {
1369 1369 cmn_err(CE_WARN, "unexpectedly short FBT table "
1370 1370 "in module %s (sym %d of %d)", modname,
1371 1371 i, nsyms);
1372 1372 break;
1373 1373 }
1374 1374
1375 1375 fbt->fbtp_patchpoint =
1376 1376 (uint32_t *)((uintptr_t)mp->textwin +
1377 1377 ((uintptr_t)instr - (uintptr_t)mp->text));
1378 1378 fbt->fbtp_savedval = *instr;
1379 1379
1380 1380 fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1381 1381 fbt->fbtp_primary = primary;
1382 1382 fbt->fbtp_symndx = i;
1383 1383 mp->fbt_nentries++;
1384 1384 }
1385 1385
1386 1386 retfbt = NULL;
1387 1387 again:
1388 1388 if (++instr == limit)
1389 1389 continue;
1390 1390
1391 1391 offset = (uintptr_t)instr - (uintptr_t)base;
1392 1392
1393 1393 if (!(*canpatch)(instr, offset, name))
1394 1394 goto again;
1395 1395
1396 1396 if (estimate) {
1397 1397 tramp.fbtt_next = (uintptr_t)faketramp;
1398 1398 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1399 1399 (void) (*patch)(instr, base, limit,
1400 1400 offset, FBT_ESTIMATE_ID, &tramp, name);
1401 1401 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1402 1402
1403 1403 goto again;
1404 1404 }
1405 1405
1406 1406 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1407 1407 fbt->fbtp_name = name;
1408 1408 fbt->fbtp_ctl = ctl;
1409 1409
1410 1410 if (retfbt == NULL) {
1411 1411 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1412 1412 name, FBT_PROBENAME_RETURN, 1, fbt);
1413 1413 } else {
1414 1414 retfbt->fbtp_next = fbt;
1415 1415 fbt->fbtp_id = retfbt->fbtp_id;
1416 1416 }
1417 1417
1418 1418 fbt->fbtp_return = 1;
1419 1419 retfbt = fbt;
1420 1420
1421 1421 if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
1422 1422 fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
1423 1423 cmn_err(CE_WARN, "unexpectedly short FBT table "
1424 1424 "in module %s (sym %d of %d)", modname, i, nsyms);
1425 1425 break;
1426 1426 }
1427 1427
1428 1428 fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
1429 1429 ((uintptr_t)instr - (uintptr_t)mp->text));
1430 1430 fbt->fbtp_savedval = *instr;
1431 1431 fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1432 1432 fbt->fbtp_primary = primary;
1433 1433 fbt->fbtp_symndx = i;
1434 1434 mp->fbt_nentries++;
1435 1435
1436 1436 goto again;
1437 1437 }
1438 1438
1439 1439 if (estimate) {
1440 1440 /*
1441 1441 * Slosh on another entry's worth...
1442 1442 */
1443 1443 fbt_size += FBT_ENT_MAXSIZE;
1444 1444 mp->fbt_size = fbt_size;
1445 1445 mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);
1446 1446
1447 1447 if (mp->fbt_tab == NULL) {
1448 1448 cmn_err(CE_WARN, "couldn't allocate FBT table "
1449 1449 "for module %s", modname);
1450 1450 } else {
1451 1451 estimate = 0;
1452 1452 goto forreal;
1453 1453 }
1454 1454 } else {
1455 1455 fbt_trampoline_unmap();
1456 1456 }
1457 1457
1458 1458 error:
1459 1459 if (fp != NULL)
1460 1460 ctf_close(fp);
1461 1461 }
1462 1462
1463 1463 /*ARGSUSED*/
1464 1464 static void
1465 1465 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
1466 1466 {
1467 1467 fbt_probe_t *fbt = parg, *next;
1468 1468 struct modctl *ctl = fbt->fbtp_ctl;
1469 1469
1470 1470 do {
1471 1471 if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
1472 1472 if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
1473 1473 ctl->mod_loaded) || fbt->fbtp_primary) {
1474 1474 ((struct module *)
1475 1475 (ctl->mod_mp))->fbt_nentries--;
1476 1476 }
1477 1477 }
1478 1478
1479 1479 next = fbt->fbtp_next;
1480 1480 kmem_free(fbt, sizeof (fbt_probe_t));
1481 1481 fbt = next;
1482 1482 } while (fbt != NULL);
1483 1483 }
1484 1484
1485 1485 /*ARGSUSED*/
1486 1486 static int
1487 1487 fbt_enable(void *arg, dtrace_id_t id, void *parg)
1488 1488 {
1489 1489 fbt_probe_t *fbt = parg, *f;
1490 1490 struct modctl *ctl = fbt->fbtp_ctl;
1491 1491
1492 1492 ctl->mod_nenabled++;
1493 1493
1494 1494 for (f = fbt; f != NULL; f = f->fbtp_next) {
1495 1495 if (f->fbtp_patchpoint == NULL) {
1496 1496 /*
1497 1497 * Due to a shortened FBT table, this entry was never
1498 1498 * completed; refuse to enable it.
1499 1499 */
1500 1500 if (fbt_verbose) {
1501 1501 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1502 1502 "(short FBT table in %s)",
1503 1503 fbt->fbtp_name, ctl->mod_modname);
1504 1504 }
1505 1505
1506 1506 return (0);
1507 1507 }
1508 1508 }
1509 1509
1510 1510 /*
1511 1511 * If this module has disappeared since we discovered its probes,
1512 1512 * refuse to enable it.
1513 1513 */
1514 1514 if (!fbt->fbtp_primary && !ctl->mod_loaded) {
1515 1515 if (fbt_verbose) {
1516 1516 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1517 1517 "(module %s unloaded)",
1518 1518 fbt->fbtp_name, ctl->mod_modname);
1519 1519 }
1520 1520
1521 1521 return (0);
1522 1522 }
1523 1523
1524 1524 /*
1525 1525 * Now check that our modctl has the expected load count. If it
1526 1526 * doesn't, this module must have been unloaded and reloaded -- and
1527 1527 * we're not going to touch it.
1528 1528 */
1529 1529 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
1530 1530 if (fbt_verbose) {
1531 1531 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1532 1532 "(module %s reloaded)",
1533 1533 fbt->fbtp_name, ctl->mod_modname);
1534 1534 }
1535 1535
1536 1536 return (0);
1537 1537 }
1538 1538
1539 1539 for (; fbt != NULL; fbt = fbt->fbtp_next)
1540 1540 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1541 1541
1542 1542 return (0);
1543 1543 }
1544 1544
1545 1545 /*ARGSUSED*/
1546 1546 static void
1547 1547 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1548 1548 {
1549 1549 fbt_probe_t *fbt = parg, *f;
1550 1550 struct modctl *ctl = fbt->fbtp_ctl;
1551 1551
1552 1552 ASSERT(ctl->mod_nenabled > 0);
1553 1553 ctl->mod_nenabled--;
1554 1554
1555 1555 for (f = fbt; f != NULL; f = f->fbtp_next) {
1556 1556 if (f->fbtp_patchpoint == NULL)
1557 1557 return;
1558 1558 }
1559 1559
1560 1560 if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
1561 1561 (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1562 1562 return;
1563 1563
1564 1564 for (; fbt != NULL; fbt = fbt->fbtp_next)
1565 1565 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1566 1566 }
1567 1567
1568 1568 /*ARGSUSED*/
1569 1569 static void
1570 1570 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1571 1571 {
1572 1572 fbt_probe_t *fbt = parg;
1573 1573 struct modctl *ctl = fbt->fbtp_ctl;
1574 1574
1575 1575 if (!fbt->fbtp_primary && !ctl->mod_loaded)
1576 1576 return;
1577 1577
1578 1578 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1579 1579 return;
1580 1580
1581 1581 ASSERT(ctl->mod_nenabled > 0);
1582 1582
1583 1583 for (; fbt != NULL; fbt = fbt->fbtp_next)
1584 1584 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1585 1585 }
1586 1586
1587 1587 /*ARGSUSED*/
1588 1588 static void
1589 1589 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1590 1590 {
1591 1591 fbt_probe_t *fbt = parg;
1592 1592 struct modctl *ctl = fbt->fbtp_ctl;
1593 1593
1594 1594 if (!fbt->fbtp_primary && !ctl->mod_loaded)
1595 1595 return;
1596 1596
1597 1597 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1598 1598 return;
1599 1599
1600 1600 ASSERT(ctl->mod_nenabled > 0);
1601 1601
1602 1602 for (; fbt != NULL; fbt = fbt->fbtp_next)
1603 1603 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1604 1604 }
1605 1605
1606 1606 /*ARGSUSED*/
1607 1607 static void
1608 1608 fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
1609 1609 {
1610 1610 fbt_probe_t *fbt = parg;
1611 1611 struct modctl *ctl = fbt->fbtp_ctl;
1612 1612 struct module *mp = ctl->mod_mp;
1613 1613 ctf_file_t *fp = NULL, *pfp;
1614 1614 ctf_funcinfo_t f;
1615 1615 int error;
1616 1616 ctf_id_t argv[32], type;
1617 1617 int argc = sizeof (argv) / sizeof (ctf_id_t);
1618 1618 const char *parent;
1619 1619
1620 1620 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1621 1621 goto err;
1622 1622
1623 1623 if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
1624 1624 (void) strcpy(desc->dtargd_native, "int");
1625 1625 return;
1626 1626 }
1627 1627
1628 1628 if ((fp = ctf_modopen(mp, &error)) == NULL) {
1629 1629 /*
1630 1630 * We have no CTF information for this module -- and therefore
1631 1631 * no args[] information.
1632 1632 */
1633 1633 goto err;
1634 1634 }
1635 1635
1636 1636 /*
1637 1637 * If we have a parent container, we must manually import it.
1638 1638 */
1639 1639 if ((parent = ctf_parent_name(fp)) != NULL) {
1640 1640 struct modctl *mp = &modules;
1641 1641 struct modctl *mod = NULL;
1642 1642
1643 1643 /*
1644 1644 * We must iterate over all modules to find the module that
1645 1645 * is our parent.
1646 1646 */
1647 1647 do {
1648 1648 if (strcmp(mp->mod_modname, parent) == 0) {
1649 1649 mod = mp;
|
↓ open down ↓ |
1649 lines elided |
↑ open up ↑ |
1650 1650 break;
1651 1651 }
1652 1652 } while ((mp = mp->mod_next) != &modules);
1653 1653
1654 1654 if (mod == NULL)
1655 1655 goto err;
1656 1656
1657 1657 if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
1658 1658 goto err;
1659 1659
1660 + /*
1661 + * If the parent module does not have the label we expect,
1662 + * ignore it and fail to avoid presenting non-sensical data.
1663 + */
1664 + if (ctf_label_info(pfp, ctf_parent_label(fp),
1665 + NULL) == CTF_ERR) {
1666 + ctf_close(pfp);
1667 + goto err;
1668 + }
1669 +
1660 1670 if (ctf_import(fp, pfp) != 0) {
1661 1671 ctf_close(pfp);
1662 1672 goto err;
1663 1673 }
1664 1674
1665 1675 ctf_close(pfp);
1666 1676 }
1667 1677
1668 1678 if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
1669 1679 goto err;
1670 1680
1671 1681 if (fbt->fbtp_return) {
1672 1682 if (desc->dtargd_ndx > 1)
1673 1683 goto err;
1674 1684
1675 1685 ASSERT(desc->dtargd_ndx == 1);
1676 1686 type = f.ctc_return;
1677 1687 } else {
1678 1688 if (desc->dtargd_ndx + 1 > f.ctc_argc)
1679 1689 goto err;
1680 1690
1681 1691 if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
1682 1692 goto err;
1683 1693
1684 1694 type = argv[desc->dtargd_ndx];
1685 1695 }
1686 1696
1687 1697 if (ctf_type_name(fp, type, desc->dtargd_native,
1688 1698 DTRACE_ARGTYPELEN) != NULL) {
1689 1699 ctf_close(fp);
1690 1700 return;
1691 1701 }
1692 1702 err:
1693 1703 if (fp != NULL)
1694 1704 ctf_close(fp);
1695 1705
1696 1706 desc->dtargd_ndx = DTRACE_ARGNONE;
1697 1707 }
1698 1708
1699 1709 static dtrace_pattr_t fbt_attr = {
1700 1710 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1701 1711 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1702 1712 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1703 1713 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1704 1714 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
1705 1715 };
1706 1716
1707 1717 static dtrace_pops_t fbt_pops = {
1708 1718 NULL,
1709 1719 fbt_provide_module,
1710 1720 fbt_enable,
1711 1721 fbt_disable,
1712 1722 fbt_suspend,
1713 1723 fbt_resume,
1714 1724 fbt_getargdesc,
1715 1725 NULL,
1716 1726 NULL,
1717 1727 fbt_destroy
1718 1728 };
1719 1729
1720 1730 static int
1721 1731 fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1722 1732 {
1723 1733 switch (cmd) {
1724 1734 case DDI_ATTACH:
1725 1735 break;
1726 1736 case DDI_RESUME:
1727 1737 return (DDI_SUCCESS);
1728 1738 default:
1729 1739 return (DDI_FAILURE);
1730 1740 }
1731 1741
1732 1742 if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
1733 1743 DDI_PSEUDO, NULL) == DDI_FAILURE ||
1734 1744 dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
1735 1745 &fbt_pops, NULL, &fbt_id) != 0) {
1736 1746 ddi_remove_minor_node(devi, NULL);
1737 1747 return (DDI_FAILURE);
1738 1748 }
1739 1749
1740 1750 ddi_report_dev(devi);
1741 1751 fbt_devi = devi;
1742 1752 return (DDI_SUCCESS);
1743 1753 }
1744 1754
1745 1755 static int
1746 1756 fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
1747 1757 {
1748 1758 switch (cmd) {
1749 1759 case DDI_DETACH:
1750 1760 break;
1751 1761 case DDI_SUSPEND:
1752 1762 return (DDI_SUCCESS);
1753 1763 default:
1754 1764 return (DDI_FAILURE);
1755 1765 }
1756 1766
1757 1767 if (dtrace_unregister(fbt_id) != 0)
1758 1768 return (DDI_FAILURE);
1759 1769
1760 1770 ddi_remove_minor_node(devi, NULL);
1761 1771 return (DDI_SUCCESS);
1762 1772 }
1763 1773
1764 1774 /*ARGSUSED*/
1765 1775 static int
1766 1776 fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1767 1777 {
1768 1778 int error;
1769 1779
1770 1780 switch (infocmd) {
1771 1781 case DDI_INFO_DEVT2DEVINFO:
1772 1782 *result = (void *)fbt_devi;
1773 1783 error = DDI_SUCCESS;
1774 1784 break;
1775 1785 case DDI_INFO_DEVT2INSTANCE:
1776 1786 *result = (void *)0;
1777 1787 error = DDI_SUCCESS;
1778 1788 break;
1779 1789 default:
1780 1790 error = DDI_FAILURE;
1781 1791 }
1782 1792 return (error);
1783 1793 }
1784 1794
1785 1795 /*ARGSUSED*/
1786 1796 static int
1787 1797 fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1788 1798 {
1789 1799 return (0);
1790 1800 }
1791 1801
1792 1802 static struct cb_ops fbt_cb_ops = {
1793 1803 fbt_open, /* open */
1794 1804 nodev, /* close */
1795 1805 nulldev, /* strategy */
1796 1806 nulldev, /* print */
1797 1807 nodev, /* dump */
1798 1808 nodev, /* read */
1799 1809 nodev, /* write */
1800 1810 nodev, /* ioctl */
1801 1811 nodev, /* devmap */
1802 1812 nodev, /* mmap */
1803 1813 nodev, /* segmap */
1804 1814 nochpoll, /* poll */
1805 1815 ddi_prop_op, /* cb_prop_op */
1806 1816 0, /* streamtab */
1807 1817 D_NEW | D_MP /* Driver compatibility flag */
1808 1818 };
1809 1819
1810 1820 static struct dev_ops fbt_ops = {
1811 1821 DEVO_REV, /* devo_rev */
1812 1822 0, /* refcnt */
1813 1823 fbt_info, /* get_dev_info */
1814 1824 nulldev, /* identify */
1815 1825 nulldev, /* probe */
1816 1826 fbt_attach, /* attach */
1817 1827 fbt_detach, /* detach */
1818 1828 nodev, /* reset */
1819 1829 &fbt_cb_ops, /* driver operations */
1820 1830 NULL, /* bus operations */
1821 1831 nodev, /* dev power */
1822 1832 ddi_quiesce_not_needed, /* quiesce */
1823 1833 };
1824 1834
1825 1835 /*
1826 1836 * Module linkage information for the kernel.
1827 1837 */
1828 1838 static struct modldrv modldrv = {
1829 1839 &mod_driverops, /* module type (this is a pseudo driver) */
1830 1840 "Function Boundary Tracing", /* name of module */
1831 1841 &fbt_ops, /* driver ops */
1832 1842 };
1833 1843
1834 1844 static struct modlinkage modlinkage = {
1835 1845 MODREV_1,
1836 1846 (void *)&modldrv,
1837 1847 NULL
1838 1848 };
1839 1849
1840 1850 int
1841 1851 _init(void)
1842 1852 {
1843 1853 return (mod_install(&modlinkage));
1844 1854 }
1845 1855
1846 1856 int
1847 1857 _info(struct modinfo *modinfop)
1848 1858 {
1849 1859 return (mod_info(&modlinkage, modinfop));
1850 1860 }
1851 1861
1852 1862 int
1853 1863 _fini(void)
1854 1864 {
1855 1865 return (mod_remove(&modlinkage));
1856 1866 }
|
↓ open down ↓ |
187 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX