1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/dtrace.h>
28 #include <sys/cmn_err.h>
29 #include <sys/tnf.h>
30 #include <sys/atomic.h>
31 #include <sys/prsystm.h>
32 #include <sys/modctl.h>
33 #include <sys/aio_impl.h>
34
35 #ifdef __sparc
36 #include <sys/privregs.h>
37 #endif
38
39 void (*dtrace_cpu_init)(processorid_t);
40 void (*dtrace_modload)(struct modctl *);
41 void (*dtrace_modunload)(struct modctl *);
42 void (*dtrace_helpers_cleanup)(void);
43 void (*dtrace_helpers_fork)(proc_t *, proc_t *);
44 void (*dtrace_cpustart_init)(void);
45 void (*dtrace_cpustart_fini)(void);
46 void (*dtrace_cpc_fire)(uint64_t);
47 void (*dtrace_closef)(void);
48
49 void (*dtrace_debugger_init)(void);
50 void (*dtrace_debugger_fini)(void);
51
52 dtrace_vtime_state_t dtrace_vtime_active = 0;
53 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
54
55 /*
56 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
57 * hardware overflow interrupt handler and the kernel cpc framework to check
58 * whether or not the DTrace cpc provider is currently in use. The variable is
59 * set before counters are enabled with the first enabling and cleared when
60 * the last enabling is disabled. Its value at any given time indicates the
61 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
62 * is held during initial setting to protect races between kcpc_open() and the
63 * first enabling. The locking provided by the DTrace subsystem, the kernel
64 * cpc framework and the cpu management framework protect consumers from race
65 * conditions on enabling and disabling probes.
66 */
67 uint32_t dtrace_cpc_in_use = 0;
68
69 typedef struct dtrace_hrestime {
70 lock_t dthr_lock; /* lock for this element */
71 timestruc_t dthr_hrestime; /* hrestime value */
72 int64_t dthr_adj; /* hrestime_adj value */
73 hrtime_t dthr_hrtime; /* hrtime value */
74 } dtrace_hrestime_t;
75
76 static dtrace_hrestime_t dtrace_hrestime[2];
77
78 /*
79 * Making available adjustable high-resolution time in DTrace is regrettably
80 * more complicated than one might think it should be. The problem is that
81 * the variables related to adjusted high-resolution time (hrestime,
82 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may
83 * be held when we enter probe context. One might think that we could address
84 * this by having a single snapshot copy that is stored under a different lock
85 * from hres_tick(), using the snapshot iff hres_lock is locked in probe
86 * context. Unfortunately, this too won't work: because hres_lock is grabbed
87 * in more than just hres_tick() context, we could enter probe context
88 * concurrently on two different CPUs with both locks (hres_lock and the
89 * snapshot lock) held. As this implies, the fundamental problem is that we
90 * need to have access to a snapshot of these variables that we _know_ will
91 * not be locked in probe context. To effect this, we have two snapshots
92 * protected by two different locks, and we mandate that these snapshots are
93 * recorded in succession by a single thread calling dtrace_hres_tick(). (We
94 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls
95 * hres_tick().) A single thread can't be in two places at once: one of the
96 * snapshot locks is guaranteed to be unheld at all times. The
97 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then
98 * the other to find the unlocked snapshot.
99 */
100 void
101 dtrace_hres_tick(void)
102 {
103 int i;
104 ushort_t spl;
105
106 for (i = 0; i < 2; i++) {
107 dtrace_hrestime_t tmp;
108
109 spl = hr_clock_lock();
110 tmp.dthr_hrestime = hrestime;
111 tmp.dthr_adj = hrestime_adj;
112 tmp.dthr_hrtime = dtrace_gethrtime();
113 hr_clock_unlock(spl);
114
115 lock_set(&dtrace_hrestime[i].dthr_lock);
116 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime;
117 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj;
118 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime;
119 dtrace_membar_producer();
120
121 /*
122 * To allow for lock-free examination of this lock, we use
123 * the same trick that is used hres_lock; for more details,
124 * see the description of this technique in sun4u/sys/clock.h.
125 */
126 dtrace_hrestime[i].dthr_lock++;
127 }
128 }
129
130 hrtime_t
131 dtrace_gethrestime(void)
132 {
133 dtrace_hrestime_t snap;
134 hrtime_t now;
135 int i = 0, adj, nslt;
136
137 for (;;) {
138 snap.dthr_lock = dtrace_hrestime[i].dthr_lock;
139 dtrace_membar_consumer();
140 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime;
141 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime;
142 snap.dthr_adj = dtrace_hrestime[i].dthr_adj;
143 dtrace_membar_consumer();
144
145 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock)
146 break;
147
148 /*
149 * If we're here, the lock was either locked, or it
150 * transitioned while we were taking the snapshot. Either
151 * way, we're going to try the other dtrace_hrestime element;
152 * we know that it isn't possible for both to be locked
153 * simultaneously, so we will ultimately get a good snapshot.
154 */
155 i ^= 1;
156 }
157
158 /*
159 * We have a good snapshot. Now perform any necessary adjustments.
160 */
161 nslt = dtrace_gethrtime() - snap.dthr_hrtime;
162 ASSERT(nslt >= 0);
163
164 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) +
165 snap.dthr_hrestime.tv_nsec;
166
167 if (snap.dthr_adj != 0) {
168 if (snap.dthr_adj > 0) {
169 adj = (nslt >> adj_shift);
170 if (adj > snap.dthr_adj)
171 adj = (int)snap.dthr_adj;
172 } else {
173 adj = -(nslt >> adj_shift);
174 if (adj < snap.dthr_adj)
175 adj = (int)snap.dthr_adj;
176 }
177 now += adj;
178 }
179
180 return (now);
181 }
182
183 void
184 dtrace_vtime_enable(void)
185 {
186 dtrace_vtime_state_t state, nstate;
187
188 do {
189 state = dtrace_vtime_active;
190
191 switch (state) {
192 case DTRACE_VTIME_INACTIVE:
193 nstate = DTRACE_VTIME_ACTIVE;
194 break;
195
196 case DTRACE_VTIME_INACTIVE_TNF:
197 nstate = DTRACE_VTIME_ACTIVE_TNF;
198 break;
199
200 case DTRACE_VTIME_ACTIVE:
201 case DTRACE_VTIME_ACTIVE_TNF:
202 panic("DTrace virtual time already enabled");
203 /*NOTREACHED*/
204 }
205
206 } while (cas32((uint32_t *)&dtrace_vtime_active,
207 state, nstate) != state);
208 }
209
210 void
211 dtrace_vtime_disable(void)
212 {
213 dtrace_vtime_state_t state, nstate;
214
215 do {
216 state = dtrace_vtime_active;
217
218 switch (state) {
219 case DTRACE_VTIME_ACTIVE:
220 nstate = DTRACE_VTIME_INACTIVE;
221 break;
222
223 case DTRACE_VTIME_ACTIVE_TNF:
224 nstate = DTRACE_VTIME_INACTIVE_TNF;
225 break;
226
227 case DTRACE_VTIME_INACTIVE:
228 case DTRACE_VTIME_INACTIVE_TNF:
229 panic("DTrace virtual time already disabled");
230 /*NOTREACHED*/
231 }
232
233 } while (cas32((uint32_t *)&dtrace_vtime_active,
234 state, nstate) != state);
235 }
236
237 void
238 dtrace_vtime_enable_tnf(void)
239 {
240 dtrace_vtime_state_t state, nstate;
241
242 do {
243 state = dtrace_vtime_active;
244
245 switch (state) {
246 case DTRACE_VTIME_ACTIVE:
247 nstate = DTRACE_VTIME_ACTIVE_TNF;
248 break;
249
250 case DTRACE_VTIME_INACTIVE:
251 nstate = DTRACE_VTIME_INACTIVE_TNF;
252 break;
253
254 case DTRACE_VTIME_ACTIVE_TNF:
255 case DTRACE_VTIME_INACTIVE_TNF:
256 panic("TNF already active");
257 /*NOTREACHED*/
258 }
259
260 } while (cas32((uint32_t *)&dtrace_vtime_active,
261 state, nstate) != state);
262 }
263
264 void
265 dtrace_vtime_disable_tnf(void)
266 {
267 dtrace_vtime_state_t state, nstate;
268
269 do {
270 state = dtrace_vtime_active;
271
272 switch (state) {
273 case DTRACE_VTIME_ACTIVE_TNF:
274 nstate = DTRACE_VTIME_ACTIVE;
275 break;
276
277 case DTRACE_VTIME_INACTIVE_TNF:
278 nstate = DTRACE_VTIME_INACTIVE;
279 break;
280
281 case DTRACE_VTIME_ACTIVE:
282 case DTRACE_VTIME_INACTIVE:
283 panic("TNF already inactive");
284 /*NOTREACHED*/
285 }
286
287 } while (cas32((uint32_t *)&dtrace_vtime_active,
288 state, nstate) != state);
289 }
290
291 void
292 dtrace_vtime_switch(kthread_t *next)
293 {
294 dtrace_icookie_t cookie;
295 hrtime_t ts;
296
297 if (tnf_tracing_active) {
298 tnf_thread_switch(next);
299
300 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF)
301 return;
302 }
303
304 cookie = dtrace_interrupt_disable();
305 ts = dtrace_gethrtime();
306
307 if (curthread->t_dtrace_start != 0) {
308 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start;
309 curthread->t_dtrace_start = 0;
310 }
311
312 next->t_dtrace_start = ts;
313
314 dtrace_interrupt_enable(cookie);
315 }
316
317 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
318 void (*dtrace_fasttrap_exec_ptr)(proc_t *);
319 void (*dtrace_fasttrap_exit_ptr)(proc_t *);
320
321 /*
322 * This function is called by cfork() in the event that it appears that
323 * there may be dtrace tracepoints active in the parent process's address
324 * space. This first confirms the existence of dtrace tracepoints in the
325 * parent process and calls into the fasttrap module to remove the
326 * corresponding tracepoints from the child. By knowing that there are
327 * existing tracepoints, and ensuring they can't be removed, we can rely
328 * on the fasttrap module remaining loaded.
329 */
330 void
331 dtrace_fasttrap_fork(proc_t *p, proc_t *cp)
332 {
333 ASSERT(p->p_proc_flag & P_PR_LOCK);
334 ASSERT(p->p_dtrace_count > 0);
335 ASSERT(dtrace_fasttrap_fork_ptr != NULL);
336
337 dtrace_fasttrap_fork_ptr(p, cp);
338 }