Print this page
OS-2366 ddi_periodic_add(9F) is entirely rubbish (MORE)
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/ddi_timer.c
+++ new/usr/src/uts/common/os/ddi_periodic.c
1 1 /*
2 - * CDDL HEADER START
2 + * This file and its contents are supplied under the terms of the
3 + * Common Development and Distribution License ("CDDL"), version 1.0.
4 + * You may only use this file in accordance with the terms of version
5 + * 1.0 of the CDDL.
3 6 *
4 - * The contents of this file are subject to the terms of the
5 - * Common Development and Distribution License (the "License").
6 - * You may not use this file except in compliance with the License.
7 - *
8 - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 - * or http://www.opensolaris.org/os/licensing.
10 - * See the License for the specific language governing permissions
11 - * and limitations under the License.
12 - *
13 - * When distributing Covered Code, include this CDDL HEADER in each
14 - * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 - * If applicable, add the following below this CDDL HEADER, with the
16 - * fields enclosed by brackets "[]" replaced with your own identifying
17 - * information: Portions Copyright [yyyy] [name of copyright owner]
18 - *
19 - * CDDL HEADER END
7 + * A full copy of the text of the CDDL should have accompanied this
8 + * source. A copy of the CDDL is also available via the Internet at
9 + * http://www.illumos.org/license/CDDL.
20 10 */
21 -
22 11 /*
23 - * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 - * Use is subject to license terms.
25 - */
26 -/*
27 12 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
28 13 */
29 14
30 15 #include <sys/cmn_err.h>
31 -#include <sys/ddi_timer.h>
16 +#include <sys/ddi_periodic.h>
32 17 #include <sys/id_space.h>
33 18 #include <sys/kobj.h>
34 19 #include <sys/sysmacros.h>
35 20 #include <sys/systm.h>
36 21 #include <sys/taskq.h>
37 22 #include <sys/taskq_impl.h>
38 23 #include <sys/time.h>
39 24 #include <sys/types.h>
40 25 #include <sys/sdt.h>
41 26
42 27 /*
43 28 * The ddi_periodic_add(9F) Implementation
44 29 *
45 30 * This file contains the implementation of the ddi_periodic_add(9F) interface.
46 31 * It is a thin wrapper around the cyclic subsystem (see documentation in
47 32 * uts/common/os/cyclic.c), providing a DDI interface for registering
48 33 * (and unregistering) callbacks for periodic invocation at arbitrary
49 34 * interrupt levels, or in kernel context.
50 35 *
51 36 * Each call to ddi_periodic_add will result in a new opaque handle, as
52 37 * allocated from an id_space, a new "periodic" object (ddi_periodic_impl_t)
53 38 * and a registered cyclic.
54 39 *
55 40 * Operation
56 41 *
57 42 * Whenever the cyclic fires, our cyclic handler checks that the particular
58 43 * periodic is not dispatched already (we do not support overlapping execution
59 44 * of the consumer's handler function), and not yet cancelled. If both of
60 45 * these conditions hold, we mark the periodic as DPF_DISPATCHED and enqueue it
61 46 * to either the taskq (for DDI_IPL_0) or to one of the soft interrupt queues
62 47 * (DDI_IPL_1 to DDI_IPL_10).
63 48 *
64 49 * While the taskq (or soft interrupt handler) is handling a particular
65 50 * periodic, we mark it as DPF_EXECUTING. When complete, we reset both
66 51 * DPF_DISPATCHED and DPF_EXECUTING.
67 52 *
68 53 * Cancellation
69 54 *
70 55 * ddi_periodic_delete(9F) historically had spectacularly loose semantics with
71 56 * respect to cancellation concurrent with handler execution. These semantics
72 57 * are now tighter:
73 58 *
74 59 * 1. At most one invocation of ddi_periodic_delete(9F) will actually
75 60 * perform the deletion, all others will return immediately.
76 61 * 2. The invocation that performs the deletion will _block_ until
77 62 * the handler is no longer running, and all resources have been
78 63 * released.
79 64 *
80 65 * We affect this model by removing the cancelling periodic from the
81 66 * global list and marking it DPF_CANCELLED. This will prevent further
82 67 * execution of the handler. We then wait on a CV until the DPF_EXECUTING
83 68 * and DPF_DISPATCHED flags are clear, which means the periodic is removed
84 69 * from all request queues, is no longer executing, and may be freed. At this
85 70 * point we return the opaque ID to the id_space and free the memory.
86 71 *
87 72 * NOTE:
88 73 * The ddi_periodic_add(9F) interface is presently limited to a minimum period
89 74 * of 10ms between firings.
90 75 */
91 76
92 77 /*
93 78 * Tuneables:
94 79 */
95 80 int ddi_periodic_max_id = 1024;
96 81 int ddi_periodic_taskq_threadcount = 4;
97 82 hrtime_t ddi_periodic_resolution = 10000000;
98 83
99 84 /*
100 85 * Globals:
101 86 */
102 87 static kmem_cache_t *periodic_cache;
103 88 static id_space_t *periodic_id_space;
104 89 static taskq_t *periodic_taskq;
105 90
106 91 /*
107 92 * periodics_lock protects the list of all periodics (periodics), and
108 93 * each of the soft interrupt request queues (periodic_softint_queue).
109 94 *
110 95 * Do not hold an individual periodic's lock while obtaining periodics_lock.
111 96 * While in the periodic_softint_queue list, the periodic will be marked
112 97 * DPF_DISPATCHED, and thus safe from frees. Only the invocation of
113 98 * i_untimeout() that removes the periodic from the global list is allowed
114 99 * to free it.
115 100 */
116 101 static kmutex_t periodics_lock;
117 102 static list_t periodics;
118 103 static list_t periodic_softint_queue[10]; /* for IPL1 up to IPL10 */
119 104
120 105 typedef enum periodic_ipl {
|
↓ open down ↓ |
79 lines elided |
↑ open up ↑ |
121 106 PERI_IPL_0 = 0,
122 107 PERI_IPL_1,
123 108 PERI_IPL_2,
124 109 PERI_IPL_3,
125 110 PERI_IPL_4,
126 111 PERI_IPL_5,
127 112 PERI_IPL_6,
128 113 PERI_IPL_7,
129 114 PERI_IPL_8,
130 115 PERI_IPL_9,
131 - PERI_IPL_10,
116 + PERI_IPL_10
132 117 } periodic_ipl_t;
133 118
119 +static char *
120 +periodic_handler_symbol(ddi_periodic_impl_t *dpr)
121 +{
122 + ulong_t off;
123 +
124 + return (kobj_getsymname((uintptr_t)dpr->dpr_handler, &off));
125 +}
126 +
134 127 /*
135 128 * This function may be called either from a soft interrupt handler
136 129 * (ddi_periodic_softintr), or as a taskq worker function.
137 130 */
138 131 static void
139 132 periodic_execute(void *arg)
140 133 {
141 134 ddi_periodic_impl_t *dpr = arg;
142 135 mutex_enter(&dpr->dpr_lock);
143 136
144 137 /*
145 138 * We must be DISPATCHED, but not yet EXECUTING:
146 139 */
147 140 VERIFY((dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) ==
148 141 DPF_DISPATCHED);
142 + VERIFY(dpr->dpr_thread == NULL);
149 143
150 144 if (!(dpr->dpr_flags & DPF_CANCELLED)) {
151 145 int level = dpr->dpr_level;
152 146 uint64_t count = dpr->dpr_fire_count;
153 147 /*
154 148 * If we have not yet been cancelled, then
155 149 * mark us executing:
156 150 */
157 151 dpr->dpr_flags |= DPF_EXECUTING;
152 + dpr->dpr_thread = curthread;
158 153 mutex_exit(&dpr->dpr_lock);
159 154
160 155 /*
161 156 * Execute the handler, without holding locks:
162 157 */
163 158 DTRACE_PROBE4(ddi__periodic__execute, void *, dpr->dpr_handler,
164 159 void *, dpr->dpr_arg, int, level, uint64_t, count);
165 160 (*dpr->dpr_handler)(dpr->dpr_arg);
166 161 DTRACE_PROBE4(ddi__periodic__done, void *, dpr->dpr_handler,
167 162 void *, dpr->dpr_arg, int, level, uint64_t, count);
168 163
169 164 mutex_enter(&dpr->dpr_lock);
165 + dpr->dpr_thread = NULL;
170 166 dpr->dpr_fire_count++;
171 167 }
172 168
173 169 /*
174 170 * We're done with this periodic for now, so release it and
175 171 * wake anybody that was waiting for us to be finished:
176 172 */
177 173 dpr->dpr_flags &= ~(DPF_DISPATCHED | DPF_EXECUTING);
178 174 cv_broadcast(&dpr->dpr_cv);
179 175 mutex_exit(&dpr->dpr_lock);
180 176 }
181 177
182 178 void
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
183 179 ddi_periodic_softintr(int level)
184 180 {
185 181 ddi_periodic_impl_t *dpr;
186 182 VERIFY(level >= PERI_IPL_1 && level <= PERI_IPL_10);
187 183
188 184 mutex_enter(&periodics_lock);
189 185 /*
190 186 * Pull the first scheduled periodic off the queue for this priority
191 187 * level:
192 188 */
193 - while ((dpr = list_remove_head(&periodic_softint_queue[level - 1]))
194 - != NULL) {
189 + while ((dpr = list_remove_head(&periodic_softint_queue[level - 1])) !=
190 + NULL) {
195 191 mutex_exit(&periodics_lock);
196 192 /*
197 193 * And execute it:
198 194 */
199 195 periodic_execute(dpr);
200 196 mutex_enter(&periodics_lock);
201 197 }
202 198 mutex_exit(&periodics_lock);
203 199 }
204 200
205 201 void
206 202 ddi_periodic_init(void)
207 203 {
208 204 int i;
209 205
210 206 /*
211 207 * Create a kmem_cache for request tracking objects, and a list
212 208 * to store them in so we can later delete based on opaque handles:
213 209 */
214 210 periodic_cache = kmem_cache_create("ddi_periodic",
215 211 sizeof (ddi_periodic_impl_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
216 212 list_create(&periodics, sizeof (ddi_periodic_impl_t),
217 213 offsetof(ddi_periodic_impl_t, dpr_link));
218 214
219 215 /*
220 216 * Initialise the identifier space for ddi_periodic_add(9F):
221 217 */
222 218 periodic_id_space = id_space_create("ddi_periodic", 1,
223 219 ddi_periodic_max_id);
224 220
225 221 /*
226 222 * Initialise the request queue for each soft interrupt level:
227 223 */
228 224 for (i = PERI_IPL_1; i <= PERI_IPL_10; i++) {
229 225 list_create(&periodic_softint_queue[i - 1],
230 226 sizeof (ddi_periodic_impl_t), offsetof(ddi_periodic_impl_t,
231 227 dpr_softint_link));
232 228 }
233 229
234 230 /*
235 231 * Create the taskq for running PERI_IPL_0 handlers. This taskq will
236 232 * _only_ be used with taskq_dispatch_ent(), and a taskq_ent_t
237 233 * pre-allocated with the ddi_periodic_impl_t.
238 234 */
|
↓ open down ↓ |
34 lines elided |
↑ open up ↑ |
239 235 periodic_taskq = taskq_create_instance("ddi_periodic_taskq", -1,
240 236 ddi_periodic_taskq_threadcount, maxclsyspri, 0, 0, 0);
241 237
242 238 /*
243 239 * Initialize the mutex lock used for the soft interrupt request
244 240 * queues.
245 241 */
246 242 mutex_init(&periodics_lock, NULL, MUTEX_ADAPTIVE, NULL);
247 243 }
248 244
245 +void
246 +ddi_periodic_fini(void)
247 +{
248 + ddi_periodic_impl_t *dpr;
249 + /*
250 + * Find all periodics that have not yet been unregistered and,
251 + * on DEBUG bits, print a warning about this resource leak.
252 + */
253 + mutex_enter(&periodics_lock);
254 + while ((dpr = list_head(&periodics)) != NULL) {
255 +#ifdef DEBUG
256 + printf("DDI periodic handler not deleted (id=%p, handler=%s)\n",
257 + dpr->dpr_id, periodic_handler_symbol(dpr));
258 +#endif
259 +
260 + mutex_exit(&periodics_lock);
261 + /*
262 + * Delete the periodic ourselves:
263 + */
264 + i_untimeout(dpr->dpr_id);
265 + mutex_enter(&periodics_lock);
266 + }
267 + mutex_exit(&periodics_lock);
268 +}
269 +
249 270 static void
250 271 periodic_cyclic_handler(void *arg)
251 272 {
252 273 extern void sir_on(int);
253 274 ddi_periodic_impl_t *dpr = arg;
254 275
255 276 mutex_enter(&dpr->dpr_lock);
256 277 /*
257 278 * If we've been cancelled, or we're already dispatched, then exit
258 279 * immediately:
259 280 */
260 281 if (dpr->dpr_flags & (DPF_CANCELLED | DPF_DISPATCHED)) {
261 282 mutex_exit(&dpr->dpr_lock);
262 283 return;
263 284 }
264 285 VERIFY(!(dpr->dpr_flags & DPF_EXECUTING));
265 286
266 287 /*
267 288 * This periodic is not presently dispatched, so dispatch it now:
268 289 */
269 290 dpr->dpr_flags |= DPF_DISPATCHED;
270 291 mutex_exit(&dpr->dpr_lock);
271 292
272 293 if (dpr->dpr_level == PERI_IPL_0) {
273 294 /*
274 295 * DDI_IPL_0 periodics are dispatched onto the taskq:
275 296 */
276 297 taskq_dispatch_ent(periodic_taskq, periodic_execute,
277 298 dpr, 0, &dpr->dpr_taskq_ent);
278 299 } else {
279 300 /*
280 301 * Higher priority periodics are handled by a soft
281 302 * interrupt handler. Enqueue us for processing and
282 303 * fire the soft interrupt:
283 304 */
284 305 mutex_enter(&periodics_lock);
285 306 list_insert_tail(&periodic_softint_queue[dpr->dpr_level - 1],
286 307 dpr);
287 308 mutex_exit(&periodics_lock);
288 309
289 310 /*
290 311 * Raise the soft interrupt level for this periodic:
291 312 */
292 313 sir_on(dpr->dpr_level);
293 314 }
294 315 }
295 316
296 317 static void
|
↓ open down ↓ |
38 lines elided |
↑ open up ↑ |
297 318 periodic_destroy(ddi_periodic_impl_t *dpr)
298 319 {
299 320 if (dpr == NULL)
300 321 return;
301 322
302 323 /*
303 324 * By now, we should have a periodic that is not busy, and has been
304 325 * cancelled:
305 326 */
306 327 VERIFY(dpr->dpr_flags == DPF_CANCELLED);
328 + VERIFY(dpr->dpr_thread == NULL);
307 329
308 330 id_free(periodic_id_space, dpr->dpr_id);
331 + cv_destroy(dpr->dpr_cv);
332 + mutex_destroy(dpr->dpr_lock);
309 333 kmem_cache_free(periodic_cache, dpr);
310 334 }
311 335
312 336 static ddi_periodic_impl_t *
313 337 periodic_create(void)
314 338 {
315 339 ddi_periodic_impl_t *dpr;
316 340
317 341 dpr = kmem_cache_alloc(periodic_cache, KM_SLEEP);
318 342 bzero(dpr, sizeof (*dpr));
319 343 dpr->dpr_id = id_alloc(periodic_id_space);
320 344 mutex_init(&dpr->dpr_lock, NULL, MUTEX_ADAPTIVE, NULL);
321 345 cv_init(&dpr->dpr_cv, NULL, CV_DEFAULT, NULL);
322 346
323 347 return (dpr);
324 348 }
325 349
350 +/*
351 + * This function provides the implementation for the ddi_periodic_add(9F)
352 + * interface. It registers a periodic handler and returns an opaque identifier
353 + * that can be unregistered via ddi_periodic_delete(9F)/i_untimeout().
354 + *
355 + * It may be called in user or kernel context, provided cpu_lock is not held.
356 + */
326 357 timeout_t
327 358 i_timeout(void (*func)(void *), void *arg, hrtime_t interval, int level)
328 359 {
329 360 cyc_handler_t cyh;
330 361 cyc_time_t cyt;
331 362 ddi_periodic_impl_t *dpr;
332 363
333 364 VERIFY(func != NULL);
334 365 VERIFY(level >= 0 && level <= 10);
335 366
336 367 /*
337 368 * Allocate object to track this periodic:
338 369 */
339 370 dpr = periodic_create();
340 371 dpr->dpr_level = level;
341 372 dpr->dpr_handler = func;
342 373 dpr->dpr_arg = arg;
343 374
344 375 /*
345 - * The resolution must be finer than or equal to
346 - * the requested interval. If it's not, set the resolution
347 - * to the interval.
348 - * Note. There is a restriction currently. Regardless of the
349 - * clock resolution used here, 10ms is set as the timer resolution.
350 - * Even on the 1ms resolution timer, the minimum interval is 10ms.
376 + * The minimum supported interval between firings of the periodic
377 + * handler is 10ms; see ddi_periodic_add(9F) for more details. If a
378 + * shorter interval is requested, round up.
351 379 */
352 380 if (ddi_periodic_resolution > interval) {
353 - uintptr_t pc = (uintptr_t)dpr->dpr_handler;
354 - ulong_t off;
355 381 cmn_err(CE_WARN,
356 382 "The periodic timeout (handler=%s, interval=%lld) "
357 383 "requests a finer interval than the supported resolution. "
358 - "It rounds up to %lld\n", kobj_getsymname(pc, &off),
384 + "It rounds up to %lld\n", periodic_handler_symbol(dpr),
359 385 interval, ddi_periodic_resolution);
360 386 interval = ddi_periodic_resolution;
361 387 }
362 388
363 389 /*
364 - * If the specified interval is already multiples of
365 - * the resolution, use it as is. Otherwise, it rounds
366 - * up to multiples of the timer resolution.
390 + * Ensure that the interval is an even multiple of the base resolution
391 + * that is at least as long as the requested interval.
367 392 */
368 393 dpr->dpr_interval = roundup(interval, ddi_periodic_resolution);
369 394
370 395 /*
371 396 * Create the underlying cyclic:
372 397 */
373 398 cyh.cyh_func = periodic_cyclic_handler;
374 399 cyh.cyh_arg = dpr;
375 400 cyh.cyh_level = CY_LOCK_LEVEL;
376 401
377 - cyt.cyt_when = roundup(gethrtime() + dpr->dpr_interval,
378 - ddi_periodic_resolution);
402 + cyt.cyt_when = 0;
379 403 cyt.cyt_interval = dpr->dpr_interval;
380 404
381 405 mutex_enter(&cpu_lock);
382 406 dpr->dpr_cyclic_id = cyclic_add(&cyh, &cyt);
383 407 mutex_exit(&cpu_lock);
384 408
385 409 /*
386 410 * Make the id visible to ddi_periodic_delete(9F) before we
387 411 * return it:
388 412 */
389 413 mutex_enter(&periodics_lock);
390 414 list_insert_tail(&periodics, dpr);
391 415 mutex_exit(&periodics_lock);
392 416
393 417 return ((timeout_t)(uintptr_t)dpr->dpr_id);
394 418 }
395 419
396 420 /*
397 - * void
398 - * i_untimeout(timeout_t req)
421 + * This function provides the implementation for the ddi_periodic_delete(9F)
422 + * interface. It cancels a periodic handler previously registered through
423 + * ddi_periodic_add(9F)/i_timeout().
399 424 *
400 - * Overview
401 - * i_untimeout() is an internal function canceling the i_timeout()
402 - * request previously issued.
403 - * This function is used for ddi_periodic_delete(9F).
404 - *
405 - * Argument
406 - * req: timeout_t opaque value i_timeout() returned previously.
407 - *
408 - * Return value
409 - * Nothing.
410 - *
411 - * Caller's context
412 - * i_untimeout() can be called in user, kernel or interrupt context.
413 - * It cannot be called in high interrupt context.
414 - *
415 - * Note. This function is used by ddi_periodic_delete(), which cannot
416 - * be called in interrupt context. As a result, this function is called
417 - * in user or kernel context only in practice.
425 + * It may be called in user or kernel context, provided cpu_lock is not held.
426 + * It may NOT be called from within a periodic handler.
418 427 */
419 428 void
420 429 i_untimeout(timeout_t id)
421 430 {
422 431 ddi_periodic_impl_t *dpr;
423 432
424 433 /*
425 434 * Find the periodic in the list of all periodics and remove it.
426 435 * If we find in (and remove it from) the global list, we have
427 436 * license to free it once it is no longer busy.
428 437 */
429 438 mutex_enter(&periodics_lock);
430 439 for (dpr = list_head(&periodics); dpr != NULL; dpr =
431 440 list_next(&periodics, dpr)) {
432 441 if (dpr->dpr_id == (id_t)(uintptr_t)id) {
433 442 list_remove(&periodics, dpr);
434 443 break;
435 444 }
436 445 }
437 446 mutex_exit(&periodics_lock);
438 447
439 448 /*
440 449 * We could not find a periodic for this id, so bail out:
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
441 450 */
442 451 if (dpr == NULL)
443 452 return;
444 453
445 454 mutex_enter(&dpr->dpr_lock);
446 455 /*
447 456 * We should be the only one trying to cancel this periodic:
448 457 */
449 458 VERIFY(!(dpr->dpr_flags & DPF_CANCELLED));
450 459 /*
460 + * Removing a periodic from within its own handler function will
461 + * cause a deadlock, so panic explicitly.
462 + */
463 + if (dpr->dpr_thread == curthread) {
464 + panic("ddi_periodic_delete(%p) called from its own handler\n",
465 + dpr->dpr_id);
466 + }
467 + /*
451 468 * Mark the periodic as cancelled:
452 469 */
453 470 dpr->dpr_flags |= DPF_CANCELLED;
454 471 mutex_exit(&dpr->dpr_lock);
455 472
456 473 /*
457 474 * Cancel our cyclic. cyclic_remove() guarantees that the cyclic
458 475 * handler will not run again after it returns. Note that the cyclic
459 476 * handler merely _dispatches_ the periodic, so this does _not_ mean
460 477 * the periodic handler is also finished running.
461 478 */
462 479 mutex_enter(&cpu_lock);
463 480 cyclic_remove(dpr->dpr_cyclic_id);
464 481 mutex_exit(&cpu_lock);
465 482
466 483 /*
467 484 * Wait until the periodic handler is no longer running:
468 485 */
469 486 mutex_enter(&dpr->dpr_lock);
470 487 while (dpr->dpr_flags & (DPF_DISPATCHED | DPF_EXECUTING)) {
471 488 cv_wait(&dpr->dpr_cv, &dpr->dpr_lock);
472 489 }
473 490 mutex_exit(&dpr->dpr_lock);
474 491
475 492 periodic_destroy(dpr);
476 493 }
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX