1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * Autovectored Interrupt Configuration and Deconfiguration
27 */
28
29 #include <sys/param.h>
30 #include <sys/cmn_err.h>
31 #include <sys/trap.h>
32 #include <sys/t_lock.h>
33 #include <sys/avintr.h>
34 #include <sys/kmem.h>
35 #include <sys/machlock.h>
36 #include <sys/systm.h>
37 #include <sys/machsystm.h>
38 #include <sys/sunddi.h>
39 #include <sys/x_call.h>
40 #include <sys/cpuvar.h>
41 #include <sys/atomic.h>
42 #include <sys/smp_impldefs.h>
43 #include <sys/sdt.h>
44 #include <sys/stack.h>
45 #include <sys/ddi_impldefs.h>
46 #ifdef __xpv
47 #include <sys/evtchn_impl.h>
48 #endif
49
50 typedef struct av_softinfo {
51 cpuset_t av_pending; /* pending bitmasks */
52 } av_softinfo_t;
53
54 static void insert_av(void *intr_id, struct av_head *vectp, avfunc f,
55 caddr_t arg1, caddr_t arg2, uint64_t *ticksp, int pri_level,
56 dev_info_t *dip);
57 static void remove_av(void *intr_id, struct av_head *vectp, avfunc f,
58 int pri_level, int vect);
59
60 /*
61 * Arrange for a driver to be called when a particular
62 * auto-vectored interrupt occurs.
63 * NOTE: if a device can generate interrupts on more than
64 * one level, or if a driver services devices that interrupt
65 * on more than one level, then the driver should install
66 * itself on each of those levels.
67 */
68 static char badsoft[] =
69 "add_avintr: bad soft interrupt level %d for driver '%s'\n";
70 static char multilevel[] =
71 "!IRQ%d is being shared by drivers with different interrupt levels.\n"
72 "This may result in reduced system performance.";
73 static char multilevel2[] =
74 "Cannot register interrupt for '%s' device at IPL %d because it\n"
75 "conflicts with another device using the same vector %d with an IPL\n"
76 "of %d. Reconfigure the conflicting devices to use different vectors.";
77
78 #ifdef __xpv
79 #define MAX_VECT NR_IRQS
80 #else
81 #define MAX_VECT 256
82 #endif
83
84 struct autovec *nmivect = NULL;
85 struct av_head autovect[MAX_VECT];
86 struct av_head softvect[LOCK_LEVEL + 1];
87 kmutex_t av_lock;
88 /*
89 * These are software interrupt handlers dedicated to ddi timer.
90 * The interrupt levels up to 10 are supported, but high interrupts
91 * must not be used there.
92 */
93 ddi_softint_hdl_impl_t softlevel_hdl[DDI_IPL_10] = {
94 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 1 */
95 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 2 */
96 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 3 */
97 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 4 */
98 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 5 */
99 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 6 */
100 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 7 */
101 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 8 */
102 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 9 */
103 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL}, /* level 10 */
104 };
105 ddi_softint_hdl_impl_t softlevel1_hdl =
106 {0, NULL, NULL, NULL, 0, NULL, NULL, NULL};
107
108 /*
109 * clear/check softint pending flag corresponding for
110 * the current CPU
111 */
112 void
113 av_clear_softint_pending(av_softinfo_t *infop)
114 {
115 CPUSET_ATOMIC_DEL(infop->av_pending, CPU->cpu_seqid);
116 }
117
118 boolean_t
119 av_check_softint_pending(av_softinfo_t *infop, boolean_t check_all)
120 {
121 if (check_all)
122 return (!CPUSET_ISNULL(infop->av_pending));
123 else
124 return (CPU_IN_SET(infop->av_pending, CPU->cpu_seqid) != 0);
125 }
126
127 /*
128 * This is the wrapper function which is generally used to set a softint
129 * pending
130 */
131 void
132 av_set_softint_pending(int pri, av_softinfo_t *infop)
133 {
134 kdi_av_set_softint_pending(pri, infop);
135 }
136
137 /*
138 * This is kmdb's private entry point to setsoftint called from kdi_siron
139 * It first sets our av softint pending bit for the current CPU,
140 * then it sets the CPU softint pending bit for pri.
141 */
142 void
143 kdi_av_set_softint_pending(int pri, av_softinfo_t *infop)
144 {
145 CPUSET_ATOMIC_ADD(infop->av_pending, CPU->cpu_seqid);
146
147 atomic_or_32((uint32_t *)&CPU->cpu_softinfo.st_pending, 1 << pri);
148 }
149
150 /*
151 * register nmi interrupt routine. The first arg is used only to order
152 * various nmi interrupt service routines in the chain. Higher lvls will
153 * be called first
154 */
155 int
156 add_nmintr(int lvl, avfunc nmintr, char *name, caddr_t arg)
157 {
158 struct autovec *mem;
159 struct autovec *p, *prev = NULL;
160
161 if (nmintr == NULL) {
162 printf("Attempt to add null vect for %s on nmi\n", name);
163 return (0);
164
165 }
166
167 mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
168 mem->av_vector = nmintr;
169 mem->av_intarg1 = arg;
170 mem->av_intarg2 = NULL;
171 mem->av_intr_id = NULL;
172 mem->av_prilevel = lvl;
173 mem->av_dip = NULL;
174 mem->av_link = NULL;
175
176 mutex_enter(&av_lock);
177
178 if (!nmivect) {
179 nmivect = mem;
180 mutex_exit(&av_lock);
181 return (1);
182 }
183 /* find where it goes in list */
184 for (p = nmivect; p != NULL; p = p->av_link) {
185 if (p->av_vector == nmintr && p->av_intarg1 == arg) {
186 /*
187 * already in list
188 * So? Somebody added the same interrupt twice.
189 */
190 cmn_err(CE_WARN, "Driver already registered '%s'",
191 name);
192 kmem_free(mem, sizeof (struct autovec));
193 mutex_exit(&av_lock);
194 return (0);
195 }
196 if (p->av_prilevel < lvl) {
197 if (p == nmivect) { /* it's at head of list */
198 mem->av_link = p;
199 nmivect = mem;
200 } else {
201 mem->av_link = p;
202 prev->av_link = mem;
203 }
204 mutex_exit(&av_lock);
205 return (1);
206 }
207 prev = p;
208
209 }
210 /* didn't find it, add it to the end */
211 prev->av_link = mem;
212 mutex_exit(&av_lock);
213 return (1);
214
215 }
216
217 /*
218 * register a hardware interrupt handler.
219 *
220 * The autovect data structure only supports globally 256 interrupts.
221 * In order to support 256 * #LocalAPIC interrupts, a new PSM module
222 * apix is introduced. It defines PSM private data structures for the
223 * interrupt handlers. The PSM module initializes addintr to a PSM
224 * private function so that it could override add_avintr() to operate
225 * on its private data structures.
226 */
227 int
228 add_avintr(void *intr_id, int lvl, avfunc xxintr, char *name, int vect,
229 caddr_t arg1, caddr_t arg2, uint64_t *ticksp, dev_info_t *dip)
230 {
231 struct av_head *vecp = (struct av_head *)0;
232 avfunc f;
233 int s, vectindex; /* save old spl value */
234 ushort_t hi_pri;
235
236 if (addintr) {
237 return ((*addintr)(intr_id, lvl, xxintr, name, vect,
238 arg1, arg2, ticksp, dip));
239 }
240
241 if ((f = xxintr) == NULL) {
242 printf("Attempt to add null vect for %s on vector %d\n",
243 name, vect);
244 return (0);
245
246 }
247 vectindex = vect % MAX_VECT;
248
249 vecp = &autovect[vectindex];
250
251 /*
252 * "hi_pri == 0" implies all entries on list are "unused",
253 * which means that it's OK to just insert this one.
254 */
255 hi_pri = vecp->avh_hi_pri;
256 if (vecp->avh_link && (hi_pri != 0)) {
257 if (((hi_pri > LOCK_LEVEL) && (lvl < LOCK_LEVEL)) ||
258 ((hi_pri < LOCK_LEVEL) && (lvl > LOCK_LEVEL))) {
259 cmn_err(CE_WARN, multilevel2, name, lvl, vect,
260 hi_pri);
261 return (0);
262 }
263 if ((vecp->avh_lo_pri != lvl) || (hi_pri != lvl))
264 cmn_err(CE_NOTE, multilevel, vect);
265 }
266
267 insert_av(intr_id, vecp, f, arg1, arg2, ticksp, lvl, dip);
268 s = splhi();
269 /*
270 * do what ever machine specific things are necessary
271 * to set priority level (e.g. set picmasks)
272 */
273 mutex_enter(&av_lock);
274 (*addspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
275 mutex_exit(&av_lock);
276 splx(s);
277 return (1);
278
279 }
280
281 void
282 update_avsoftintr_args(void *intr_id, int lvl, caddr_t arg2)
283 {
284 struct autovec *p;
285 struct autovec *target = NULL;
286 struct av_head *vectp = (struct av_head *)&softvect[lvl];
287
288 for (p = vectp->avh_link; p && p->av_vector; p = p->av_link) {
289 if (p->av_intr_id == intr_id) {
290 target = p;
291 break;
292 }
293 }
294
295 if (target == NULL)
296 return;
297 target->av_intarg2 = arg2;
298 }
299
300 /*
301 * Register a software interrupt handler
302 */
303 int
304 add_avsoftintr(void *intr_id, int lvl, avfunc xxintr, char *name,
305 caddr_t arg1, caddr_t arg2)
306 {
307 int slvl;
308 ddi_softint_hdl_impl_t *hdlp = (ddi_softint_hdl_impl_t *)intr_id;
309
310 if ((slvl = slvltovect(lvl)) != -1)
311 return (add_avintr(intr_id, lvl, xxintr,
312 name, slvl, arg1, arg2, NULL, NULL));
313
314 if (intr_id == NULL) {
315 printf("Attempt to add null intr_id for %s on level %d\n",
316 name, lvl);
317 return (0);
318 }
319
320 if (xxintr == NULL) {
321 printf("Attempt to add null handler for %s on level %d\n",
322 name, lvl);
323 return (0);
324 }
325
326 if (lvl <= 0 || lvl > LOCK_LEVEL) {
327 printf(badsoft, lvl, name);
328 return (0);
329 }
330
331 if (hdlp->ih_pending == NULL) {
332 hdlp->ih_pending =
333 kmem_zalloc(sizeof (av_softinfo_t), KM_SLEEP);
334 }
335
336 insert_av(intr_id, &softvect[lvl], xxintr, arg1, arg2, NULL, lvl, NULL);
337
338 return (1);
339 }
340
341 /*
342 * insert an interrupt vector into chain by its priority from high
343 * to low
344 */
345 static void
346 insert_av(void *intr_id, struct av_head *vectp, avfunc f, caddr_t arg1,
347 caddr_t arg2, uint64_t *ticksp, int pri_level, dev_info_t *dip)
348 {
349 /*
350 * Protect rewrites of the list
351 */
352 struct autovec *p, *prep, *mem;
353
354 mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
355 mem->av_vector = f;
356 mem->av_intarg1 = arg1;
357 mem->av_intarg2 = arg2;
358 mem->av_ticksp = ticksp;
359 mem->av_intr_id = intr_id;
360 mem->av_prilevel = pri_level;
361 mem->av_dip = dip;
362 mem->av_link = NULL;
363
364 mutex_enter(&av_lock);
365
366 if (vectp->avh_link == NULL) { /* Nothing on list - put it at head */
367 vectp->avh_link = mem;
368 vectp->avh_hi_pri = vectp->avh_lo_pri = (ushort_t)pri_level;
369
370 mutex_exit(&av_lock);
371 return;
372 }
373
374 /* find where it goes in list */
375 prep = NULL;
376 for (p = vectp->avh_link; p != NULL; p = p->av_link) {
377 if (p->av_vector && p->av_prilevel <= pri_level)
378 break;
379 prep = p;
380 }
381 if (prep != NULL) {
382 if (prep->av_vector == NULL) { /* freed struct available */
383 p = prep;
384 p->av_intarg1 = arg1;
385 p->av_intarg2 = arg2;
386 p->av_ticksp = ticksp;
387 p->av_intr_id = intr_id;
388 p->av_prilevel = pri_level;
389 p->av_dip = dip;
390 if (pri_level > (int)vectp->avh_hi_pri) {
391 vectp->avh_hi_pri = (ushort_t)pri_level;
392 }
393 if (pri_level < (int)vectp->avh_lo_pri) {
394 vectp->avh_lo_pri = (ushort_t)pri_level;
395 }
396 /*
397 * To prevent calling service routine before args
398 * and ticksp are ready fill in vector last.
399 */
400 p->av_vector = f;
401 mutex_exit(&av_lock);
402 kmem_free(mem, sizeof (struct autovec));
403 return;
404 }
405
406 mem->av_link = prep->av_link;
407 prep->av_link = mem;
408 } else {
409 /* insert new intpt at beginning of chain */
410 mem->av_link = vectp->avh_link;
411 vectp->avh_link = mem;
412 }
413 if (pri_level > (int)vectp->avh_hi_pri) {
414 vectp->avh_hi_pri = (ushort_t)pri_level;
415 }
416 if (pri_level < (int)vectp->avh_lo_pri) {
417 vectp->avh_lo_pri = (ushort_t)pri_level;
418 }
419 mutex_exit(&av_lock);
420 }
421
422 static int
423 av_rem_softintr(void *intr_id, int lvl, avfunc xxintr, boolean_t rem_softinfo)
424 {
425 struct av_head *vecp = (struct av_head *)0;
426 int slvl;
427 ddi_softint_hdl_impl_t *hdlp = (ddi_softint_hdl_impl_t *)intr_id;
428 av_softinfo_t *infop = (av_softinfo_t *)hdlp->ih_pending;
429
430 if (xxintr == NULL)
431 return (0);
432
433 if ((slvl = slvltovect(lvl)) != -1) {
434 rem_avintr(intr_id, lvl, xxintr, slvl);
435 return (1);
436 }
437
438 if (lvl <= 0 && lvl >= LOCK_LEVEL) {
439 return (0);
440 }
441 vecp = &softvect[lvl];
442 remove_av(intr_id, vecp, xxintr, lvl, 0);
443
444 if (rem_softinfo) {
445 kmem_free(infop, sizeof (av_softinfo_t));
446 hdlp->ih_pending = NULL;
447 }
448
449 return (1);
450 }
451
452 int
453 av_softint_movepri(void *intr_id, int old_lvl)
454 {
455 int ret;
456 ddi_softint_hdl_impl_t *hdlp = (ddi_softint_hdl_impl_t *)intr_id;
457
458 ret = add_avsoftintr(intr_id, hdlp->ih_pri, hdlp->ih_cb_func,
459 DEVI(hdlp->ih_dip)->devi_name, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2);
460
461 if (ret) {
462 (void) av_rem_softintr(intr_id, old_lvl, hdlp->ih_cb_func,
463 B_FALSE);
464 }
465
466 return (ret);
467 }
468
469 /*
470 * Remove a driver from the autovector list.
471 */
472 int
473 rem_avsoftintr(void *intr_id, int lvl, avfunc xxintr)
474 {
475 return (av_rem_softintr(intr_id, lvl, xxintr, B_TRUE));
476 }
477
478 /*
479 * Remove specified interrupt handler.
480 *
481 * PSM module could initialize remintr to some PSM private function
482 * so that it could override rem_avintr() to operate on its private
483 * data structures.
484 */
485 void
486 rem_avintr(void *intr_id, int lvl, avfunc xxintr, int vect)
487 {
488 struct av_head *vecp = (struct av_head *)0;
489 avfunc f;
490 int s, vectindex; /* save old spl value */
491
492 if (remintr) {
493 (*remintr)(intr_id, lvl, xxintr, vect);
494 return;
495 }
496
497 if ((f = xxintr) == NULL)
498 return;
499
500 vectindex = vect % MAX_VECT;
501 vecp = &autovect[vectindex];
502 remove_av(intr_id, vecp, f, lvl, vect);
503 s = splhi();
504 mutex_enter(&av_lock);
505 (*delspl)(vect, lvl, vecp->avh_lo_pri, vecp->avh_hi_pri);
506 mutex_exit(&av_lock);
507 splx(s);
508 }
509
510
511 /*
512 * After having made a change to an autovector list, wait until we have
513 * seen each cpu not executing an interrupt at that level--so we know our
514 * change has taken effect completely (no old state in registers, etc).
515 */
516 void
517 wait_till_seen(int ipl)
518 {
519 int cpu_in_chain, cix;
520 struct cpu *cpup;
521 cpuset_t cpus_to_check;
522
523 CPUSET_ALL(cpus_to_check);
524 do {
525 cpu_in_chain = 0;
526 for (cix = 0; cix < NCPU; cix++) {
527 cpup = cpu[cix];
528 if (cpup != NULL && CPU_IN_SET(cpus_to_check, cix)) {
529 if (INTR_ACTIVE(cpup, ipl)) {
530 cpu_in_chain = 1;
531 } else {
532 CPUSET_DEL(cpus_to_check, cix);
533 }
534 }
535 }
536 } while (cpu_in_chain);
537 }
538
539 static uint64_t dummy_tick;
540
541 /* remove an interrupt vector from the chain */
542 static void
543 remove_av(void *intr_id, struct av_head *vectp, avfunc f, int pri_level,
544 int vect)
545 {
546 struct autovec *p, *target;
547 int lo_pri, hi_pri;
548 int ipl;
549 /*
550 * Protect rewrites of the list
551 */
552 target = NULL;
553
554 mutex_enter(&av_lock);
555 ipl = pri_level;
556 lo_pri = MAXIPL;
557 hi_pri = 0;
558 for (p = vectp->avh_link; p; p = p->av_link) {
559 if ((p->av_vector == f) && (p->av_intr_id == intr_id)) {
560 /* found the handler */
561 target = p;
562 continue;
563 }
564 if (p->av_vector != NULL) {
565 if (p->av_prilevel > hi_pri)
566 hi_pri = p->av_prilevel;
567 if (p->av_prilevel < lo_pri)
568 lo_pri = p->av_prilevel;
569 }
570 }
571 if (ipl < hi_pri)
572 ipl = hi_pri;
573 if (target == NULL) { /* not found */
574 printf("Couldn't remove function %p at %d, %d\n",
575 (void *)f, vect, pri_level);
576 mutex_exit(&av_lock);
577 return;
578 }
579
580 /*
581 * This drops the handler from the chain, it can no longer be called.
582 * However, there is no guarantee that the handler is not currently
583 * still executing.
584 */
585 target->av_vector = NULL;
586 /*
587 * There is a race where we could be just about to pick up the ticksp
588 * pointer to increment it after returning from the service routine
589 * in av_dispatch_autovect. Rather than NULL it out let's just point
590 * it off to something safe so that any final tick update attempt
591 * won't fault.
592 */
593 target->av_ticksp = &dummy_tick;
594 wait_till_seen(ipl);
595
596 if (lo_pri > hi_pri) { /* the chain is now empty */
597 /* Leave the unused entries here for probable future use */
598 vectp->avh_lo_pri = MAXIPL;
599 vectp->avh_hi_pri = 0;
600 } else {
601 if ((int)vectp->avh_lo_pri < lo_pri)
602 vectp->avh_lo_pri = (ushort_t)lo_pri;
603 if ((int)vectp->avh_hi_pri > hi_pri)
604 vectp->avh_hi_pri = (ushort_t)hi_pri;
605 }
606 mutex_exit(&av_lock);
607 wait_till_seen(ipl);
608 }
609
610 /*
611 * kmdb uses siron (and thus setsoftint) while the world is stopped in order to
612 * inform its driver component that there's work to be done. We need to keep
613 * DTrace from instrumenting kmdb's siron and setsoftint. We duplicate siron,
614 * giving kmdb's version a kdi prefix to keep DTrace at bay. We also
615 * provide a version of the various setsoftint functions available for kmdb to
616 * use using a kdi_ prefix while the main *setsoftint() functionality is
617 * implemented as a wrapper. This allows tracing, while still providing a
618 * way for kmdb to sneak in unmolested.
619 */
620 void
621 kdi_siron(void)
622 {
623 (*kdisetsoftint)(1, softlevel1_hdl.ih_pending);
624 }
625
626 /*
627 * Trigger a soft interrupt.
628 */
629 void
630 siron(void)
631 {
632 /* Level 1 software interrupt */
633 (*setsoftint)(1, softlevel1_hdl.ih_pending);
634 }
635
636 /*
637 * Trigger software interrupts dedicated to ddi timer.
638 */
639 void
640 sir_on(int level)
641 {
642 ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10);
643 (*setsoftint)(level, softlevel_hdl[level-1].ih_pending);
644 }
645
646 /*
647 * The handler which is executed on the target CPU.
648 */
649 /*ARGSUSED*/
650 static int
651 siron_poke_intr(xc_arg_t a1, xc_arg_t a2, xc_arg_t a3)
652 {
653 siron();
654 return (0);
655 }
656
657 /*
658 * May get called from softcall to poke CPUs.
659 */
660 void
661 siron_poke_cpu(cpuset_t poke)
662 {
663 int cpuid = CPU->cpu_id;
664
665 /*
666 * If we are poking to ourself then we can simply
667 * generate level1 using siron()
668 */
669 if (CPU_IN_SET(poke, cpuid)) {
670 siron();
671 CPUSET_DEL(poke, cpuid);
672 if (CPUSET_ISNULL(poke))
673 return;
674 }
675
676 xc_call(0, 0, 0, CPUSET2BV(poke), (xc_func_t)siron_poke_intr);
677 }
678
679 /*
680 * Walk the autovector table for this vector, invoking each
681 * interrupt handler as we go.
682 */
683
684 extern uint64_t intr_get_time(void);
685
686 void
687 av_dispatch_autovect(uint_t vec)
688 {
689 struct autovec *av;
690
691 ASSERT_STACK_ALIGNED();
692
693 while ((av = autovect[vec].avh_link) != NULL) {
694 uint_t numcalled = 0;
695 uint_t claimed = 0;
696
697 for (; av; av = av->av_link) {
698 uint_t r;
699 uint_t (*intr)() = av->av_vector;
700 caddr_t arg1 = av->av_intarg1;
701 caddr_t arg2 = av->av_intarg2;
702 dev_info_t *dip = av->av_dip;
703
704 /*
705 * We must walk the entire chain. Removed handlers
706 * may be anywhere in the chain.
707 */
708 if (intr == NULL)
709 continue;
710
711 DTRACE_PROBE4(interrupt__start, dev_info_t *, dip,
712 void *, intr, caddr_t, arg1, caddr_t, arg2);
713 r = (*intr)(arg1, arg2);
714 DTRACE_PROBE4(interrupt__complete, dev_info_t *, dip,
715 void *, intr, caddr_t, arg1, uint_t, r);
716 numcalled++;
717 claimed |= r;
718 if (av->av_ticksp && av->av_prilevel <= LOCK_LEVEL)
719 atomic_add_64(av->av_ticksp, intr_get_time());
720 }
721
722 /*
723 * If there's only one interrupt handler in the chain,
724 * or if no-one claimed the interrupt at all give up now.
725 */
726 if (numcalled == 1 || claimed == 0)
727 break;
728 }
729 }
730
731 /*
732 * Call every soft interrupt handler we can find at this level once.
733 */
734 void
735 av_dispatch_softvect(uint_t pil)
736 {
737 struct autovec *av;
738 ddi_softint_hdl_impl_t *hdlp;
739 uint_t (*intr)();
740 caddr_t arg1;
741 caddr_t arg2;
742
743 ASSERT_STACK_ALIGNED();
744 ASSERT(pil >= 0 && pil <= PIL_MAX);
745
746 for (av = softvect[pil].avh_link; av; av = av->av_link) {
747 /*
748 * We must walk the entire chain. Removed handlers
749 * may be anywhere in the chain.
750 */
751 if ((intr = av->av_vector) == NULL)
752 continue;
753 arg1 = av->av_intarg1;
754 arg2 = av->av_intarg2;
755
756 hdlp = (ddi_softint_hdl_impl_t *)av->av_intr_id;
757 ASSERT(hdlp);
758
759 /*
760 * Each cpu has its own pending bit in hdlp->ih_pending,
761 * here av_check/clear_softint_pending is just checking
762 * and clearing the pending bit for the current cpu, who
763 * has just triggered a softint.
764 */
765 if (av_check_softint_pending(hdlp->ih_pending, B_FALSE)) {
766 av_clear_softint_pending(hdlp->ih_pending);
767 (void) (*intr)(arg1, arg2);
768 }
769 }
770 }
771
772 struct regs;
773
774 /*
775 * Call every NMI handler we know of once.
776 */
777 void
778 av_dispatch_nmivect(struct regs *rp)
779 {
780 struct autovec *av;
781
782 ASSERT_STACK_ALIGNED();
783
784 for (av = nmivect; av; av = av->av_link)
785 (void) (av->av_vector)(av->av_intarg1, rp);
786 }