1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * Copyright (c) 2010, Intel Corporation.
27 * All rights reserved.
28 */
29 /*
30 * Copyright (c) 2017, Joyent, Inc. All rights reserved.
31 */
32
33 /*
34 * To understand how the apix module interacts with the interrupt subsystem read
35 * the theory statement in uts/i86pc/os/intr.c.
36 */
37
38 /*
39 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
40 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
41 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
42 * PSMI 1.5 extensions are supported in Solaris Nevada.
43 * PSMI 1.6 extensions are supported in Solaris Nevada.
44 * PSMI 1.7 extensions are supported in Solaris Nevada.
45 */
46 #define PSMI_1_7
47
48 #include <sys/processor.h>
49 #include <sys/time.h>
50 #include <sys/psm.h>
51 #include <sys/smp_impldefs.h>
52 #include <sys/cram.h>
53 #include <sys/acpi/acpi.h>
54 #include <sys/acpica.h>
55 #include <sys/psm_common.h>
56 #include <sys/pit.h>
57 #include <sys/ddi.h>
58 #include <sys/sunddi.h>
59 #include <sys/ddi_impldefs.h>
60 #include <sys/pci.h>
61 #include <sys/promif.h>
62 #include <sys/x86_archext.h>
63 #include <sys/cpc_impl.h>
64 #include <sys/uadmin.h>
65 #include <sys/panic.h>
66 #include <sys/debug.h>
67 #include <sys/archsystm.h>
68 #include <sys/trap.h>
69 #include <sys/machsystm.h>
70 #include <sys/sysmacros.h>
71 #include <sys/cpuvar.h>
72 #include <sys/rm_platter.h>
73 #include <sys/privregs.h>
74 #include <sys/note.h>
75 #include <sys/pci_intr_lib.h>
76 #include <sys/spl.h>
77 #include <sys/clock.h>
78 #include <sys/cyclic.h>
79 #include <sys/dditypes.h>
80 #include <sys/sunddi.h>
81 #include <sys/x_call.h>
82 #include <sys/reboot.h>
83 #include <sys/mach_intr.h>
84 #include <sys/apix.h>
85 #include <sys/apix_irm_impl.h>
86
87 static int apix_probe();
88 static void apix_init();
89 static void apix_picinit(void);
90 static int apix_intr_enter(int, int *);
91 static void apix_intr_exit(int, int);
92 static void apix_setspl(int);
93 static int apix_disable_intr(processorid_t);
94 static void apix_enable_intr(processorid_t);
95 static int apix_get_clkvect(int);
96 static int apix_get_ipivect(int, int);
97 static void apix_post_cyclic_setup(void *);
98 static int apix_post_cpu_start();
99 static int apix_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
100 psm_intr_op_t, int *);
101
102 /*
103 * Helper functions for apix_intr_ops()
104 */
105 static void apix_redistribute_compute(void);
106 static int apix_get_pending(apix_vector_t *);
107 static apix_vector_t *apix_get_req_vector(ddi_intr_handle_impl_t *, ushort_t);
108 static int apix_get_intr_info(ddi_intr_handle_impl_t *, apic_get_intr_t *);
109 static char *apix_get_apic_type(void);
110 static int apix_intx_get_pending(int);
111 static void apix_intx_set_mask(int irqno);
112 static void apix_intx_clear_mask(int irqno);
113 static int apix_intx_get_shared(int irqno);
114 static void apix_intx_set_shared(int irqno, int delta);
115 static apix_vector_t *apix_intx_xlate_vector(dev_info_t *, int,
116 struct intrspec *);
117 static int apix_intx_alloc_vector(dev_info_t *, int, struct intrspec *);
118
119 extern int apic_clkinit(int);
120
121 /* IRM initialization for APIX PSM module */
122 extern void apix_irm_init(void);
123
124 extern int irm_enable;
125
126 /*
127 * Local static data
128 */
129 static struct psm_ops apix_ops = {
130 apix_probe,
131
132 apix_init,
133 apix_picinit,
134 apix_intr_enter,
135 apix_intr_exit,
136 apix_setspl,
137 apix_addspl,
138 apix_delspl,
139 apix_disable_intr,
140 apix_enable_intr,
141 NULL, /* psm_softlvl_to_irq */
142 NULL, /* psm_set_softintr */
143
144 apic_set_idlecpu,
145 apic_unset_idlecpu,
146
147 apic_clkinit,
148 apix_get_clkvect,
149 NULL, /* psm_hrtimeinit */
150 apic_gethrtime,
151
152 apic_get_next_processorid,
153 apic_cpu_start,
154 apix_post_cpu_start,
155 apic_shutdown,
156 apix_get_ipivect,
157 apic_send_ipi,
158
159 NULL, /* psm_translate_irq */
160 NULL, /* psm_notify_error */
161 NULL, /* psm_notify_func */
162 apic_timer_reprogram,
163 apic_timer_enable,
164 apic_timer_disable,
165 apix_post_cyclic_setup,
166 apic_preshutdown,
167 apix_intr_ops, /* Advanced DDI Interrupt framework */
168 apic_state, /* save, restore apic state for S3 */
169 apic_cpu_ops, /* CPU control interface. */
170 };
171
172 struct psm_ops *psmops = &apix_ops;
173
174 static struct psm_info apix_psm_info = {
175 PSM_INFO_VER01_7, /* version */
176 PSM_OWN_EXCLUSIVE, /* ownership */
177 &apix_ops, /* operation */
178 APIX_NAME, /* machine name */
179 "apix MPv1.4 compatible",
180 };
181
182 static void *apix_hdlp;
183
184 static int apix_is_enabled = 0;
185
186 /*
187 * Flag to indicate if APIX is to be enabled only for platforms
188 * with specific hw feature(s).
189 */
190 int apix_hw_chk_enable = 1;
191
192 /*
193 * Hw features that are checked for enabling APIX support.
194 */
195 #define APIX_SUPPORT_X2APIC 0x00000001
196 uint_t apix_supported_hw = APIX_SUPPORT_X2APIC;
197
198 /*
199 * apix_lock is used for cpu selection and vector re-binding
200 */
201 lock_t apix_lock;
202 apix_impl_t *apixs[NCPU];
203 /*
204 * Mapping between device interrupt and the allocated vector. Indexed
205 * by major number.
206 */
207 apix_dev_vector_t **apix_dev_vector;
208 /*
209 * Mapping between device major number and cpu id. It gets used
210 * when interrupt binding policy round robin with affinity is
211 * applied. With that policy, devices with the same major number
212 * will be bound to the same CPU.
213 */
214 processorid_t *apix_major_to_cpu; /* major to cpu mapping */
215 kmutex_t apix_mutex; /* for apix_dev_vector & apix_major_to_cpu */
216
217 int apix_nipis = 16; /* Maximum number of IPIs */
218 /*
219 * Maximum number of vectors in a CPU that can be used for interrupt
220 * allocation (including IPIs and the reserved vectors).
221 */
222 int apix_cpu_nvectors = APIX_NVECTOR;
223
224 /* number of CPUs in power-on transition state */
225 static int apic_poweron_cnt = 0;
226
227 /* gcpu.h */
228
229 extern void apic_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
230 extern void apic_change_eoi();
231
232 /*
233 * This is the loadable module wrapper
234 */
235
236 int
237 _init(void)
238 {
239 if (apic_coarse_hrtime)
240 apix_ops.psm_gethrtime = &apic_gettime;
241 return (psm_mod_init(&apix_hdlp, &apix_psm_info));
242 }
243
244 int
245 _fini(void)
246 {
247 return (psm_mod_fini(&apix_hdlp, &apix_psm_info));
248 }
249
250 int
251 _info(struct modinfo *modinfop)
252 {
253 return (psm_mod_info(&apix_hdlp, &apix_psm_info, modinfop));
254 }
255
256 static int
257 apix_probe()
258 {
259 int rval;
260
261 if (apix_enable == 0)
262 return (PSM_FAILURE);
263
264 /*
265 * FIXME Temporarily disable apix module on Xen HVM platform due to
266 * known hang during boot (see #3605).
267 *
268 * Please remove when/if the issue is resolved.
269 */
270 if (get_hwenv() == HW_XEN_HVM)
271 return (PSM_FAILURE);
272
273 /* check for hw features if specified */
274 if (apix_hw_chk_enable) {
275 /* check if x2APIC mode is supported */
276 if ((apix_supported_hw & APIX_SUPPORT_X2APIC) ==
277 APIX_SUPPORT_X2APIC) {
278 if (apic_local_mode() == LOCAL_X2APIC) {
279 /* x2APIC mode activated by BIOS, switch ops */
280 apic_mode = LOCAL_X2APIC;
281 apic_change_ops();
282 } else if (!apic_detect_x2apic()) {
283 /* x2APIC mode is not supported in the hw */
284 apix_enable = 0;
285 }
286 }
287 if (apix_enable == 0)
288 return (PSM_FAILURE);
289 }
290
291 rval = apic_probe_common(apix_psm_info.p_mach_idstring);
292 if (rval == PSM_SUCCESS)
293 apix_is_enabled = 1;
294 else
295 apix_is_enabled = 0;
296 return (rval);
297 }
298
299 /*
300 * Initialize the data structures needed by pcplusmpx module.
301 * Specifically, the data structures used by addspl() and delspl()
302 * routines.
303 */
304 static void
305 apix_softinit()
306 {
307 int i, *iptr;
308 apix_impl_t *hdlp;
309 int nproc;
310
311 nproc = max(apic_nproc, apic_max_nproc);
312
313 hdlp = kmem_zalloc(nproc * sizeof (apix_impl_t), KM_SLEEP);
314 for (i = 0; i < nproc; i++) {
315 apixs[i] = &hdlp[i];
316 apixs[i]->x_cpuid = i;
317 LOCK_INIT_CLEAR(&apixs[i]->x_lock);
318 }
319
320 /* cpu 0 is always up (for now) */
321 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
322
323 iptr = (int *)&apic_irq_table[0];
324 for (i = 0; i <= APIC_MAX_VECTOR; i++) {
325 apic_level_intr[i] = 0;
326 *iptr++ = NULL;
327 }
328 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
329
330 apix_dev_vector = kmem_zalloc(sizeof (apix_dev_vector_t *) * devcnt,
331 KM_SLEEP);
332
333 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
334 apix_major_to_cpu = kmem_zalloc(sizeof (int) * devcnt,
335 KM_SLEEP);
336 for (i = 0; i < devcnt; i++)
337 apix_major_to_cpu[i] = IRQ_UNINIT;
338 }
339
340 mutex_init(&apix_mutex, NULL, MUTEX_DEFAULT, NULL);
341 }
342
343 static int
344 apix_get_pending_spl(void)
345 {
346 int cpuid = CPU->cpu_id;
347
348 return (bsrw_insn(apixs[cpuid]->x_intr_pending));
349 }
350
351 static uintptr_t
352 apix_get_intr_handler(int cpu, short vec)
353 {
354 apix_vector_t *apix_vector;
355
356 ASSERT(cpu < apic_nproc && vec < APIX_NVECTOR);
357 if (cpu >= apic_nproc)
358 return (NULL);
359
360 apix_vector = apixs[cpu]->x_vectbl[vec];
361
362 return ((uintptr_t)(apix_vector->v_autovect));
363 }
364
365 static void
366 apix_init()
367 {
368 extern void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *);
369
370 APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_softinit\n"));
371
372 do_interrupt_common = apix_do_interrupt;
373 addintr = apix_add_avintr;
374 remintr = apix_rem_avintr;
375 get_pending_spl = apix_get_pending_spl;
376 get_intr_handler = apix_get_intr_handler;
377 psm_get_localapicid = apic_get_localapicid;
378 psm_get_ioapicid = apic_get_ioapicid;
379
380 apix_softinit();
381
382 #if !defined(__amd64)
383 if (cpuid_have_cr8access(CPU))
384 apic_have_32bit_cr8 = 1;
385 #endif
386
387 /*
388 * Initialize IRM pool parameters
389 */
390 if (irm_enable) {
391 int i;
392 int lowest_irq;
393 int highest_irq;
394
395 /* number of CPUs present */
396 apix_irminfo.apix_ncpus = apic_nproc;
397 /* total number of entries in all of the IOAPICs present */
398 lowest_irq = apic_io_vectbase[0];
399 highest_irq = apic_io_vectend[0];
400 for (i = 1; i < apic_io_max; i++) {
401 if (apic_io_vectbase[i] < lowest_irq)
402 lowest_irq = apic_io_vectbase[i];
403 if (apic_io_vectend[i] > highest_irq)
404 highest_irq = apic_io_vectend[i];
405 }
406 apix_irminfo.apix_ioapic_max_vectors =
407 highest_irq - lowest_irq + 1;
408 /*
409 * Number of available per-CPU vectors excluding
410 * reserved vectors for Dtrace, int80, system-call,
411 * fast-trap, etc.
412 */
413 apix_irminfo.apix_per_cpu_vectors = APIX_NAVINTR -
414 APIX_SW_RESERVED_VECTORS;
415
416 /* Number of vectors (pre) allocated (SCI and HPET) */
417 apix_irminfo.apix_vectors_allocated = 0;
418 if (apic_hpet_vect != -1)
419 apix_irminfo.apix_vectors_allocated++;
420 if (apic_sci_vect != -1)
421 apix_irminfo.apix_vectors_allocated++;
422 }
423 }
424
425 static void
426 apix_init_intr()
427 {
428 processorid_t cpun = psm_get_cpu_id();
429 uint_t nlvt;
430 uint32_t svr = AV_UNIT_ENABLE | APIC_SPUR_INTR;
431 extern void cmi_cmci_trap(void);
432
433 apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
434
435 if (apic_mode == LOCAL_APIC) {
436 /*
437 * We are running APIC in MMIO mode.
438 */
439 if (apic_flat_model) {
440 apic_reg_ops->apic_write(APIC_FORMAT_REG,
441 APIC_FLAT_MODEL);
442 } else {
443 apic_reg_ops->apic_write(APIC_FORMAT_REG,
444 APIC_CLUSTER_MODEL);
445 }
446
447 apic_reg_ops->apic_write(APIC_DEST_REG,
448 AV_HIGH_ORDER >> cpun);
449 }
450
451 if (apic_directed_EOI_supported()) {
452 /*
453 * Setting the 12th bit in the Spurious Interrupt Vector
454 * Register suppresses broadcast EOIs generated by the local
455 * APIC. The suppression of broadcast EOIs happens only when
456 * interrupts are level-triggered.
457 */
458 svr |= APIC_SVR_SUPPRESS_BROADCAST_EOI;
459 }
460
461 /* need to enable APIC before unmasking NMI */
462 apic_reg_ops->apic_write(APIC_SPUR_INT_REG, svr);
463
464 /*
465 * Presence of an invalid vector with delivery mode AV_FIXED can
466 * cause an error interrupt, even if the entry is masked...so
467 * write a valid vector to LVT entries along with the mask bit
468 */
469
470 /* All APICs have timer and LINT0/1 */
471 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK|APIC_RESV_IRQ);
472 apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK|APIC_RESV_IRQ);
473 apic_reg_ops->apic_write(APIC_INT_VECT1, AV_NMI); /* enable NMI */
474
475 /*
476 * On integrated APICs, the number of LVT entries is
477 * 'Max LVT entry' + 1; on 82489DX's (non-integrated
478 * APICs), nlvt is "3" (LINT0, LINT1, and timer)
479 */
480
481 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
482 nlvt = 3;
483 } else {
484 nlvt = ((apic_reg_ops->apic_read(APIC_VERS_REG) >> 16) &
485 0xFF) + 1;
486 }
487
488 if (nlvt >= 5) {
489 /* Enable performance counter overflow interrupt */
490
491 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
492 apic_enable_cpcovf_intr = 0;
493 if (apic_enable_cpcovf_intr) {
494 if (apic_cpcovf_vect == 0) {
495 int ipl = APIC_PCINT_IPL;
496
497 apic_cpcovf_vect = apix_get_ipivect(ipl, -1);
498 ASSERT(apic_cpcovf_vect);
499
500 (void) add_avintr(NULL, ipl,
501 (avfunc)kcpc_hw_overflow_intr,
502 "apic pcint", apic_cpcovf_vect,
503 NULL, NULL, NULL, NULL);
504 kcpc_hw_overflow_intr_installed = 1;
505 kcpc_hw_enable_cpc_intr =
506 apic_cpcovf_mask_clear;
507 }
508 apic_reg_ops->apic_write(APIC_PCINT_VECT,
509 apic_cpcovf_vect);
510 }
511 }
512
513 if (nlvt >= 6) {
514 /* Only mask TM intr if the BIOS apparently doesn't use it */
515
516 uint32_t lvtval;
517
518 lvtval = apic_reg_ops->apic_read(APIC_THERM_VECT);
519 if (((lvtval & AV_MASK) == AV_MASK) ||
520 ((lvtval & AV_DELIV_MODE) != AV_SMI)) {
521 apic_reg_ops->apic_write(APIC_THERM_VECT,
522 AV_MASK|APIC_RESV_IRQ);
523 }
524 }
525
526 /* Enable error interrupt */
527
528 if (nlvt >= 4 && apic_enable_error_intr) {
529 if (apic_errvect == 0) {
530 int ipl = 0xf; /* get highest priority intr */
531 apic_errvect = apix_get_ipivect(ipl, -1);
532 ASSERT(apic_errvect);
533 /*
534 * Not PSMI compliant, but we are going to merge
535 * with ON anyway
536 */
537 (void) add_avintr(NULL, ipl,
538 (avfunc)apic_error_intr, "apic error intr",
539 apic_errvect, NULL, NULL, NULL, NULL);
540 }
541 apic_reg_ops->apic_write(APIC_ERR_VECT, apic_errvect);
542 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
543 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
544 }
545
546 /* Enable CMCI interrupt */
547 if (cmi_enable_cmci) {
548 mutex_enter(&cmci_cpu_setup_lock);
549 if (cmci_cpu_setup_registered == 0) {
550 mutex_enter(&cpu_lock);
551 register_cpu_setup_func(cmci_cpu_setup, NULL);
552 mutex_exit(&cpu_lock);
553 cmci_cpu_setup_registered = 1;
554 }
555 mutex_exit(&cmci_cpu_setup_lock);
556
557 if (apic_cmci_vect == 0) {
558 int ipl = 0x2;
559 apic_cmci_vect = apix_get_ipivect(ipl, -1);
560 ASSERT(apic_cmci_vect);
561
562 (void) add_avintr(NULL, ipl,
563 (avfunc)cmi_cmci_trap, "apic cmci intr",
564 apic_cmci_vect, NULL, NULL, NULL, NULL);
565 }
566 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
567 }
568
569 apic_reg_ops->apic_write_task_reg(0);
570 }
571
572 static void
573 apix_picinit(void)
574 {
575 int i, j;
576 uint_t isr;
577
578 APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_picinit\n"));
579
580 /*
581 * initialize interrupt remapping before apic
582 * hardware initialization
583 */
584 apic_intrmap_init(apic_mode);
585 if (apic_vt_ops == psm_vt_ops)
586 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IIR;
587
588 /*
589 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
590 * bit on without clearing it with EOI. Since softint
591 * uses vector 0x20 to interrupt itself, so softint will
592 * not work on this machine. In order to fix this problem
593 * a check is made to verify all the isr bits are clear.
594 * If not, EOIs are issued to clear the bits.
595 */
596 for (i = 7; i >= 1; i--) {
597 isr = apic_reg_ops->apic_read(APIC_ISR_REG + (i * 4));
598 if (isr != 0)
599 for (j = 0; ((j < 32) && (isr != 0)); j++)
600 if (isr & (1 << j)) {
601 apic_reg_ops->apic_write(
602 APIC_EOI_REG, 0);
603 isr &= ~(1 << j);
604 apic_error |= APIC_ERR_BOOT_EOI;
605 }
606 }
607
608 /* set a flag so we know we have run apic_picinit() */
609 apic_picinit_called = 1;
610 LOCK_INIT_CLEAR(&apic_gethrtime_lock);
611 LOCK_INIT_CLEAR(&apic_ioapic_lock);
612 LOCK_INIT_CLEAR(&apic_error_lock);
613 LOCK_INIT_CLEAR(&apic_mode_switch_lock);
614
615 picsetup(); /* initialise the 8259 */
616
617 /* add nmi handler - least priority nmi handler */
618 LOCK_INIT_CLEAR(&apic_nmi_lock);
619
620 if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr,
621 "apix NMI handler", (caddr_t)NULL))
622 cmn_err(CE_WARN, "apix: Unable to add nmi handler");
623
624 apix_init_intr();
625
626 /* enable apic mode if imcr present */
627 if (apic_imcrp) {
628 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
629 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC);
630 }
631
632 ioapix_init_intr(IOAPIC_MASK);
633
634 /* setup global IRM pool if applicable */
635 if (irm_enable)
636 apix_irm_init();
637 }
638
639 static __inline__ void
640 apix_send_eoi(void)
641 {
642 if (apic_mode == LOCAL_APIC)
643 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0);
644 else
645 X2APIC_WRITE(APIC_EOI_REG, 0);
646 }
647
648 /*
649 * platform_intr_enter
650 *
651 * Called at the beginning of the interrupt service routine, but unlike
652 * pcplusmp, does not mask interrupts. An EOI is given to the interrupt
653 * controller to enable other HW interrupts but interrupts are still
654 * masked by the IF flag.
655 *
656 * Return -1 for spurious interrupts
657 *
658 */
659 static int
660 apix_intr_enter(int ipl, int *vectorp)
661 {
662 struct cpu *cpu = CPU;
663 uint32_t cpuid = CPU->cpu_id;
664 apic_cpus_info_t *cpu_infop;
665 uchar_t vector;
666 apix_vector_t *vecp;
667 int nipl = -1;
668
669 /*
670 * The real vector delivered is (*vectorp + 0x20), but our caller
671 * subtracts 0x20 from the vector before passing it to us.
672 * (That's why APIC_BASE_VECT is 0x20.)
673 */
674 vector = *vectorp = (uchar_t)*vectorp + APIC_BASE_VECT;
675
676 cpu_infop = &apic_cpus[cpuid];
677 if (vector == APIC_SPUR_INTR) {
678 cpu_infop->aci_spur_cnt++;
679 return (APIC_INT_SPURIOUS);
680 }
681
682 vecp = xv_vector(cpuid, vector);
683 if (vecp == NULL) {
684 if (APIX_IS_FAKE_INTR(vector))
685 nipl = apix_rebindinfo.i_pri;
686 apix_send_eoi();
687 return (nipl);
688 }
689 nipl = vecp->v_pri;
690
691 /* if interrupted by the clock, increment apic_nsec_since_boot */
692 if (vector == (apic_clkvect + APIC_BASE_VECT)) {
693 if (!apic_oneshot) {
694 /* NOTE: this is not MT aware */
695 apic_hrtime_stamp++;
696 apic_nsec_since_boot += apic_nsec_per_intr;
697 apic_hrtime_stamp++;
698 last_count_read = apic_hertz_count;
699 apix_redistribute_compute();
700 }
701
702 apix_send_eoi();
703
704 return (nipl);
705 }
706
707 ASSERT(vecp->v_state != APIX_STATE_OBSOLETED);
708
709 /* pre-EOI handling for level-triggered interrupts */
710 if (!APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method) &&
711 (vecp->v_type & APIX_TYPE_FIXED) && apic_level_intr[vecp->v_inum])
712 apix_level_intr_pre_eoi(vecp->v_inum);
713
714 /* send back EOI */
715 apix_send_eoi();
716
717 cpu_infop->aci_current[nipl] = vector;
718 if ((nipl > ipl) && (nipl > cpu->cpu_base_spl)) {
719 cpu_infop->aci_curipl = (uchar_t)nipl;
720 cpu_infop->aci_ISR_in_progress |= 1 << nipl;
721 }
722
723 #ifdef DEBUG
724 if (vector >= APIX_IPI_MIN)
725 return (nipl); /* skip IPI */
726
727 APIC_DEBUG_BUF_PUT(vector);
728 APIC_DEBUG_BUF_PUT(vecp->v_inum);
729 APIC_DEBUG_BUF_PUT(nipl);
730 APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
731 if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl)))
732 drv_usecwait(apic_stretch_interrupts);
733 #endif /* DEBUG */
734
735 return (nipl);
736 }
737
738 /*
739 * Any changes made to this function must also change X2APIC
740 * version of intr_exit.
741 */
742 static void
743 apix_intr_exit(int prev_ipl, int arg2)
744 {
745 int cpuid = psm_get_cpu_id();
746 apic_cpus_info_t *cpu_infop = &apic_cpus[cpuid];
747 apix_impl_t *apixp = apixs[cpuid];
748
749 UNREFERENCED_1PARAMETER(arg2);
750
751 cpu_infop->aci_curipl = (uchar_t)prev_ipl;
752 /* ISR above current pri could not be in progress */
753 cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1;
754
755 if (apixp->x_obsoletes != NULL) {
756 if (APIX_CPU_LOCK_HELD(cpuid))
757 return;
758
759 APIX_ENTER_CPU_LOCK(cpuid);
760 (void) apix_obsolete_vector(apixp->x_obsoletes);
761 APIX_LEAVE_CPU_LOCK(cpuid);
762 }
763 }
764
765 /*
766 * The pcplusmp setspl code uses the TPR to mask all interrupts at or below the
767 * given ipl, but apix never uses the TPR and we never mask a subset of the
768 * interrupts. They are either all blocked by the IF flag or all can come in.
769 *
770 * For setspl, we mask all interrupts for XC_HI_PIL (15), otherwise, interrupts
771 * can come in if currently enabled by the IF flag. This table shows the state
772 * of the IF flag when we leave this function.
773 *
774 * curr IF | ipl == 15 ipl != 15
775 * --------+---------------------------
776 * 0 | 0 0
777 * 1 | 0 1
778 */
779 static void
780 apix_setspl(int ipl)
781 {
782 /*
783 * Interrupts at ipl above this cannot be in progress, so the following
784 * mask is ok.
785 */
786 apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1;
787
788 if (ipl == XC_HI_PIL)
789 cli();
790 }
791
792 int
793 apix_addspl(int virtvec, int ipl, int min_ipl, int max_ipl)
794 {
795 uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
796 uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
797 apix_vector_t *vecp = xv_vector(cpuid, vector);
798
799 UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
800 ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
801
802 if (vecp->v_type == APIX_TYPE_FIXED)
803 apix_intx_set_shared(vecp->v_inum, 1);
804
805 /* There are more interrupts, so it's already been enabled */
806 if (vecp->v_share > 1)
807 return (PSM_SUCCESS);
808
809 /* return if it is not hardware interrupt */
810 if (vecp->v_type == APIX_TYPE_IPI)
811 return (PSM_SUCCESS);
812
813 /*
814 * if apix_picinit() has not been called yet, just return.
815 * At the end of apic_picinit(), we will call setup_io_intr().
816 */
817 if (!apic_picinit_called)
818 return (PSM_SUCCESS);
819
820 (void) apix_setup_io_intr(vecp);
821
822 return (PSM_SUCCESS);
823 }
824
825 int
826 apix_delspl(int virtvec, int ipl, int min_ipl, int max_ipl)
827 {
828 uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
829 uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
830 apix_vector_t *vecp = xv_vector(cpuid, vector);
831
832 UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
833 ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
834
835 if (vecp->v_type == APIX_TYPE_FIXED)
836 apix_intx_set_shared(vecp->v_inum, -1);
837
838 /* There are more interrupts */
839 if (vecp->v_share > 1)
840 return (PSM_SUCCESS);
841
842 /* return if it is not hardware interrupt */
843 if (vecp->v_type == APIX_TYPE_IPI)
844 return (PSM_SUCCESS);
845
846 if (!apic_picinit_called) {
847 cmn_err(CE_WARN, "apix: delete 0x%x before apic init",
848 virtvec);
849 return (PSM_SUCCESS);
850 }
851
852 apix_disable_vector(vecp);
853
854 return (PSM_SUCCESS);
855 }
856
857 /*
858 * Try and disable all interrupts. We just assign interrupts to other
859 * processors based on policy. If any were bound by user request, we
860 * let them continue and return failure. We do not bother to check
861 * for cache affinity while rebinding.
862 */
863 static int
864 apix_disable_intr(processorid_t cpun)
865 {
866 apix_impl_t *apixp = apixs[cpun];
867 apix_vector_t *vecp, *newp;
868 int bindcpu, i, hardbound = 0, errbound = 0, ret, loop, type;
869
870 lock_set(&apix_lock);
871
872 apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE;
873 apic_cpus[cpun].aci_curipl = 0;
874
875 /* if this is for SUSPEND operation, skip rebinding */
876 if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
877 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
878 vecp = apixp->x_vectbl[i];
879 if (!IS_VECT_ENABLED(vecp))
880 continue;
881
882 apix_disable_vector(vecp);
883 }
884 lock_clear(&apix_lock);
885 return (PSM_SUCCESS);
886 }
887
888 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
889 vecp = apixp->x_vectbl[i];
890 if (!IS_VECT_ENABLED(vecp))
891 continue;
892
893 if (vecp->v_flags & APIX_VECT_USER_BOUND) {
894 hardbound++;
895 continue;
896 }
897 type = vecp->v_type;
898
899 /*
900 * If there are bound interrupts on this cpu, then
901 * rebind them to other processors.
902 */
903 loop = 0;
904 do {
905 bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
906
907 if (type != APIX_TYPE_MSI)
908 newp = apix_set_cpu(vecp, bindcpu, &ret);
909 else
910 newp = apix_grp_set_cpu(vecp, bindcpu, &ret);
911 } while ((newp == NULL) && (loop++ < apic_nproc));
912
913 if (loop >= apic_nproc) {
914 errbound++;
915 cmn_err(CE_WARN, "apix: failed to rebind vector %x/%x",
916 vecp->v_cpuid, vecp->v_vector);
917 }
918 }
919
920 lock_clear(&apix_lock);
921
922 if (hardbound || errbound) {
923 cmn_err(CE_WARN, "Could not disable interrupts on %d"
924 "due to user bound interrupts or failed operation",
925 cpun);
926 return (PSM_FAILURE);
927 }
928
929 return (PSM_SUCCESS);
930 }
931
932 /*
933 * Bind interrupts to specified CPU
934 */
935 static void
936 apix_enable_intr(processorid_t cpun)
937 {
938 apix_vector_t *vecp;
939 int i, ret;
940 processorid_t n;
941
942 lock_set(&apix_lock);
943
944 apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE;
945
946 /* interrupt enabling for system resume */
947 if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
948 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
949 vecp = xv_vector(cpun, i);
950 if (!IS_VECT_ENABLED(vecp))
951 continue;
952
953 apix_enable_vector(vecp);
954 }
955 apic_cpus[cpun].aci_status &= ~APIC_CPU_SUSPEND;
956 }
957
958 for (n = 0; n < apic_nproc; n++) {
959 if (!apic_cpu_in_range(n) || n == cpun ||
960 (apic_cpus[n].aci_status & APIC_CPU_INTR_ENABLE) == 0)
961 continue;
962
963 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
964 vecp = xv_vector(n, i);
965 if (!IS_VECT_ENABLED(vecp) ||
966 vecp->v_bound_cpuid != cpun)
967 continue;
968
969 if (vecp->v_type != APIX_TYPE_MSI)
970 (void) apix_set_cpu(vecp, cpun, &ret);
971 else
972 (void) apix_grp_set_cpu(vecp, cpun, &ret);
973 }
974 }
975
976 lock_clear(&apix_lock);
977 }
978
979 /*
980 * Allocate vector for IPI
981 * type == -1 indicates it is an internal request. Do not change
982 * resv_vector for these requests.
983 */
984 static int
985 apix_get_ipivect(int ipl, int type)
986 {
987 uchar_t vector;
988
989 if ((vector = apix_alloc_ipi(ipl)) > 0) {
990 if (type != -1)
991 apic_resv_vector[ipl] = vector;
992 return (vector);
993 }
994 apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
995 return (-1); /* shouldn't happen */
996 }
997
998 static int
999 apix_get_clkvect(int ipl)
1000 {
1001 int vector;
1002
1003 if ((vector = apix_get_ipivect(ipl, -1)) == -1)
1004 return (-1);
1005
1006 apic_clkvect = vector - APIC_BASE_VECT;
1007 APIC_VERBOSE(IPI, (CE_CONT, "apix: clock vector = %x\n",
1008 apic_clkvect));
1009 return (vector);
1010 }
1011
1012 static int
1013 apix_post_cpu_start()
1014 {
1015 int cpun;
1016 static int cpus_started = 1;
1017
1018 /* We know this CPU + BSP started successfully. */
1019 cpus_started++;
1020
1021 /*
1022 * On BSP we would have enabled X2APIC, if supported by processor,
1023 * in acpi_probe(), but on AP we do it here.
1024 *
1025 * We enable X2APIC mode only if BSP is running in X2APIC & the
1026 * local APIC mode of the current CPU is MMIO (xAPIC).
1027 */
1028 if (apic_mode == LOCAL_X2APIC && apic_detect_x2apic() &&
1029 apic_local_mode() == LOCAL_APIC) {
1030 apic_enable_x2apic();
1031 }
1032
1033 /*
1034 * Switch back to x2apic IPI sending method for performance when target
1035 * CPU has entered x2apic mode.
1036 */
1037 if (apic_mode == LOCAL_X2APIC) {
1038 apic_switch_ipi_callback(B_FALSE);
1039 }
1040
1041 splx(ipltospl(LOCK_LEVEL));
1042 apix_init_intr();
1043
1044 /*
1045 * since some systems don't enable the internal cache on the non-boot
1046 * cpus, so we have to enable them here
1047 */
1048 setcr0(getcr0() & ~(CR0_CD | CR0_NW));
1049
1050 #ifdef DEBUG
1051 APIC_AV_PENDING_SET();
1052 #else
1053 if (apic_mode == LOCAL_APIC)
1054 APIC_AV_PENDING_SET();
1055 #endif /* DEBUG */
1056
1057 /*
1058 * We may be booting, or resuming from suspend; aci_status will
1059 * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the
1060 * APIC_CPU_ONLINE flag here rather than setting aci_status completely.
1061 */
1062 cpun = psm_get_cpu_id();
1063 apic_cpus[cpun].aci_status |= APIC_CPU_ONLINE;
1064
1065 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1066
1067 return (PSM_SUCCESS);
1068 }
1069
1070 /*
1071 * If this module needs a periodic handler for the interrupt distribution, it
1072 * can be added here. The argument to the periodic handler is not currently
1073 * used, but is reserved for future.
1074 */
1075 static void
1076 apix_post_cyclic_setup(void *arg)
1077 {
1078 UNREFERENCED_1PARAMETER(arg);
1079
1080 cyc_handler_t cyh;
1081 cyc_time_t cyt;
1082
1083 /* cpu_lock is held */
1084 /* set up a periodic handler for intr redistribution */
1085
1086 /*
1087 * In peridoc mode intr redistribution processing is done in
1088 * apic_intr_enter during clk intr processing
1089 */
1090 if (!apic_oneshot)
1091 return;
1092
1093 /*
1094 * Register a periodical handler for the redistribution processing.
1095 * Though we would generally prefer to use the DDI interface for
1096 * periodic handler invocation, ddi_periodic_add(9F), we are
1097 * unfortunately already holding cpu_lock, which ddi_periodic_add will
1098 * attempt to take for us. Thus, we add our own cyclic directly:
1099 */
1100 cyh.cyh_func = (void (*)(void *))apix_redistribute_compute;
1101 cyh.cyh_arg = NULL;
1102 cyh.cyh_level = CY_LOW_LEVEL;
1103
1104 cyt.cyt_when = 0;
1105 cyt.cyt_interval = apic_redistribute_sample_interval;
1106
1107 apic_cyclic_id = cyclic_add(&cyh, &cyt);
1108 }
1109
1110 /*
1111 * Called the first time we enable x2apic mode on this cpu.
1112 * Update some of the function pointers to use x2apic routines.
1113 */
1114 void
1115 x2apic_update_psm()
1116 {
1117 struct psm_ops *pops = &apix_ops;
1118
1119 ASSERT(pops != NULL);
1120
1121 /*
1122 * The pcplusmp module's version of x2apic_update_psm makes additional
1123 * changes that we do not have to make here. It needs to make those
1124 * changes because pcplusmp relies on the TPR register and the means of
1125 * addressing that changes when using the local apic versus the x2apic.
1126 * It's also worth noting that the apix driver specific function end up
1127 * being apix_foo as opposed to apic_foo and x2apic_foo.
1128 */
1129 pops->psm_send_ipi = x2apic_send_ipi;
1130
1131 send_dirintf = pops->psm_send_ipi;
1132
1133 apic_mode = LOCAL_X2APIC;
1134 apic_change_ops();
1135 }
1136
1137 /*
1138 * This function provides external interface to the nexus for all
1139 * functionalities related to the new DDI interrupt framework.
1140 *
1141 * Input:
1142 * dip - pointer to the dev_info structure of the requested device
1143 * hdlp - pointer to the internal interrupt handle structure for the
1144 * requested interrupt
1145 * intr_op - opcode for this call
1146 * result - pointer to the integer that will hold the result to be
1147 * passed back if return value is PSM_SUCCESS
1148 *
1149 * Output:
1150 * return value is either PSM_SUCCESS or PSM_FAILURE
1151 */
1152 static int
1153 apix_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1154 psm_intr_op_t intr_op, int *result)
1155 {
1156 int cap;
1157 apix_vector_t *vecp, *newvecp;
1158 struct intrspec *ispec, intr_spec;
1159 processorid_t target;
1160
1161 ispec = &intr_spec;
1162 ispec->intrspec_pri = hdlp->ih_pri;
1163 ispec->intrspec_vec = hdlp->ih_inum;
1164 ispec->intrspec_func = hdlp->ih_cb_func;
1165
1166 switch (intr_op) {
1167 case PSM_INTR_OP_ALLOC_VECTORS:
1168 switch (hdlp->ih_type) {
1169 case DDI_INTR_TYPE_MSI:
1170 /* allocate MSI vectors */
1171 *result = apix_alloc_msi(dip, hdlp->ih_inum,
1172 hdlp->ih_scratch1,
1173 (int)(uintptr_t)hdlp->ih_scratch2);
1174 break;
1175 case DDI_INTR_TYPE_MSIX:
1176 /* allocate MSI-X vectors */
1177 *result = apix_alloc_msix(dip, hdlp->ih_inum,
1178 hdlp->ih_scratch1,
1179 (int)(uintptr_t)hdlp->ih_scratch2);
1180 break;
1181 case DDI_INTR_TYPE_FIXED:
1182 /* allocate or share vector for fixed */
1183 if ((ihdl_plat_t *)hdlp->ih_private == NULL) {
1184 return (PSM_FAILURE);
1185 }
1186 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1187 *result = apix_intx_alloc_vector(dip, hdlp->ih_inum,
1188 ispec);
1189 break;
1190 default:
1191 return (PSM_FAILURE);
1192 }
1193 break;
1194 case PSM_INTR_OP_FREE_VECTORS:
1195 apix_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1,
1196 hdlp->ih_type);
1197 break;
1198 case PSM_INTR_OP_XLATE_VECTOR:
1199 /*
1200 * Vectors are allocated by ALLOC and freed by FREE.
1201 * XLATE finds and returns APIX_VIRTVEC_VECTOR(cpu, vector).
1202 */
1203 *result = APIX_INVALID_VECT;
1204 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1205 if (vecp != NULL) {
1206 *result = APIX_VIRTVECTOR(vecp->v_cpuid,
1207 vecp->v_vector);
1208 break;
1209 }
1210
1211 /*
1212 * No vector to device mapping exists. If this is FIXED type
1213 * then check if this IRQ is already mapped for another device
1214 * then return the vector number for it (i.e. shared IRQ case).
1215 * Otherwise, return PSM_FAILURE.
1216 */
1217 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
1218 vecp = apix_intx_xlate_vector(dip, hdlp->ih_inum,
1219 ispec);
1220 *result = (vecp == NULL) ? APIX_INVALID_VECT :
1221 APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1222 }
1223 if (*result == APIX_INVALID_VECT)
1224 return (PSM_FAILURE);
1225 break;
1226 case PSM_INTR_OP_GET_PENDING:
1227 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1228 if (vecp == NULL)
1229 return (PSM_FAILURE);
1230
1231 *result = apix_get_pending(vecp);
1232 break;
1233 case PSM_INTR_OP_CLEAR_MASK:
1234 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1235 return (PSM_FAILURE);
1236
1237 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1238 if (vecp == NULL)
1239 return (PSM_FAILURE);
1240
1241 apix_intx_clear_mask(vecp->v_inum);
1242 break;
1243 case PSM_INTR_OP_SET_MASK:
1244 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1245 return (PSM_FAILURE);
1246
1247 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1248 if (vecp == NULL)
1249 return (PSM_FAILURE);
1250
1251 apix_intx_set_mask(vecp->v_inum);
1252 break;
1253 case PSM_INTR_OP_GET_SHARED:
1254 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1255 return (PSM_FAILURE);
1256
1257 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1258 if (vecp == NULL)
1259 return (PSM_FAILURE);
1260
1261 *result = apix_intx_get_shared(vecp->v_inum);
1262 break;
1263 case PSM_INTR_OP_SET_PRI:
1264 /*
1265 * Called prior to adding the interrupt handler or when
1266 * an interrupt handler is unassigned.
1267 */
1268 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1269 return (PSM_SUCCESS);
1270
1271 if (apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type) == NULL)
1272 return (PSM_FAILURE);
1273
1274 break;
1275 case PSM_INTR_OP_SET_CPU:
1276 case PSM_INTR_OP_GRP_SET_CPU:
1277 /*
1278 * The interrupt handle given here has been allocated
1279 * specifically for this command, and ih_private carries
1280 * a CPU value.
1281 */
1282 *result = EINVAL;
1283 target = (int)(intptr_t)hdlp->ih_private;
1284 if (!apic_cpu_in_range(target)) {
1285 DDI_INTR_IMPLDBG((CE_WARN,
1286 "[grp_]set_cpu: cpu out of range: %d\n", target));
1287 return (PSM_FAILURE);
1288 }
1289
1290 lock_set(&apix_lock);
1291
1292 vecp = apix_get_req_vector(hdlp, hdlp->ih_flags);
1293 if (!IS_VECT_ENABLED(vecp)) {
1294 DDI_INTR_IMPLDBG((CE_WARN,
1295 "[grp]_set_cpu: invalid vector 0x%x\n",
1296 hdlp->ih_vector));
1297 lock_clear(&apix_lock);
1298 return (PSM_FAILURE);
1299 }
1300
1301 *result = 0;
1302
1303 if (intr_op == PSM_INTR_OP_SET_CPU)
1304 newvecp = apix_set_cpu(vecp, target, result);
1305 else
1306 newvecp = apix_grp_set_cpu(vecp, target, result);
1307
1308 lock_clear(&apix_lock);
1309
1310 if (newvecp == NULL) {
1311 *result = EIO;
1312 return (PSM_FAILURE);
1313 }
1314 newvecp->v_bound_cpuid = target;
1315 hdlp->ih_vector = APIX_VIRTVECTOR(newvecp->v_cpuid,
1316 newvecp->v_vector);
1317 break;
1318
1319 case PSM_INTR_OP_GET_INTR:
1320 /*
1321 * The interrupt handle given here has been allocated
1322 * specifically for this command, and ih_private carries
1323 * a pointer to a apic_get_intr_t.
1324 */
1325 if (apix_get_intr_info(hdlp, hdlp->ih_private) != PSM_SUCCESS)
1326 return (PSM_FAILURE);
1327 break;
1328
1329 case PSM_INTR_OP_CHECK_MSI:
1330 /*
1331 * Check MSI/X is supported or not at APIC level and
1332 * masked off the MSI/X bits in hdlp->ih_type if not
1333 * supported before return. If MSI/X is supported,
1334 * leave the ih_type unchanged and return.
1335 *
1336 * hdlp->ih_type passed in from the nexus has all the
1337 * interrupt types supported by the device.
1338 */
1339 if (apic_support_msi == 0) { /* uninitialized */
1340 /*
1341 * if apic_support_msi is not set, call
1342 * apic_check_msi_support() to check whether msi
1343 * is supported first
1344 */
1345 if (apic_check_msi_support() == PSM_SUCCESS)
1346 apic_support_msi = 1; /* supported */
1347 else
1348 apic_support_msi = -1; /* not-supported */
1349 }
1350 if (apic_support_msi == 1) {
1351 if (apic_msix_enable)
1352 *result = hdlp->ih_type;
1353 else
1354 *result = hdlp->ih_type & ~DDI_INTR_TYPE_MSIX;
1355 } else
1356 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1357 DDI_INTR_TYPE_MSIX);
1358 break;
1359 case PSM_INTR_OP_GET_CAP:
1360 cap = DDI_INTR_FLAG_PENDING;
1361 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1362 cap |= DDI_INTR_FLAG_MASKABLE;
1363 *result = cap;
1364 break;
1365 case PSM_INTR_OP_APIC_TYPE:
1366 ((apic_get_type_t *)(hdlp->ih_private))->avgi_type =
1367 apix_get_apic_type();
1368 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_intr =
1369 APIX_IPI_MIN;
1370 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_cpu =
1371 apic_nproc;
1372 hdlp->ih_ver = apic_get_apic_version();
1373 break;
1374 case PSM_INTR_OP_SET_CAP:
1375 default:
1376 return (PSM_FAILURE);
1377 }
1378
1379 return (PSM_SUCCESS);
1380 }
1381
1382 static void
1383 apix_cleanup_busy(void)
1384 {
1385 int i, j;
1386 apix_vector_t *vecp;
1387
1388 for (i = 0; i < apic_nproc; i++) {
1389 if (!apic_cpu_in_range(i))
1390 continue;
1391 apic_cpus[i].aci_busy = 0;
1392 for (j = APIX_AVINTR_MIN; j < APIX_AVINTR_MAX; j++) {
1393 if ((vecp = xv_vector(i, j)) != NULL)
1394 vecp->v_busy = 0;
1395 }
1396 }
1397 }
1398
1399 static void
1400 apix_redistribute_compute(void)
1401 {
1402 int i, j, max_busy;
1403
1404 if (!apic_enable_dynamic_migration)
1405 return;
1406
1407 if (++apic_nticks == apic_sample_factor_redistribution) {
1408 /*
1409 * Time to call apic_intr_redistribute().
1410 * reset apic_nticks. This will cause max_busy
1411 * to be calculated below and if it is more than
1412 * apic_int_busy, we will do the whole thing
1413 */
1414 apic_nticks = 0;
1415 }
1416 max_busy = 0;
1417 for (i = 0; i < apic_nproc; i++) {
1418 if (!apic_cpu_in_range(i))
1419 continue;
1420 /*
1421 * Check if curipl is non zero & if ISR is in
1422 * progress
1423 */
1424 if (((j = apic_cpus[i].aci_curipl) != 0) &&
1425 (apic_cpus[i].aci_ISR_in_progress & (1 << j))) {
1426
1427 int vect;
1428 apic_cpus[i].aci_busy++;
1429 vect = apic_cpus[i].aci_current[j];
1430 apixs[i]->x_vectbl[vect]->v_busy++;
1431 }
1432
1433 if (!apic_nticks &&
1434 (apic_cpus[i].aci_busy > max_busy))
1435 max_busy = apic_cpus[i].aci_busy;
1436 }
1437 if (!apic_nticks) {
1438 if (max_busy > apic_int_busy_mark) {
1439 /*
1440 * We could make the following check be
1441 * skipped > 1 in which case, we get a
1442 * redistribution at half the busy mark (due to
1443 * double interval). Need to be able to collect
1444 * more empirical data to decide if that is a
1445 * good strategy. Punt for now.
1446 */
1447 apix_cleanup_busy();
1448 apic_skipped_redistribute = 0;
1449 } else
1450 apic_skipped_redistribute++;
1451 }
1452 }
1453
1454 /*
1455 * intr_ops() service routines
1456 */
1457
1458 static int
1459 apix_get_pending(apix_vector_t *vecp)
1460 {
1461 int bit, index, irr, pending;
1462
1463 /* need to get on the bound cpu */
1464 mutex_enter(&cpu_lock);
1465 affinity_set(vecp->v_cpuid);
1466
1467 index = vecp->v_vector / 32;
1468 bit = vecp->v_vector % 32;
1469 irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
1470
1471 affinity_clear();
1472 mutex_exit(&cpu_lock);
1473
1474 pending = (irr & (1 << bit)) ? 1 : 0;
1475 if (!pending && vecp->v_type == APIX_TYPE_FIXED)
1476 pending = apix_intx_get_pending(vecp->v_inum);
1477
1478 return (pending);
1479 }
1480
1481 static apix_vector_t *
1482 apix_get_req_vector(ddi_intr_handle_impl_t *hdlp, ushort_t flags)
1483 {
1484 apix_vector_t *vecp;
1485 processorid_t cpuid;
1486 int32_t virt_vec = 0;
1487
1488 switch (flags & PSMGI_INTRBY_FLAGS) {
1489 case PSMGI_INTRBY_IRQ:
1490 return (apix_intx_get_vector(hdlp->ih_vector));
1491 case PSMGI_INTRBY_VEC:
1492 virt_vec = (virt_vec == 0) ? hdlp->ih_vector : virt_vec;
1493
1494 cpuid = APIX_VIRTVEC_CPU(virt_vec);
1495 if (!apic_cpu_in_range(cpuid))
1496 return (NULL);
1497
1498 vecp = xv_vector(cpuid, APIX_VIRTVEC_VECTOR(virt_vec));
1499 break;
1500 case PSMGI_INTRBY_DEFAULT:
1501 vecp = apix_get_dev_map(hdlp->ih_dip, hdlp->ih_inum,
1502 hdlp->ih_type);
1503 break;
1504 default:
1505 return (NULL);
1506 }
1507
1508 return (vecp);
1509 }
1510
1511 static int
1512 apix_get_intr_info(ddi_intr_handle_impl_t *hdlp,
1513 apic_get_intr_t *intr_params_p)
1514 {
1515 apix_vector_t *vecp;
1516 struct autovec *av_dev;
1517 int i;
1518
1519 vecp = apix_get_req_vector(hdlp, intr_params_p->avgi_req_flags);
1520 if (IS_VECT_FREE(vecp)) {
1521 intr_params_p->avgi_num_devs = 0;
1522 intr_params_p->avgi_cpu_id = 0;
1523 intr_params_p->avgi_req_flags = 0;
1524 return (PSM_SUCCESS);
1525 }
1526
1527 if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) {
1528 intr_params_p->avgi_cpu_id = vecp->v_cpuid;
1529
1530 /* Return user bound info for intrd. */
1531 if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) {
1532 intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND;
1533 intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND;
1534 }
1535 }
1536
1537 if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR)
1538 intr_params_p->avgi_vector = vecp->v_vector;
1539
1540 if (intr_params_p->avgi_req_flags &
1541 (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS))
1542 /* Get number of devices from apic_irq table shared field. */
1543 intr_params_p->avgi_num_devs = vecp->v_share;
1544
1545 if (intr_params_p->avgi_req_flags & PSMGI_REQ_GET_DEVS) {
1546
1547 intr_params_p->avgi_req_flags |= PSMGI_REQ_NUM_DEVS;
1548
1549 /* Some devices have NULL dip. Don't count these. */
1550 if (intr_params_p->avgi_num_devs > 0) {
1551 for (i = 0, av_dev = vecp->v_autovect; av_dev;
1552 av_dev = av_dev->av_link) {
1553 if (av_dev->av_vector && av_dev->av_dip)
1554 i++;
1555 }
1556 intr_params_p->avgi_num_devs =
1557 (uint8_t)MIN(intr_params_p->avgi_num_devs, i);
1558 }
1559
1560 /* There are no viable dips to return. */
1561 if (intr_params_p->avgi_num_devs == 0) {
1562 intr_params_p->avgi_dip_list = NULL;
1563
1564 } else { /* Return list of dips */
1565
1566 /* Allocate space in array for that number of devs. */
1567 intr_params_p->avgi_dip_list = kmem_zalloc(
1568 intr_params_p->avgi_num_devs *
1569 sizeof (dev_info_t *),
1570 KM_NOSLEEP);
1571 if (intr_params_p->avgi_dip_list == NULL) {
1572 DDI_INTR_IMPLDBG((CE_WARN,
1573 "apix_get_vector_intr_info: no memory"));
1574 return (PSM_FAILURE);
1575 }
1576
1577 /*
1578 * Loop through the device list of the autovec table
1579 * filling in the dip array.
1580 *
1581 * Note that the autovect table may have some special
1582 * entries which contain NULL dips. These will be
1583 * ignored.
1584 */
1585 for (i = 0, av_dev = vecp->v_autovect; av_dev;
1586 av_dev = av_dev->av_link) {
1587 if (av_dev->av_vector && av_dev->av_dip)
1588 intr_params_p->avgi_dip_list[i++] =
1589 av_dev->av_dip;
1590 }
1591 }
1592 }
1593
1594 return (PSM_SUCCESS);
1595 }
1596
1597 static char *
1598 apix_get_apic_type(void)
1599 {
1600 return (apix_psm_info.p_mach_idstring);
1601 }
1602
1603 apix_vector_t *
1604 apix_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1605 {
1606 apix_vector_t *newp = NULL;
1607 dev_info_t *dip;
1608 int inum, cap_ptr;
1609 ddi_acc_handle_t handle;
1610 ddi_intr_msix_t *msix_p = NULL;
1611 ushort_t msix_ctrl;
1612 uintptr_t off = 0;
1613 uint32_t mask = 0;
1614
1615 ASSERT(LOCK_HELD(&apix_lock));
1616 *result = ENXIO;
1617
1618 /* Fail if this is an MSI intr and is part of a group. */
1619 if (vecp->v_type == APIX_TYPE_MSI) {
1620 if (i_ddi_intr_get_current_nintrs(APIX_GET_DIP(vecp)) > 1)
1621 return (NULL);
1622 else
1623 return (apix_grp_set_cpu(vecp, new_cpu, result));
1624 }
1625
1626 /*
1627 * Mask MSI-X. It's unmasked when MSI-X gets enabled.
1628 */
1629 if (vecp->v_type == APIX_TYPE_MSIX && IS_VECT_ENABLED(vecp)) {
1630 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1631 return (NULL);
1632 inum = vecp->v_devp->dv_inum;
1633
1634 handle = i_ddi_get_pci_config_handle(dip);
1635 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1636 msix_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1637 if ((msix_ctrl & PCI_MSIX_FUNCTION_MASK) == 0) {
1638 /*
1639 * Function is not masked, then mask "inum"th
1640 * entry in the MSI-X table
1641 */
1642 msix_p = i_ddi_get_msix(dip);
1643 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1644 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1645 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1646 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off,
1647 mask | 1);
1648 }
1649 }
1650
1651 *result = 0;
1652 if ((newp = apix_rebind(vecp, new_cpu, 1)) == NULL)
1653 *result = EIO;
1654
1655 /* Restore mask bit */
1656 if (msix_p != NULL)
1657 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, mask);
1658
1659 return (newp);
1660 }
1661
1662 /*
1663 * Set cpu for MSIs
1664 */
1665 apix_vector_t *
1666 apix_grp_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1667 {
1668 apix_vector_t *newp, *vp;
1669 uint32_t orig_cpu = vecp->v_cpuid;
1670 int orig_vect = vecp->v_vector;
1671 int i, num_vectors, cap_ptr, msi_mask_off = 0;
1672 uint32_t msi_pvm = 0;
1673 ushort_t msi_ctrl;
1674 ddi_acc_handle_t handle;
1675 dev_info_t *dip;
1676
1677 APIC_VERBOSE(INTR, (CE_CONT, "apix_grp_set_cpu: oldcpu: %x, vector: %x,"
1678 " newcpu:%x\n", vecp->v_cpuid, vecp->v_vector, new_cpu));
1679
1680 ASSERT(LOCK_HELD(&apix_lock));
1681
1682 *result = ENXIO;
1683
1684 if (vecp->v_type != APIX_TYPE_MSI) {
1685 DDI_INTR_IMPLDBG((CE_WARN, "set_grp: intr not MSI\n"));
1686 return (NULL);
1687 }
1688
1689 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1690 return (NULL);
1691
1692 num_vectors = i_ddi_intr_get_current_nintrs(dip);
1693 if ((num_vectors < 1) || ((num_vectors - 1) & orig_vect)) {
1694 APIC_VERBOSE(INTR, (CE_WARN,
1695 "set_grp: base vec not part of a grp or not aligned: "
1696 "vec:0x%x, num_vec:0x%x\n", orig_vect, num_vectors));
1697 return (NULL);
1698 }
1699
1700 if (vecp->v_inum != apix_get_min_dev_inum(dip, vecp->v_type))
1701 return (NULL);
1702
1703 *result = EIO;
1704 for (i = 1; i < num_vectors; i++) {
1705 if ((vp = xv_vector(orig_cpu, orig_vect + i)) == NULL)
1706 return (NULL);
1707 #ifdef DEBUG
1708 /*
1709 * Sanity check: CPU and dip is the same for all entries.
1710 * May be called when first msi to be enabled, at this time
1711 * add_avintr() is not called for other msi
1712 */
1713 if ((vp->v_share != 0) &&
1714 ((APIX_GET_DIP(vp) != dip) ||
1715 (vp->v_cpuid != vecp->v_cpuid))) {
1716 APIC_VERBOSE(INTR, (CE_WARN,
1717 "set_grp: cpu or dip for vec 0x%x difft than for "
1718 "vec 0x%x\n", orig_vect, orig_vect + i));
1719 APIC_VERBOSE(INTR, (CE_WARN,
1720 " cpu: %d vs %d, dip: 0x%p vs 0x%p\n", orig_cpu,
1721 vp->v_cpuid, (void *)dip,
1722 (void *)APIX_GET_DIP(vp)));
1723 return (NULL);
1724 }
1725 #endif /* DEBUG */
1726 }
1727
1728 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1729 handle = i_ddi_get_pci_config_handle(dip);
1730 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1731
1732 /* MSI Per vector masking is supported. */
1733 if (msi_ctrl & PCI_MSI_PVM_MASK) {
1734 if (msi_ctrl & PCI_MSI_64BIT_MASK)
1735 msi_mask_off = cap_ptr + PCI_MSI_64BIT_MASKBITS;
1736 else
1737 msi_mask_off = cap_ptr + PCI_MSI_32BIT_MASK;
1738 msi_pvm = pci_config_get32(handle, msi_mask_off);
1739 pci_config_put32(handle, msi_mask_off, (uint32_t)-1);
1740 APIC_VERBOSE(INTR, (CE_CONT,
1741 "set_grp: pvm supported. Mask set to 0x%x\n",
1742 pci_config_get32(handle, msi_mask_off)));
1743 }
1744
1745 if ((newp = apix_rebind(vecp, new_cpu, num_vectors)) != NULL)
1746 *result = 0;
1747
1748 /* Reenable vectors if per vector masking is supported. */
1749 if (msi_ctrl & PCI_MSI_PVM_MASK) {
1750 pci_config_put32(handle, msi_mask_off, msi_pvm);
1751 APIC_VERBOSE(INTR, (CE_CONT,
1752 "set_grp: pvm supported. Mask restored to 0x%x\n",
1753 pci_config_get32(handle, msi_mask_off)));
1754 }
1755
1756 return (newp);
1757 }
1758
1759 void
1760 apix_intx_set_vector(int irqno, uint32_t cpuid, uchar_t vector)
1761 {
1762 apic_irq_t *irqp;
1763
1764 mutex_enter(&airq_mutex);
1765 irqp = apic_irq_table[irqno];
1766 irqp->airq_cpu = cpuid;
1767 irqp->airq_vector = vector;
1768 apic_record_rdt_entry(irqp, irqno);
1769 mutex_exit(&airq_mutex);
1770 }
1771
1772 apix_vector_t *
1773 apix_intx_get_vector(int irqno)
1774 {
1775 apic_irq_t *irqp;
1776 uint32_t cpuid;
1777 uchar_t vector;
1778
1779 mutex_enter(&airq_mutex);
1780 irqp = apic_irq_table[irqno & 0xff];
1781 if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
1782 mutex_exit(&airq_mutex);
1783 return (NULL);
1784 }
1785 cpuid = irqp->airq_cpu;
1786 vector = irqp->airq_vector;
1787 mutex_exit(&airq_mutex);
1788
1789 return (xv_vector(cpuid, vector));
1790 }
1791
1792 /*
1793 * Must called with interrupts disabled and apic_ioapic_lock held
1794 */
1795 void
1796 apix_intx_enable(int irqno)
1797 {
1798 uchar_t ioapicindex, intin;
1799 apic_irq_t *irqp = apic_irq_table[irqno];
1800 ioapic_rdt_t irdt;
1801 apic_cpus_info_t *cpu_infop;
1802 apix_vector_t *vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1803
1804 ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1805
1806 ioapicindex = irqp->airq_ioapicindex;
1807 intin = irqp->airq_intin_no;
1808 cpu_infop = &apic_cpus[irqp->airq_cpu];
1809
1810 irdt.ir_lo = AV_PDEST | AV_FIXED | irqp->airq_rdt_entry;
1811 irdt.ir_hi = cpu_infop->aci_local_id;
1812
1813 apic_vt_ops->apic_intrmap_alloc_entry(&vecp->v_intrmap_private, NULL,
1814 vecp->v_type, 1, ioapicindex);
1815 apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
1816 (void *)&irdt, vecp->v_type, 1);
1817 apic_vt_ops->apic_intrmap_record_rdt(vecp->v_intrmap_private, &irdt);
1818
1819 /* write RDT entry high dword - destination */
1820 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin,
1821 irdt.ir_hi);
1822
1823 /* Write the vector, trigger, and polarity portion of the RDT */
1824 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin, irdt.ir_lo);
1825
1826 vecp->v_state = APIX_STATE_ENABLED;
1827
1828 APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_enable: ioapic 0x%x"
1829 " intin 0x%x rdt_low 0x%x rdt_high 0x%x\n",
1830 ioapicindex, intin, irdt.ir_lo, irdt.ir_hi));
1831 }
1832
1833 /*
1834 * Must called with interrupts disabled and apic_ioapic_lock held
1835 */
1836 void
1837 apix_intx_disable(int irqno)
1838 {
1839 apic_irq_t *irqp = apic_irq_table[irqno];
1840 int ioapicindex, intin;
1841
1842 ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1843 /*
1844 * The assumption here is that this is safe, even for
1845 * systems with IOAPICs that suffer from the hardware
1846 * erratum because all devices have been quiesced before
1847 * they unregister their interrupt handlers. If that
1848 * assumption turns out to be false, this mask operation
1849 * can induce the same erratum result we're trying to
1850 * avoid.
1851 */
1852 ioapicindex = irqp->airq_ioapicindex;
1853 intin = irqp->airq_intin_no;
1854 ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * intin, AV_MASK);
1855
1856 APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_disable: ioapic 0x%x"
1857 " intin 0x%x\n", ioapicindex, intin));
1858 }
1859
1860 void
1861 apix_intx_free(int irqno)
1862 {
1863 apic_irq_t *irqp;
1864
1865 mutex_enter(&airq_mutex);
1866 irqp = apic_irq_table[irqno];
1867
1868 if (IS_IRQ_FREE(irqp)) {
1869 mutex_exit(&airq_mutex);
1870 return;
1871 }
1872
1873 irqp->airq_mps_intr_index = FREE_INDEX;
1874 irqp->airq_cpu = IRQ_UNINIT;
1875 irqp->airq_vector = APIX_INVALID_VECT;
1876 mutex_exit(&airq_mutex);
1877 }
1878
1879 #ifdef DEBUG
1880 int apix_intr_deliver_timeouts = 0;
1881 int apix_intr_rirr_timeouts = 0;
1882 int apix_intr_rirr_reset_failure = 0;
1883 #endif
1884 int apix_max_reps_irr_pending = 10;
1885
1886 #define GET_RDT_BITS(ioapic, intin, bits) \
1887 (READ_IOAPIC_RDT_ENTRY_LOW_DWORD((ioapic), (intin)) & (bits))
1888 #define APIX_CHECK_IRR_DELAY drv_usectohz(5000)
1889
1890 int
1891 apix_intx_rebind(int irqno, processorid_t cpuid, uchar_t vector)
1892 {
1893 apic_irq_t *irqp = apic_irq_table[irqno];
1894 ulong_t iflag;
1895 int waited, ioapic_ix, intin_no, level, repeats, rdt_entry, masked;
1896
1897 ASSERT(irqp != NULL);
1898
1899 iflag = intr_clear();
1900 lock_set(&apic_ioapic_lock);
1901
1902 ioapic_ix = irqp->airq_ioapicindex;
1903 intin_no = irqp->airq_intin_no;
1904 level = apic_level_intr[irqno];
1905
1906 /*
1907 * Wait for the delivery status bit to be cleared. This should
1908 * be a very small amount of time.
1909 */
1910 repeats = 0;
1911 do {
1912 repeats++;
1913
1914 for (waited = 0; waited < apic_max_reps_clear_pending;
1915 waited++) {
1916 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) == 0)
1917 break;
1918 }
1919 if (!level)
1920 break;
1921
1922 /*
1923 * Mask the RDT entry for level-triggered interrupts.
1924 */
1925 irqp->airq_rdt_entry |= AV_MASK;
1926 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1927 intin_no);
1928 if ((masked = (rdt_entry & AV_MASK)) == 0) {
1929 /* Mask it */
1930 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1931 AV_MASK | rdt_entry);
1932 }
1933
1934 /*
1935 * If there was a race and an interrupt was injected
1936 * just before we masked, check for that case here.
1937 * Then, unmask the RDT entry and try again. If we're
1938 * on our last try, don't unmask (because we want the
1939 * RDT entry to remain masked for the rest of the
1940 * function).
1941 */
1942 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1943 intin_no);
1944 if ((masked == 0) && ((rdt_entry & AV_PENDING) != 0) &&
1945 (repeats < apic_max_reps_clear_pending)) {
1946 /* Unmask it */
1947 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1948 intin_no, rdt_entry & ~AV_MASK);
1949 irqp->airq_rdt_entry &= ~AV_MASK;
1950 }
1951 } while ((rdt_entry & AV_PENDING) &&
1952 (repeats < apic_max_reps_clear_pending));
1953
1954 #ifdef DEBUG
1955 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) != 0)
1956 apix_intr_deliver_timeouts++;
1957 #endif
1958
1959 if (!level || !APIX_IS_MASK_RDT(apix_mul_ioapic_method))
1960 goto done;
1961
1962 /*
1963 * wait for remote IRR to be cleared for level-triggered
1964 * interrupts
1965 */
1966 repeats = 0;
1967 do {
1968 repeats++;
1969
1970 for (waited = 0; waited < apic_max_reps_clear_pending;
1971 waited++) {
1972 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR)
1973 == 0)
1974 break;
1975 }
1976
1977 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1978 lock_clear(&apic_ioapic_lock);
1979 intr_restore(iflag);
1980
1981 delay(APIX_CHECK_IRR_DELAY);
1982
1983 iflag = intr_clear();
1984 lock_set(&apic_ioapic_lock);
1985 }
1986 } while (repeats < apix_max_reps_irr_pending);
1987
1988 if (repeats >= apix_max_reps_irr_pending) {
1989 #ifdef DEBUG
1990 apix_intr_rirr_timeouts++;
1991 #endif
1992
1993 /*
1994 * If we waited and the Remote IRR bit is still not cleared,
1995 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1996 * times for this interrupt, try the last-ditch workaround:
1997 */
1998 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1999 /*
2000 * Trying to clear the bit through normal
2001 * channels has failed. So as a last-ditch
2002 * effort, try to set the trigger mode to
2003 * edge, then to level. This has been
2004 * observed to work on many systems.
2005 */
2006 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2007 intin_no,
2008 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2009 intin_no) & ~AV_LEVEL);
2010 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2011 intin_no,
2012 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2013 intin_no) | AV_LEVEL);
2014 }
2015
2016 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
2017 #ifdef DEBUG
2018 apix_intr_rirr_reset_failure++;
2019 #endif
2020 lock_clear(&apic_ioapic_lock);
2021 intr_restore(iflag);
2022 prom_printf("apix: Remote IRR still "
2023 "not clear for IOAPIC %d intin %d.\n"
2024 "\tInterrupts to this pin may cease "
2025 "functioning.\n", ioapic_ix, intin_no);
2026 return (1); /* return failure */
2027 }
2028 }
2029
2030 done:
2031 /* change apic_irq_table */
2032 lock_clear(&apic_ioapic_lock);
2033 intr_restore(iflag);
2034 apix_intx_set_vector(irqno, cpuid, vector);
2035 iflag = intr_clear();
2036 lock_set(&apic_ioapic_lock);
2037
2038 /* reprogramme IO-APIC RDT entry */
2039 apix_intx_enable(irqno);
2040
2041 lock_clear(&apic_ioapic_lock);
2042 intr_restore(iflag);
2043
2044 return (0);
2045 }
2046
2047 static int
2048 apix_intx_get_pending(int irqno)
2049 {
2050 apic_irq_t *irqp;
2051 int intin, ioapicindex, pending;
2052 ulong_t iflag;
2053
2054 mutex_enter(&airq_mutex);
2055 irqp = apic_irq_table[irqno];
2056 if (IS_IRQ_FREE(irqp)) {
2057 mutex_exit(&airq_mutex);
2058 return (0);
2059 }
2060
2061 /* check IO-APIC delivery status */
2062 intin = irqp->airq_intin_no;
2063 ioapicindex = irqp->airq_ioapicindex;
2064 mutex_exit(&airq_mutex);
2065
2066 iflag = intr_clear();
2067 lock_set(&apic_ioapic_lock);
2068
2069 pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin) &
2070 AV_PENDING) ? 1 : 0;
2071
2072 lock_clear(&apic_ioapic_lock);
2073 intr_restore(iflag);
2074
2075 return (pending);
2076 }
2077
2078 /*
2079 * This function will mask the interrupt on the I/O APIC
2080 */
2081 static void
2082 apix_intx_set_mask(int irqno)
2083 {
2084 int intin, ioapixindex, rdt_entry;
2085 ulong_t iflag;
2086 apic_irq_t *irqp;
2087
2088 mutex_enter(&airq_mutex);
2089 irqp = apic_irq_table[irqno];
2090
2091 ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2092
2093 intin = irqp->airq_intin_no;
2094 ioapixindex = irqp->airq_ioapicindex;
2095 mutex_exit(&airq_mutex);
2096
2097 iflag = intr_clear();
2098 lock_set(&apic_ioapic_lock);
2099
2100 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2101
2102 /* clear mask */
2103 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2104 (AV_MASK | rdt_entry));
2105
2106 lock_clear(&apic_ioapic_lock);
2107 intr_restore(iflag);
2108 }
2109
2110 /*
2111 * This function will clear the mask for the interrupt on the I/O APIC
2112 */
2113 static void
2114 apix_intx_clear_mask(int irqno)
2115 {
2116 int intin, ioapixindex, rdt_entry;
2117 ulong_t iflag;
2118 apic_irq_t *irqp;
2119
2120 mutex_enter(&airq_mutex);
2121 irqp = apic_irq_table[irqno];
2122
2123 ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2124
2125 intin = irqp->airq_intin_no;
2126 ioapixindex = irqp->airq_ioapicindex;
2127 mutex_exit(&airq_mutex);
2128
2129 iflag = intr_clear();
2130 lock_set(&apic_ioapic_lock);
2131
2132 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2133
2134 /* clear mask */
2135 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2136 ((~AV_MASK) & rdt_entry));
2137
2138 lock_clear(&apic_ioapic_lock);
2139 intr_restore(iflag);
2140 }
2141
2142 /*
2143 * For level-triggered interrupt, mask the IRQ line. Mask means
2144 * new interrupts will not be delivered. The interrupt already
2145 * accepted by a local APIC is not affected
2146 */
2147 void
2148 apix_level_intr_pre_eoi(int irq)
2149 {
2150 apic_irq_t *irqp = apic_irq_table[irq];
2151 int apic_ix, intin_ix;
2152
2153 if (irqp == NULL)
2154 return;
2155
2156 ASSERT(apic_level_intr[irq] == TRIGGER_MODE_LEVEL);
2157
2158 lock_set(&apic_ioapic_lock);
2159
2160 intin_ix = irqp->airq_intin_no;
2161 apic_ix = irqp->airq_ioapicindex;
2162
2163 if (irqp->airq_cpu != CPU->cpu_id) {
2164 if (!APIX_IS_MASK_RDT(apix_mul_ioapic_method))
2165 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2166 lock_clear(&apic_ioapic_lock);
2167 return;
2168 }
2169
2170 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC) {
2171 /*
2172 * This is a IOxAPIC and there is EOI register:
2173 * Change the vector to reserved unused vector, so that
2174 * the EOI from Local APIC won't clear the Remote IRR for
2175 * this level trigger interrupt. Instead, we'll manually
2176 * clear it in apix_post_hardint() after ISR handling.
2177 */
2178 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2179 (irqp->airq_rdt_entry & (~0xff)) | APIX_RESV_VECTOR);
2180 } else {
2181 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2182 AV_MASK | irqp->airq_rdt_entry);
2183 }
2184
2185 lock_clear(&apic_ioapic_lock);
2186 }
2187
2188 /*
2189 * For level-triggered interrupt, unmask the IRQ line
2190 * or restore the original vector number.
2191 */
2192 void
2193 apix_level_intr_post_dispatch(int irq)
2194 {
2195 apic_irq_t *irqp = apic_irq_table[irq];
2196 int apic_ix, intin_ix;
2197
2198 if (irqp == NULL)
2199 return;
2200
2201 lock_set(&apic_ioapic_lock);
2202
2203 intin_ix = irqp->airq_intin_no;
2204 apic_ix = irqp->airq_ioapicindex;
2205
2206 if (APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method)) {
2207 /*
2208 * Already sent EOI back to Local APIC.
2209 * Send EOI to IO-APIC
2210 */
2211 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2212 } else {
2213 /* clear the mask or restore the vector */
2214 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2215 irqp->airq_rdt_entry);
2216
2217 /* send EOI to IOxAPIC */
2218 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC)
2219 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2220 }
2221
2222 lock_clear(&apic_ioapic_lock);
2223 }
2224
2225 static int
2226 apix_intx_get_shared(int irqno)
2227 {
2228 apic_irq_t *irqp;
2229 int share;
2230
2231 mutex_enter(&airq_mutex);
2232 irqp = apic_irq_table[irqno];
2233 if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
2234 mutex_exit(&airq_mutex);
2235 return (0);
2236 }
2237 share = irqp->airq_share;
2238 mutex_exit(&airq_mutex);
2239
2240 return (share);
2241 }
2242
2243 static void
2244 apix_intx_set_shared(int irqno, int delta)
2245 {
2246 apic_irq_t *irqp;
2247
2248 mutex_enter(&airq_mutex);
2249 irqp = apic_irq_table[irqno];
2250 if (IS_IRQ_FREE(irqp)) {
2251 mutex_exit(&airq_mutex);
2252 return;
2253 }
2254 irqp->airq_share += delta;
2255 mutex_exit(&airq_mutex);
2256 }
2257
2258 /*
2259 * Setup IRQ table. Return IRQ no or -1 on failure
2260 */
2261 static int
2262 apix_intx_setup(dev_info_t *dip, int inum, int irqno,
2263 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *iflagp)
2264 {
2265 int origirq = ispec->intrspec_vec;
2266 int newirq;
2267 short intr_index;
2268 uchar_t ipin, ioapic, ioapicindex;
2269 apic_irq_t *irqp;
2270
2271 UNREFERENCED_1PARAMETER(inum);
2272
2273 if (intrp != NULL) {
2274 intr_index = (short)(intrp - apic_io_intrp);
2275 ioapic = intrp->intr_destid;
2276 ipin = intrp->intr_destintin;
2277
2278 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
2279 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
2280 if (apic_io_id[ioapicindex] == ioapic)
2281 break;
2282 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
2283 (ioapic == INTR_ALL_APIC));
2284
2285 /* check whether this intin# has been used by another irqno */
2286 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1)
2287 return (newirq);
2288
2289 } else if (iflagp != NULL) { /* ACPI */
2290 intr_index = ACPI_INDEX;
2291 ioapicindex = acpi_find_ioapic(irqno);
2292 ASSERT(ioapicindex != 0xFF);
2293 ioapic = apic_io_id[ioapicindex];
2294 ipin = irqno - apic_io_vectbase[ioapicindex];
2295
2296 if (apic_irq_table[irqno] &&
2297 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
2298 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
2299 apic_irq_table[irqno]->airq_ioapicindex ==
2300 ioapicindex);
2301 return (irqno);
2302 }
2303
2304 } else { /* default configuration */
2305 intr_index = DEFAULT_INDEX;
2306 ioapicindex = 0;
2307 ioapic = apic_io_id[ioapicindex];
2308 ipin = (uchar_t)irqno;
2309 }
2310
2311 /* allocate a new IRQ no */
2312 if ((irqp = apic_irq_table[irqno]) == NULL) {
2313 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2314 apic_irq_table[irqno] = irqp;
2315 } else {
2316 if (irqp->airq_mps_intr_index != FREE_INDEX) {
2317 newirq = apic_allocate_irq(apic_first_avail_irq);
2318 if (newirq == -1) {
2319 return (-1);
2320 }
2321 irqno = newirq;
2322 irqp = apic_irq_table[irqno];
2323 ASSERT(irqp != NULL);
2324 }
2325 }
2326 apic_max_device_irq = max(irqno, apic_max_device_irq);
2327 apic_min_device_irq = min(irqno, apic_min_device_irq);
2328
2329 irqp->airq_mps_intr_index = intr_index;
2330 irqp->airq_ioapicindex = ioapicindex;
2331 irqp->airq_intin_no = ipin;
2332 irqp->airq_dip = dip;
2333 irqp->airq_origirq = (uchar_t)origirq;
2334 if (iflagp != NULL)
2335 irqp->airq_iflag = *iflagp;
2336 irqp->airq_cpu = IRQ_UNINIT;
2337 irqp->airq_vector = 0;
2338
2339 return (irqno);
2340 }
2341
2342 /*
2343 * Setup IRQ table for non-pci devices. Return IRQ no or -1 on error
2344 */
2345 static int
2346 apix_intx_setup_nonpci(dev_info_t *dip, int inum, int bustype,
2347 struct intrspec *ispec)
2348 {
2349 int irqno = ispec->intrspec_vec;
2350 int newirq, i;
2351 iflag_t intr_flag;
2352 ACPI_SUBTABLE_HEADER *hp;
2353 ACPI_MADT_INTERRUPT_OVERRIDE *isop;
2354 struct apic_io_intr *intrp;
2355
2356 if (!apic_enable_acpi || apic_use_acpi_madt_only) {
2357 int busid;
2358
2359 if (bustype == 0)
2360 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
2361
2362 /* loop checking BUS_ISA/BUS_EISA */
2363 for (i = 0; i < 2; i++) {
2364 if (((busid = apic_find_bus_id(bustype)) != -1) &&
2365 ((intrp = apic_find_io_intr_w_busid(irqno, busid))
2366 != NULL)) {
2367 return (apix_intx_setup(dip, inum, irqno,
2368 intrp, ispec, NULL));
2369 }
2370 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
2371 }
2372
2373 /* fall back to default configuration */
2374 return (-1);
2375 }
2376
2377 /* search iso entries first */
2378 if (acpi_iso_cnt != 0) {
2379 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
2380 i = 0;
2381 while (i < acpi_iso_cnt) {
2382 if (hp->Type == ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
2383 isop = (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
2384 if (isop->Bus == 0 &&
2385 isop->SourceIrq == irqno) {
2386 newirq = isop->GlobalIrq;
2387 intr_flag.intr_po = isop->IntiFlags &
2388 ACPI_MADT_POLARITY_MASK;
2389 intr_flag.intr_el = (isop->IntiFlags &
2390 ACPI_MADT_TRIGGER_MASK) >> 2;
2391 intr_flag.bustype = BUS_ISA;
2392
2393 return (apix_intx_setup(dip, inum,
2394 newirq, NULL, ispec, &intr_flag));
2395 }
2396 i++;
2397 }
2398 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
2399 hp->Length);
2400 }
2401 }
2402 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
2403 intr_flag.intr_el = INTR_EL_EDGE;
2404 intr_flag.bustype = BUS_ISA;
2405 return (apix_intx_setup(dip, inum, irqno, NULL, ispec, &intr_flag));
2406 }
2407
2408
2409 /*
2410 * Setup IRQ table for pci devices. Return IRQ no or -1 on error
2411 */
2412 static int
2413 apix_intx_setup_pci(dev_info_t *dip, int inum, int bustype,
2414 struct intrspec *ispec)
2415 {
2416 int busid, devid, pci_irq;
2417 ddi_acc_handle_t cfg_handle;
2418 uchar_t ipin;
2419 iflag_t intr_flag;
2420 struct apic_io_intr *intrp;
2421
2422 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
2423 return (-1);
2424
2425 if (busid == 0 && apic_pci_bus_total == 1)
2426 busid = (int)apic_single_pci_busid;
2427
2428 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
2429 return (-1);
2430 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
2431 pci_config_teardown(&cfg_handle);
2432
2433 if (apic_enable_acpi && !apic_use_acpi_madt_only) { /* ACPI */
2434 if (apic_acpi_translate_pci_irq(dip, busid, devid,
2435 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
2436 return (-1);
2437
2438 intr_flag.bustype = (uchar_t)bustype;
2439 return (apix_intx_setup(dip, inum, pci_irq, NULL, ispec,
2440 &intr_flag));
2441 }
2442
2443 /* MP configuration table */
2444 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
2445 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) == NULL) {
2446 pci_irq = apic_handle_pci_pci_bridge(dip, devid, ipin, &intrp);
2447 if (pci_irq == -1)
2448 return (-1);
2449 }
2450
2451 return (apix_intx_setup(dip, inum, pci_irq, intrp, ispec, NULL));
2452 }
2453
2454 /*
2455 * Translate and return IRQ no
2456 */
2457 static int
2458 apix_intx_xlate_irq(dev_info_t *dip, int inum, struct intrspec *ispec)
2459 {
2460 int newirq, irqno = ispec->intrspec_vec;
2461 int parent_is_pci_or_pciex = 0, child_is_pciex = 0;
2462 int bustype = 0, dev_len;
2463 char dev_type[16];
2464
2465 if (apic_defconf) {
2466 mutex_enter(&airq_mutex);
2467 goto defconf;
2468 }
2469
2470 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) {
2471 mutex_enter(&airq_mutex);
2472 goto nonpci;
2473 }
2474
2475 /*
2476 * use ddi_getlongprop_buf() instead of ddi_prop_lookup_string()
2477 * to avoid extra buffer allocation.
2478 */
2479 dev_len = sizeof (dev_type);
2480 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
2481 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
2482 &dev_len) == DDI_PROP_SUCCESS) {
2483 if ((strcmp(dev_type, "pci") == 0) ||
2484 (strcmp(dev_type, "pciex") == 0))
2485 parent_is_pci_or_pciex = 1;
2486 }
2487
2488 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
2489 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
2490 &dev_len) == DDI_PROP_SUCCESS) {
2491 if (strstr(dev_type, "pciex"))
2492 child_is_pciex = 1;
2493 }
2494
2495 mutex_enter(&airq_mutex);
2496
2497 if (parent_is_pci_or_pciex) {
2498 bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
2499 newirq = apix_intx_setup_pci(dip, inum, bustype, ispec);
2500 if (newirq != -1)
2501 goto done;
2502 bustype = 0;
2503 } else if (strcmp(dev_type, "isa") == 0)
2504 bustype = BUS_ISA;
2505 else if (strcmp(dev_type, "eisa") == 0)
2506 bustype = BUS_EISA;
2507
2508 nonpci:
2509 newirq = apix_intx_setup_nonpci(dip, inum, bustype, ispec);
2510 if (newirq != -1)
2511 goto done;
2512
2513 defconf:
2514 newirq = apix_intx_setup(dip, inum, irqno, NULL, ispec, NULL);
2515 if (newirq == -1) {
2516 mutex_exit(&airq_mutex);
2517 return (-1);
2518 }
2519 done:
2520 ASSERT(apic_irq_table[newirq]);
2521 mutex_exit(&airq_mutex);
2522 return (newirq);
2523 }
2524
2525 static int
2526 apix_intx_alloc_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2527 {
2528 int irqno;
2529 apix_vector_t *vecp;
2530
2531 if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2532 return (0);
2533
2534 if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
2535 return (0);
2536
2537 DDI_INTR_IMPLDBG((CE_CONT, "apix_intx_alloc_vector: dip=0x%p name=%s "
2538 "irqno=0x%x cpuid=%d vector=0x%x\n",
2539 (void *)dip, ddi_driver_name(dip), irqno,
2540 vecp->v_cpuid, vecp->v_vector));
2541
2542 return (1);
2543 }
2544
2545 /*
2546 * Return the vector number if the translated IRQ for this device
2547 * has a vector mapping setup. If no IRQ setup exists or no vector is
2548 * allocated to it then return 0.
2549 */
2550 static apix_vector_t *
2551 apix_intx_xlate_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2552 {
2553 int irqno;
2554 apix_vector_t *vecp;
2555
2556 /* get the IRQ number */
2557 if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2558 return (NULL);
2559
2560 /* get the vector number if a vector is allocated to this irqno */
2561 vecp = apix_intx_get_vector(irqno);
2562
2563 return (vecp);
2564 }
2565
2566 /*
2567 * Switch between safe and x2APIC IPI sending method.
2568 * The CPU may power on in xapic mode or x2apic mode. If the CPU needs to send
2569 * an IPI to other CPUs before entering x2APIC mode, it still needs to use the
2570 * xAPIC method. Before sending a StartIPI to the target CPU, psm_send_ipi will
2571 * be changed to apic_common_send_ipi, which detects current local APIC mode and
2572 * use the right method to send an IPI. If some CPUs fail to start up,
2573 * apic_poweron_cnt won't return to zero, so apic_common_send_ipi will always be
2574 * used. psm_send_ipi can't be simply changed back to x2apic_send_ipi if some
2575 * CPUs failed to start up because those failed CPUs may recover itself later at
2576 * unpredictable time.
2577 */
2578 void
2579 apic_switch_ipi_callback(boolean_t enter)
2580 {
2581 ulong_t iflag;
2582 struct psm_ops *pops = psmops;
2583
2584 iflag = intr_clear();
2585 lock_set(&apic_mode_switch_lock);
2586 if (enter) {
2587 ASSERT(apic_poweron_cnt >= 0);
2588 if (apic_poweron_cnt == 0) {
2589 pops->psm_send_ipi = apic_common_send_ipi;
2590 send_dirintf = pops->psm_send_ipi;
2591 }
2592 apic_poweron_cnt++;
2593 } else {
2594 ASSERT(apic_poweron_cnt > 0);
2595 apic_poweron_cnt--;
2596 if (apic_poweron_cnt == 0) {
2597 pops->psm_send_ipi = x2apic_send_ipi;
2598 send_dirintf = pops->psm_send_ipi;
2599 }
2600 }
2601 lock_clear(&apic_mode_switch_lock);
2602 intr_restore(iflag);
2603 }
2604
2605 /* stub function */
2606 int
2607 apix_loaded(void)
2608 {
2609 return (apix_is_enabled);
2610 }