1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright (c) 2010, Intel Corporation.
26 * All rights reserved.
27 */
28
29 /*
30 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33 * PSMI 1.5 extensions are supported in Solaris Nevada.
34 * PSMI 1.6 extensions are supported in Solaris Nevada.
35 * PSMI 1.7 extensions are supported in Solaris Nevada.
36 */
37 #define PSMI_1_7
38
39 #include <sys/processor.h>
40 #include <sys/time.h>
41 #include <sys/psm.h>
42 #include <sys/smp_impldefs.h>
43 #include <sys/inttypes.h>
44 #include <sys/cram.h>
45 #include <acpica/include/acpi.h>
46 #include <sys/acpica.h>
47 #include <sys/psm_common.h>
48 #include <sys/apic.h>
49 #include <sys/apic_common.h>
50 #include <sys/pit.h>
51 #include <sys/ddi.h>
52 #include <sys/sunddi.h>
53 #include <sys/ddi_impldefs.h>
54 #include <sys/pci.h>
55 #include <sys/promif.h>
56 #include <sys/x86_archext.h>
57 #include <sys/cpc_impl.h>
58 #include <sys/uadmin.h>
59 #include <sys/panic.h>
60 #include <sys/debug.h>
61 #include <sys/archsystm.h>
62 #include <sys/trap.h>
63 #include <sys/machsystm.h>
64 #include <sys/cpuvar.h>
65 #include <sys/rm_platter.h>
66 #include <sys/privregs.h>
67 #include <sys/cyclic.h>
68 #include <sys/note.h>
69 #include <sys/pci_intr_lib.h>
70 #include <sys/sunndi.h>
71 #include <sys/hpet.h>
72 #include <sys/clock.h>
73
74 /*
75 * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm
76 * but not apix.
77 * These functions may be moved to xpv_psm later when apix and pcplusmp
78 * are merged together
79 */
80
81 /*
82 * Local Function Prototypes
83 */
84 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
85 static void apic_xlate_vector_free_timeout_handler(void *arg);
86 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
87 int new_bind_cpu, int apicindex, int intin_no, int which_irq,
88 struct ioapic_reprogram_data *drep);
89 static int apic_setup_irq_table(dev_info_t *dip, int irqno,
90 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
91 int type);
92 static void apic_try_deferred_reprogram(int ipl, int vect);
93 static void delete_defer_repro_ent(int which_irq);
94 static void apic_ioapic_wait_pending_clear(int ioapicindex,
95 int intin_no);
96
97 extern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
98 int ipin, int *pci_irqp, iflag_t *intr_flagp);
99 extern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
100 int child_ipin, struct apic_io_intr **intrp);
101 extern uchar_t acpi_find_ioapic(int irq);
102 extern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
103 extern int apic_find_bus_id(int bustype);
104 extern int apic_find_intin(uchar_t ioapic, uchar_t intin);
105 extern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
106
107 extern int apic_sci_vect;
108 extern iflag_t apic_sci_flags;
109 /* ACPI HPET interrupt configuration; -1 if HPET not used */
110 extern int apic_hpet_vect;
111 extern iflag_t apic_hpet_flags;
112 extern int apic_intr_policy;
113 extern char *psm_name;
114
115 /*
116 * number of bits per byte, from <sys/param.h>
117 */
118 #define UCHAR_MAX UINT8_MAX
119
120 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
121 extern int apic_max_reps_clear_pending;
122
123 /* The irq # is implicit in the array index: */
124 struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
125 /*
126 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
127 * is indexed by IRQ number, NOT by vector number.
128 */
129
130 extern int apic_int_busy_mark;
131 extern int apic_int_free_mark;
132 extern int apic_diff_for_redistribution;
133 extern int apic_sample_factor_redistribution;
134 extern int apic_redist_cpu_skip;
135 extern int apic_num_imbalance;
136 extern int apic_num_rebind;
137
138 /* timeout for xlate_vector, mark_vector */
139 int apic_revector_timeout = 16 * 10000; /* 160 millisec */
140
141 extern int apic_defconf;
142 extern int apic_irq_translate;
143
144 extern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */
145
146 extern uchar_t apic_io_vectbase[MAX_IO_APIC];
147
148 extern boolean_t ioapic_mask_workaround[MAX_IO_APIC];
149
150 /*
151 * First available slot to be used as IRQ index into the apic_irq_table
152 * for those interrupts (like MSI/X) that don't have a physical IRQ.
153 */
154 extern int apic_first_avail_irq;
155
156 /*
157 * apic_defer_reprogram_lock ensures that only one processor is handling
158 * deferred interrupt programming at *_intr_exit time.
159 */
160 static lock_t apic_defer_reprogram_lock;
161
162 /*
163 * The current number of deferred reprogrammings outstanding
164 */
165 uint_t apic_reprogram_outstanding = 0;
166
167 #ifdef DEBUG
168 /*
169 * Counters that keep track of deferred reprogramming stats
170 */
171 uint_t apic_intr_deferrals = 0;
172 uint_t apic_intr_deliver_timeouts = 0;
173 uint_t apic_last_ditch_reprogram_failures = 0;
174 uint_t apic_deferred_setup_failures = 0;
175 uint_t apic_defer_repro_total_retries = 0;
176 uint_t apic_defer_repro_successes = 0;
177 uint_t apic_deferred_spurious_enters = 0;
178 #endif
179
180 extern int apic_io_max;
181 extern struct apic_io_intr *apic_io_intrp;
182
183 uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1];
184
185 extern uint32_t eisa_level_intr_mask;
186 /* At least MSB will be set if EISA bus */
187
188 extern int apic_pci_bus_total;
189 extern uchar_t apic_single_pci_busid;
190
191 /*
192 * Following declarations are for revectoring; used when ISRs at different
193 * IPLs share an irq.
194 */
195 static lock_t apic_revector_lock;
196 int apic_revector_pending = 0;
197 static uchar_t *apic_oldvec_to_newvec;
198 static uchar_t *apic_newvec_to_oldvec;
199
200 /* ACPI Interrupt Source Override Structure ptr */
201 extern ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop;
202 extern int acpi_iso_cnt;
203
204 /*
205 * Auto-configuration routines
206 */
207
208 /*
209 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
210 * are also set to NULL. vector->irq is set to a value which cannot map
211 * to a real irq to show that it is free.
212 */
213 void
214 apic_init_common(void)
215 {
216 int i, j, indx;
217 int *iptr;
218
219 /*
220 * Initialize apic_ipls from apic_vectortoipl. This array is
221 * used in apic_intr_enter to determine the IPL to use for the
222 * corresponding vector. On some systems, due to hardware errata
223 * and interrupt sharing, the IPL may not correspond to the IPL listed
224 * in apic_vectortoipl (see apic_addspl and apic_delspl).
225 */
226 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
227 indx = i * APIC_VECTOR_PER_IPL;
228
229 for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
230 apic_ipls[indx] = apic_vectortoipl[i];
231 }
232
233 /* cpu 0 is always up (for now) */
234 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
235
236 iptr = (int *)&apic_irq_table[0];
237 for (i = 0; i <= APIC_MAX_VECTOR; i++) {
238 apic_level_intr[i] = 0;
239 *iptr++ = NULL;
240 apic_vector_to_irq[i] = APIC_RESV_IRQ;
241
242 /* These *must* be initted to B_TRUE! */
243 apic_reprogram_info[i].done = B_TRUE;
244 apic_reprogram_info[i].irqp = NULL;
245 apic_reprogram_info[i].tries = 0;
246 apic_reprogram_info[i].bindcpu = 0;
247 }
248
249 /*
250 * Allocate a dummy irq table entry for the reserved entry.
251 * This takes care of the race between removing an irq and
252 * clock detecting a CPU in that irq during interrupt load
253 * sampling.
254 */
255 apic_irq_table[APIC_RESV_IRQ] =
256 kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
257
258 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
259 }
260
261 void
262 ioapic_init_intr(int mask_apic)
263 {
264 int ioapic_ix;
265 struct intrspec ispec;
266 apic_irq_t *irqptr;
267 int i, j;
268 ulong_t iflag;
269
270 LOCK_INIT_CLEAR(&apic_revector_lock);
271 LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
272
273 /* mask interrupt vectors */
274 for (j = 0; j < apic_io_max && mask_apic; j++) {
275 int intin_max;
276
277 ioapic_ix = j;
278 /* Bits 23-16 define the maximum redirection entries */
279 intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
280 & 0xff;
281 for (i = 0; i <= intin_max; i++)
282 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
283 }
284
285 /*
286 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
287 */
288 if (apic_sci_vect > 0) {
289 /*
290 * acpica has already done add_avintr(); we just
291 * to finish the job by mimicing translate_irq()
292 *
293 * Fake up an intrspec and setup the tables
294 */
295 ispec.intrspec_vec = apic_sci_vect;
296 ispec.intrspec_pri = SCI_IPL;
297
298 if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
299 &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
300 cmn_err(CE_WARN, "!apic: SCI setup failed");
301 return;
302 }
303 irqptr = apic_irq_table[apic_sci_vect];
304
305 iflag = intr_clear();
306 lock_set(&apic_ioapic_lock);
307
308 /* Program I/O APIC */
309 (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
310
311 lock_clear(&apic_ioapic_lock);
312 intr_restore(iflag);
313
314 irqptr->airq_share++;
315 }
316
317 /*
318 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
319 */
320 if (apic_hpet_vect > 0) {
321 /*
322 * hpet has already done add_avintr(); we just need
323 * to finish the job by mimicing translate_irq()
324 *
325 * Fake up an intrspec and setup the tables
326 */
327 ispec.intrspec_vec = apic_hpet_vect;
328 ispec.intrspec_pri = CBE_HIGH_PIL;
329
330 if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL,
331 &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) {
332 cmn_err(CE_WARN, "!apic: HPET setup failed");
333 return;
334 }
335 irqptr = apic_irq_table[apic_hpet_vect];
336
337 iflag = intr_clear();
338 lock_set(&apic_ioapic_lock);
339
340 /* Program I/O APIC */
341 (void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE);
342
343 lock_clear(&apic_ioapic_lock);
344 intr_restore(iflag);
345
346 irqptr->airq_share++;
347 }
348 }
349
350 /*
351 * Add mask bits to disable interrupt vector from happening
352 * at or above IPL. In addition, it should remove mask bits
353 * to enable interrupt vectors below the given IPL.
354 *
355 * Both add and delspl are complicated by the fact that different interrupts
356 * may share IRQs. This can happen in two ways.
357 * 1. The same H/W line is shared by more than 1 device
358 * 1a. with interrupts at different IPLs
359 * 1b. with interrupts at same IPL
360 * 2. We ran out of vectors at a given IPL and started sharing vectors.
361 * 1b and 2 should be handled gracefully, except for the fact some ISRs
362 * will get called often when no interrupt is pending for the device.
363 * For 1a, we handle it at the higher IPL.
364 */
365 /*ARGSUSED*/
366 int
367 apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
368 {
369 uchar_t vector;
370 ulong_t iflag;
371 apic_irq_t *irqptr, *irqheadptr;
372 int irqindex;
373
374 ASSERT(max_ipl <= UCHAR_MAX);
375 irqindex = IRQINDEX(irqno);
376
377 if ((irqindex == -1) || (!apic_irq_table[irqindex]))
378 return (PSM_FAILURE);
379
380 mutex_enter(&airq_mutex);
381 irqptr = irqheadptr = apic_irq_table[irqindex];
382
383 DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
384 "vector=0x%x\n", (void *)irqptr->airq_dip,
385 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
386
387 while (irqptr) {
388 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
389 break;
390 irqptr = irqptr->airq_next;
391 }
392 irqptr->airq_share++;
393
394 mutex_exit(&airq_mutex);
395
396 /* return if it is not hardware interrupt */
397 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
398 return (PSM_SUCCESS);
399
400 /* Or if there are more interupts at a higher IPL */
401 if (ipl != max_ipl)
402 return (PSM_SUCCESS);
403
404 /*
405 * if apic_picinit() has not been called yet, just return.
406 * At the end of apic_picinit(), we will call setup_io_intr().
407 */
408
409 if (!apic_picinit_called)
410 return (PSM_SUCCESS);
411
412 /*
413 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
414 * return failure.
415 */
416 if (irqptr->airq_ipl != max_ipl &&
417 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
418
419 vector = apic_allocate_vector(max_ipl, irqindex, 1);
420 if (vector == 0) {
421 irqptr->airq_share--;
422 return (PSM_FAILURE);
423 }
424 irqptr = irqheadptr;
425 apic_mark_vector(irqptr->airq_vector, vector);
426 while (irqptr) {
427 irqptr->airq_vector = vector;
428 irqptr->airq_ipl = (uchar_t)max_ipl;
429 /*
430 * reprogram irq being added and every one else
431 * who is not in the UNINIT state
432 */
433 if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
434 irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
435 apic_record_rdt_entry(irqptr, irqindex);
436
437 iflag = intr_clear();
438 lock_set(&apic_ioapic_lock);
439
440 (void) apic_setup_io_intr(irqptr, irqindex,
441 B_FALSE);
442
443 lock_clear(&apic_ioapic_lock);
444 intr_restore(iflag);
445 }
446 irqptr = irqptr->airq_next;
447 }
448 return (PSM_SUCCESS);
449
450 } else if (irqptr->airq_ipl != max_ipl &&
451 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
452 /*
453 * We cannot upgrade the vector, but we can change
454 * the IPL that this vector induces.
455 *
456 * Note that we subtract APIC_BASE_VECT from the vector
457 * here because this array is used in apic_intr_enter
458 * (no need to add APIC_BASE_VECT in that hot code
459 * path since we can do it in the rarely-executed path
460 * here).
461 */
462 apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
463 (uchar_t)max_ipl;
464
465 irqptr = irqheadptr;
466 while (irqptr) {
467 irqptr->airq_ipl = (uchar_t)max_ipl;
468 irqptr = irqptr->airq_next;
469 }
470
471 return (PSM_SUCCESS);
472 }
473
474 ASSERT(irqptr);
475
476 iflag = intr_clear();
477 lock_set(&apic_ioapic_lock);
478
479 (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
480
481 lock_clear(&apic_ioapic_lock);
482 intr_restore(iflag);
483
484 return (PSM_SUCCESS);
485 }
486
487 /*
488 * Recompute mask bits for the given interrupt vector.
489 * If there is no interrupt servicing routine for this
490 * vector, this function should disable interrupt vector
491 * from happening at all IPLs. If there are still
492 * handlers using the given vector, this function should
493 * disable the given vector from happening below the lowest
494 * IPL of the remaining hadlers.
495 */
496 /*ARGSUSED*/
497 int
498 apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
499 {
500 uchar_t vector;
501 uint32_t bind_cpu;
502 int intin, irqindex;
503 int ioapic_ix;
504 apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp;
505 ulong_t iflag;
506
507 mutex_enter(&airq_mutex);
508 irqindex = IRQINDEX(irqno);
509 irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex];
510
511 DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
512 "vector=0x%x\n", (void *)irqptr->airq_dip,
513 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
514
515 while (irqptr) {
516 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
517 break;
518 preirqptr = irqptr;
519 irqptr = irqptr->airq_next;
520 }
521 ASSERT(irqptr);
522
523 irqptr->airq_share--;
524
525 mutex_exit(&airq_mutex);
526
527 /*
528 * If there are more interrupts at a higher IPL, we don't need
529 * to disable anything.
530 */
531 if (ipl < max_ipl)
532 return (PSM_SUCCESS);
533
534 /* return if it is not hardware interrupt */
535 if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
536 return (PSM_SUCCESS);
537
538 if (!apic_picinit_called) {
539 /*
540 * Clear irq_struct. If two devices shared an intpt
541 * line & 1 unloaded before picinit, we are hosed. But, then
542 * we hope the machine survive.
543 */
544 irqptr->airq_mps_intr_index = FREE_INDEX;
545 irqptr->airq_temp_cpu = IRQ_UNINIT;
546 apic_free_vector(irqptr->airq_vector);
547 return (PSM_SUCCESS);
548 }
549 /*
550 * Downgrade vector to new max_ipl if needed. If we cannot allocate,
551 * use old IPL. Not very elegant, but it should work.
552 */
553 if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
554 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
555 apic_irq_t *irqp;
556 if (vector = apic_allocate_vector(max_ipl, irqno, 1)) {
557 apic_mark_vector(irqheadptr->airq_vector, vector);
558 irqp = irqheadptr;
559 while (irqp) {
560 irqp->airq_vector = vector;
561 irqp->airq_ipl = (uchar_t)max_ipl;
562 if (irqp->airq_temp_cpu != IRQ_UNINIT) {
563 apic_record_rdt_entry(irqp, irqindex);
564
565 iflag = intr_clear();
566 lock_set(&apic_ioapic_lock);
567
568 (void) apic_setup_io_intr(irqp,
569 irqindex, B_FALSE);
570
571 lock_clear(&apic_ioapic_lock);
572 intr_restore(iflag);
573 }
574 irqp = irqp->airq_next;
575 }
576 }
577
578 } else if (irqptr->airq_ipl != max_ipl &&
579 max_ipl != PSM_INVALID_IPL &&
580 ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
581
582 /*
583 * We cannot downgrade the IPL of the vector below the vector's
584 * hardware priority. If we did, it would be possible for a
585 * higher-priority hardware vector to interrupt a CPU running at an IPL
586 * lower than the hardware priority of the interrupting vector (but
587 * higher than the soft IPL of this IRQ). When this happens, we would
588 * then try to drop the IPL BELOW what it was (effectively dropping
589 * below base_spl) which would be potentially catastrophic.
590 *
591 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
592 * (hardware IPL of 4). Further assume that the old IPL of this IRQ
593 * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
594 * an IPL of 1, it would be possible for the processor to be executing
595 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
596 * the currently-executing ISR. When apic_intr_enter consults
597 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
598 * so even though the processor was running at IPL 4, an IPL 1
599 * interrupt will have interrupted it, which must not happen)).
600 *
601 * Effectively, this means that the hardware priority corresponding to
602 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
603 * hardware priority.
604 *
605 * (In the above example, then, after removal of the IPL 4 device's
606 * interrupt handler, the new IPL will continue to be 4 because the
607 * hardware priority that IPL 1 implies is lower than the hardware
608 * priority of the vector used.)
609 */
610 /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
611 const int apic_ipls_index = irqptr->airq_vector -
612 APIC_BASE_VECT;
613 const int vect_inherent_hwpri = irqptr->airq_vector >>
614 APIC_IPL_SHIFT;
615
616 /*
617 * If there are still devices using this IRQ, determine the
618 * new ipl to use.
619 */
620 if (irqptr->airq_share) {
621 int vect_desired_hwpri, hwpri;
622
623 ASSERT(max_ipl < MAXIPL);
624 vect_desired_hwpri = apic_ipltopri[max_ipl] >>
625 APIC_IPL_SHIFT;
626
627 /*
628 * If the desired IPL's hardware priority is lower
629 * than that of the vector, use the hardware priority
630 * of the vector to determine the new IPL.
631 */
632 hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
633 vect_inherent_hwpri : vect_desired_hwpri;
634
635 /*
636 * Now, to get the right index for apic_vectortoipl,
637 * we need to subtract APIC_BASE_VECT from the
638 * hardware-vector-equivalent (in hwpri). Since hwpri
639 * is already shifted, we shift APIC_BASE_VECT before
640 * doing the subtraction.
641 */
642 hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
643
644 ASSERT(hwpri >= 0);
645 ASSERT(hwpri < MAXIPL);
646 max_ipl = apic_vectortoipl[hwpri];
647 apic_ipls[apic_ipls_index] = max_ipl;
648
649 irqp = irqheadptr;
650 while (irqp) {
651 irqp->airq_ipl = (uchar_t)max_ipl;
652 irqp = irqp->airq_next;
653 }
654 } else {
655 /*
656 * No more devices on this IRQ, so reset this vector's
657 * element in apic_ipls to the original IPL for this
658 * vector
659 */
660 apic_ipls[apic_ipls_index] =
661 apic_vectortoipl[vect_inherent_hwpri];
662 }
663 }
664
665 /*
666 * If there are still active interrupts, we are done.
667 */
668 if (irqptr->airq_share)
669 return (PSM_SUCCESS);
670
671 iflag = intr_clear();
672 lock_set(&apic_ioapic_lock);
673
674 if (irqptr->airq_mps_intr_index == MSI_INDEX) {
675 /*
676 * Disable the MSI vector
677 * Make sure we only disable on the last
678 * of the multi-MSI support
679 */
680 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
681 apic_pci_msi_disable_mode(irqptr->airq_dip,
682 DDI_INTR_TYPE_MSI);
683 }
684 } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
685 /*
686 * Disable the MSI-X vector
687 * needs to clear its mask and addr/data for each MSI-X
688 */
689 apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
690 irqptr->airq_origirq);
691 /*
692 * Make sure we only disable on the last MSI-X
693 */
694 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
695 apic_pci_msi_disable_mode(irqptr->airq_dip,
696 DDI_INTR_TYPE_MSIX);
697 }
698 } else {
699 /*
700 * The assumption here is that this is safe, even for
701 * systems with IOAPICs that suffer from the hardware
702 * erratum because all devices have been quiesced before
703 * they unregister their interrupt handlers. If that
704 * assumption turns out to be false, this mask operation
705 * can induce the same erratum result we're trying to
706 * avoid.
707 */
708 ioapic_ix = irqptr->airq_ioapicindex;
709 intin = irqptr->airq_intin_no;
710 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
711 }
712
713 apic_vt_ops->apic_intrmap_free_entry(&irqptr->airq_intrmap_private);
714
715 /*
716 * This irq entry is the only one in the chain.
717 */
718 if (irqheadptr->airq_next == NULL) {
719 ASSERT(irqheadptr == irqptr);
720 bind_cpu = irqptr->airq_temp_cpu;
721 if (((uint32_t)bind_cpu != IRQ_UNBOUND) &&
722 ((uint32_t)bind_cpu != IRQ_UNINIT)) {
723 ASSERT(apic_cpu_in_range(bind_cpu));
724 if (bind_cpu & IRQ_USER_BOUND) {
725 /* If hardbound, temp_cpu == cpu */
726 bind_cpu &= ~IRQ_USER_BOUND;
727 apic_cpus[bind_cpu].aci_bound--;
728 } else
729 apic_cpus[bind_cpu].aci_temp_bound--;
730 }
731 irqptr->airq_temp_cpu = IRQ_UNINIT;
732 irqptr->airq_mps_intr_index = FREE_INDEX;
733 lock_clear(&apic_ioapic_lock);
734 intr_restore(iflag);
735 apic_free_vector(irqptr->airq_vector);
736 return (PSM_SUCCESS);
737 }
738
739 /*
740 * If we get here, we are sharing the vector and there are more than
741 * one active irq entries in the chain.
742 */
743 lock_clear(&apic_ioapic_lock);
744 intr_restore(iflag);
745
746 mutex_enter(&airq_mutex);
747 /* Remove the irq entry from the chain */
748 if (irqptr == irqheadptr) { /* The irq entry is at the head */
749 apic_irq_table[irqindex] = irqptr->airq_next;
750 } else {
751 preirqptr->airq_next = irqptr->airq_next;
752 }
753 /* Free the irq entry */
754 kmem_free(irqptr, sizeof (apic_irq_t));
755 mutex_exit(&airq_mutex);
756
757 return (PSM_SUCCESS);
758 }
759
760 /*
761 * apic_introp_xlate() replaces apic_translate_irq() and is
762 * called only from apic_intr_ops(). With the new ADII framework,
763 * the priority can no longer be retrieved through i_ddi_get_intrspec().
764 * It has to be passed in from the caller.
765 *
766 * Return value:
767 * Success: irqno for the given device
768 * Failure: -1
769 */
770 int
771 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
772 {
773 char dev_type[16];
774 int dev_len, pci_irq, newirq, bustype, devid, busid, i;
775 int irqno = ispec->intrspec_vec;
776 ddi_acc_handle_t cfg_handle;
777 uchar_t ipin;
778 struct apic_io_intr *intrp;
779 iflag_t intr_flag;
780 ACPI_SUBTABLE_HEADER *hp;
781 ACPI_MADT_INTERRUPT_OVERRIDE *isop;
782 apic_irq_t *airqp;
783 int parent_is_pci_or_pciex = 0;
784 int child_is_pciex = 0;
785
786 DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
787 "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
788 irqno));
789
790 dev_len = sizeof (dev_type);
791 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
792 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
793 &dev_len) == DDI_PROP_SUCCESS) {
794 if ((strcmp(dev_type, "pci") == 0) ||
795 (strcmp(dev_type, "pciex") == 0))
796 parent_is_pci_or_pciex = 1;
797 }
798
799 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
800 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
801 &dev_len) == DDI_PROP_SUCCESS) {
802 if (strstr(dev_type, "pciex"))
803 child_is_pciex = 1;
804 }
805
806 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
807 if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
808 airqp->airq_iflag.bustype =
809 child_is_pciex ? BUS_PCIE : BUS_PCI;
810 return (apic_vector_to_irq[airqp->airq_vector]);
811 }
812 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
813 NULL, type));
814 }
815
816 bustype = 0;
817
818 /* check if we have already translated this irq */
819 mutex_enter(&airq_mutex);
820 newirq = apic_min_device_irq;
821 for (; newirq <= apic_max_device_irq; newirq++) {
822 airqp = apic_irq_table[newirq];
823 while (airqp) {
824 if ((airqp->airq_dip == dip) &&
825 (airqp->airq_origirq == irqno) &&
826 (airqp->airq_mps_intr_index != FREE_INDEX)) {
827
828 mutex_exit(&airq_mutex);
829 return (VIRTIRQ(newirq, airqp->airq_share_id));
830 }
831 airqp = airqp->airq_next;
832 }
833 }
834 mutex_exit(&airq_mutex);
835
836 if (apic_defconf)
837 goto defconf;
838
839 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
840 goto nonpci;
841
842 if (parent_is_pci_or_pciex) {
843 /* pci device */
844 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
845 goto nonpci;
846 if (busid == 0 && apic_pci_bus_total == 1)
847 busid = (int)apic_single_pci_busid;
848
849 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
850 return (-1);
851 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
852 pci_config_teardown(&cfg_handle);
853 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
854 if (apic_acpi_translate_pci_irq(dip, busid, devid,
855 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
856 return (-1);
857
858 intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
859 return (apic_setup_irq_table(dip, pci_irq, NULL, ispec,
860 &intr_flag, type));
861 } else {
862 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
863 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
864 == NULL) {
865 if ((pci_irq = apic_handle_pci_pci_bridge(dip,
866 devid, ipin, &intrp)) == -1)
867 return (-1);
868 }
869 return (apic_setup_irq_table(dip, pci_irq, intrp, ispec,
870 NULL, type));
871 }
872 } else if (strcmp(dev_type, "isa") == 0)
873 bustype = BUS_ISA;
874 else if (strcmp(dev_type, "eisa") == 0)
875 bustype = BUS_EISA;
876
877 nonpci:
878 if (apic_enable_acpi && !apic_use_acpi_madt_only) {
879 /* search iso entries first */
880 if (acpi_iso_cnt != 0) {
881 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
882 i = 0;
883 while (i < acpi_iso_cnt) {
884 if (hp->Type ==
885 ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
886 isop =
887 (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
888 if (isop->Bus == 0 &&
889 isop->SourceIrq == irqno) {
890 newirq = isop->GlobalIrq;
891 intr_flag.intr_po =
892 isop->IntiFlags &
893 ACPI_MADT_POLARITY_MASK;
894 intr_flag.intr_el =
895 (isop->IntiFlags &
896 ACPI_MADT_TRIGGER_MASK)
897 >> 2;
898 intr_flag.bustype = BUS_ISA;
899
900 return (apic_setup_irq_table(
901 dip, newirq, NULL, ispec,
902 &intr_flag, type));
903
904 }
905 i++;
906 }
907 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
908 hp->Length);
909 }
910 }
911 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
912 intr_flag.intr_el = INTR_EL_EDGE;
913 intr_flag.bustype = BUS_ISA;
914 return (apic_setup_irq_table(dip, irqno, NULL, ispec,
915 &intr_flag, type));
916 } else {
917 if (bustype == 0) /* not initialized */
918 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
919 for (i = 0; i < 2; i++) {
920 if (((busid = apic_find_bus_id(bustype)) != -1) &&
921 ((intrp = apic_find_io_intr_w_busid(irqno, busid))
922 != NULL)) {
923 if ((newirq = apic_setup_irq_table(dip, irqno,
924 intrp, ispec, NULL, type)) != -1) {
925 return (newirq);
926 }
927 goto defconf;
928 }
929 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
930 }
931 }
932
933 /* MPS default configuration */
934 defconf:
935 newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
936 if (newirq == -1)
937 return (-1);
938 ASSERT(IRQINDEX(newirq) == irqno);
939 ASSERT(apic_irq_table[irqno]);
940 return (newirq);
941 }
942
943 /*
944 * Attempt to share vector with someone else
945 */
946 static int
947 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
948 uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
949 {
950 #ifdef DEBUG
951 apic_irq_t *tmpirqp = NULL;
952 #endif /* DEBUG */
953 apic_irq_t *irqptr, dummyirq;
954 int newirq, chosen_irq = -1, share = 127;
955 int lowest, highest, i;
956 uchar_t share_id;
957
958 DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
959 "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
960
961 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
962 lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
963
964 if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
965 lowest -= APIC_VECTOR_PER_IPL;
966 dummyirq.airq_mps_intr_index = intr_index;
967 dummyirq.airq_ioapicindex = ioapicindex;
968 dummyirq.airq_intin_no = ipin;
969 if (intr_flagp)
970 dummyirq.airq_iflag = *intr_flagp;
971 apic_record_rdt_entry(&dummyirq, irqno);
972 for (i = lowest; i <= highest; i++) {
973 newirq = apic_vector_to_irq[i];
974 if (newirq == APIC_RESV_IRQ)
975 continue;
976 irqptr = apic_irq_table[newirq];
977
978 if ((dummyirq.airq_rdt_entry & 0xFF00) !=
979 (irqptr->airq_rdt_entry & 0xFF00))
980 /* not compatible */
981 continue;
982
983 if (irqptr->airq_share < share) {
984 share = irqptr->airq_share;
985 chosen_irq = newirq;
986 }
987 }
988 if (chosen_irq != -1) {
989 /*
990 * Assign a share id which is free or which is larger
991 * than the largest one.
992 */
993 share_id = 1;
994 mutex_enter(&airq_mutex);
995 irqptr = apic_irq_table[chosen_irq];
996 while (irqptr) {
997 if (irqptr->airq_mps_intr_index == FREE_INDEX) {
998 share_id = irqptr->airq_share_id;
999 break;
1000 }
1001 if (share_id <= irqptr->airq_share_id)
1002 share_id = irqptr->airq_share_id + 1;
1003 #ifdef DEBUG
1004 tmpirqp = irqptr;
1005 #endif /* DEBUG */
1006 irqptr = irqptr->airq_next;
1007 }
1008 if (!irqptr) {
1009 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1010 irqptr->airq_temp_cpu = IRQ_UNINIT;
1011 irqptr->airq_next =
1012 apic_irq_table[chosen_irq]->airq_next;
1013 apic_irq_table[chosen_irq]->airq_next = irqptr;
1014 #ifdef DEBUG
1015 tmpirqp = apic_irq_table[chosen_irq];
1016 #endif /* DEBUG */
1017 }
1018 irqptr->airq_mps_intr_index = intr_index;
1019 irqptr->airq_ioapicindex = ioapicindex;
1020 irqptr->airq_intin_no = ipin;
1021 if (intr_flagp)
1022 irqptr->airq_iflag = *intr_flagp;
1023 irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
1024 irqptr->airq_share_id = share_id;
1025 apic_record_rdt_entry(irqptr, irqno);
1026 *irqptrp = irqptr;
1027 #ifdef DEBUG
1028 /* shuffle the pointers to test apic_delspl path */
1029 if (tmpirqp) {
1030 tmpirqp->airq_next = irqptr->airq_next;
1031 irqptr->airq_next = apic_irq_table[chosen_irq];
1032 apic_irq_table[chosen_irq] = irqptr;
1033 }
1034 #endif /* DEBUG */
1035 mutex_exit(&airq_mutex);
1036 return (VIRTIRQ(chosen_irq, share_id));
1037 }
1038 return (-1);
1039 }
1040
1041 /*
1042 * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
1043 * is used already, we will try to allocate a new irqno.
1044 *
1045 * Return value:
1046 * Success: irqno
1047 * Failure: -1
1048 */
1049 static int
1050 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
1051 struct intrspec *ispec, iflag_t *intr_flagp, int type)
1052 {
1053 int origirq = ispec->intrspec_vec;
1054 uchar_t ipl = ispec->intrspec_pri;
1055 int newirq, intr_index;
1056 uchar_t ipin, ioapic, ioapicindex, vector;
1057 apic_irq_t *irqptr;
1058 major_t major;
1059 dev_info_t *sdip;
1060
1061 DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
1062 "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
1063
1064 ASSERT(ispec != NULL);
1065
1066 major = (dip != NULL) ? ddi_driver_major(dip) : 0;
1067
1068 if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
1069 /* MSI/X doesn't need to setup ioapic stuffs */
1070 ioapicindex = 0xff;
1071 ioapic = 0xff;
1072 ipin = (uchar_t)0xff;
1073 intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
1074 MSIX_INDEX;
1075 mutex_enter(&airq_mutex);
1076 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
1077 mutex_exit(&airq_mutex);
1078 /* need an irq for MSI/X to index into autovect[] */
1079 cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
1080 ddi_get_name(dip), ddi_get_instance(dip));
1081 return (-1);
1082 }
1083 mutex_exit(&airq_mutex);
1084
1085 } else if (intrp != NULL) {
1086 intr_index = (int)(intrp - apic_io_intrp);
1087 ioapic = intrp->intr_destid;
1088 ipin = intrp->intr_destintin;
1089 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
1090 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
1091 if (apic_io_id[ioapicindex] == ioapic)
1092 break;
1093 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
1094 (ioapic == INTR_ALL_APIC));
1095
1096 /* check whether this intin# has been used by another irqno */
1097 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
1098 return (newirq);
1099 }
1100
1101 } else if (intr_flagp != NULL) {
1102 /* ACPI case */
1103 intr_index = ACPI_INDEX;
1104 ioapicindex = acpi_find_ioapic(irqno);
1105 ASSERT(ioapicindex != 0xFF);
1106 ioapic = apic_io_id[ioapicindex];
1107 ipin = irqno - apic_io_vectbase[ioapicindex];
1108 if (apic_irq_table[irqno] &&
1109 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
1110 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
1111 apic_irq_table[irqno]->airq_ioapicindex ==
1112 ioapicindex);
1113 return (irqno);
1114 }
1115
1116 } else {
1117 /* default configuration */
1118 ioapicindex = 0;
1119 ioapic = apic_io_id[ioapicindex];
1120 ipin = (uchar_t)irqno;
1121 intr_index = DEFAULT_INDEX;
1122 }
1123
1124 if (ispec == NULL) {
1125 APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n",
1126 irqno));
1127 } else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
1128 if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
1129 ipl, ioapicindex, ipin, &irqptr)) != -1) {
1130 irqptr->airq_ipl = ipl;
1131 irqptr->airq_origirq = (uchar_t)origirq;
1132 irqptr->airq_dip = dip;
1133 irqptr->airq_major = major;
1134 sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
1135 /* This is OK to do really */
1136 if (sdip == NULL) {
1137 cmn_err(CE_WARN, "Sharing vectors: %s"
1138 " instance %d and SCI",
1139 ddi_get_name(dip), ddi_get_instance(dip));
1140 } else {
1141 cmn_err(CE_WARN, "Sharing vectors: %s"
1142 " instance %d and %s instance %d",
1143 ddi_get_name(sdip), ddi_get_instance(sdip),
1144 ddi_get_name(dip), ddi_get_instance(dip));
1145 }
1146 return (newirq);
1147 }
1148 /* try high priority allocation now that share has failed */
1149 if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
1150 cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
1151 ddi_get_name(dip), ddi_get_instance(dip));
1152 return (-1);
1153 }
1154 }
1155
1156 mutex_enter(&airq_mutex);
1157 if (apic_irq_table[irqno] == NULL) {
1158 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1159 irqptr->airq_temp_cpu = IRQ_UNINIT;
1160 apic_irq_table[irqno] = irqptr;
1161 } else {
1162 irqptr = apic_irq_table[irqno];
1163 if (irqptr->airq_mps_intr_index != FREE_INDEX) {
1164 /*
1165 * The slot is used by another irqno, so allocate
1166 * a free irqno for this interrupt
1167 */
1168 newirq = apic_allocate_irq(apic_first_avail_irq);
1169 if (newirq == -1) {
1170 mutex_exit(&airq_mutex);
1171 return (-1);
1172 }
1173 irqno = newirq;
1174 irqptr = apic_irq_table[irqno];
1175 if (irqptr == NULL) {
1176 irqptr = kmem_zalloc(sizeof (apic_irq_t),
1177 KM_SLEEP);
1178 irqptr->airq_temp_cpu = IRQ_UNINIT;
1179 apic_irq_table[irqno] = irqptr;
1180 }
1181 vector = apic_modify_vector(vector, newirq);
1182 }
1183 }
1184 apic_max_device_irq = max(irqno, apic_max_device_irq);
1185 apic_min_device_irq = min(irqno, apic_min_device_irq);
1186 mutex_exit(&airq_mutex);
1187 irqptr->airq_ioapicindex = ioapicindex;
1188 irqptr->airq_intin_no = ipin;
1189 irqptr->airq_ipl = ipl;
1190 irqptr->airq_vector = vector;
1191 irqptr->airq_origirq = (uchar_t)origirq;
1192 irqptr->airq_share_id = 0;
1193 irqptr->airq_mps_intr_index = (short)intr_index;
1194 irqptr->airq_dip = dip;
1195 irqptr->airq_major = major;
1196 irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
1197 if (intr_flagp)
1198 irqptr->airq_iflag = *intr_flagp;
1199
1200 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
1201 /* setup I/O APIC entry for non-MSI/X interrupts */
1202 apic_record_rdt_entry(irqptr, irqno);
1203 }
1204 return (irqno);
1205 }
1206
1207 /*
1208 * return the cpu to which this intr should be bound.
1209 * Check properties or any other mechanism to see if user wants it
1210 * bound to a specific CPU. If so, return the cpu id with high bit set.
1211 * If not, use the policy to choose a cpu and return the id.
1212 */
1213 uint32_t
1214 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
1215 {
1216 int instance, instno, prop_len, bind_cpu, count;
1217 uint_t i, rc;
1218 uint32_t cpu;
1219 major_t major;
1220 char *name, *drv_name, *prop_val, *cptr;
1221 char prop_name[32];
1222 ulong_t iflag;
1223
1224
1225 if (apic_intr_policy == INTR_LOWEST_PRIORITY)
1226 return (IRQ_UNBOUND);
1227
1228 if (apic_nproc == 1)
1229 return (0);
1230
1231 drv_name = NULL;
1232 rc = DDI_PROP_NOT_FOUND;
1233 major = (major_t)-1;
1234 if (dip != NULL) {
1235 name = ddi_get_name(dip);
1236 major = ddi_name_to_major(name);
1237 drv_name = ddi_major_to_name(major);
1238 instance = ddi_get_instance(dip);
1239 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1240 i = apic_min_device_irq;
1241 for (; i <= apic_max_device_irq; i++) {
1242
1243 if ((i == irq) || (apic_irq_table[i] == NULL) ||
1244 (apic_irq_table[i]->airq_mps_intr_index
1245 == FREE_INDEX))
1246 continue;
1247
1248 if ((apic_irq_table[i]->airq_major == major) &&
1249 (!(apic_irq_table[i]->airq_cpu &
1250 IRQ_USER_BOUND))) {
1251
1252 cpu = apic_irq_table[i]->airq_cpu;
1253
1254 cmn_err(CE_CONT,
1255 "!%s: %s (%s) instance #%d "
1256 "irq 0x%x vector 0x%x ioapic 0x%x "
1257 "intin 0x%x is bound to cpu %d\n",
1258 psm_name,
1259 name, drv_name, instance, irq,
1260 apic_irq_table[irq]->airq_vector,
1261 ioapicid, intin, cpu);
1262 return (cpu);
1263 }
1264 }
1265 }
1266 /*
1267 * search for "drvname"_intpt_bind_cpus property first, the
1268 * syntax of the property should be "a[,b,c,...]" where
1269 * instance 0 binds to cpu a, instance 1 binds to cpu b,
1270 * instance 3 binds to cpu c...
1271 * ddi_getlongprop() will search /option first, then /
1272 * if "drvname"_intpt_bind_cpus doesn't exist, then find
1273 * intpt_bind_cpus property. The syntax is the same, and
1274 * it applies to all the devices if its "drvname" specific
1275 * property doesn't exist
1276 */
1277 (void) strcpy(prop_name, drv_name);
1278 (void) strcat(prop_name, "_intpt_bind_cpus");
1279 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1280 (caddr_t)&prop_val, &prop_len);
1281 if (rc != DDI_PROP_SUCCESS) {
1282 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1283 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1284 }
1285 }
1286 if (rc == DDI_PROP_SUCCESS) {
1287 for (i = count = 0; i < (prop_len - 1); i++)
1288 if (prop_val[i] == ',')
1289 count++;
1290 if (prop_val[i-1] != ',')
1291 count++;
1292 /*
1293 * if somehow the binding instances defined in the
1294 * property are not enough for this instno., then
1295 * reuse the pattern for the next instance until
1296 * it reaches the requested instno
1297 */
1298 instno = instance % count;
1299 i = 0;
1300 cptr = prop_val;
1301 while (i < instno)
1302 if (*cptr++ == ',')
1303 i++;
1304 bind_cpu = stoi(&cptr);
1305 kmem_free(prop_val, prop_len);
1306 /* if specific CPU is bogus, then default to next cpu */
1307 if (!apic_cpu_in_range(bind_cpu)) {
1308 cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
1309 psm_name, prop_name, prop_val, bind_cpu);
1310 rc = DDI_PROP_NOT_FOUND;
1311 } else {
1312 /* indicate that we are bound at user request */
1313 bind_cpu |= IRQ_USER_BOUND;
1314 }
1315 /*
1316 * no need to check apic_cpus[].aci_status, if specific CPU is
1317 * not up, then post_cpu_start will handle it.
1318 */
1319 }
1320 if (rc != DDI_PROP_SUCCESS) {
1321 iflag = intr_clear();
1322 lock_set(&apic_ioapic_lock);
1323 bind_cpu = apic_get_next_bind_cpu();
1324 lock_clear(&apic_ioapic_lock);
1325 intr_restore(iflag);
1326 }
1327
1328 if (drv_name != NULL)
1329 cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
1330 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1331 psm_name, name, drv_name, instance, irq,
1332 apic_irq_table[irq]->airq_vector, ioapicid, intin,
1333 bind_cpu & ~IRQ_USER_BOUND);
1334 else
1335 cmn_err(CE_CONT, "!%s: irq 0x%x "
1336 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
1337 psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
1338 intin, bind_cpu & ~IRQ_USER_BOUND);
1339
1340 return ((uint32_t)bind_cpu);
1341 }
1342
1343 /*
1344 * Mark vector as being in the process of being deleted. Interrupts
1345 * may still come in on some CPU. The moment an interrupt comes with
1346 * the new vector, we know we can free the old one. Called only from
1347 * addspl and delspl with interrupts disabled. Because an interrupt
1348 * can be shared, but no interrupt from either device may come in,
1349 * we also use a timeout mechanism, which we arbitrarily set to
1350 * apic_revector_timeout microseconds.
1351 */
1352 static void
1353 apic_mark_vector(uchar_t oldvector, uchar_t newvector)
1354 {
1355 ulong_t iflag;
1356
1357 iflag = intr_clear();
1358 lock_set(&apic_revector_lock);
1359 if (!apic_oldvec_to_newvec) {
1360 apic_oldvec_to_newvec =
1361 kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
1362 KM_NOSLEEP);
1363
1364 if (!apic_oldvec_to_newvec) {
1365 /*
1366 * This failure is not catastrophic.
1367 * But, the oldvec will never be freed.
1368 */
1369 apic_error |= APIC_ERR_MARK_VECTOR_FAIL;
1370 lock_clear(&apic_revector_lock);
1371 intr_restore(iflag);
1372 return;
1373 }
1374 apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
1375 }
1376
1377 /* See if we already did this for drivers which do double addintrs */
1378 if (apic_oldvec_to_newvec[oldvector] != newvector) {
1379 apic_oldvec_to_newvec[oldvector] = newvector;
1380 apic_newvec_to_oldvec[newvector] = oldvector;
1381 apic_revector_pending++;
1382 }
1383 lock_clear(&apic_revector_lock);
1384 intr_restore(iflag);
1385 (void) timeout(apic_xlate_vector_free_timeout_handler,
1386 (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
1387 }
1388
1389 /*
1390 * xlate_vector is called from intr_enter if revector_pending is set.
1391 * It will xlate it if needed and mark the old vector as free.
1392 */
1393 uchar_t
1394 apic_xlate_vector(uchar_t vector)
1395 {
1396 uchar_t newvector, oldvector = 0;
1397
1398 lock_set(&apic_revector_lock);
1399 /* Do we really need to do this ? */
1400 if (!apic_revector_pending) {
1401 lock_clear(&apic_revector_lock);
1402 return (vector);
1403 }
1404 if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
1405 oldvector = vector;
1406 else {
1407 /*
1408 * The incoming vector is new . See if a stale entry is
1409 * remaining
1410 */
1411 if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
1412 newvector = vector;
1413 }
1414
1415 if (oldvector) {
1416 apic_revector_pending--;
1417 apic_oldvec_to_newvec[oldvector] = 0;
1418 apic_newvec_to_oldvec[newvector] = 0;
1419 apic_free_vector(oldvector);
1420 lock_clear(&apic_revector_lock);
1421 /* There could have been more than one reprogramming! */
1422 return (apic_xlate_vector(newvector));
1423 }
1424 lock_clear(&apic_revector_lock);
1425 return (vector);
1426 }
1427
1428 void
1429 apic_xlate_vector_free_timeout_handler(void *arg)
1430 {
1431 ulong_t iflag;
1432 uchar_t oldvector, newvector;
1433
1434 oldvector = (uchar_t)(uintptr_t)arg;
1435 iflag = intr_clear();
1436 lock_set(&apic_revector_lock);
1437 if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
1438 apic_free_vector(oldvector);
1439 apic_oldvec_to_newvec[oldvector] = 0;
1440 apic_newvec_to_oldvec[newvector] = 0;
1441 apic_revector_pending--;
1442 }
1443
1444 lock_clear(&apic_revector_lock);
1445 intr_restore(iflag);
1446 }
1447
1448 /*
1449 * Bind interrupt corresponding to irq_ptr to bind_cpu.
1450 * Must be called with interrupts disabled and apic_ioapic_lock held
1451 */
1452 int
1453 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
1454 struct ioapic_reprogram_data *drep)
1455 {
1456 int ioapicindex, intin_no;
1457 uint32_t airq_temp_cpu;
1458 apic_cpus_info_t *cpu_infop;
1459 uint32_t rdt_entry;
1460 int which_irq;
1461 ioapic_rdt_t irdt;
1462
1463 which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
1464
1465 intin_no = irq_ptr->airq_intin_no;
1466 ioapicindex = irq_ptr->airq_ioapicindex;
1467 airq_temp_cpu = irq_ptr->airq_temp_cpu;
1468 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
1469 if (airq_temp_cpu & IRQ_USER_BOUND)
1470 /* Mask off high bit so it can be used as array index */
1471 airq_temp_cpu &= ~IRQ_USER_BOUND;
1472
1473 ASSERT(apic_cpu_in_range(airq_temp_cpu));
1474 }
1475
1476 /*
1477 * Can't bind to a CPU that's not accepting interrupts:
1478 */
1479 cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
1480 if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
1481 return (1);
1482
1483 /*
1484 * If we are about to change the interrupt vector for this interrupt,
1485 * and this interrupt is level-triggered, attached to an IOAPIC,
1486 * has been delivered to a CPU and that CPU has not handled it
1487 * yet, we cannot reprogram the IOAPIC now.
1488 */
1489 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1490
1491 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
1492 intin_no);
1493
1494 if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
1495 apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
1496 bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
1497
1498 return (0);
1499 }
1500
1501 /*
1502 * NOTE: We do not unmask the RDT here, as an interrupt MAY
1503 * still come in before we have a chance to reprogram it below.
1504 * The reprogramming below will simultaneously change and
1505 * unmask the RDT entry.
1506 */
1507
1508 if ((uint32_t)bind_cpu == IRQ_UNBOUND) {
1509 irdt.ir_lo = AV_LDEST | AV_LOPRI |
1510 irq_ptr->airq_rdt_entry;
1511
1512 irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET;
1513
1514 apic_vt_ops->apic_intrmap_alloc_entry(
1515 &irq_ptr->airq_intrmap_private, NULL,
1516 DDI_INTR_TYPE_FIXED, 1, ioapicindex);
1517 apic_vt_ops->apic_intrmap_map_entry(
1518 irq_ptr->airq_intrmap_private, (void *)&irdt,
1519 DDI_INTR_TYPE_FIXED, 1);
1520 apic_vt_ops->apic_intrmap_record_rdt(
1521 irq_ptr->airq_intrmap_private, &irdt);
1522
1523 /* Write the RDT entry -- no specific CPU binding */
1524 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1525 irdt.ir_hi | AV_TOALL);
1526
1527 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
1528 IRQ_UNBOUND)
1529 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1530
1531 /*
1532 * Write the vector, trigger, and polarity portion of
1533 * the RDT
1534 */
1535 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1536 irdt.ir_lo);
1537
1538 irq_ptr->airq_temp_cpu = IRQ_UNBOUND;
1539 return (0);
1540 }
1541 }
1542
1543 if (bind_cpu & IRQ_USER_BOUND) {
1544 cpu_infop->aci_bound++;
1545 } else {
1546 cpu_infop->aci_temp_bound++;
1547 }
1548 ASSERT(apic_cpu_in_range(bind_cpu));
1549
1550 if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
1551 apic_cpus[airq_temp_cpu].aci_temp_bound--;
1552 }
1553 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
1554
1555 irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
1556 irdt.ir_hi = cpu_infop->aci_local_id;
1557
1558 apic_vt_ops->apic_intrmap_alloc_entry(
1559 &irq_ptr->airq_intrmap_private, NULL, DDI_INTR_TYPE_FIXED,
1560 1, ioapicindex);
1561 apic_vt_ops->apic_intrmap_map_entry(
1562 irq_ptr->airq_intrmap_private,
1563 (void *)&irdt, DDI_INTR_TYPE_FIXED, 1);
1564 apic_vt_ops->apic_intrmap_record_rdt(
1565 irq_ptr->airq_intrmap_private, &irdt);
1566
1567 /* Write the RDT entry -- bind to a specific CPU: */
1568 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
1569 irdt.ir_hi);
1570
1571 /* Write the vector, trigger, and polarity portion of the RDT */
1572 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
1573 irdt.ir_lo);
1574
1575 } else {
1576 int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
1577 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX;
1578 if (type == DDI_INTR_TYPE_MSI) {
1579 if (irq_ptr->airq_ioapicindex ==
1580 irq_ptr->airq_origirq) {
1581 /* first one */
1582 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1583 "apic_pci_msi_enable_vector\n"));
1584 apic_pci_msi_enable_vector(irq_ptr,
1585 type, which_irq, irq_ptr->airq_vector,
1586 irq_ptr->airq_intin_no,
1587 cpu_infop->aci_local_id);
1588 }
1589 if ((irq_ptr->airq_ioapicindex +
1590 irq_ptr->airq_intin_no - 1) ==
1591 irq_ptr->airq_origirq) { /* last one */
1592 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
1593 "apic_pci_msi_enable_mode\n"));
1594 apic_pci_msi_enable_mode(irq_ptr->airq_dip,
1595 type, which_irq);
1596 }
1597 } else { /* MSI-X */
1598 apic_pci_msi_enable_vector(irq_ptr, type,
1599 irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
1600 cpu_infop->aci_local_id);
1601 apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
1602 irq_ptr->airq_origirq);
1603 }
1604 }
1605 irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
1606 apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
1607 return (0);
1608 }
1609
1610 static void
1611 apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
1612 {
1613 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
1614 & AV_REMOTE_IRR) != 0) {
1615 /*
1616 * Trying to clear the bit through normal
1617 * channels has failed. So as a last-ditch
1618 * effort, try to set the trigger mode to
1619 * edge, then to level. This has been
1620 * observed to work on many systems.
1621 */
1622 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1623 intin_no,
1624 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1625 intin_no) & ~AV_LEVEL);
1626
1627 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1628 intin_no,
1629 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1630 intin_no) | AV_LEVEL);
1631
1632 /*
1633 * If the bit's STILL set, this interrupt may
1634 * be hosed.
1635 */
1636 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1637 intin_no) & AV_REMOTE_IRR) != 0) {
1638
1639 prom_printf("%s: Remote IRR still "
1640 "not clear for IOAPIC %d intin %d.\n"
1641 "\tInterrupts to this pin may cease "
1642 "functioning.\n", psm_name, ioapic_ix,
1643 intin_no);
1644 #ifdef DEBUG
1645 apic_last_ditch_reprogram_failures++;
1646 #endif
1647 }
1648 }
1649 }
1650
1651 /*
1652 * This function is protected by apic_ioapic_lock coupled with the
1653 * fact that interrupts are disabled.
1654 */
1655 static void
1656 delete_defer_repro_ent(int which_irq)
1657 {
1658 ASSERT(which_irq >= 0);
1659 ASSERT(which_irq <= 255);
1660 ASSERT(LOCK_HELD(&apic_ioapic_lock));
1661
1662 if (apic_reprogram_info[which_irq].done)
1663 return;
1664
1665 apic_reprogram_info[which_irq].done = B_TRUE;
1666
1667 #ifdef DEBUG
1668 apic_defer_repro_total_retries +=
1669 apic_reprogram_info[which_irq].tries;
1670
1671 apic_defer_repro_successes++;
1672 #endif
1673
1674 if (--apic_reprogram_outstanding == 0) {
1675
1676 setlvlx = psm_intr_exit_fn();
1677 }
1678 }
1679
1680
1681 /*
1682 * Interrupts must be disabled during this function to prevent
1683 * self-deadlock. Interrupts are disabled because this function
1684 * is called from apic_check_stuck_interrupt(), which is called
1685 * from apic_rebind(), which requires its caller to disable interrupts.
1686 */
1687 static void
1688 add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
1689 {
1690 ASSERT(which_irq >= 0);
1691 ASSERT(which_irq <= 255);
1692 ASSERT(!interrupts_enabled());
1693
1694 /*
1695 * On the off-chance that there's already a deferred
1696 * reprogramming on this irq, check, and if so, just update the
1697 * CPU and irq pointer to which the interrupt is targeted, then return.
1698 */
1699 if (!apic_reprogram_info[which_irq].done) {
1700 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1701 apic_reprogram_info[which_irq].irqp = irq_ptr;
1702 return;
1703 }
1704
1705 apic_reprogram_info[which_irq].irqp = irq_ptr;
1706 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
1707 apic_reprogram_info[which_irq].tries = 0;
1708 /*
1709 * This must be the last thing set, since we're not
1710 * grabbing any locks, apic_try_deferred_reprogram() will
1711 * make its decision about using this entry iff done
1712 * is false.
1713 */
1714 apic_reprogram_info[which_irq].done = B_FALSE;
1715
1716 /*
1717 * If there were previously no deferred reprogrammings, change
1718 * setlvlx to call apic_try_deferred_reprogram()
1719 */
1720 if (++apic_reprogram_outstanding == 1) {
1721
1722 setlvlx = apic_try_deferred_reprogram;
1723 }
1724 }
1725
1726 static void
1727 apic_try_deferred_reprogram(int prev_ipl, int irq)
1728 {
1729 int reproirq;
1730 ulong_t iflag;
1731 struct ioapic_reprogram_data *drep;
1732
1733 (*psm_intr_exit_fn())(prev_ipl, irq);
1734
1735 if (!lock_try(&apic_defer_reprogram_lock)) {
1736 return;
1737 }
1738
1739 /*
1740 * Acquire the apic_ioapic_lock so that any other operations that
1741 * may affect the apic_reprogram_info state are serialized.
1742 * It's still possible for the last deferred reprogramming to clear
1743 * between the time we entered this function and the time we get to
1744 * the for loop below. In that case, *setlvlx will have been set
1745 * back to *_intr_exit and drep will be NULL. (There's no way to
1746 * stop that from happening -- we would need to grab a lock before
1747 * calling *setlvlx, which is neither realistic nor prudent).
1748 */
1749 iflag = intr_clear();
1750 lock_set(&apic_ioapic_lock);
1751
1752 /*
1753 * For each deferred RDT entry, try to reprogram it now. Note that
1754 * there is no lock acquisition to read apic_reprogram_info because
1755 * '.done' is set only after the other fields in the structure are set.
1756 */
1757
1758 drep = NULL;
1759 for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
1760 if (apic_reprogram_info[reproirq].done == B_FALSE) {
1761 drep = &apic_reprogram_info[reproirq];
1762 break;
1763 }
1764 }
1765
1766 /*
1767 * Either we found a deferred action to perform, or
1768 * we entered this function spuriously, after *setlvlx
1769 * was restored to point to *_intr_exit. Any other
1770 * permutation is invalid.
1771 */
1772 ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
1773
1774 /*
1775 * Though we can't really do anything about errors
1776 * at this point, keep track of them for reporting.
1777 * Note that it is very possible for apic_setup_io_intr
1778 * to re-register this very timeout if the Remote IRR bit
1779 * has not yet cleared.
1780 */
1781
1782 #ifdef DEBUG
1783 if (drep != NULL) {
1784 if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
1785 apic_deferred_setup_failures++;
1786 }
1787 } else {
1788 apic_deferred_spurious_enters++;
1789 }
1790 #else
1791 if (drep != NULL)
1792 (void) apic_setup_io_intr(drep, reproirq, B_TRUE);
1793 #endif
1794
1795 lock_clear(&apic_ioapic_lock);
1796 intr_restore(iflag);
1797
1798 lock_clear(&apic_defer_reprogram_lock);
1799 }
1800
1801 static void
1802 apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
1803 {
1804 int waited;
1805
1806 /*
1807 * Wait for the delivery pending bit to clear.
1808 */
1809 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1810 (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
1811
1812 /*
1813 * If we're still waiting on the delivery of this interrupt,
1814 * continue to wait here until it is delivered (this should be
1815 * a very small amount of time, but include a timeout just in
1816 * case).
1817 */
1818 for (waited = 0; waited < apic_max_reps_clear_pending;
1819 waited++) {
1820 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1821 intin_no) & AV_PENDING) == 0) {
1822 break;
1823 }
1824 }
1825 }
1826 }
1827
1828
1829 /*
1830 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
1831 * bit set. Calls functions that modify the function that setlvlx points to,
1832 * so that the reprogramming can be retried very shortly.
1833 *
1834 * This function will mask the RDT entry if the interrupt is level-triggered.
1835 * (The caller is responsible for unmasking the RDT entry.)
1836 *
1837 * Returns non-zero if the caller should defer IOAPIC reprogramming.
1838 */
1839 static int
1840 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
1841 int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
1842 struct ioapic_reprogram_data *drep)
1843 {
1844 int32_t rdt_entry;
1845 int waited;
1846 int reps = 0;
1847
1848 /*
1849 * Wait for the delivery pending bit to clear.
1850 */
1851 do {
1852 ++reps;
1853
1854 apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
1855
1856 /*
1857 * Mask the RDT entry, but only if it's a level-triggered
1858 * interrupt
1859 */
1860 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1861 intin_no);
1862 if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
1863
1864 /* Mask it */
1865 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1866 AV_MASK | rdt_entry);
1867 }
1868
1869 if ((rdt_entry & AV_LEVEL) == AV_LEVEL) {
1870 /*
1871 * If there was a race and an interrupt was injected
1872 * just before we masked, check for that case here.
1873 * Then, unmask the RDT entry and try again. If we're
1874 * on our last try, don't unmask (because we want the
1875 * RDT entry to remain masked for the rest of the
1876 * function).
1877 */
1878 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1879 intin_no);
1880 if ((rdt_entry & AV_PENDING) &&
1881 (reps < apic_max_reps_clear_pending)) {
1882 /* Unmask it */
1883 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1884 intin_no, rdt_entry & ~AV_MASK);
1885 }
1886 }
1887
1888 } while ((rdt_entry & AV_PENDING) &&
1889 (reps < apic_max_reps_clear_pending));
1890
1891 #ifdef DEBUG
1892 if (rdt_entry & AV_PENDING)
1893 apic_intr_deliver_timeouts++;
1894 #endif
1895
1896 /*
1897 * If the remote IRR bit is set, then the interrupt has been sent
1898 * to a CPU for processing. We have no choice but to wait for
1899 * that CPU to process the interrupt, at which point the remote IRR
1900 * bit will be cleared.
1901 */
1902 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
1903 (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
1904
1905 /*
1906 * If the CPU that this RDT is bound to is NOT the current
1907 * CPU, wait until that CPU handles the interrupt and ACKs
1908 * it. If this interrupt is not bound to any CPU (that is,
1909 * if it's bound to the logical destination of "anyone"), it
1910 * may have been delivered to the current CPU so handle that
1911 * case by deferring the reprogramming (below).
1912 */
1913 if ((old_bind_cpu != IRQ_UNBOUND) &&
1914 (old_bind_cpu != IRQ_UNINIT) &&
1915 (old_bind_cpu != psm_get_cpu_id())) {
1916 for (waited = 0; waited < apic_max_reps_clear_pending;
1917 waited++) {
1918 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1919 intin_no) & AV_REMOTE_IRR) == 0) {
1920
1921 delete_defer_repro_ent(which_irq);
1922
1923 /* Remote IRR has cleared! */
1924 return (0);
1925 }
1926 }
1927 }
1928
1929 /*
1930 * If we waited and the Remote IRR bit is still not cleared,
1931 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1932 * times for this interrupt, try the last-ditch workaround:
1933 */
1934 if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
1935
1936 apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
1937
1938 /* Mark this one as reprogrammed: */
1939 delete_defer_repro_ent(which_irq);
1940
1941 return (0);
1942 } else {
1943 #ifdef DEBUG
1944 apic_intr_deferrals++;
1945 #endif
1946
1947 /*
1948 * If waiting for the Remote IRR bit (above) didn't
1949 * allow it to clear, defer the reprogramming.
1950 * Add a new deferred-programming entry if the
1951 * caller passed a NULL one (and update the existing one
1952 * in case anything changed).
1953 */
1954 add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
1955 if (drep)
1956 drep->tries++;
1957
1958 /* Inform caller to defer IOAPIC programming: */
1959 return (1);
1960 }
1961
1962 }
1963
1964 /* Remote IRR is clear */
1965 delete_defer_repro_ent(which_irq);
1966
1967 return (0);
1968 }
1969
1970 /*
1971 * Called to migrate all interrupts at an irq to another cpu.
1972 * Must be called with interrupts disabled and apic_ioapic_lock held
1973 */
1974 int
1975 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
1976 {
1977 apic_irq_t *irqptr = irq_ptr;
1978 int retval = 0;
1979
1980 while (irqptr) {
1981 if (irqptr->airq_temp_cpu != IRQ_UNINIT)
1982 retval |= apic_rebind(irqptr, bind_cpu, NULL);
1983 irqptr = irqptr->airq_next;
1984 }
1985
1986 return (retval);
1987 }
1988
1989 /*
1990 * apic_intr_redistribute does all the messy computations for identifying
1991 * which interrupt to move to which CPU. Currently we do just one interrupt
1992 * at a time. This reduces the time we spent doing all this within clock
1993 * interrupt. When it is done in idle, we could do more than 1.
1994 * First we find the most busy and the most free CPU (time in ISR only)
1995 * skipping those CPUs that has been identified as being ineligible (cpu_skip)
1996 * Then we look for IRQs which are closest to the difference between the
1997 * most busy CPU and the average ISR load. We try to find one whose load
1998 * is less than difference.If none exists, then we chose one larger than the
1999 * difference, provided it does not make the most idle CPU worse than the
2000 * most busy one. In the end, we clear all the busy fields for CPUs. For
2001 * IRQs, they are cleared as they are scanned.
2002 */
2003 void
2004 apic_intr_redistribute(void)
2005 {
2006 int busiest_cpu, most_free_cpu;
2007 int cpu_free, cpu_busy, max_busy, min_busy;
2008 int min_free, diff;
2009 int average_busy, cpus_online;
2010 int i, busy;
2011 ulong_t iflag;
2012 apic_cpus_info_t *cpu_infop;
2013 apic_irq_t *min_busy_irq = NULL;
2014 apic_irq_t *max_busy_irq = NULL;
2015
2016 busiest_cpu = most_free_cpu = -1;
2017 cpu_free = cpu_busy = max_busy = average_busy = 0;
2018 min_free = apic_sample_factor_redistribution;
2019 cpus_online = 0;
2020 /*
2021 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
2022 * without ioapic_lock. That is OK as we are just doing statistical
2023 * sampling anyway and any inaccuracy now will get corrected next time
2024 * The call to rebind which actually changes things will make sure
2025 * we are consistent.
2026 */
2027 for (i = 0; i < apic_nproc; i++) {
2028 if (apic_cpu_in_range(i) &&
2029 !(apic_redist_cpu_skip & (1 << i)) &&
2030 (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
2031
2032 cpu_infop = &apic_cpus[i];
2033 /*
2034 * If no unbound interrupts or only 1 total on this
2035 * CPU, skip
2036 */
2037 if (!cpu_infop->aci_temp_bound ||
2038 (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
2039 == 1) {
2040 apic_redist_cpu_skip |= 1 << i;
2041 continue;
2042 }
2043
2044 busy = cpu_infop->aci_busy;
2045 average_busy += busy;
2046 cpus_online++;
2047 if (max_busy < busy) {
2048 max_busy = busy;
2049 busiest_cpu = i;
2050 }
2051 if (min_free > busy) {
2052 min_free = busy;
2053 most_free_cpu = i;
2054 }
2055 if (busy > apic_int_busy_mark) {
2056 cpu_busy |= 1 << i;
2057 } else {
2058 if (busy < apic_int_free_mark)
2059 cpu_free |= 1 << i;
2060 }
2061 }
2062 }
2063 if ((cpu_busy && cpu_free) ||
2064 (max_busy >= (min_free + apic_diff_for_redistribution))) {
2065
2066 apic_num_imbalance++;
2067 #ifdef DEBUG
2068 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2069 prom_printf(
2070 "redistribute busy=%x free=%x max=%x min=%x",
2071 cpu_busy, cpu_free, max_busy, min_free);
2072 }
2073 #endif /* DEBUG */
2074
2075
2076 average_busy /= cpus_online;
2077
2078 diff = max_busy - average_busy;
2079 min_busy = max_busy; /* start with the max possible value */
2080 max_busy = 0;
2081 min_busy_irq = max_busy_irq = NULL;
2082 i = apic_min_device_irq;
2083 for (; i <= apic_max_device_irq; i++) {
2084 apic_irq_t *irq_ptr;
2085 /* Change to linked list per CPU ? */
2086 if ((irq_ptr = apic_irq_table[i]) == NULL)
2087 continue;
2088 /* Check for irq_busy & decide which one to move */
2089 /* Also zero them for next round */
2090 if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
2091 irq_ptr->airq_busy) {
2092 if (irq_ptr->airq_busy < diff) {
2093 /*
2094 * Check for least busy CPU,
2095 * best fit or what ?
2096 */
2097 if (max_busy < irq_ptr->airq_busy) {
2098 /*
2099 * Most busy within the
2100 * required differential
2101 */
2102 max_busy = irq_ptr->airq_busy;
2103 max_busy_irq = irq_ptr;
2104 }
2105 } else {
2106 if (min_busy > irq_ptr->airq_busy) {
2107 /*
2108 * least busy, but more than
2109 * the reqd diff
2110 */
2111 if (min_busy <
2112 (diff + average_busy -
2113 min_free)) {
2114 /*
2115 * Making sure new cpu
2116 * will not end up
2117 * worse
2118 */
2119 min_busy =
2120 irq_ptr->airq_busy;
2121
2122 min_busy_irq = irq_ptr;
2123 }
2124 }
2125 }
2126 }
2127 irq_ptr->airq_busy = 0;
2128 }
2129
2130 if (max_busy_irq != NULL) {
2131 #ifdef DEBUG
2132 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2133 prom_printf("rebinding %x to %x",
2134 max_busy_irq->airq_vector, most_free_cpu);
2135 }
2136 #endif /* DEBUG */
2137 iflag = intr_clear();
2138 if (lock_try(&apic_ioapic_lock)) {
2139 if (apic_rebind_all(max_busy_irq,
2140 most_free_cpu) == 0) {
2141 /* Make change permenant */
2142 max_busy_irq->airq_cpu =
2143 (uint32_t)most_free_cpu;
2144 }
2145 lock_clear(&apic_ioapic_lock);
2146 }
2147 intr_restore(iflag);
2148
2149 } else if (min_busy_irq != NULL) {
2150 #ifdef DEBUG
2151 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
2152 prom_printf("rebinding %x to %x",
2153 min_busy_irq->airq_vector, most_free_cpu);
2154 }
2155 #endif /* DEBUG */
2156
2157 iflag = intr_clear();
2158 if (lock_try(&apic_ioapic_lock)) {
2159 if (apic_rebind_all(min_busy_irq,
2160 most_free_cpu) == 0) {
2161 /* Make change permenant */
2162 min_busy_irq->airq_cpu =
2163 (uint32_t)most_free_cpu;
2164 }
2165 lock_clear(&apic_ioapic_lock);
2166 }
2167 intr_restore(iflag);
2168
2169 } else {
2170 if (cpu_busy != (1 << busiest_cpu)) {
2171 apic_redist_cpu_skip |= 1 << busiest_cpu;
2172 /*
2173 * We leave cpu_skip set so that next time we
2174 * can choose another cpu
2175 */
2176 }
2177 }
2178 apic_num_rebind++;
2179 } else {
2180 /*
2181 * found nothing. Could be that we skipped over valid CPUs
2182 * or we have balanced everything. If we had a variable
2183 * ticks_for_redistribution, it could be increased here.
2184 * apic_int_busy, int_free etc would also need to be
2185 * changed.
2186 */
2187 if (apic_redist_cpu_skip)
2188 apic_redist_cpu_skip = 0;
2189 }
2190 for (i = 0; i < apic_nproc; i++) {
2191 if (apic_cpu_in_range(i)) {
2192 apic_cpus[i].aci_busy = 0;
2193 }
2194 }
2195 }
2196
2197 void
2198 apic_cleanup_busy(void)
2199 {
2200 int i;
2201 apic_irq_t *irq_ptr;
2202
2203 for (i = 0; i < apic_nproc; i++) {
2204 if (apic_cpu_in_range(i)) {
2205 apic_cpus[i].aci_busy = 0;
2206 }
2207 }
2208
2209 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
2210 if ((irq_ptr = apic_irq_table[i]) != NULL)
2211 irq_ptr->airq_busy = 0;
2212 }
2213 }