Print this page
OS-2366 ddi_periodic_add(9F) is entirely rubbish
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/i86pc/io/apix/apix.c
+++ new/usr/src/uts/i86pc/io/apix/apix.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright (c) 2010, Intel Corporation.
27 27 * All rights reserved.
28 28 */
29 29 /*
30 - * Copyright (c) 2012, Joyent, Inc. All rights reserved.
30 + * Copyright (c) 2013, Joyent, Inc. All rights reserved.
31 31 */
32 32
33 33 /*
34 34 * To understand how the apix module interacts with the interrupt subsystem read
35 35 * the theory statement in uts/i86pc/os/intr.c.
36 36 */
37 37
38 38 /*
39 39 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
40 40 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
41 41 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
42 42 * PSMI 1.5 extensions are supported in Solaris Nevada.
43 43 * PSMI 1.6 extensions are supported in Solaris Nevada.
44 44 * PSMI 1.7 extensions are supported in Solaris Nevada.
45 45 */
46 46 #define PSMI_1_7
47 47
48 48 #include <sys/processor.h>
49 49 #include <sys/time.h>
50 50 #include <sys/psm.h>
51 51 #include <sys/smp_impldefs.h>
52 52 #include <sys/cram.h>
53 53 #include <sys/acpi/acpi.h>
54 54 #include <sys/acpica.h>
55 55 #include <sys/psm_common.h>
56 56 #include <sys/pit.h>
57 57 #include <sys/ddi.h>
58 58 #include <sys/sunddi.h>
59 59 #include <sys/ddi_impldefs.h>
60 60 #include <sys/pci.h>
61 61 #include <sys/promif.h>
62 62 #include <sys/x86_archext.h>
63 63 #include <sys/cpc_impl.h>
64 64 #include <sys/uadmin.h>
65 65 #include <sys/panic.h>
66 66 #include <sys/debug.h>
67 67 #include <sys/archsystm.h>
↓ open down ↓ |
27 lines elided |
↑ open up ↑ |
68 68 #include <sys/trap.h>
69 69 #include <sys/machsystm.h>
70 70 #include <sys/sysmacros.h>
71 71 #include <sys/cpuvar.h>
72 72 #include <sys/rm_platter.h>
73 73 #include <sys/privregs.h>
74 74 #include <sys/note.h>
75 75 #include <sys/pci_intr_lib.h>
76 76 #include <sys/spl.h>
77 77 #include <sys/clock.h>
78 +#include <sys/cyclic.h>
78 79 #include <sys/dditypes.h>
79 80 #include <sys/sunddi.h>
80 81 #include <sys/x_call.h>
81 82 #include <sys/reboot.h>
82 83 #include <sys/mach_intr.h>
83 84 #include <sys/apix.h>
84 85 #include <sys/apix_irm_impl.h>
85 86
86 87 static int apix_probe();
87 88 static void apix_init();
88 89 static void apix_picinit(void);
89 90 static int apix_intr_enter(int, int *);
90 91 static void apix_intr_exit(int, int);
91 92 static void apix_setspl(int);
92 93 static int apix_disable_intr(processorid_t);
93 94 static void apix_enable_intr(processorid_t);
94 95 static int apix_get_clkvect(int);
95 96 static int apix_get_ipivect(int, int);
96 97 static void apix_post_cyclic_setup(void *);
97 98 static int apix_post_cpu_start();
98 99 static int apix_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
99 100 psm_intr_op_t, int *);
100 101
101 102 /*
102 103 * Helper functions for apix_intr_ops()
103 104 */
104 105 static void apix_redistribute_compute(void);
105 106 static int apix_get_pending(apix_vector_t *);
106 107 static apix_vector_t *apix_get_req_vector(ddi_intr_handle_impl_t *, ushort_t);
107 108 static int apix_get_intr_info(ddi_intr_handle_impl_t *, apic_get_intr_t *);
108 109 static char *apix_get_apic_type(void);
109 110 static int apix_intx_get_pending(int);
110 111 static void apix_intx_set_mask(int irqno);
111 112 static void apix_intx_clear_mask(int irqno);
112 113 static int apix_intx_get_shared(int irqno);
113 114 static void apix_intx_set_shared(int irqno, int delta);
114 115 static apix_vector_t *apix_intx_xlate_vector(dev_info_t *, int,
115 116 struct intrspec *);
116 117 static int apix_intx_alloc_vector(dev_info_t *, int, struct intrspec *);
117 118
118 119 extern int apic_clkinit(int);
119 120
120 121 /* IRM initialization for APIX PSM module */
121 122 extern void apix_irm_init(void);
122 123
123 124 extern int irm_enable;
124 125
125 126 /*
126 127 * Local static data
127 128 */
128 129 static struct psm_ops apix_ops = {
129 130 apix_probe,
130 131
131 132 apix_init,
132 133 apix_picinit,
133 134 apix_intr_enter,
134 135 apix_intr_exit,
135 136 apix_setspl,
136 137 apix_addspl,
137 138 apix_delspl,
138 139 apix_disable_intr,
139 140 apix_enable_intr,
140 141 NULL, /* psm_softlvl_to_irq */
141 142 NULL, /* psm_set_softintr */
142 143
143 144 apic_set_idlecpu,
144 145 apic_unset_idlecpu,
145 146
146 147 apic_clkinit,
147 148 apix_get_clkvect,
148 149 NULL, /* psm_hrtimeinit */
149 150 apic_gethrtime,
150 151
151 152 apic_get_next_processorid,
152 153 apic_cpu_start,
153 154 apix_post_cpu_start,
154 155 apic_shutdown,
155 156 apix_get_ipivect,
156 157 apic_send_ipi,
157 158
158 159 NULL, /* psm_translate_irq */
159 160 NULL, /* psm_notify_error */
160 161 NULL, /* psm_notify_func */
161 162 apic_timer_reprogram,
162 163 apic_timer_enable,
163 164 apic_timer_disable,
164 165 apix_post_cyclic_setup,
165 166 apic_preshutdown,
166 167 apix_intr_ops, /* Advanced DDI Interrupt framework */
167 168 apic_state, /* save, restore apic state for S3 */
168 169 apic_cpu_ops, /* CPU control interface. */
169 170 };
170 171
171 172 struct psm_ops *psmops = &apix_ops;
172 173
173 174 static struct psm_info apix_psm_info = {
174 175 PSM_INFO_VER01_7, /* version */
175 176 PSM_OWN_EXCLUSIVE, /* ownership */
176 177 &apix_ops, /* operation */
177 178 APIX_NAME, /* machine name */
178 179 "apix MPv1.4 compatible",
179 180 };
180 181
181 182 static void *apix_hdlp;
182 183
183 184 static int apix_is_enabled = 0;
184 185
185 186 /*
186 187 * Flag to indicate if APIX is to be enabled only for platforms
187 188 * with specific hw feature(s).
188 189 */
189 190 int apix_hw_chk_enable = 1;
190 191
191 192 /*
192 193 * Hw features that are checked for enabling APIX support.
193 194 */
194 195 #define APIX_SUPPORT_X2APIC 0x00000001
195 196 uint_t apix_supported_hw = APIX_SUPPORT_X2APIC;
196 197
197 198 /*
198 199 * apix_lock is used for cpu selection and vector re-binding
199 200 */
200 201 lock_t apix_lock;
201 202 apix_impl_t *apixs[NCPU];
202 203 /*
203 204 * Mapping between device interrupt and the allocated vector. Indexed
204 205 * by major number.
205 206 */
206 207 apix_dev_vector_t **apix_dev_vector;
207 208 /*
208 209 * Mapping between device major number and cpu id. It gets used
209 210 * when interrupt binding policy round robin with affinity is
210 211 * applied. With that policy, devices with the same major number
211 212 * will be bound to the same CPU.
212 213 */
213 214 processorid_t *apix_major_to_cpu; /* major to cpu mapping */
214 215 kmutex_t apix_mutex; /* for apix_dev_vector & apix_major_to_cpu */
215 216
216 217 int apix_nipis = 16; /* Maximum number of IPIs */
217 218 /*
218 219 * Maximum number of vectors in a CPU that can be used for interrupt
219 220 * allocation (including IPIs and the reserved vectors).
220 221 */
221 222 int apix_cpu_nvectors = APIX_NVECTOR;
222 223
223 224 /* gcpu.h */
224 225
225 226 extern void apic_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
226 227 extern void apic_change_eoi();
227 228
228 229 /*
229 230 * This is the loadable module wrapper
230 231 */
231 232
232 233 int
233 234 _init(void)
234 235 {
235 236 if (apic_coarse_hrtime)
236 237 apix_ops.psm_gethrtime = &apic_gettime;
237 238 return (psm_mod_init(&apix_hdlp, &apix_psm_info));
238 239 }
239 240
240 241 int
241 242 _fini(void)
242 243 {
243 244 return (psm_mod_fini(&apix_hdlp, &apix_psm_info));
244 245 }
245 246
246 247 int
247 248 _info(struct modinfo *modinfop)
248 249 {
249 250 return (psm_mod_info(&apix_hdlp, &apix_psm_info, modinfop));
250 251 }
251 252
252 253 static int
253 254 apix_probe()
254 255 {
255 256 int rval;
256 257
257 258 if (apix_enable == 0)
258 259 return (PSM_FAILURE);
259 260
260 261 /* check for hw features if specified */
261 262 if (apix_hw_chk_enable) {
262 263 /* check if x2APIC mode is supported */
263 264 if ((apix_supported_hw & APIX_SUPPORT_X2APIC) ==
264 265 APIX_SUPPORT_X2APIC) {
265 266 if (!((apic_local_mode() == LOCAL_X2APIC) ||
266 267 apic_detect_x2apic())) {
267 268 /* x2APIC mode is not supported in the hw */
268 269 apix_enable = 0;
269 270 }
270 271 }
271 272 if (apix_enable == 0)
272 273 return (PSM_FAILURE);
273 274 }
274 275
275 276 rval = apic_probe_common(apix_psm_info.p_mach_idstring);
276 277 if (rval == PSM_SUCCESS)
277 278 apix_is_enabled = 1;
278 279 else
279 280 apix_is_enabled = 0;
280 281 return (rval);
281 282 }
282 283
283 284 /*
284 285 * Initialize the data structures needed by pcplusmpx module.
285 286 * Specifically, the data structures used by addspl() and delspl()
286 287 * routines.
287 288 */
288 289 static void
289 290 apix_softinit()
290 291 {
291 292 int i, *iptr;
292 293 apix_impl_t *hdlp;
293 294 int nproc;
294 295
295 296 nproc = max(apic_nproc, apic_max_nproc);
296 297
297 298 hdlp = kmem_zalloc(nproc * sizeof (apix_impl_t), KM_SLEEP);
298 299 for (i = 0; i < nproc; i++) {
299 300 apixs[i] = &hdlp[i];
300 301 apixs[i]->x_cpuid = i;
301 302 LOCK_INIT_CLEAR(&apixs[i]->x_lock);
302 303 }
303 304
304 305 /* cpu 0 is always up (for now) */
305 306 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
306 307
307 308 iptr = (int *)&apic_irq_table[0];
308 309 for (i = 0; i <= APIC_MAX_VECTOR; i++) {
309 310 apic_level_intr[i] = 0;
310 311 *iptr++ = NULL;
311 312 }
312 313 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
313 314
314 315 apix_dev_vector = kmem_zalloc(sizeof (apix_dev_vector_t *) * devcnt,
315 316 KM_SLEEP);
316 317
317 318 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
318 319 apix_major_to_cpu = kmem_zalloc(sizeof (int) * devcnt,
319 320 KM_SLEEP);
320 321 for (i = 0; i < devcnt; i++)
321 322 apix_major_to_cpu[i] = IRQ_UNINIT;
322 323 }
323 324
324 325 mutex_init(&apix_mutex, NULL, MUTEX_DEFAULT, NULL);
325 326 }
326 327
327 328 static int
328 329 apix_get_pending_spl(void)
329 330 {
330 331 int cpuid = CPU->cpu_id;
331 332
332 333 return (bsrw_insn(apixs[cpuid]->x_intr_pending));
333 334 }
334 335
335 336 static uintptr_t
336 337 apix_get_intr_handler(int cpu, short vec)
337 338 {
338 339 apix_vector_t *apix_vector;
339 340
340 341 ASSERT(cpu < apic_nproc && vec < APIX_NVECTOR);
341 342 if (cpu >= apic_nproc)
342 343 return (NULL);
343 344
344 345 apix_vector = apixs[cpu]->x_vectbl[vec];
345 346
346 347 return ((uintptr_t)(apix_vector->v_autovect));
347 348 }
348 349
349 350 #if defined(__amd64)
350 351 static unsigned char dummy_cpu_pri[MAXIPL + 1] = {
351 352 0, 0, 0, 0, 0, 0, 0, 0,
352 353 0, 0, 0, 0, 0, 0, 0, 0, 0
353 354 };
354 355 #endif
355 356
356 357 static void
357 358 apix_init()
358 359 {
359 360 extern void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *);
360 361
361 362 APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_softinit\n"));
362 363
363 364 do_interrupt_common = apix_do_interrupt;
364 365 addintr = apix_add_avintr;
365 366 remintr = apix_rem_avintr;
366 367 get_pending_spl = apix_get_pending_spl;
367 368 get_intr_handler = apix_get_intr_handler;
368 369 psm_get_localapicid = apic_get_localapicid;
369 370 psm_get_ioapicid = apic_get_ioapicid;
370 371
371 372 apix_softinit();
372 373 #if defined(__amd64)
373 374 /*
374 375 * Make cpu-specific interrupt info point to cr8pri vector
375 376 */
376 377 CPU->cpu_pri_data = dummy_cpu_pri;
377 378 #else
378 379 if (cpuid_have_cr8access(CPU))
379 380 apic_have_32bit_cr8 = 1;
380 381 #endif /* __amd64 */
381 382
382 383 /*
383 384 * Initialize IRM pool parameters
384 385 */
385 386 if (irm_enable) {
386 387 int i;
387 388 int lowest_irq;
388 389 int highest_irq;
389 390
390 391 /* number of CPUs present */
391 392 apix_irminfo.apix_ncpus = apic_nproc;
392 393 /* total number of entries in all of the IOAPICs present */
393 394 lowest_irq = apic_io_vectbase[0];
394 395 highest_irq = apic_io_vectend[0];
395 396 for (i = 1; i < apic_io_max; i++) {
396 397 if (apic_io_vectbase[i] < lowest_irq)
397 398 lowest_irq = apic_io_vectbase[i];
398 399 if (apic_io_vectend[i] > highest_irq)
399 400 highest_irq = apic_io_vectend[i];
400 401 }
401 402 apix_irminfo.apix_ioapic_max_vectors =
402 403 highest_irq - lowest_irq + 1;
403 404 /*
404 405 * Number of available per-CPU vectors excluding
405 406 * reserved vectors for Dtrace, int80, system-call,
406 407 * fast-trap, etc.
407 408 */
408 409 apix_irminfo.apix_per_cpu_vectors = APIX_NAVINTR -
409 410 APIX_SW_RESERVED_VECTORS;
410 411
411 412 /* Number of vectors (pre) allocated (SCI and HPET) */
412 413 apix_irminfo.apix_vectors_allocated = 0;
413 414 if (apic_hpet_vect != -1)
414 415 apix_irminfo.apix_vectors_allocated++;
415 416 if (apic_sci_vect != -1)
416 417 apix_irminfo.apix_vectors_allocated++;
417 418 }
418 419 }
419 420
420 421 static void
421 422 apix_init_intr()
422 423 {
423 424 processorid_t cpun = psm_get_cpu_id();
424 425 uint_t nlvt;
425 426 uint32_t svr = AV_UNIT_ENABLE | APIC_SPUR_INTR;
426 427 extern void cmi_cmci_trap(void);
427 428
428 429 apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
429 430
430 431 if (apic_mode == LOCAL_APIC) {
431 432 /*
432 433 * We are running APIC in MMIO mode.
433 434 */
434 435 if (apic_flat_model) {
435 436 apic_reg_ops->apic_write(APIC_FORMAT_REG,
436 437 APIC_FLAT_MODEL);
437 438 } else {
438 439 apic_reg_ops->apic_write(APIC_FORMAT_REG,
439 440 APIC_CLUSTER_MODEL);
440 441 }
441 442
442 443 apic_reg_ops->apic_write(APIC_DEST_REG,
443 444 AV_HIGH_ORDER >> cpun);
444 445 }
445 446
446 447 if (apic_directed_EOI_supported()) {
447 448 /*
448 449 * Setting the 12th bit in the Spurious Interrupt Vector
449 450 * Register suppresses broadcast EOIs generated by the local
450 451 * APIC. The suppression of broadcast EOIs happens only when
451 452 * interrupts are level-triggered.
452 453 */
453 454 svr |= APIC_SVR_SUPPRESS_BROADCAST_EOI;
454 455 }
455 456
456 457 /* need to enable APIC before unmasking NMI */
457 458 apic_reg_ops->apic_write(APIC_SPUR_INT_REG, svr);
458 459
459 460 /*
460 461 * Presence of an invalid vector with delivery mode AV_FIXED can
461 462 * cause an error interrupt, even if the entry is masked...so
462 463 * write a valid vector to LVT entries along with the mask bit
463 464 */
464 465
465 466 /* All APICs have timer and LINT0/1 */
466 467 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK|APIC_RESV_IRQ);
467 468 apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK|APIC_RESV_IRQ);
468 469 apic_reg_ops->apic_write(APIC_INT_VECT1, AV_NMI); /* enable NMI */
469 470
470 471 /*
471 472 * On integrated APICs, the number of LVT entries is
472 473 * 'Max LVT entry' + 1; on 82489DX's (non-integrated
473 474 * APICs), nlvt is "3" (LINT0, LINT1, and timer)
474 475 */
475 476
476 477 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
477 478 nlvt = 3;
478 479 } else {
479 480 nlvt = ((apic_reg_ops->apic_read(APIC_VERS_REG) >> 16) &
480 481 0xFF) + 1;
481 482 }
482 483
483 484 if (nlvt >= 5) {
484 485 /* Enable performance counter overflow interrupt */
485 486
486 487 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
487 488 apic_enable_cpcovf_intr = 0;
488 489 if (apic_enable_cpcovf_intr) {
489 490 if (apic_cpcovf_vect == 0) {
490 491 int ipl = APIC_PCINT_IPL;
491 492
492 493 apic_cpcovf_vect = apix_get_ipivect(ipl, -1);
493 494 ASSERT(apic_cpcovf_vect);
494 495
495 496 (void) add_avintr(NULL, ipl,
496 497 (avfunc)kcpc_hw_overflow_intr,
497 498 "apic pcint", apic_cpcovf_vect,
498 499 NULL, NULL, NULL, NULL);
499 500 kcpc_hw_overflow_intr_installed = 1;
500 501 kcpc_hw_enable_cpc_intr =
501 502 apic_cpcovf_mask_clear;
502 503 }
503 504 apic_reg_ops->apic_write(APIC_PCINT_VECT,
504 505 apic_cpcovf_vect);
505 506 }
506 507 }
507 508
508 509 if (nlvt >= 6) {
509 510 /* Only mask TM intr if the BIOS apparently doesn't use it */
510 511
511 512 uint32_t lvtval;
512 513
513 514 lvtval = apic_reg_ops->apic_read(APIC_THERM_VECT);
514 515 if (((lvtval & AV_MASK) == AV_MASK) ||
515 516 ((lvtval & AV_DELIV_MODE) != AV_SMI)) {
516 517 apic_reg_ops->apic_write(APIC_THERM_VECT,
517 518 AV_MASK|APIC_RESV_IRQ);
518 519 }
519 520 }
520 521
521 522 /* Enable error interrupt */
522 523
523 524 if (nlvt >= 4 && apic_enable_error_intr) {
524 525 if (apic_errvect == 0) {
525 526 int ipl = 0xf; /* get highest priority intr */
526 527 apic_errvect = apix_get_ipivect(ipl, -1);
527 528 ASSERT(apic_errvect);
528 529 /*
529 530 * Not PSMI compliant, but we are going to merge
530 531 * with ON anyway
531 532 */
532 533 (void) add_avintr(NULL, ipl,
533 534 (avfunc)apic_error_intr, "apic error intr",
534 535 apic_errvect, NULL, NULL, NULL, NULL);
535 536 }
536 537 apic_reg_ops->apic_write(APIC_ERR_VECT, apic_errvect);
537 538 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
538 539 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
539 540 }
540 541
541 542 /* Enable CMCI interrupt */
542 543 if (cmi_enable_cmci) {
543 544 mutex_enter(&cmci_cpu_setup_lock);
544 545 if (cmci_cpu_setup_registered == 0) {
545 546 mutex_enter(&cpu_lock);
546 547 register_cpu_setup_func(cmci_cpu_setup, NULL);
547 548 mutex_exit(&cpu_lock);
548 549 cmci_cpu_setup_registered = 1;
549 550 }
550 551 mutex_exit(&cmci_cpu_setup_lock);
551 552
552 553 if (apic_cmci_vect == 0) {
553 554 int ipl = 0x2;
554 555 apic_cmci_vect = apix_get_ipivect(ipl, -1);
555 556 ASSERT(apic_cmci_vect);
556 557
557 558 (void) add_avintr(NULL, ipl,
558 559 (avfunc)cmi_cmci_trap, "apic cmci intr",
559 560 apic_cmci_vect, NULL, NULL, NULL, NULL);
560 561 }
561 562 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
562 563 }
563 564
564 565 apic_reg_ops->apic_write_task_reg(0);
565 566 }
566 567
567 568 static void
568 569 apix_picinit(void)
569 570 {
570 571 int i, j;
571 572 uint_t isr;
572 573
573 574 APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_picinit\n"));
574 575
575 576 /*
576 577 * initialize interrupt remapping before apic
577 578 * hardware initialization
578 579 */
579 580 apic_intrmap_init(apic_mode);
580 581 if (apic_vt_ops == psm_vt_ops)
581 582 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IIR;
582 583
583 584 /*
584 585 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
585 586 * bit on without clearing it with EOI. Since softint
586 587 * uses vector 0x20 to interrupt itself, so softint will
587 588 * not work on this machine. In order to fix this problem
588 589 * a check is made to verify all the isr bits are clear.
589 590 * If not, EOIs are issued to clear the bits.
590 591 */
591 592 for (i = 7; i >= 1; i--) {
592 593 isr = apic_reg_ops->apic_read(APIC_ISR_REG + (i * 4));
593 594 if (isr != 0)
594 595 for (j = 0; ((j < 32) && (isr != 0)); j++)
595 596 if (isr & (1 << j)) {
596 597 apic_reg_ops->apic_write(
597 598 APIC_EOI_REG, 0);
598 599 isr &= ~(1 << j);
599 600 apic_error |= APIC_ERR_BOOT_EOI;
600 601 }
601 602 }
602 603
603 604 /* set a flag so we know we have run apic_picinit() */
604 605 apic_picinit_called = 1;
605 606 LOCK_INIT_CLEAR(&apic_gethrtime_lock);
606 607 LOCK_INIT_CLEAR(&apic_ioapic_lock);
607 608 LOCK_INIT_CLEAR(&apic_error_lock);
608 609 LOCK_INIT_CLEAR(&apic_mode_switch_lock);
609 610
610 611 picsetup(); /* initialise the 8259 */
611 612
612 613 /* add nmi handler - least priority nmi handler */
613 614 LOCK_INIT_CLEAR(&apic_nmi_lock);
614 615
615 616 if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr,
616 617 "apix NMI handler", (caddr_t)NULL))
617 618 cmn_err(CE_WARN, "apix: Unable to add nmi handler");
618 619
619 620 apix_init_intr();
620 621
621 622 /* enable apic mode if imcr present */
622 623 if (apic_imcrp) {
623 624 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
624 625 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC);
625 626 }
626 627
627 628 ioapix_init_intr(IOAPIC_MASK);
628 629
629 630 /* setup global IRM pool if applicable */
630 631 if (irm_enable)
631 632 apix_irm_init();
632 633 }
633 634
634 635 static __inline__ void
635 636 apix_send_eoi(void)
636 637 {
637 638 if (apic_mode == LOCAL_APIC)
638 639 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0);
639 640 else
640 641 X2APIC_WRITE(APIC_EOI_REG, 0);
641 642 }
642 643
643 644 /*
644 645 * platform_intr_enter
645 646 *
646 647 * Called at the beginning of the interrupt service routine, but unlike
647 648 * pcplusmp, does not mask interrupts. An EOI is given to the interrupt
648 649 * controller to enable other HW interrupts but interrupts are still
649 650 * masked by the IF flag.
650 651 *
651 652 * Return -1 for spurious interrupts
652 653 *
653 654 */
654 655 static int
655 656 apix_intr_enter(int ipl, int *vectorp)
656 657 {
657 658 struct cpu *cpu = CPU;
658 659 uint32_t cpuid = CPU->cpu_id;
659 660 apic_cpus_info_t *cpu_infop;
660 661 uchar_t vector;
661 662 apix_vector_t *vecp;
662 663 int nipl = -1;
663 664
664 665 /*
665 666 * The real vector delivered is (*vectorp + 0x20), but our caller
666 667 * subtracts 0x20 from the vector before passing it to us.
667 668 * (That's why APIC_BASE_VECT is 0x20.)
668 669 */
669 670 vector = *vectorp = (uchar_t)*vectorp + APIC_BASE_VECT;
670 671
671 672 cpu_infop = &apic_cpus[cpuid];
672 673 if (vector == APIC_SPUR_INTR) {
673 674 cpu_infop->aci_spur_cnt++;
674 675 return (APIC_INT_SPURIOUS);
675 676 }
676 677
677 678 vecp = xv_vector(cpuid, vector);
678 679 if (vecp == NULL) {
679 680 if (APIX_IS_FAKE_INTR(vector))
680 681 nipl = apix_rebindinfo.i_pri;
681 682 apix_send_eoi();
682 683 return (nipl);
683 684 }
684 685 nipl = vecp->v_pri;
685 686
686 687 /* if interrupted by the clock, increment apic_nsec_since_boot */
687 688 if (vector == (apic_clkvect + APIC_BASE_VECT)) {
688 689 if (!apic_oneshot) {
689 690 /* NOTE: this is not MT aware */
690 691 apic_hrtime_stamp++;
691 692 apic_nsec_since_boot += apic_nsec_per_intr;
692 693 apic_hrtime_stamp++;
693 694 last_count_read = apic_hertz_count;
694 695 apix_redistribute_compute();
695 696 }
696 697
697 698 apix_send_eoi();
698 699
699 700 return (nipl);
700 701 }
701 702
702 703 ASSERT(vecp->v_state != APIX_STATE_OBSOLETED);
703 704
704 705 /* pre-EOI handling for level-triggered interrupts */
705 706 if (!APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method) &&
706 707 (vecp->v_type & APIX_TYPE_FIXED) && apic_level_intr[vecp->v_inum])
707 708 apix_level_intr_pre_eoi(vecp->v_inum);
708 709
709 710 /* send back EOI */
710 711 apix_send_eoi();
711 712
712 713 cpu_infop->aci_current[nipl] = vector;
713 714 if ((nipl > ipl) && (nipl > cpu->cpu_base_spl)) {
714 715 cpu_infop->aci_curipl = (uchar_t)nipl;
715 716 cpu_infop->aci_ISR_in_progress |= 1 << nipl;
716 717 }
717 718
718 719 #ifdef DEBUG
719 720 if (vector >= APIX_IPI_MIN)
720 721 return (nipl); /* skip IPI */
721 722
722 723 APIC_DEBUG_BUF_PUT(vector);
723 724 APIC_DEBUG_BUF_PUT(vecp->v_inum);
724 725 APIC_DEBUG_BUF_PUT(nipl);
725 726 APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
726 727 if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl)))
727 728 drv_usecwait(apic_stretch_interrupts);
728 729 #endif /* DEBUG */
729 730
730 731 return (nipl);
731 732 }
732 733
733 734 /*
734 735 * Any changes made to this function must also change X2APIC
735 736 * version of intr_exit.
736 737 */
737 738 static void
738 739 apix_intr_exit(int prev_ipl, int arg2)
739 740 {
740 741 int cpuid = psm_get_cpu_id();
741 742 apic_cpus_info_t *cpu_infop = &apic_cpus[cpuid];
742 743 apix_impl_t *apixp = apixs[cpuid];
743 744
744 745 UNREFERENCED_1PARAMETER(arg2);
745 746
746 747 cpu_infop->aci_curipl = (uchar_t)prev_ipl;
747 748 /* ISR above current pri could not be in progress */
748 749 cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1;
749 750
750 751 if (apixp->x_obsoletes != NULL) {
751 752 if (APIX_CPU_LOCK_HELD(cpuid))
752 753 return;
753 754
754 755 APIX_ENTER_CPU_LOCK(cpuid);
755 756 (void) apix_obsolete_vector(apixp->x_obsoletes);
756 757 APIX_LEAVE_CPU_LOCK(cpuid);
757 758 }
758 759 }
759 760
760 761 /*
761 762 * The pcplusmp setspl code uses the TPR to mask all interrupts at or below the
762 763 * given ipl, but apix never uses the TPR and we never mask a subset of the
763 764 * interrupts. They are either all blocked by the IF flag or all can come in.
764 765 *
765 766 * For setspl, we mask all interrupts for XC_HI_PIL (15), otherwise, interrupts
766 767 * can come in if currently enabled by the IF flag. This table shows the state
767 768 * of the IF flag when we leave this function.
768 769 *
769 770 * curr IF | ipl == 15 ipl != 15
770 771 * --------+---------------------------
771 772 * 0 | 0 0
772 773 * 1 | 0 1
773 774 */
774 775 static void
775 776 apix_setspl(int ipl)
776 777 {
777 778 /*
778 779 * Interrupts at ipl above this cannot be in progress, so the following
779 780 * mask is ok.
780 781 */
781 782 apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1;
782 783
783 784 if (ipl == XC_HI_PIL)
784 785 cli();
785 786 }
786 787
787 788 int
788 789 apix_addspl(int virtvec, int ipl, int min_ipl, int max_ipl)
789 790 {
790 791 uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
791 792 uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
792 793 apix_vector_t *vecp = xv_vector(cpuid, vector);
793 794
794 795 UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
795 796 ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
796 797
797 798 if (vecp->v_type == APIX_TYPE_FIXED)
798 799 apix_intx_set_shared(vecp->v_inum, 1);
799 800
800 801 /* There are more interrupts, so it's already been enabled */
801 802 if (vecp->v_share > 1)
802 803 return (PSM_SUCCESS);
803 804
804 805 /* return if it is not hardware interrupt */
805 806 if (vecp->v_type == APIX_TYPE_IPI)
806 807 return (PSM_SUCCESS);
807 808
808 809 /*
809 810 * if apix_picinit() has not been called yet, just return.
810 811 * At the end of apic_picinit(), we will call setup_io_intr().
811 812 */
812 813 if (!apic_picinit_called)
813 814 return (PSM_SUCCESS);
814 815
815 816 (void) apix_setup_io_intr(vecp);
816 817
817 818 return (PSM_SUCCESS);
818 819 }
819 820
820 821 int
821 822 apix_delspl(int virtvec, int ipl, int min_ipl, int max_ipl)
822 823 {
823 824 uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
824 825 uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
825 826 apix_vector_t *vecp = xv_vector(cpuid, vector);
826 827
827 828 UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
828 829 ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
829 830
830 831 if (vecp->v_type == APIX_TYPE_FIXED)
831 832 apix_intx_set_shared(vecp->v_inum, -1);
832 833
833 834 /* There are more interrupts */
834 835 if (vecp->v_share > 1)
835 836 return (PSM_SUCCESS);
836 837
837 838 /* return if it is not hardware interrupt */
838 839 if (vecp->v_type == APIX_TYPE_IPI)
839 840 return (PSM_SUCCESS);
840 841
841 842 if (!apic_picinit_called) {
842 843 cmn_err(CE_WARN, "apix: delete 0x%x before apic init",
843 844 virtvec);
844 845 return (PSM_SUCCESS);
845 846 }
846 847
847 848 apix_disable_vector(vecp);
848 849
849 850 return (PSM_SUCCESS);
850 851 }
851 852
852 853 /*
853 854 * Try and disable all interrupts. We just assign interrupts to other
854 855 * processors based on policy. If any were bound by user request, we
855 856 * let them continue and return failure. We do not bother to check
856 857 * for cache affinity while rebinding.
857 858 */
858 859 static int
859 860 apix_disable_intr(processorid_t cpun)
860 861 {
861 862 apix_impl_t *apixp = apixs[cpun];
862 863 apix_vector_t *vecp, *newp;
863 864 int bindcpu, i, hardbound = 0, errbound = 0, ret, loop, type;
864 865
865 866 lock_set(&apix_lock);
866 867
867 868 apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE;
868 869 apic_cpus[cpun].aci_curipl = 0;
869 870
870 871 /* if this is for SUSPEND operation, skip rebinding */
871 872 if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
872 873 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
873 874 vecp = apixp->x_vectbl[i];
874 875 if (!IS_VECT_ENABLED(vecp))
875 876 continue;
876 877
877 878 apix_disable_vector(vecp);
878 879 }
879 880 lock_clear(&apix_lock);
880 881 return (PSM_SUCCESS);
881 882 }
882 883
883 884 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
884 885 vecp = apixp->x_vectbl[i];
885 886 if (!IS_VECT_ENABLED(vecp))
886 887 continue;
887 888
888 889 if (vecp->v_flags & APIX_VECT_USER_BOUND) {
889 890 hardbound++;
890 891 continue;
891 892 }
892 893 type = vecp->v_type;
893 894
894 895 /*
895 896 * If there are bound interrupts on this cpu, then
896 897 * rebind them to other processors.
897 898 */
898 899 loop = 0;
899 900 do {
900 901 bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
901 902
902 903 if (type != APIX_TYPE_MSI)
903 904 newp = apix_set_cpu(vecp, bindcpu, &ret);
904 905 else
905 906 newp = apix_grp_set_cpu(vecp, bindcpu, &ret);
906 907 } while ((newp == NULL) && (loop++ < apic_nproc));
907 908
908 909 if (loop >= apic_nproc) {
909 910 errbound++;
910 911 cmn_err(CE_WARN, "apix: failed to rebind vector %x/%x",
911 912 vecp->v_cpuid, vecp->v_vector);
912 913 }
913 914 }
914 915
915 916 lock_clear(&apix_lock);
916 917
917 918 if (hardbound || errbound) {
918 919 cmn_err(CE_WARN, "Could not disable interrupts on %d"
919 920 "due to user bound interrupts or failed operation",
920 921 cpun);
921 922 return (PSM_FAILURE);
922 923 }
923 924
924 925 return (PSM_SUCCESS);
925 926 }
926 927
927 928 /*
928 929 * Bind interrupts to specified CPU
929 930 */
930 931 static void
931 932 apix_enable_intr(processorid_t cpun)
932 933 {
933 934 apix_vector_t *vecp;
934 935 int i, ret;
935 936 processorid_t n;
936 937
937 938 lock_set(&apix_lock);
938 939
939 940 apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE;
940 941
941 942 /* interrupt enabling for system resume */
942 943 if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
943 944 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
944 945 vecp = xv_vector(cpun, i);
945 946 if (!IS_VECT_ENABLED(vecp))
946 947 continue;
947 948
948 949 apix_enable_vector(vecp);
949 950 }
950 951 apic_cpus[cpun].aci_status &= ~APIC_CPU_SUSPEND;
951 952 }
952 953
953 954 for (n = 0; n < apic_nproc; n++) {
954 955 if (!apic_cpu_in_range(n) || n == cpun ||
955 956 (apic_cpus[n].aci_status & APIC_CPU_INTR_ENABLE) == 0)
956 957 continue;
957 958
958 959 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
959 960 vecp = xv_vector(n, i);
960 961 if (!IS_VECT_ENABLED(vecp) ||
961 962 vecp->v_bound_cpuid != cpun)
962 963 continue;
963 964
964 965 if (vecp->v_type != APIX_TYPE_MSI)
965 966 (void) apix_set_cpu(vecp, cpun, &ret);
966 967 else
967 968 (void) apix_grp_set_cpu(vecp, cpun, &ret);
968 969 }
969 970 }
970 971
971 972 lock_clear(&apix_lock);
972 973 }
973 974
974 975 /*
975 976 * Allocate vector for IPI
976 977 * type == -1 indicates it is an internal request. Do not change
977 978 * resv_vector for these requests.
978 979 */
979 980 static int
980 981 apix_get_ipivect(int ipl, int type)
981 982 {
982 983 uchar_t vector;
983 984
984 985 if ((vector = apix_alloc_ipi(ipl)) > 0) {
985 986 if (type != -1)
986 987 apic_resv_vector[ipl] = vector;
987 988 return (vector);
988 989 }
989 990 apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
990 991 return (-1); /* shouldn't happen */
991 992 }
992 993
993 994 static int
994 995 apix_get_clkvect(int ipl)
995 996 {
996 997 int vector;
997 998
998 999 if ((vector = apix_get_ipivect(ipl, -1)) == -1)
999 1000 return (-1);
1000 1001
1001 1002 apic_clkvect = vector - APIC_BASE_VECT;
1002 1003 APIC_VERBOSE(IPI, (CE_CONT, "apix: clock vector = %x\n",
1003 1004 apic_clkvect));
1004 1005 return (vector);
1005 1006 }
1006 1007
1007 1008 static int
1008 1009 apix_post_cpu_start()
1009 1010 {
1010 1011 int cpun;
1011 1012 static int cpus_started = 1;
1012 1013
1013 1014 /* We know this CPU + BSP started successfully. */
1014 1015 cpus_started++;
1015 1016
1016 1017 /*
1017 1018 * On BSP we would have enabled X2APIC, if supported by processor,
1018 1019 * in acpi_probe(), but on AP we do it here.
1019 1020 *
1020 1021 * We enable X2APIC mode only if BSP is running in X2APIC & the
1021 1022 * local APIC mode of the current CPU is MMIO (xAPIC).
1022 1023 */
1023 1024 if (apic_mode == LOCAL_X2APIC && apic_detect_x2apic() &&
1024 1025 apic_local_mode() == LOCAL_APIC) {
1025 1026 apic_enable_x2apic();
1026 1027 }
1027 1028
1028 1029 /*
1029 1030 * Switch back to x2apic IPI sending method for performance when target
1030 1031 * CPU has entered x2apic mode.
1031 1032 */
1032 1033 if (apic_mode == LOCAL_X2APIC) {
1033 1034 apic_switch_ipi_callback(B_FALSE);
1034 1035 }
1035 1036
1036 1037 splx(ipltospl(LOCK_LEVEL));
1037 1038 apix_init_intr();
1038 1039
1039 1040 /*
1040 1041 * since some systems don't enable the internal cache on the non-boot
1041 1042 * cpus, so we have to enable them here
1042 1043 */
1043 1044 setcr0(getcr0() & ~(CR0_CD | CR0_NW));
1044 1045
1045 1046 #ifdef DEBUG
1046 1047 APIC_AV_PENDING_SET();
1047 1048 #else
1048 1049 if (apic_mode == LOCAL_APIC)
1049 1050 APIC_AV_PENDING_SET();
1050 1051 #endif /* DEBUG */
1051 1052
1052 1053 /*
1053 1054 * We may be booting, or resuming from suspend; aci_status will
1054 1055 * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the
1055 1056 * APIC_CPU_ONLINE flag here rather than setting aci_status completely.
1056 1057 */
1057 1058 cpun = psm_get_cpu_id();
1058 1059 apic_cpus[cpun].aci_status |= APIC_CPU_ONLINE;
1059 1060
1060 1061 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1061 1062
1062 1063 return (PSM_SUCCESS);
1063 1064 }
1064 1065
↓ open down ↓ |
977 lines elided |
↑ open up ↑ |
1065 1066 /*
1066 1067 * If this module needs a periodic handler for the interrupt distribution, it
1067 1068 * can be added here. The argument to the periodic handler is not currently
1068 1069 * used, but is reserved for future.
1069 1070 */
1070 1071 static void
1071 1072 apix_post_cyclic_setup(void *arg)
1072 1073 {
1073 1074 UNREFERENCED_1PARAMETER(arg);
1074 1075
1076 + cyc_handler_t cyh;
1077 + cyc_time_t cyt;
1078 +
1075 1079 /* cpu_lock is held */
1076 1080 /* set up a periodic handler for intr redistribution */
1077 1081
1078 1082 /*
1079 1083 * In peridoc mode intr redistribution processing is done in
1080 1084 * apic_intr_enter during clk intr processing
1081 1085 */
1082 1086 if (!apic_oneshot)
1083 1087 return;
1084 1088
1085 1089 /*
1086 1090 * Register a periodical handler for the redistribution processing.
1087 - * On X86, CY_LOW_LEVEL is mapped to the level 2 interrupt, so
1088 - * DDI_IPL_2 should be passed to ddi_periodic_add() here.
1091 + * Though we would generally prefer to use the DDI interface for
1092 + * periodic handler invocation, ddi_periodic_add(9F), we are
1093 + * unfortunately already holding cpu_lock, which ddi_periodic_add will
1094 + * attempt to take for us. Thus, we add our own cyclic directly:
1089 1095 */
1090 - apic_periodic_id = ddi_periodic_add(
1091 - (void (*)(void *))apix_redistribute_compute, NULL,
1092 - apic_redistribute_sample_interval, DDI_IPL_2);
1096 + cyh.cyh_func = (void (*)(void *))apix_redistribute_compute;
1097 + cyh.cyh_arg = NULL;
1098 + cyh.cyh_level = CY_LOW_LEVEL;
1099 +
1100 + cyt.cyt_when = 0;
1101 + cyt.cyt_interval = apic_redistribute_sample_interval;
1102 +
1103 + apic_cyclic_id = cyclic_add(&cyh, &cyt);
1093 1104 }
1094 1105
1095 1106 /*
1096 1107 * Called the first time we enable x2apic mode on this cpu.
1097 1108 * Update some of the function pointers to use x2apic routines.
1098 1109 */
1099 1110 void
1100 1111 x2apic_update_psm()
1101 1112 {
1102 1113 struct psm_ops *pops = &apix_ops;
1103 1114
1104 1115 ASSERT(pops != NULL);
1105 1116
1106 1117 /*
1107 1118 * The pcplusmp module's version of x2apic_update_psm makes additional
1108 1119 * changes that we do not have to make here. It needs to make those
1109 1120 * changes because pcplusmp relies on the TPR register and the means of
1110 1121 * addressing that changes when using the local apic versus the x2apic.
1111 1122 * It's also worth noting that the apix driver specific function end up
1112 1123 * being apix_foo as opposed to apic_foo and x2apic_foo.
1113 1124 */
1114 1125 pops->psm_send_ipi = x2apic_send_ipi;
1115 1126
1116 1127 send_dirintf = pops->psm_send_ipi;
1117 1128
1118 1129 apic_mode = LOCAL_X2APIC;
1119 1130 apic_change_ops();
1120 1131 }
1121 1132
1122 1133 /*
1123 1134 * This function provides external interface to the nexus for all
1124 1135 * functionalities related to the new DDI interrupt framework.
1125 1136 *
1126 1137 * Input:
1127 1138 * dip - pointer to the dev_info structure of the requested device
1128 1139 * hdlp - pointer to the internal interrupt handle structure for the
1129 1140 * requested interrupt
1130 1141 * intr_op - opcode for this call
1131 1142 * result - pointer to the integer that will hold the result to be
1132 1143 * passed back if return value is PSM_SUCCESS
1133 1144 *
1134 1145 * Output:
1135 1146 * return value is either PSM_SUCCESS or PSM_FAILURE
1136 1147 */
1137 1148 static int
1138 1149 apix_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1139 1150 psm_intr_op_t intr_op, int *result)
1140 1151 {
1141 1152 int cap;
1142 1153 apix_vector_t *vecp, *newvecp;
1143 1154 struct intrspec *ispec, intr_spec;
1144 1155 processorid_t target;
1145 1156
1146 1157 ispec = &intr_spec;
1147 1158 ispec->intrspec_pri = hdlp->ih_pri;
1148 1159 ispec->intrspec_vec = hdlp->ih_inum;
1149 1160 ispec->intrspec_func = hdlp->ih_cb_func;
1150 1161
1151 1162 switch (intr_op) {
1152 1163 case PSM_INTR_OP_ALLOC_VECTORS:
1153 1164 switch (hdlp->ih_type) {
1154 1165 case DDI_INTR_TYPE_MSI:
1155 1166 /* allocate MSI vectors */
1156 1167 *result = apix_alloc_msi(dip, hdlp->ih_inum,
1157 1168 hdlp->ih_scratch1,
1158 1169 (int)(uintptr_t)hdlp->ih_scratch2);
1159 1170 break;
1160 1171 case DDI_INTR_TYPE_MSIX:
1161 1172 /* allocate MSI-X vectors */
1162 1173 *result = apix_alloc_msix(dip, hdlp->ih_inum,
1163 1174 hdlp->ih_scratch1,
1164 1175 (int)(uintptr_t)hdlp->ih_scratch2);
1165 1176 break;
1166 1177 case DDI_INTR_TYPE_FIXED:
1167 1178 /* allocate or share vector for fixed */
1168 1179 if ((ihdl_plat_t *)hdlp->ih_private == NULL) {
1169 1180 return (PSM_FAILURE);
1170 1181 }
1171 1182 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1172 1183 *result = apix_intx_alloc_vector(dip, hdlp->ih_inum,
1173 1184 ispec);
1174 1185 break;
1175 1186 default:
1176 1187 return (PSM_FAILURE);
1177 1188 }
1178 1189 break;
1179 1190 case PSM_INTR_OP_FREE_VECTORS:
1180 1191 apix_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1,
1181 1192 hdlp->ih_type);
1182 1193 break;
1183 1194 case PSM_INTR_OP_XLATE_VECTOR:
1184 1195 /*
1185 1196 * Vectors are allocated by ALLOC and freed by FREE.
1186 1197 * XLATE finds and returns APIX_VIRTVEC_VECTOR(cpu, vector).
1187 1198 */
1188 1199 *result = APIX_INVALID_VECT;
1189 1200 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1190 1201 if (vecp != NULL) {
1191 1202 *result = APIX_VIRTVECTOR(vecp->v_cpuid,
1192 1203 vecp->v_vector);
1193 1204 break;
1194 1205 }
1195 1206
1196 1207 /*
1197 1208 * No vector to device mapping exists. If this is FIXED type
1198 1209 * then check if this IRQ is already mapped for another device
1199 1210 * then return the vector number for it (i.e. shared IRQ case).
1200 1211 * Otherwise, return PSM_FAILURE.
1201 1212 */
1202 1213 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
1203 1214 vecp = apix_intx_xlate_vector(dip, hdlp->ih_inum,
1204 1215 ispec);
1205 1216 *result = (vecp == NULL) ? APIX_INVALID_VECT :
1206 1217 APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1207 1218 }
1208 1219 if (*result == APIX_INVALID_VECT)
1209 1220 return (PSM_FAILURE);
1210 1221 break;
1211 1222 case PSM_INTR_OP_GET_PENDING:
1212 1223 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1213 1224 if (vecp == NULL)
1214 1225 return (PSM_FAILURE);
1215 1226
1216 1227 *result = apix_get_pending(vecp);
1217 1228 break;
1218 1229 case PSM_INTR_OP_CLEAR_MASK:
1219 1230 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1220 1231 return (PSM_FAILURE);
1221 1232
1222 1233 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1223 1234 if (vecp == NULL)
1224 1235 return (PSM_FAILURE);
1225 1236
1226 1237 apix_intx_clear_mask(vecp->v_inum);
1227 1238 break;
1228 1239 case PSM_INTR_OP_SET_MASK:
1229 1240 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1230 1241 return (PSM_FAILURE);
1231 1242
1232 1243 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1233 1244 if (vecp == NULL)
1234 1245 return (PSM_FAILURE);
1235 1246
1236 1247 apix_intx_set_mask(vecp->v_inum);
1237 1248 break;
1238 1249 case PSM_INTR_OP_GET_SHARED:
1239 1250 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1240 1251 return (PSM_FAILURE);
1241 1252
1242 1253 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1243 1254 if (vecp == NULL)
1244 1255 return (PSM_FAILURE);
1245 1256
1246 1257 *result = apix_intx_get_shared(vecp->v_inum);
1247 1258 break;
1248 1259 case PSM_INTR_OP_SET_PRI:
1249 1260 /*
1250 1261 * Called prior to adding the interrupt handler or when
1251 1262 * an interrupt handler is unassigned.
1252 1263 */
1253 1264 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1254 1265 return (PSM_SUCCESS);
1255 1266
1256 1267 if (apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type) == NULL)
1257 1268 return (PSM_FAILURE);
1258 1269
1259 1270 break;
1260 1271 case PSM_INTR_OP_SET_CPU:
1261 1272 case PSM_INTR_OP_GRP_SET_CPU:
1262 1273 /*
1263 1274 * The interrupt handle given here has been allocated
1264 1275 * specifically for this command, and ih_private carries
1265 1276 * a CPU value.
1266 1277 */
1267 1278 *result = EINVAL;
1268 1279 target = (int)(intptr_t)hdlp->ih_private;
1269 1280 if (!apic_cpu_in_range(target)) {
1270 1281 DDI_INTR_IMPLDBG((CE_WARN,
1271 1282 "[grp_]set_cpu: cpu out of range: %d\n", target));
1272 1283 return (PSM_FAILURE);
1273 1284 }
1274 1285
1275 1286 lock_set(&apix_lock);
1276 1287
1277 1288 vecp = apix_get_req_vector(hdlp, hdlp->ih_flags);
1278 1289 if (!IS_VECT_ENABLED(vecp)) {
1279 1290 DDI_INTR_IMPLDBG((CE_WARN,
1280 1291 "[grp]_set_cpu: invalid vector 0x%x\n",
1281 1292 hdlp->ih_vector));
1282 1293 lock_clear(&apix_lock);
1283 1294 return (PSM_FAILURE);
1284 1295 }
1285 1296
1286 1297 *result = 0;
1287 1298
1288 1299 if (intr_op == PSM_INTR_OP_SET_CPU)
1289 1300 newvecp = apix_set_cpu(vecp, target, result);
1290 1301 else
1291 1302 newvecp = apix_grp_set_cpu(vecp, target, result);
1292 1303
1293 1304 lock_clear(&apix_lock);
1294 1305
1295 1306 if (newvecp == NULL) {
1296 1307 *result = EIO;
1297 1308 return (PSM_FAILURE);
1298 1309 }
1299 1310 newvecp->v_bound_cpuid = target;
1300 1311 hdlp->ih_vector = APIX_VIRTVECTOR(newvecp->v_cpuid,
1301 1312 newvecp->v_vector);
1302 1313 break;
1303 1314
1304 1315 case PSM_INTR_OP_GET_INTR:
1305 1316 /*
1306 1317 * The interrupt handle given here has been allocated
1307 1318 * specifically for this command, and ih_private carries
1308 1319 * a pointer to a apic_get_intr_t.
1309 1320 */
1310 1321 if (apix_get_intr_info(hdlp, hdlp->ih_private) != PSM_SUCCESS)
1311 1322 return (PSM_FAILURE);
1312 1323 break;
1313 1324
1314 1325 case PSM_INTR_OP_CHECK_MSI:
1315 1326 /*
1316 1327 * Check MSI/X is supported or not at APIC level and
1317 1328 * masked off the MSI/X bits in hdlp->ih_type if not
1318 1329 * supported before return. If MSI/X is supported,
1319 1330 * leave the ih_type unchanged and return.
1320 1331 *
1321 1332 * hdlp->ih_type passed in from the nexus has all the
1322 1333 * interrupt types supported by the device.
1323 1334 */
1324 1335 if (apic_support_msi == 0) { /* uninitialized */
1325 1336 /*
1326 1337 * if apic_support_msi is not set, call
1327 1338 * apic_check_msi_support() to check whether msi
1328 1339 * is supported first
1329 1340 */
1330 1341 if (apic_check_msi_support() == PSM_SUCCESS)
1331 1342 apic_support_msi = 1; /* supported */
1332 1343 else
1333 1344 apic_support_msi = -1; /* not-supported */
1334 1345 }
1335 1346 if (apic_support_msi == 1) {
1336 1347 if (apic_msix_enable)
1337 1348 *result = hdlp->ih_type;
1338 1349 else
1339 1350 *result = hdlp->ih_type & ~DDI_INTR_TYPE_MSIX;
1340 1351 } else
1341 1352 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1342 1353 DDI_INTR_TYPE_MSIX);
1343 1354 break;
1344 1355 case PSM_INTR_OP_GET_CAP:
1345 1356 cap = DDI_INTR_FLAG_PENDING;
1346 1357 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1347 1358 cap |= DDI_INTR_FLAG_MASKABLE;
1348 1359 *result = cap;
1349 1360 break;
1350 1361 case PSM_INTR_OP_APIC_TYPE:
1351 1362 ((apic_get_type_t *)(hdlp->ih_private))->avgi_type =
1352 1363 apix_get_apic_type();
1353 1364 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_intr =
1354 1365 APIX_IPI_MIN;
1355 1366 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_cpu =
1356 1367 apic_nproc;
1357 1368 hdlp->ih_ver = apic_get_apic_version();
1358 1369 break;
1359 1370 case PSM_INTR_OP_SET_CAP:
1360 1371 default:
1361 1372 return (PSM_FAILURE);
1362 1373 }
1363 1374
1364 1375 return (PSM_SUCCESS);
1365 1376 }
1366 1377
1367 1378 static void
1368 1379 apix_cleanup_busy(void)
1369 1380 {
1370 1381 int i, j;
1371 1382 apix_vector_t *vecp;
1372 1383
1373 1384 for (i = 0; i < apic_nproc; i++) {
1374 1385 if (!apic_cpu_in_range(i))
1375 1386 continue;
1376 1387 apic_cpus[i].aci_busy = 0;
1377 1388 for (j = APIX_AVINTR_MIN; j < APIX_AVINTR_MAX; j++) {
1378 1389 if ((vecp = xv_vector(i, j)) != NULL)
1379 1390 vecp->v_busy = 0;
1380 1391 }
1381 1392 }
1382 1393 }
1383 1394
1384 1395 static void
1385 1396 apix_redistribute_compute(void)
1386 1397 {
1387 1398 int i, j, max_busy;
1388 1399
1389 1400 if (!apic_enable_dynamic_migration)
1390 1401 return;
1391 1402
1392 1403 if (++apic_nticks == apic_sample_factor_redistribution) {
1393 1404 /*
1394 1405 * Time to call apic_intr_redistribute().
1395 1406 * reset apic_nticks. This will cause max_busy
1396 1407 * to be calculated below and if it is more than
1397 1408 * apic_int_busy, we will do the whole thing
1398 1409 */
1399 1410 apic_nticks = 0;
1400 1411 }
1401 1412 max_busy = 0;
1402 1413 for (i = 0; i < apic_nproc; i++) {
1403 1414 if (!apic_cpu_in_range(i))
1404 1415 continue;
1405 1416 /*
1406 1417 * Check if curipl is non zero & if ISR is in
1407 1418 * progress
1408 1419 */
1409 1420 if (((j = apic_cpus[i].aci_curipl) != 0) &&
1410 1421 (apic_cpus[i].aci_ISR_in_progress & (1 << j))) {
1411 1422
1412 1423 int vect;
1413 1424 apic_cpus[i].aci_busy++;
1414 1425 vect = apic_cpus[i].aci_current[j];
1415 1426 apixs[i]->x_vectbl[vect]->v_busy++;
1416 1427 }
1417 1428
1418 1429 if (!apic_nticks &&
1419 1430 (apic_cpus[i].aci_busy > max_busy))
1420 1431 max_busy = apic_cpus[i].aci_busy;
1421 1432 }
1422 1433 if (!apic_nticks) {
1423 1434 if (max_busy > apic_int_busy_mark) {
1424 1435 /*
1425 1436 * We could make the following check be
1426 1437 * skipped > 1 in which case, we get a
1427 1438 * redistribution at half the busy mark (due to
1428 1439 * double interval). Need to be able to collect
1429 1440 * more empirical data to decide if that is a
1430 1441 * good strategy. Punt for now.
1431 1442 */
1432 1443 apix_cleanup_busy();
1433 1444 apic_skipped_redistribute = 0;
1434 1445 } else
1435 1446 apic_skipped_redistribute++;
1436 1447 }
1437 1448 }
1438 1449
1439 1450 /*
1440 1451 * intr_ops() service routines
1441 1452 */
1442 1453
1443 1454 static int
1444 1455 apix_get_pending(apix_vector_t *vecp)
1445 1456 {
1446 1457 int bit, index, irr, pending;
1447 1458
1448 1459 /* need to get on the bound cpu */
1449 1460 mutex_enter(&cpu_lock);
1450 1461 affinity_set(vecp->v_cpuid);
1451 1462
1452 1463 index = vecp->v_vector / 32;
1453 1464 bit = vecp->v_vector % 32;
1454 1465 irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
1455 1466
1456 1467 affinity_clear();
1457 1468 mutex_exit(&cpu_lock);
1458 1469
1459 1470 pending = (irr & (1 << bit)) ? 1 : 0;
1460 1471 if (!pending && vecp->v_type == APIX_TYPE_FIXED)
1461 1472 pending = apix_intx_get_pending(vecp->v_inum);
1462 1473
1463 1474 return (pending);
1464 1475 }
1465 1476
1466 1477 static apix_vector_t *
1467 1478 apix_get_req_vector(ddi_intr_handle_impl_t *hdlp, ushort_t flags)
1468 1479 {
1469 1480 apix_vector_t *vecp;
1470 1481 processorid_t cpuid;
1471 1482 int32_t virt_vec = 0;
1472 1483
1473 1484 switch (flags & PSMGI_INTRBY_FLAGS) {
1474 1485 case PSMGI_INTRBY_IRQ:
1475 1486 return (apix_intx_get_vector(hdlp->ih_vector));
1476 1487 case PSMGI_INTRBY_VEC:
1477 1488 virt_vec = (virt_vec == 0) ? hdlp->ih_vector : virt_vec;
1478 1489
1479 1490 cpuid = APIX_VIRTVEC_CPU(virt_vec);
1480 1491 if (!apic_cpu_in_range(cpuid))
1481 1492 return (NULL);
1482 1493
1483 1494 vecp = xv_vector(cpuid, APIX_VIRTVEC_VECTOR(virt_vec));
1484 1495 break;
1485 1496 case PSMGI_INTRBY_DEFAULT:
1486 1497 vecp = apix_get_dev_map(hdlp->ih_dip, hdlp->ih_inum,
1487 1498 hdlp->ih_type);
1488 1499 break;
1489 1500 default:
1490 1501 return (NULL);
1491 1502 }
1492 1503
1493 1504 return (vecp);
1494 1505 }
1495 1506
1496 1507 static int
1497 1508 apix_get_intr_info(ddi_intr_handle_impl_t *hdlp,
1498 1509 apic_get_intr_t *intr_params_p)
1499 1510 {
1500 1511 apix_vector_t *vecp;
1501 1512 struct autovec *av_dev;
1502 1513 int i;
1503 1514
1504 1515 vecp = apix_get_req_vector(hdlp, intr_params_p->avgi_req_flags);
1505 1516 if (IS_VECT_FREE(vecp)) {
1506 1517 intr_params_p->avgi_num_devs = 0;
1507 1518 intr_params_p->avgi_cpu_id = 0;
1508 1519 intr_params_p->avgi_req_flags = 0;
1509 1520 return (PSM_SUCCESS);
1510 1521 }
1511 1522
1512 1523 if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) {
1513 1524 intr_params_p->avgi_cpu_id = vecp->v_cpuid;
1514 1525
1515 1526 /* Return user bound info for intrd. */
1516 1527 if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) {
1517 1528 intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND;
1518 1529 intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND;
1519 1530 }
1520 1531 }
1521 1532
1522 1533 if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR)
1523 1534 intr_params_p->avgi_vector = vecp->v_vector;
1524 1535
1525 1536 if (intr_params_p->avgi_req_flags &
1526 1537 (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS))
1527 1538 /* Get number of devices from apic_irq table shared field. */
1528 1539 intr_params_p->avgi_num_devs = vecp->v_share;
1529 1540
1530 1541 if (intr_params_p->avgi_req_flags & PSMGI_REQ_GET_DEVS) {
1531 1542
1532 1543 intr_params_p->avgi_req_flags |= PSMGI_REQ_NUM_DEVS;
1533 1544
1534 1545 /* Some devices have NULL dip. Don't count these. */
1535 1546 if (intr_params_p->avgi_num_devs > 0) {
1536 1547 for (i = 0, av_dev = vecp->v_autovect; av_dev;
1537 1548 av_dev = av_dev->av_link) {
1538 1549 if (av_dev->av_vector && av_dev->av_dip)
1539 1550 i++;
1540 1551 }
1541 1552 intr_params_p->avgi_num_devs =
1542 1553 (uint8_t)MIN(intr_params_p->avgi_num_devs, i);
1543 1554 }
1544 1555
1545 1556 /* There are no viable dips to return. */
1546 1557 if (intr_params_p->avgi_num_devs == 0) {
1547 1558 intr_params_p->avgi_dip_list = NULL;
1548 1559
1549 1560 } else { /* Return list of dips */
1550 1561
1551 1562 /* Allocate space in array for that number of devs. */
1552 1563 intr_params_p->avgi_dip_list = kmem_zalloc(
1553 1564 intr_params_p->avgi_num_devs *
1554 1565 sizeof (dev_info_t *),
1555 1566 KM_NOSLEEP);
1556 1567 if (intr_params_p->avgi_dip_list == NULL) {
1557 1568 DDI_INTR_IMPLDBG((CE_WARN,
1558 1569 "apix_get_vector_intr_info: no memory"));
1559 1570 return (PSM_FAILURE);
1560 1571 }
1561 1572
1562 1573 /*
1563 1574 * Loop through the device list of the autovec table
1564 1575 * filling in the dip array.
1565 1576 *
1566 1577 * Note that the autovect table may have some special
1567 1578 * entries which contain NULL dips. These will be
1568 1579 * ignored.
1569 1580 */
1570 1581 for (i = 0, av_dev = vecp->v_autovect; av_dev;
1571 1582 av_dev = av_dev->av_link) {
1572 1583 if (av_dev->av_vector && av_dev->av_dip)
1573 1584 intr_params_p->avgi_dip_list[i++] =
1574 1585 av_dev->av_dip;
1575 1586 }
1576 1587 }
1577 1588 }
1578 1589
1579 1590 return (PSM_SUCCESS);
1580 1591 }
1581 1592
1582 1593 static char *
1583 1594 apix_get_apic_type(void)
1584 1595 {
1585 1596 return (apix_psm_info.p_mach_idstring);
1586 1597 }
1587 1598
1588 1599 apix_vector_t *
1589 1600 apix_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1590 1601 {
1591 1602 apix_vector_t *newp = NULL;
1592 1603 dev_info_t *dip;
1593 1604 int inum, cap_ptr;
1594 1605 ddi_acc_handle_t handle;
1595 1606 ddi_intr_msix_t *msix_p = NULL;
1596 1607 ushort_t msix_ctrl;
1597 1608 uintptr_t off;
1598 1609 uint32_t mask;
1599 1610
1600 1611 ASSERT(LOCK_HELD(&apix_lock));
1601 1612 *result = ENXIO;
1602 1613
1603 1614 /* Fail if this is an MSI intr and is part of a group. */
1604 1615 if (vecp->v_type == APIX_TYPE_MSI) {
1605 1616 if (i_ddi_intr_get_current_nintrs(APIX_GET_DIP(vecp)) > 1)
1606 1617 return (NULL);
1607 1618 else
1608 1619 return (apix_grp_set_cpu(vecp, new_cpu, result));
1609 1620 }
1610 1621
1611 1622 /*
1612 1623 * Mask MSI-X. It's unmasked when MSI-X gets enabled.
1613 1624 */
1614 1625 if (vecp->v_type == APIX_TYPE_MSIX && IS_VECT_ENABLED(vecp)) {
1615 1626 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1616 1627 return (NULL);
1617 1628 inum = vecp->v_devp->dv_inum;
1618 1629
1619 1630 handle = i_ddi_get_pci_config_handle(dip);
1620 1631 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1621 1632 msix_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1622 1633 if ((msix_ctrl & PCI_MSIX_FUNCTION_MASK) == 0) {
1623 1634 /*
1624 1635 * Function is not masked, then mask "inum"th
1625 1636 * entry in the MSI-X table
1626 1637 */
1627 1638 msix_p = i_ddi_get_msix(dip);
1628 1639 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1629 1640 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1630 1641 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1631 1642 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off,
1632 1643 mask | 1);
1633 1644 }
1634 1645 }
1635 1646
1636 1647 *result = 0;
1637 1648 if ((newp = apix_rebind(vecp, new_cpu, 1)) == NULL)
1638 1649 *result = EIO;
1639 1650
1640 1651 /* Restore mask bit */
1641 1652 if (msix_p != NULL)
1642 1653 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, mask);
1643 1654
1644 1655 return (newp);
1645 1656 }
1646 1657
1647 1658 /*
1648 1659 * Set cpu for MSIs
1649 1660 */
1650 1661 apix_vector_t *
1651 1662 apix_grp_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1652 1663 {
1653 1664 apix_vector_t *newp, *vp;
1654 1665 uint32_t orig_cpu = vecp->v_cpuid;
1655 1666 int orig_vect = vecp->v_vector;
1656 1667 int i, num_vectors, cap_ptr, msi_mask_off;
1657 1668 uint32_t msi_pvm;
1658 1669 ushort_t msi_ctrl;
1659 1670 ddi_acc_handle_t handle;
1660 1671 dev_info_t *dip;
1661 1672
1662 1673 APIC_VERBOSE(INTR, (CE_CONT, "apix_grp_set_cpu: oldcpu: %x, vector: %x,"
1663 1674 " newcpu:%x\n", vecp->v_cpuid, vecp->v_vector, new_cpu));
1664 1675
1665 1676 ASSERT(LOCK_HELD(&apix_lock));
1666 1677
1667 1678 *result = ENXIO;
1668 1679
1669 1680 if (vecp->v_type != APIX_TYPE_MSI) {
1670 1681 DDI_INTR_IMPLDBG((CE_WARN, "set_grp: intr not MSI\n"));
1671 1682 return (NULL);
1672 1683 }
1673 1684
1674 1685 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1675 1686 return (NULL);
1676 1687
1677 1688 num_vectors = i_ddi_intr_get_current_nintrs(dip);
1678 1689 if ((num_vectors < 1) || ((num_vectors - 1) & orig_vect)) {
1679 1690 APIC_VERBOSE(INTR, (CE_WARN,
1680 1691 "set_grp: base vec not part of a grp or not aligned: "
1681 1692 "vec:0x%x, num_vec:0x%x\n", orig_vect, num_vectors));
1682 1693 return (NULL);
1683 1694 }
1684 1695
1685 1696 if (vecp->v_inum != apix_get_min_dev_inum(dip, vecp->v_type))
1686 1697 return (NULL);
1687 1698
1688 1699 *result = EIO;
1689 1700 for (i = 1; i < num_vectors; i++) {
1690 1701 if ((vp = xv_vector(orig_cpu, orig_vect + i)) == NULL)
1691 1702 return (NULL);
1692 1703 #ifdef DEBUG
1693 1704 /*
1694 1705 * Sanity check: CPU and dip is the same for all entries.
1695 1706 * May be called when first msi to be enabled, at this time
1696 1707 * add_avintr() is not called for other msi
1697 1708 */
1698 1709 if ((vp->v_share != 0) &&
1699 1710 ((APIX_GET_DIP(vp) != dip) ||
1700 1711 (vp->v_cpuid != vecp->v_cpuid))) {
1701 1712 APIC_VERBOSE(INTR, (CE_WARN,
1702 1713 "set_grp: cpu or dip for vec 0x%x difft than for "
1703 1714 "vec 0x%x\n", orig_vect, orig_vect + i));
1704 1715 APIC_VERBOSE(INTR, (CE_WARN,
1705 1716 " cpu: %d vs %d, dip: 0x%p vs 0x%p\n", orig_cpu,
1706 1717 vp->v_cpuid, (void *)dip,
1707 1718 (void *)APIX_GET_DIP(vp)));
1708 1719 return (NULL);
1709 1720 }
1710 1721 #endif /* DEBUG */
1711 1722 }
1712 1723
1713 1724 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1714 1725 handle = i_ddi_get_pci_config_handle(dip);
1715 1726 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1716 1727
1717 1728 /* MSI Per vector masking is supported. */
1718 1729 if (msi_ctrl & PCI_MSI_PVM_MASK) {
1719 1730 if (msi_ctrl & PCI_MSI_64BIT_MASK)
1720 1731 msi_mask_off = cap_ptr + PCI_MSI_64BIT_MASKBITS;
1721 1732 else
1722 1733 msi_mask_off = cap_ptr + PCI_MSI_32BIT_MASK;
1723 1734 msi_pvm = pci_config_get32(handle, msi_mask_off);
1724 1735 pci_config_put32(handle, msi_mask_off, (uint32_t)-1);
1725 1736 APIC_VERBOSE(INTR, (CE_CONT,
1726 1737 "set_grp: pvm supported. Mask set to 0x%x\n",
1727 1738 pci_config_get32(handle, msi_mask_off)));
1728 1739 }
1729 1740
1730 1741 if ((newp = apix_rebind(vecp, new_cpu, num_vectors)) != NULL)
1731 1742 *result = 0;
1732 1743
1733 1744 /* Reenable vectors if per vector masking is supported. */
1734 1745 if (msi_ctrl & PCI_MSI_PVM_MASK) {
1735 1746 pci_config_put32(handle, msi_mask_off, msi_pvm);
1736 1747 APIC_VERBOSE(INTR, (CE_CONT,
1737 1748 "set_grp: pvm supported. Mask restored to 0x%x\n",
1738 1749 pci_config_get32(handle, msi_mask_off)));
1739 1750 }
1740 1751
1741 1752 return (newp);
1742 1753 }
1743 1754
1744 1755 void
1745 1756 apix_intx_set_vector(int irqno, uint32_t cpuid, uchar_t vector)
1746 1757 {
1747 1758 apic_irq_t *irqp;
1748 1759
1749 1760 mutex_enter(&airq_mutex);
1750 1761 irqp = apic_irq_table[irqno];
1751 1762 irqp->airq_cpu = cpuid;
1752 1763 irqp->airq_vector = vector;
1753 1764 apic_record_rdt_entry(irqp, irqno);
1754 1765 mutex_exit(&airq_mutex);
1755 1766 }
1756 1767
1757 1768 apix_vector_t *
1758 1769 apix_intx_get_vector(int irqno)
1759 1770 {
1760 1771 apic_irq_t *irqp;
1761 1772 uint32_t cpuid;
1762 1773 uchar_t vector;
1763 1774
1764 1775 mutex_enter(&airq_mutex);
1765 1776 irqp = apic_irq_table[irqno & 0xff];
1766 1777 if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
1767 1778 mutex_exit(&airq_mutex);
1768 1779 return (NULL);
1769 1780 }
1770 1781 cpuid = irqp->airq_cpu;
1771 1782 vector = irqp->airq_vector;
1772 1783 mutex_exit(&airq_mutex);
1773 1784
1774 1785 return (xv_vector(cpuid, vector));
1775 1786 }
1776 1787
1777 1788 /*
1778 1789 * Must called with interrupts disabled and apic_ioapic_lock held
1779 1790 */
1780 1791 void
1781 1792 apix_intx_enable(int irqno)
1782 1793 {
1783 1794 uchar_t ioapicindex, intin;
1784 1795 apic_irq_t *irqp = apic_irq_table[irqno];
1785 1796 ioapic_rdt_t irdt;
1786 1797 apic_cpus_info_t *cpu_infop;
1787 1798 apix_vector_t *vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1788 1799
1789 1800 ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1790 1801
1791 1802 ioapicindex = irqp->airq_ioapicindex;
1792 1803 intin = irqp->airq_intin_no;
1793 1804 cpu_infop = &apic_cpus[irqp->airq_cpu];
1794 1805
1795 1806 irdt.ir_lo = AV_PDEST | AV_FIXED | irqp->airq_rdt_entry;
1796 1807 irdt.ir_hi = cpu_infop->aci_local_id;
1797 1808
1798 1809 apic_vt_ops->apic_intrmap_alloc_entry(&vecp->v_intrmap_private, NULL,
1799 1810 vecp->v_type, 1, ioapicindex);
1800 1811 apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
1801 1812 (void *)&irdt, vecp->v_type, 1);
1802 1813 apic_vt_ops->apic_intrmap_record_rdt(vecp->v_intrmap_private, &irdt);
1803 1814
1804 1815 /* write RDT entry high dword - destination */
1805 1816 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin,
1806 1817 irdt.ir_hi);
1807 1818
1808 1819 /* Write the vector, trigger, and polarity portion of the RDT */
1809 1820 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin, irdt.ir_lo);
1810 1821
1811 1822 vecp->v_state = APIX_STATE_ENABLED;
1812 1823
1813 1824 APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_enable: ioapic 0x%x"
1814 1825 " intin 0x%x rdt_low 0x%x rdt_high 0x%x\n",
1815 1826 ioapicindex, intin, irdt.ir_lo, irdt.ir_hi));
1816 1827 }
1817 1828
1818 1829 /*
1819 1830 * Must called with interrupts disabled and apic_ioapic_lock held
1820 1831 */
1821 1832 void
1822 1833 apix_intx_disable(int irqno)
1823 1834 {
1824 1835 apic_irq_t *irqp = apic_irq_table[irqno];
1825 1836 int ioapicindex, intin;
1826 1837
1827 1838 ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1828 1839 /*
1829 1840 * The assumption here is that this is safe, even for
1830 1841 * systems with IOAPICs that suffer from the hardware
1831 1842 * erratum because all devices have been quiesced before
1832 1843 * they unregister their interrupt handlers. If that
1833 1844 * assumption turns out to be false, this mask operation
1834 1845 * can induce the same erratum result we're trying to
1835 1846 * avoid.
1836 1847 */
1837 1848 ioapicindex = irqp->airq_ioapicindex;
1838 1849 intin = irqp->airq_intin_no;
1839 1850 ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * intin, AV_MASK);
1840 1851
1841 1852 APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_disable: ioapic 0x%x"
1842 1853 " intin 0x%x\n", ioapicindex, intin));
1843 1854 }
1844 1855
1845 1856 void
1846 1857 apix_intx_free(int irqno)
1847 1858 {
1848 1859 apic_irq_t *irqp;
1849 1860
1850 1861 mutex_enter(&airq_mutex);
1851 1862 irqp = apic_irq_table[irqno];
1852 1863
1853 1864 if (IS_IRQ_FREE(irqp)) {
1854 1865 mutex_exit(&airq_mutex);
1855 1866 return;
1856 1867 }
1857 1868
1858 1869 irqp->airq_mps_intr_index = FREE_INDEX;
1859 1870 irqp->airq_cpu = IRQ_UNINIT;
1860 1871 irqp->airq_vector = APIX_INVALID_VECT;
1861 1872 mutex_exit(&airq_mutex);
1862 1873 }
1863 1874
1864 1875 #ifdef DEBUG
1865 1876 int apix_intr_deliver_timeouts = 0;
1866 1877 int apix_intr_rirr_timeouts = 0;
1867 1878 int apix_intr_rirr_reset_failure = 0;
1868 1879 #endif
1869 1880 int apix_max_reps_irr_pending = 10;
1870 1881
1871 1882 #define GET_RDT_BITS(ioapic, intin, bits) \
1872 1883 (READ_IOAPIC_RDT_ENTRY_LOW_DWORD((ioapic), (intin)) & (bits))
1873 1884 #define APIX_CHECK_IRR_DELAY drv_usectohz(5000)
1874 1885
1875 1886 int
1876 1887 apix_intx_rebind(int irqno, processorid_t cpuid, uchar_t vector)
1877 1888 {
1878 1889 apic_irq_t *irqp = apic_irq_table[irqno];
1879 1890 ulong_t iflag;
1880 1891 int waited, ioapic_ix, intin_no, level, repeats, rdt_entry, masked;
1881 1892
1882 1893 ASSERT(irqp != NULL);
1883 1894
1884 1895 iflag = intr_clear();
1885 1896 lock_set(&apic_ioapic_lock);
1886 1897
1887 1898 ioapic_ix = irqp->airq_ioapicindex;
1888 1899 intin_no = irqp->airq_intin_no;
1889 1900 level = apic_level_intr[irqno];
1890 1901
1891 1902 /*
1892 1903 * Wait for the delivery status bit to be cleared. This should
1893 1904 * be a very small amount of time.
1894 1905 */
1895 1906 repeats = 0;
1896 1907 do {
1897 1908 repeats++;
1898 1909
1899 1910 for (waited = 0; waited < apic_max_reps_clear_pending;
1900 1911 waited++) {
1901 1912 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) == 0)
1902 1913 break;
1903 1914 }
1904 1915 if (!level)
1905 1916 break;
1906 1917
1907 1918 /*
1908 1919 * Mask the RDT entry for level-triggered interrupts.
1909 1920 */
1910 1921 irqp->airq_rdt_entry |= AV_MASK;
1911 1922 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1912 1923 intin_no);
1913 1924 if ((masked = (rdt_entry & AV_MASK)) == 0) {
1914 1925 /* Mask it */
1915 1926 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1916 1927 AV_MASK | rdt_entry);
1917 1928 }
1918 1929
1919 1930 /*
1920 1931 * If there was a race and an interrupt was injected
1921 1932 * just before we masked, check for that case here.
1922 1933 * Then, unmask the RDT entry and try again. If we're
1923 1934 * on our last try, don't unmask (because we want the
1924 1935 * RDT entry to remain masked for the rest of the
1925 1936 * function).
1926 1937 */
1927 1938 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1928 1939 intin_no);
1929 1940 if ((masked == 0) && ((rdt_entry & AV_PENDING) != 0) &&
1930 1941 (repeats < apic_max_reps_clear_pending)) {
1931 1942 /* Unmask it */
1932 1943 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1933 1944 intin_no, rdt_entry & ~AV_MASK);
1934 1945 irqp->airq_rdt_entry &= ~AV_MASK;
1935 1946 }
1936 1947 } while ((rdt_entry & AV_PENDING) &&
1937 1948 (repeats < apic_max_reps_clear_pending));
1938 1949
1939 1950 #ifdef DEBUG
1940 1951 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) != 0)
1941 1952 apix_intr_deliver_timeouts++;
1942 1953 #endif
1943 1954
1944 1955 if (!level || !APIX_IS_MASK_RDT(apix_mul_ioapic_method))
1945 1956 goto done;
1946 1957
1947 1958 /*
1948 1959 * wait for remote IRR to be cleared for level-triggered
1949 1960 * interrupts
1950 1961 */
1951 1962 repeats = 0;
1952 1963 do {
1953 1964 repeats++;
1954 1965
1955 1966 for (waited = 0; waited < apic_max_reps_clear_pending;
1956 1967 waited++) {
1957 1968 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR)
1958 1969 == 0)
1959 1970 break;
1960 1971 }
1961 1972
1962 1973 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1963 1974 lock_clear(&apic_ioapic_lock);
1964 1975 intr_restore(iflag);
1965 1976
1966 1977 delay(APIX_CHECK_IRR_DELAY);
1967 1978
1968 1979 iflag = intr_clear();
1969 1980 lock_set(&apic_ioapic_lock);
1970 1981 }
1971 1982 } while (repeats < apix_max_reps_irr_pending);
1972 1983
1973 1984 if (repeats >= apix_max_reps_irr_pending) {
1974 1985 #ifdef DEBUG
1975 1986 apix_intr_rirr_timeouts++;
1976 1987 #endif
1977 1988
1978 1989 /*
1979 1990 * If we waited and the Remote IRR bit is still not cleared,
1980 1991 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1981 1992 * times for this interrupt, try the last-ditch workaround:
1982 1993 */
1983 1994 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1984 1995 /*
1985 1996 * Trying to clear the bit through normal
1986 1997 * channels has failed. So as a last-ditch
1987 1998 * effort, try to set the trigger mode to
1988 1999 * edge, then to level. This has been
1989 2000 * observed to work on many systems.
1990 2001 */
1991 2002 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1992 2003 intin_no,
1993 2004 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1994 2005 intin_no) & ~AV_LEVEL);
1995 2006 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1996 2007 intin_no,
1997 2008 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1998 2009 intin_no) | AV_LEVEL);
1999 2010 }
2000 2011
2001 2012 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
2002 2013 #ifdef DEBUG
2003 2014 apix_intr_rirr_reset_failure++;
2004 2015 #endif
2005 2016 lock_clear(&apic_ioapic_lock);
2006 2017 intr_restore(iflag);
2007 2018 prom_printf("apix: Remote IRR still "
2008 2019 "not clear for IOAPIC %d intin %d.\n"
2009 2020 "\tInterrupts to this pin may cease "
2010 2021 "functioning.\n", ioapic_ix, intin_no);
2011 2022 return (1); /* return failure */
2012 2023 }
2013 2024 }
2014 2025
2015 2026 done:
2016 2027 /* change apic_irq_table */
2017 2028 lock_clear(&apic_ioapic_lock);
2018 2029 intr_restore(iflag);
2019 2030 apix_intx_set_vector(irqno, cpuid, vector);
2020 2031 iflag = intr_clear();
2021 2032 lock_set(&apic_ioapic_lock);
2022 2033
2023 2034 /* reprogramme IO-APIC RDT entry */
2024 2035 apix_intx_enable(irqno);
2025 2036
2026 2037 lock_clear(&apic_ioapic_lock);
2027 2038 intr_restore(iflag);
2028 2039
2029 2040 return (0);
2030 2041 }
2031 2042
2032 2043 static int
2033 2044 apix_intx_get_pending(int irqno)
2034 2045 {
2035 2046 apic_irq_t *irqp;
2036 2047 int intin, ioapicindex, pending;
2037 2048 ulong_t iflag;
2038 2049
2039 2050 mutex_enter(&airq_mutex);
2040 2051 irqp = apic_irq_table[irqno];
2041 2052 if (IS_IRQ_FREE(irqp)) {
2042 2053 mutex_exit(&airq_mutex);
2043 2054 return (0);
2044 2055 }
2045 2056
2046 2057 /* check IO-APIC delivery status */
2047 2058 intin = irqp->airq_intin_no;
2048 2059 ioapicindex = irqp->airq_ioapicindex;
2049 2060 mutex_exit(&airq_mutex);
2050 2061
2051 2062 iflag = intr_clear();
2052 2063 lock_set(&apic_ioapic_lock);
2053 2064
2054 2065 pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin) &
2055 2066 AV_PENDING) ? 1 : 0;
2056 2067
2057 2068 lock_clear(&apic_ioapic_lock);
2058 2069 intr_restore(iflag);
2059 2070
2060 2071 return (pending);
2061 2072 }
2062 2073
2063 2074 /*
2064 2075 * This function will mask the interrupt on the I/O APIC
2065 2076 */
2066 2077 static void
2067 2078 apix_intx_set_mask(int irqno)
2068 2079 {
2069 2080 int intin, ioapixindex, rdt_entry;
2070 2081 ulong_t iflag;
2071 2082 apic_irq_t *irqp;
2072 2083
2073 2084 mutex_enter(&airq_mutex);
2074 2085 irqp = apic_irq_table[irqno];
2075 2086
2076 2087 ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2077 2088
2078 2089 intin = irqp->airq_intin_no;
2079 2090 ioapixindex = irqp->airq_ioapicindex;
2080 2091 mutex_exit(&airq_mutex);
2081 2092
2082 2093 iflag = intr_clear();
2083 2094 lock_set(&apic_ioapic_lock);
2084 2095
2085 2096 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2086 2097
2087 2098 /* clear mask */
2088 2099 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2089 2100 (AV_MASK | rdt_entry));
2090 2101
2091 2102 lock_clear(&apic_ioapic_lock);
2092 2103 intr_restore(iflag);
2093 2104 }
2094 2105
2095 2106 /*
2096 2107 * This function will clear the mask for the interrupt on the I/O APIC
2097 2108 */
2098 2109 static void
2099 2110 apix_intx_clear_mask(int irqno)
2100 2111 {
2101 2112 int intin, ioapixindex, rdt_entry;
2102 2113 ulong_t iflag;
2103 2114 apic_irq_t *irqp;
2104 2115
2105 2116 mutex_enter(&airq_mutex);
2106 2117 irqp = apic_irq_table[irqno];
2107 2118
2108 2119 ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2109 2120
2110 2121 intin = irqp->airq_intin_no;
2111 2122 ioapixindex = irqp->airq_ioapicindex;
2112 2123 mutex_exit(&airq_mutex);
2113 2124
2114 2125 iflag = intr_clear();
2115 2126 lock_set(&apic_ioapic_lock);
2116 2127
2117 2128 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2118 2129
2119 2130 /* clear mask */
2120 2131 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2121 2132 ((~AV_MASK) & rdt_entry));
2122 2133
2123 2134 lock_clear(&apic_ioapic_lock);
2124 2135 intr_restore(iflag);
2125 2136 }
2126 2137
2127 2138 /*
2128 2139 * For level-triggered interrupt, mask the IRQ line. Mask means
2129 2140 * new interrupts will not be delivered. The interrupt already
2130 2141 * accepted by a local APIC is not affected
2131 2142 */
2132 2143 void
2133 2144 apix_level_intr_pre_eoi(int irq)
2134 2145 {
2135 2146 apic_irq_t *irqp = apic_irq_table[irq];
2136 2147 int apic_ix, intin_ix;
2137 2148
2138 2149 if (irqp == NULL)
2139 2150 return;
2140 2151
2141 2152 ASSERT(apic_level_intr[irq] == TRIGGER_MODE_LEVEL);
2142 2153
2143 2154 lock_set(&apic_ioapic_lock);
2144 2155
2145 2156 intin_ix = irqp->airq_intin_no;
2146 2157 apic_ix = irqp->airq_ioapicindex;
2147 2158
2148 2159 if (irqp->airq_cpu != CPU->cpu_id) {
2149 2160 if (!APIX_IS_MASK_RDT(apix_mul_ioapic_method))
2150 2161 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2151 2162 lock_clear(&apic_ioapic_lock);
2152 2163 return;
2153 2164 }
2154 2165
2155 2166 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC) {
2156 2167 /*
2157 2168 * This is a IOxAPIC and there is EOI register:
2158 2169 * Change the vector to reserved unused vector, so that
2159 2170 * the EOI from Local APIC won't clear the Remote IRR for
2160 2171 * this level trigger interrupt. Instead, we'll manually
2161 2172 * clear it in apix_post_hardint() after ISR handling.
2162 2173 */
2163 2174 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2164 2175 (irqp->airq_rdt_entry & (~0xff)) | APIX_RESV_VECTOR);
2165 2176 } else {
2166 2177 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2167 2178 AV_MASK | irqp->airq_rdt_entry);
2168 2179 }
2169 2180
2170 2181 lock_clear(&apic_ioapic_lock);
2171 2182 }
2172 2183
2173 2184 /*
2174 2185 * For level-triggered interrupt, unmask the IRQ line
2175 2186 * or restore the original vector number.
2176 2187 */
2177 2188 void
2178 2189 apix_level_intr_post_dispatch(int irq)
2179 2190 {
2180 2191 apic_irq_t *irqp = apic_irq_table[irq];
2181 2192 int apic_ix, intin_ix;
2182 2193
2183 2194 if (irqp == NULL)
2184 2195 return;
2185 2196
2186 2197 lock_set(&apic_ioapic_lock);
2187 2198
2188 2199 intin_ix = irqp->airq_intin_no;
2189 2200 apic_ix = irqp->airq_ioapicindex;
2190 2201
2191 2202 if (APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method)) {
2192 2203 /*
2193 2204 * Already sent EOI back to Local APIC.
2194 2205 * Send EOI to IO-APIC
2195 2206 */
2196 2207 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2197 2208 } else {
2198 2209 /* clear the mask or restore the vector */
2199 2210 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2200 2211 irqp->airq_rdt_entry);
2201 2212
2202 2213 /* send EOI to IOxAPIC */
2203 2214 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC)
2204 2215 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2205 2216 }
2206 2217
2207 2218 lock_clear(&apic_ioapic_lock);
2208 2219 }
2209 2220
2210 2221 static int
2211 2222 apix_intx_get_shared(int irqno)
2212 2223 {
2213 2224 apic_irq_t *irqp;
2214 2225 int share;
2215 2226
2216 2227 mutex_enter(&airq_mutex);
2217 2228 irqp = apic_irq_table[irqno];
2218 2229 if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
2219 2230 mutex_exit(&airq_mutex);
2220 2231 return (0);
2221 2232 }
2222 2233 share = irqp->airq_share;
2223 2234 mutex_exit(&airq_mutex);
2224 2235
2225 2236 return (share);
2226 2237 }
2227 2238
2228 2239 static void
2229 2240 apix_intx_set_shared(int irqno, int delta)
2230 2241 {
2231 2242 apic_irq_t *irqp;
2232 2243
2233 2244 mutex_enter(&airq_mutex);
2234 2245 irqp = apic_irq_table[irqno];
2235 2246 if (IS_IRQ_FREE(irqp)) {
2236 2247 mutex_exit(&airq_mutex);
2237 2248 return;
2238 2249 }
2239 2250 irqp->airq_share += delta;
2240 2251 mutex_exit(&airq_mutex);
2241 2252 }
2242 2253
2243 2254 /*
2244 2255 * Setup IRQ table. Return IRQ no or -1 on failure
2245 2256 */
2246 2257 static int
2247 2258 apix_intx_setup(dev_info_t *dip, int inum, int irqno,
2248 2259 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *iflagp)
2249 2260 {
2250 2261 int origirq = ispec->intrspec_vec;
2251 2262 int newirq;
2252 2263 short intr_index;
2253 2264 uchar_t ipin, ioapic, ioapicindex;
2254 2265 apic_irq_t *irqp;
2255 2266
2256 2267 UNREFERENCED_1PARAMETER(inum);
2257 2268
2258 2269 if (intrp != NULL) {
2259 2270 intr_index = (short)(intrp - apic_io_intrp);
2260 2271 ioapic = intrp->intr_destid;
2261 2272 ipin = intrp->intr_destintin;
2262 2273
2263 2274 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
2264 2275 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
2265 2276 if (apic_io_id[ioapicindex] == ioapic)
2266 2277 break;
2267 2278 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
2268 2279 (ioapic == INTR_ALL_APIC));
2269 2280
2270 2281 /* check whether this intin# has been used by another irqno */
2271 2282 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1)
2272 2283 return (newirq);
2273 2284
2274 2285 } else if (iflagp != NULL) { /* ACPI */
2275 2286 intr_index = ACPI_INDEX;
2276 2287 ioapicindex = acpi_find_ioapic(irqno);
2277 2288 ASSERT(ioapicindex != 0xFF);
2278 2289 ioapic = apic_io_id[ioapicindex];
2279 2290 ipin = irqno - apic_io_vectbase[ioapicindex];
2280 2291
2281 2292 if (apic_irq_table[irqno] &&
2282 2293 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
2283 2294 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
2284 2295 apic_irq_table[irqno]->airq_ioapicindex ==
2285 2296 ioapicindex);
2286 2297 return (irqno);
2287 2298 }
2288 2299
2289 2300 } else { /* default configuration */
2290 2301 intr_index = DEFAULT_INDEX;
2291 2302 ioapicindex = 0;
2292 2303 ioapic = apic_io_id[ioapicindex];
2293 2304 ipin = (uchar_t)irqno;
2294 2305 }
2295 2306
2296 2307 /* allocate a new IRQ no */
2297 2308 if ((irqp = apic_irq_table[irqno]) == NULL) {
2298 2309 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2299 2310 apic_irq_table[irqno] = irqp;
2300 2311 } else {
2301 2312 if (irqp->airq_mps_intr_index != FREE_INDEX) {
2302 2313 newirq = apic_allocate_irq(apic_first_avail_irq);
2303 2314 if (newirq == -1) {
2304 2315 return (-1);
2305 2316 }
2306 2317 irqno = newirq;
2307 2318 irqp = apic_irq_table[irqno];
2308 2319 ASSERT(irqp != NULL);
2309 2320 }
2310 2321 }
2311 2322 apic_max_device_irq = max(irqno, apic_max_device_irq);
2312 2323 apic_min_device_irq = min(irqno, apic_min_device_irq);
2313 2324
2314 2325 irqp->airq_mps_intr_index = intr_index;
2315 2326 irqp->airq_ioapicindex = ioapicindex;
2316 2327 irqp->airq_intin_no = ipin;
2317 2328 irqp->airq_dip = dip;
2318 2329 irqp->airq_origirq = (uchar_t)origirq;
2319 2330 if (iflagp != NULL)
2320 2331 irqp->airq_iflag = *iflagp;
2321 2332 irqp->airq_cpu = IRQ_UNINIT;
2322 2333 irqp->airq_vector = 0;
2323 2334
2324 2335 return (irqno);
2325 2336 }
2326 2337
2327 2338 /*
2328 2339 * Setup IRQ table for non-pci devices. Return IRQ no or -1 on error
2329 2340 */
2330 2341 static int
2331 2342 apix_intx_setup_nonpci(dev_info_t *dip, int inum, int bustype,
2332 2343 struct intrspec *ispec)
2333 2344 {
2334 2345 int irqno = ispec->intrspec_vec;
2335 2346 int newirq, i;
2336 2347 iflag_t intr_flag;
2337 2348 ACPI_SUBTABLE_HEADER *hp;
2338 2349 ACPI_MADT_INTERRUPT_OVERRIDE *isop;
2339 2350 struct apic_io_intr *intrp;
2340 2351
2341 2352 if (!apic_enable_acpi || apic_use_acpi_madt_only) {
2342 2353 int busid;
2343 2354
2344 2355 if (bustype == 0)
2345 2356 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
2346 2357
2347 2358 /* loop checking BUS_ISA/BUS_EISA */
2348 2359 for (i = 0; i < 2; i++) {
2349 2360 if (((busid = apic_find_bus_id(bustype)) != -1) &&
2350 2361 ((intrp = apic_find_io_intr_w_busid(irqno, busid))
2351 2362 != NULL)) {
2352 2363 return (apix_intx_setup(dip, inum, irqno,
2353 2364 intrp, ispec, NULL));
2354 2365 }
2355 2366 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
2356 2367 }
2357 2368
2358 2369 /* fall back to default configuration */
2359 2370 return (-1);
2360 2371 }
2361 2372
2362 2373 /* search iso entries first */
2363 2374 if (acpi_iso_cnt != 0) {
2364 2375 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
2365 2376 i = 0;
2366 2377 while (i < acpi_iso_cnt) {
2367 2378 if (hp->Type == ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
2368 2379 isop = (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
2369 2380 if (isop->Bus == 0 &&
2370 2381 isop->SourceIrq == irqno) {
2371 2382 newirq = isop->GlobalIrq;
2372 2383 intr_flag.intr_po = isop->IntiFlags &
2373 2384 ACPI_MADT_POLARITY_MASK;
2374 2385 intr_flag.intr_el = (isop->IntiFlags &
2375 2386 ACPI_MADT_TRIGGER_MASK) >> 2;
2376 2387 intr_flag.bustype = BUS_ISA;
2377 2388
2378 2389 return (apix_intx_setup(dip, inum,
2379 2390 newirq, NULL, ispec, &intr_flag));
2380 2391 }
2381 2392 i++;
2382 2393 }
2383 2394 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
2384 2395 hp->Length);
2385 2396 }
2386 2397 }
2387 2398 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
2388 2399 intr_flag.intr_el = INTR_EL_EDGE;
2389 2400 intr_flag.bustype = BUS_ISA;
2390 2401 return (apix_intx_setup(dip, inum, irqno, NULL, ispec, &intr_flag));
2391 2402 }
2392 2403
2393 2404
2394 2405 /*
2395 2406 * Setup IRQ table for pci devices. Return IRQ no or -1 on error
2396 2407 */
2397 2408 static int
2398 2409 apix_intx_setup_pci(dev_info_t *dip, int inum, int bustype,
2399 2410 struct intrspec *ispec)
2400 2411 {
2401 2412 int busid, devid, pci_irq;
2402 2413 ddi_acc_handle_t cfg_handle;
2403 2414 uchar_t ipin;
2404 2415 iflag_t intr_flag;
2405 2416 struct apic_io_intr *intrp;
2406 2417
2407 2418 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
2408 2419 return (-1);
2409 2420
2410 2421 if (busid == 0 && apic_pci_bus_total == 1)
2411 2422 busid = (int)apic_single_pci_busid;
2412 2423
2413 2424 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
2414 2425 return (-1);
2415 2426 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
2416 2427 pci_config_teardown(&cfg_handle);
2417 2428
2418 2429 if (apic_enable_acpi && !apic_use_acpi_madt_only) { /* ACPI */
2419 2430 if (apic_acpi_translate_pci_irq(dip, busid, devid,
2420 2431 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
2421 2432 return (-1);
2422 2433
2423 2434 intr_flag.bustype = (uchar_t)bustype;
2424 2435 return (apix_intx_setup(dip, inum, pci_irq, NULL, ispec,
2425 2436 &intr_flag));
2426 2437 }
2427 2438
2428 2439 /* MP configuration table */
2429 2440 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
2430 2441 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) == NULL) {
2431 2442 pci_irq = apic_handle_pci_pci_bridge(dip, devid, ipin, &intrp);
2432 2443 if (pci_irq == -1)
2433 2444 return (-1);
2434 2445 }
2435 2446
2436 2447 return (apix_intx_setup(dip, inum, pci_irq, intrp, ispec, NULL));
2437 2448 }
2438 2449
2439 2450 /*
2440 2451 * Translate and return IRQ no
2441 2452 */
2442 2453 static int
2443 2454 apix_intx_xlate_irq(dev_info_t *dip, int inum, struct intrspec *ispec)
2444 2455 {
2445 2456 int newirq, irqno = ispec->intrspec_vec;
2446 2457 int parent_is_pci_or_pciex = 0, child_is_pciex = 0;
2447 2458 int bustype = 0, dev_len;
2448 2459 char dev_type[16];
2449 2460
2450 2461 if (apic_defconf) {
2451 2462 mutex_enter(&airq_mutex);
2452 2463 goto defconf;
2453 2464 }
2454 2465
2455 2466 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) {
2456 2467 mutex_enter(&airq_mutex);
2457 2468 goto nonpci;
2458 2469 }
2459 2470
2460 2471 /*
2461 2472 * use ddi_getlongprop_buf() instead of ddi_prop_lookup_string()
2462 2473 * to avoid extra buffer allocation.
2463 2474 */
2464 2475 dev_len = sizeof (dev_type);
2465 2476 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
2466 2477 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
2467 2478 &dev_len) == DDI_PROP_SUCCESS) {
2468 2479 if ((strcmp(dev_type, "pci") == 0) ||
2469 2480 (strcmp(dev_type, "pciex") == 0))
2470 2481 parent_is_pci_or_pciex = 1;
2471 2482 }
2472 2483
2473 2484 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
2474 2485 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
2475 2486 &dev_len) == DDI_PROP_SUCCESS) {
2476 2487 if (strstr(dev_type, "pciex"))
2477 2488 child_is_pciex = 1;
2478 2489 }
2479 2490
2480 2491 mutex_enter(&airq_mutex);
2481 2492
2482 2493 if (parent_is_pci_or_pciex) {
2483 2494 bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
2484 2495 newirq = apix_intx_setup_pci(dip, inum, bustype, ispec);
2485 2496 if (newirq != -1)
2486 2497 goto done;
2487 2498 bustype = 0;
2488 2499 } else if (strcmp(dev_type, "isa") == 0)
2489 2500 bustype = BUS_ISA;
2490 2501 else if (strcmp(dev_type, "eisa") == 0)
2491 2502 bustype = BUS_EISA;
2492 2503
2493 2504 nonpci:
2494 2505 newirq = apix_intx_setup_nonpci(dip, inum, bustype, ispec);
2495 2506 if (newirq != -1)
2496 2507 goto done;
2497 2508
2498 2509 defconf:
2499 2510 newirq = apix_intx_setup(dip, inum, irqno, NULL, ispec, NULL);
2500 2511 if (newirq == -1) {
2501 2512 mutex_exit(&airq_mutex);
2502 2513 return (-1);
2503 2514 }
2504 2515 done:
2505 2516 ASSERT(apic_irq_table[newirq]);
2506 2517 mutex_exit(&airq_mutex);
2507 2518 return (newirq);
2508 2519 }
2509 2520
2510 2521 static int
2511 2522 apix_intx_alloc_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2512 2523 {
2513 2524 int irqno;
2514 2525 apix_vector_t *vecp;
2515 2526
2516 2527 if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2517 2528 return (0);
2518 2529
2519 2530 if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
2520 2531 return (0);
2521 2532
2522 2533 DDI_INTR_IMPLDBG((CE_CONT, "apix_intx_alloc_vector: dip=0x%p name=%s "
2523 2534 "irqno=0x%x cpuid=%d vector=0x%x\n",
2524 2535 (void *)dip, ddi_driver_name(dip), irqno,
2525 2536 vecp->v_cpuid, vecp->v_vector));
2526 2537
2527 2538 return (1);
2528 2539 }
2529 2540
2530 2541 /*
2531 2542 * Return the vector number if the translated IRQ for this device
2532 2543 * has a vector mapping setup. If no IRQ setup exists or no vector is
2533 2544 * allocated to it then return 0.
2534 2545 */
2535 2546 static apix_vector_t *
2536 2547 apix_intx_xlate_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2537 2548 {
2538 2549 int irqno;
2539 2550 apix_vector_t *vecp;
2540 2551
2541 2552 /* get the IRQ number */
2542 2553 if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2543 2554 return (NULL);
2544 2555
2545 2556 /* get the vector number if a vector is allocated to this irqno */
2546 2557 vecp = apix_intx_get_vector(irqno);
2547 2558
2548 2559 return (vecp);
2549 2560 }
2550 2561
2551 2562 /* stub function */
2552 2563 int
2553 2564 apix_loaded(void)
2554 2565 {
2555 2566 return (apix_is_enabled);
2556 2567 }
↓ open down ↓ |
1454 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX