Print this page
3426 assertion failed: irq < 16 on VMware hardware version 9 (apix related)
Reviewed by: Albert Lee <trisk@nexenta.com>
Reviewed by: Dan McDonald <danmcd@nexenta.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/apix/apix_utils.c
+++ new/usr/src/uts/i86pc/io/apix/apix_utils.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
|
↓ open down ↓ |
18 lines elided |
↑ open up ↑ |
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright (c) 2010, Intel Corporation.
27 27 * All rights reserved.
28 28 */
29 +/*
30 + * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
31 + */
29 32
30 33 #include <sys/processor.h>
31 34 #include <sys/time.h>
32 35 #include <sys/psm.h>
33 36 #include <sys/smp_impldefs.h>
34 37 #include <sys/cram.h>
35 38 #include <sys/acpi/acpi.h>
36 39 #include <sys/acpica.h>
37 40 #include <sys/psm_common.h>
38 41 #include <sys/pit.h>
39 42 #include <sys/ddi.h>
40 43 #include <sys/sunddi.h>
41 44 #include <sys/ddi_impldefs.h>
42 45 #include <sys/pci.h>
43 46 #include <sys/promif.h>
44 47 #include <sys/x86_archext.h>
45 48 #include <sys/cpc_impl.h>
46 49 #include <sys/uadmin.h>
47 50 #include <sys/panic.h>
48 51 #include <sys/debug.h>
49 52 #include <sys/archsystm.h>
50 53 #include <sys/trap.h>
51 54 #include <sys/machsystm.h>
52 55 #include <sys/sysmacros.h>
53 56 #include <sys/cpuvar.h>
54 57 #include <sys/rm_platter.h>
55 58 #include <sys/privregs.h>
56 59 #include <sys/note.h>
57 60 #include <sys/pci_intr_lib.h>
58 61 #include <sys/spl.h>
59 62 #include <sys/clock.h>
60 63 #include <sys/dditypes.h>
61 64 #include <sys/sunddi.h>
62 65 #include <sys/x_call.h>
63 66 #include <sys/reboot.h>
64 67 #include <sys/apix.h>
65 68
66 69 static int apix_get_avail_vector_oncpu(uint32_t, int, int);
67 70 static apix_vector_t *apix_init_vector(processorid_t, uchar_t);
68 71 static void apix_cleanup_vector(apix_vector_t *);
69 72 static void apix_insert_av(apix_vector_t *, void *, avfunc, caddr_t, caddr_t,
70 73 uint64_t *, int, dev_info_t *);
71 74 static void apix_remove_av(apix_vector_t *, struct autovec *);
72 75 static void apix_clear_dev_map(dev_info_t *, int, int);
73 76 static boolean_t apix_is_cpu_enabled(processorid_t);
74 77 static void apix_wait_till_seen(processorid_t, int);
75 78
76 79 #define GET_INTR_INUM(ihdlp) \
77 80 (((ihdlp) != NULL) ? ((ddi_intr_handle_impl_t *)(ihdlp))->ih_inum : 0)
78 81
79 82 apix_rebind_info_t apix_rebindinfo = {0, 0, 0, NULL, 0, NULL};
80 83
81 84 /*
82 85 * Allocate IPI
83 86 *
84 87 * Return vector number or 0 on error
85 88 */
86 89 uchar_t
87 90 apix_alloc_ipi(int ipl)
88 91 {
89 92 apix_vector_t *vecp;
90 93 uchar_t vector;
91 94 int cpun;
92 95 int nproc;
93 96
94 97 APIX_ENTER_CPU_LOCK(0);
95 98
96 99 vector = apix_get_avail_vector_oncpu(0, APIX_IPI_MIN, APIX_IPI_MAX);
97 100 if (vector == 0) {
98 101 APIX_LEAVE_CPU_LOCK(0);
99 102 cmn_err(CE_WARN, "apix: no available IPI\n");
100 103 apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
101 104 return (0);
102 105 }
103 106
104 107 nproc = max(apic_nproc, apic_max_nproc);
105 108 for (cpun = 0; cpun < nproc; cpun++) {
106 109 vecp = xv_vector(cpun, vector);
107 110 if (vecp == NULL) {
108 111 vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP);
109 112 if (vecp == NULL) {
110 113 cmn_err(CE_WARN, "apix: No memory for ipi");
111 114 goto fail;
112 115 }
113 116 xv_vector(cpun, vector) = vecp;
114 117 }
115 118 vecp->v_state = APIX_STATE_ALLOCED;
116 119 vecp->v_type = APIX_TYPE_IPI;
117 120 vecp->v_cpuid = vecp->v_bound_cpuid = cpun;
118 121 vecp->v_vector = vector;
119 122 vecp->v_pri = ipl;
120 123 }
121 124 APIX_LEAVE_CPU_LOCK(0);
122 125 return (vector);
123 126
124 127 fail:
125 128 while (--cpun >= 0)
126 129 apix_cleanup_vector(xv_vector(cpun, vector));
127 130 APIX_LEAVE_CPU_LOCK(0);
128 131 return (0);
129 132 }
130 133
131 134 /*
132 135 * Add IPI service routine
133 136 */
134 137 static int
135 138 apix_add_ipi(int ipl, avfunc xxintr, char *name, int vector,
136 139 caddr_t arg1, caddr_t arg2)
137 140 {
138 141 int cpun;
139 142 apix_vector_t *vecp;
140 143 int nproc;
141 144
142 145 ASSERT(vector >= APIX_IPI_MIN && vector <= APIX_IPI_MAX);
143 146
144 147 nproc = max(apic_nproc, apic_max_nproc);
145 148 for (cpun = 0; cpun < nproc; cpun++) {
146 149 APIX_ENTER_CPU_LOCK(cpun);
147 150 vecp = xv_vector(cpun, vector);
148 151 apix_insert_av(vecp, NULL, xxintr, arg1, arg2, NULL, ipl, NULL);
149 152 vecp->v_state = APIX_STATE_ENABLED;
150 153 APIX_LEAVE_CPU_LOCK(cpun);
151 154 }
152 155
153 156 APIC_VERBOSE(IPI, (CE_CONT, "apix: add ipi for %s, vector %x "
154 157 "ipl %x\n", name, vector, ipl));
155 158
156 159 return (1);
157 160 }
158 161
159 162 /*
160 163 * Find and return first free vector in range (start, end)
161 164 */
162 165 static int
163 166 apix_get_avail_vector_oncpu(uint32_t cpuid, int start, int end)
164 167 {
165 168 int i;
166 169 apix_impl_t *apixp = apixs[cpuid];
167 170
168 171 for (i = start; i <= end; i++) {
169 172 if (APIC_CHECK_RESERVE_VECTORS(i))
170 173 continue;
171 174 if (IS_VECT_FREE(apixp->x_vectbl[i]))
172 175 return (i);
173 176 }
174 177
175 178 return (0);
176 179 }
177 180
178 181 /*
179 182 * Allocate a vector on specified cpu
180 183 *
181 184 * Return NULL on error
182 185 */
183 186 static apix_vector_t *
184 187 apix_alloc_vector_oncpu(uint32_t cpuid, dev_info_t *dip, int inum, int type)
185 188 {
186 189 processorid_t tocpu = cpuid & ~IRQ_USER_BOUND;
187 190 apix_vector_t *vecp;
188 191 int vector;
189 192
190 193 ASSERT(APIX_CPU_LOCK_HELD(tocpu));
191 194
192 195 /* find free vector */
193 196 vector = apix_get_avail_vector_oncpu(tocpu, APIX_AVINTR_MIN,
194 197 APIX_AVINTR_MAX);
195 198 if (vector == 0)
196 199 return (NULL);
197 200
198 201 vecp = apix_init_vector(tocpu, vector);
199 202 vecp->v_type = (ushort_t)type;
200 203 vecp->v_inum = inum;
201 204 vecp->v_flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0;
202 205
203 206 if (dip != NULL)
204 207 apix_set_dev_map(vecp, dip, inum);
205 208
206 209 return (vecp);
207 210 }
208 211
209 212 /*
210 213 * Allocates "count" contiguous MSI vectors starting at the proper alignment.
211 214 * Caller needs to make sure that count has to be power of 2 and should not
212 215 * be < 1.
213 216 *
214 217 * Return first vector number
215 218 */
216 219 apix_vector_t *
217 220 apix_alloc_nvectors_oncpu(uint32_t cpuid, dev_info_t *dip, int inum,
218 221 int count, int type)
219 222 {
220 223 int i, msibits, start = 0, navail = 0;
221 224 apix_vector_t *vecp, *startp = NULL;
222 225 processorid_t tocpu = cpuid & ~IRQ_USER_BOUND;
223 226 uint_t flags;
224 227
225 228 ASSERT(APIX_CPU_LOCK_HELD(tocpu));
226 229
227 230 /*
228 231 * msibits is the no. of lower order message data bits for the
229 232 * allocated MSI vectors and is used to calculate the aligned
230 233 * starting vector
231 234 */
232 235 msibits = count - 1;
233 236
234 237 /* It has to be contiguous */
235 238 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
236 239 if (!IS_VECT_FREE(xv_vector(tocpu, i)))
237 240 continue;
238 241
239 242 /*
240 243 * starting vector has to be aligned accordingly for
241 244 * multiple MSIs
242 245 */
243 246 if (msibits)
244 247 i = (i + msibits) & ~msibits;
245 248
246 249 for (navail = 0, start = i; i <= APIX_AVINTR_MAX; i++) {
247 250 if (!IS_VECT_FREE(xv_vector(tocpu, i)))
248 251 break;
249 252 if (APIC_CHECK_RESERVE_VECTORS(i))
250 253 break;
251 254 if (++navail == count)
252 255 goto done;
253 256 }
254 257 }
255 258
256 259 return (NULL);
257 260
258 261 done:
259 262 flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0;
260 263
261 264 for (i = 0; i < count; i++) {
262 265 if ((vecp = apix_init_vector(tocpu, start + i)) == NULL)
263 266 goto fail;
264 267
265 268 vecp->v_type = (ushort_t)type;
266 269 vecp->v_inum = inum + i;
267 270 vecp->v_flags = flags;
268 271
269 272 if (dip != NULL)
270 273 apix_set_dev_map(vecp, dip, inum + i);
271 274
272 275 if (i == 0)
273 276 startp = vecp;
274 277 }
275 278
276 279 return (startp);
277 280
278 281 fail:
279 282 while (i-- > 0) { /* Free allocated vectors */
280 283 vecp = xv_vector(tocpu, start + i);
281 284 apix_clear_dev_map(dip, inum + i, type);
282 285 apix_cleanup_vector(vecp);
283 286 }
284 287 return (NULL);
285 288 }
286 289
287 290 #define APIX_WRITE_MSI_DATA(_hdl, _cap, _ctrl, _v)\
288 291 do {\
289 292 if ((_ctrl) & PCI_MSI_64BIT_MASK)\
290 293 pci_config_put16((_hdl), (_cap) + PCI_MSI_64BIT_DATA, (_v));\
291 294 else\
292 295 pci_config_put16((_hdl), (_cap) + PCI_MSI_32BIT_DATA, (_v));\
293 296 _NOTE(CONSTCOND)} while (0)
294 297
295 298 static void
296 299 apix_pci_msi_enable_vector(apix_vector_t *vecp, dev_info_t *dip, int type,
297 300 int inum, int count, uchar_t vector, int target_apic_id)
298 301 {
299 302 uint64_t msi_addr, msi_data;
300 303 ushort_t msi_ctrl;
301 304 int i, cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
302 305 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip);
303 306 msi_regs_t msi_regs;
304 307 void *intrmap_tbl[PCI_MSI_MAX_INTRS];
305 308
306 309 DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: dip=0x%p\n"
307 310 "\tdriver = %s, inum=0x%x vector=0x%x apicid=0x%x\n", (void *)dip,
308 311 ddi_driver_name(dip), inum, vector, target_apic_id));
309 312
310 313 ASSERT((handle != NULL) && (cap_ptr != 0));
311 314
312 315 msi_regs.mr_data = vector;
313 316 msi_regs.mr_addr = target_apic_id;
314 317
315 318 for (i = 0; i < count; i++)
316 319 intrmap_tbl[i] = xv_intrmap_private(vecp->v_cpuid, vector + i);
317 320 apic_vt_ops->apic_intrmap_alloc_entry(intrmap_tbl, dip, type,
318 321 count, 0xff);
319 322 for (i = 0; i < count; i++)
320 323 xv_intrmap_private(vecp->v_cpuid, vector + i) = intrmap_tbl[i];
321 324
322 325 apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
323 326 (void *)&msi_regs, type, count);
324 327 apic_vt_ops->apic_intrmap_record_msi(vecp->v_intrmap_private,
325 328 &msi_regs);
326 329
327 330 /* MSI Address */
328 331 msi_addr = msi_regs.mr_addr;
329 332
330 333 /* MSI Data: MSI is edge triggered according to spec */
331 334 msi_data = msi_regs.mr_data;
332 335
333 336 DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: addr=0x%lx "
334 337 "data=0x%lx\n", (long)msi_addr, (long)msi_data));
335 338
336 339 if (type == APIX_TYPE_MSI) {
337 340 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
338 341
339 342 /* Set the bits to inform how many MSIs are enabled */
340 343 msi_ctrl |= ((highbit(count) - 1) << PCI_MSI_MME_SHIFT);
341 344 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
342 345
343 346 if ((vecp->v_flags & APIX_VECT_MASKABLE) == 0)
344 347 APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl,
345 348 APIX_RESV_VECTOR);
346 349
347 350 pci_config_put32(handle,
348 351 cap_ptr + PCI_MSI_ADDR_OFFSET, msi_addr);
349 352 if (msi_ctrl & PCI_MSI_64BIT_MASK)
350 353 pci_config_put32(handle,
351 354 cap_ptr + PCI_MSI_ADDR_OFFSET + 4, msi_addr >> 32);
352 355
353 356 APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl, msi_data);
354 357 } else if (type == APIX_TYPE_MSIX) {
355 358 uintptr_t off;
356 359 ddi_intr_msix_t *msix_p = i_ddi_get_msix(dip);
357 360
358 361 /* Offset into the "inum"th entry in the MSI-X table */
359 362 off = (uintptr_t)msix_p->msix_tbl_addr +
360 363 (inum * PCI_MSIX_VECTOR_SIZE);
361 364
362 365 ddi_put32(msix_p->msix_tbl_hdl,
363 366 (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), msi_data);
364 367 ddi_put64(msix_p->msix_tbl_hdl,
365 368 (uint64_t *)(off + PCI_MSIX_LOWER_ADDR_OFFSET), msi_addr);
366 369 }
367 370 }
368 371
369 372 static void
370 373 apix_pci_msi_enable_mode(dev_info_t *dip, int type, int inum)
371 374 {
372 375 ushort_t msi_ctrl;
373 376 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
374 377 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip);
375 378
376 379 ASSERT((handle != NULL) && (cap_ptr != 0));
377 380
378 381 if (type == APIX_TYPE_MSI) {
379 382 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
380 383 if ((msi_ctrl & PCI_MSI_ENABLE_BIT))
381 384 return;
382 385
383 386 msi_ctrl |= PCI_MSI_ENABLE_BIT;
384 387 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
385 388
386 389 } else if (type == DDI_INTR_TYPE_MSIX) {
387 390 uintptr_t off;
388 391 uint32_t mask;
389 392 ddi_intr_msix_t *msix_p;
390 393
391 394 msix_p = i_ddi_get_msix(dip);
392 395
393 396 /* Offset into "inum"th entry in the MSI-X table & clear mask */
394 397 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
395 398 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
396 399
397 400 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
398 401
399 402 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask & ~1));
400 403
401 404 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
402 405
403 406 if (!(msi_ctrl & PCI_MSIX_ENABLE_BIT)) {
404 407 msi_ctrl |= PCI_MSIX_ENABLE_BIT;
405 408 pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
406 409 msi_ctrl);
407 410 }
408 411 }
409 412 }
410 413
411 414 /*
412 415 * Setup interrupt, pogramming IO-APIC or MSI/X address/data.
413 416 */
414 417 void
415 418 apix_enable_vector(apix_vector_t *vecp)
416 419 {
417 420 int tocpu = vecp->v_cpuid, type = vecp->v_type;
418 421 apic_cpus_info_t *cpu_infop;
419 422 ulong_t iflag;
420 423
421 424 ASSERT(tocpu < apic_nproc);
422 425
423 426 cpu_infop = &apic_cpus[tocpu];
424 427 if (vecp->v_flags & APIX_VECT_USER_BOUND)
425 428 cpu_infop->aci_bound++;
426 429 else
427 430 cpu_infop->aci_temp_bound++;
428 431
429 432 iflag = intr_clear();
430 433 lock_set(&apic_ioapic_lock);
431 434
432 435 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) { /* fixed */
433 436 apix_intx_enable(vecp->v_inum);
434 437 } else {
435 438 int inum = vecp->v_inum;
436 439 dev_info_t *dip = APIX_GET_DIP(vecp);
437 440 int count = i_ddi_intr_get_current_nintrs(dip);
438 441
439 442 if (type == APIX_TYPE_MSI) { /* MSI */
440 443 if (inum == apix_get_max_dev_inum(dip, type)) {
441 444 /* last one */
442 445 uchar_t start_inum = inum + 1 - count;
443 446 uchar_t start_vect = vecp->v_vector + 1 - count;
444 447 apix_vector_t *start_vecp =
445 448 xv_vector(vecp->v_cpuid, start_vect);
446 449
447 450 APIC_VERBOSE(INTR, (CE_CONT, "apix: call "
448 451 "apix_pci_msi_enable_vector\n"));
449 452 apix_pci_msi_enable_vector(start_vecp, dip,
450 453 type, start_inum, count, start_vect,
451 454 cpu_infop->aci_local_id);
452 455
453 456 APIC_VERBOSE(INTR, (CE_CONT, "apix: call "
454 457 "apix_pci_msi_enable_mode\n"));
455 458 apix_pci_msi_enable_mode(dip, type, inum);
456 459 }
457 460 } else { /* MSI-X */
458 461 apix_pci_msi_enable_vector(vecp, dip,
459 462 type, inum, 1, vecp->v_vector,
460 463 cpu_infop->aci_local_id);
461 464 apix_pci_msi_enable_mode(dip, type, inum);
462 465 }
463 466 }
464 467 vecp->v_state = APIX_STATE_ENABLED;
465 468 apic_redist_cpu_skip &= ~(1 << tocpu);
466 469
467 470 lock_clear(&apic_ioapic_lock);
468 471 intr_restore(iflag);
469 472 }
470 473
471 474 /*
472 475 * Disable the interrupt
473 476 */
474 477 void
475 478 apix_disable_vector(apix_vector_t *vecp)
476 479 {
477 480 struct autovec *avp = vecp->v_autovect;
478 481 ulong_t iflag;
479 482
480 483 ASSERT(avp != NULL);
481 484
482 485 iflag = intr_clear();
483 486 lock_set(&apic_ioapic_lock);
484 487
485 488 switch (vecp->v_type) {
486 489 case APIX_TYPE_MSI:
487 490 ASSERT(avp->av_vector != NULL && avp->av_dip != NULL);
488 491 /*
489 492 * Disable the MSI vector
490 493 * Make sure we only disable on the last
491 494 * of the multi-MSI support
492 495 */
493 496 if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) {
494 497 apic_pci_msi_disable_mode(avp->av_dip,
495 498 DDI_INTR_TYPE_MSI);
496 499 }
497 500 break;
498 501 case APIX_TYPE_MSIX:
499 502 ASSERT(avp->av_vector != NULL && avp->av_dip != NULL);
500 503 /*
501 504 * Disable the MSI-X vector
502 505 * needs to clear its mask and addr/data for each MSI-X
503 506 */
504 507 apic_pci_msi_unconfigure(avp->av_dip, DDI_INTR_TYPE_MSIX,
505 508 vecp->v_inum);
506 509 /*
507 510 * Make sure we only disable on the last MSI-X
508 511 */
509 512 if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) {
510 513 apic_pci_msi_disable_mode(avp->av_dip,
511 514 DDI_INTR_TYPE_MSIX);
512 515 }
513 516 break;
514 517 default:
515 518 apix_intx_disable(vecp->v_inum);
516 519 break;
517 520 }
518 521
519 522 if (!(apic_cpus[vecp->v_cpuid].aci_status & APIC_CPU_SUSPEND))
520 523 vecp->v_state = APIX_STATE_DISABLED;
521 524 apic_vt_ops->apic_intrmap_free_entry(&vecp->v_intrmap_private);
522 525 vecp->v_intrmap_private = NULL;
523 526
524 527 lock_clear(&apic_ioapic_lock);
525 528 intr_restore(iflag);
526 529 }
527 530
528 531 /*
529 532 * Mark vector as obsoleted or freed. The vector is marked
530 533 * obsoleted if there are pending requests on it. Otherwise,
531 534 * free the vector. The obsoleted vectors get freed after
532 535 * being serviced.
533 536 *
534 537 * Return 1 on being obosoleted and 0 on being freed.
535 538 */
536 539 #define INTR_BUSY(_avp)\
537 540 ((((volatile ushort_t)(_avp)->av_flags) &\
538 541 (AV_PENTRY_PEND | AV_PENTRY_ONPROC)) != 0)
539 542 #define LOCAL_WITH_INTR_DISABLED(_cpuid)\
540 543 ((_cpuid) == psm_get_cpu_id() && !interrupts_enabled())
541 544 static uint64_t dummy_tick;
542 545
543 546 int
544 547 apix_obsolete_vector(apix_vector_t *vecp)
545 548 {
546 549 struct autovec *avp = vecp->v_autovect;
547 550 int repeats, tries, ipl, busy = 0, cpuid = vecp->v_cpuid;
548 551 apix_impl_t *apixp = apixs[cpuid];
549 552
550 553 ASSERT(APIX_CPU_LOCK_HELD(cpuid));
551 554
552 555 for (avp = vecp->v_autovect; avp != NULL; avp = avp->av_link) {
553 556 if (avp->av_vector == NULL)
554 557 continue;
555 558
556 559 if (LOCAL_WITH_INTR_DISABLED(cpuid)) {
557 560 int bit, index, irr;
558 561
559 562 if (INTR_BUSY(avp)) {
560 563 busy++;
561 564 continue;
562 565 }
563 566
564 567 /* check IRR for pending interrupts */
565 568 index = vecp->v_vector / 32;
566 569 bit = vecp->v_vector % 32;
567 570 irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
568 571 if ((irr & (1 << bit)) != 0)
569 572 busy++;
570 573
571 574 if (!busy)
572 575 apix_remove_av(vecp, avp);
573 576
574 577 continue;
575 578 }
576 579
577 580 repeats = 0;
578 581 do {
579 582 repeats++;
580 583 for (tries = 0; tries < apic_max_reps_clear_pending;
581 584 tries++)
582 585 if (!INTR_BUSY(avp))
583 586 break;
584 587 } while (INTR_BUSY(avp) &&
585 588 (repeats < apic_max_reps_clear_pending));
586 589
587 590 if (INTR_BUSY(avp))
588 591 busy++;
589 592 else {
590 593 /*
591 594 * Interrupt is not in pending list or being serviced.
592 595 * However it might be cached in Local APIC's IRR
593 596 * register. It's impossible to check another CPU's
594 597 * IRR register. Then wait till lower levels finish
595 598 * running.
596 599 */
597 600 for (ipl = 1; ipl < MIN(LOCK_LEVEL, vecp->v_pri); ipl++)
598 601 apix_wait_till_seen(cpuid, ipl);
599 602 if (INTR_BUSY(avp))
600 603 busy++;
601 604 }
602 605
603 606 if (!busy)
604 607 apix_remove_av(vecp, avp);
605 608 }
606 609
607 610 if (busy) {
608 611 apix_vector_t *tp = apixp->x_obsoletes;
609 612
610 613 if (vecp->v_state == APIX_STATE_OBSOLETED)
611 614 return (1);
612 615
613 616 vecp->v_state = APIX_STATE_OBSOLETED;
614 617 vecp->v_next = NULL;
615 618 if (tp == NULL)
616 619 apixp->x_obsoletes = vecp;
617 620 else {
618 621 while (tp->v_next != NULL)
619 622 tp = tp->v_next;
620 623 tp->v_next = vecp;
621 624 }
622 625 return (1);
623 626 }
624 627
625 628 /* interrupt is not busy */
626 629 if (vecp->v_state == APIX_STATE_OBSOLETED) {
627 630 /* remove from obsoleted list */
628 631 apixp->x_obsoletes = vecp->v_next;
629 632 vecp->v_next = NULL;
630 633 }
631 634 apix_cleanup_vector(vecp);
632 635 return (0);
633 636 }
634 637
635 638 /*
636 639 * Duplicate number of continuous vectors to specified target vectors.
637 640 */
638 641 static void
639 642 apix_dup_vectors(apix_vector_t *oldp, apix_vector_t *newp, int count)
640 643 {
641 644 struct autovec *avp;
642 645 apix_vector_t *fromp, *top;
643 646 processorid_t oldcpu = oldp->v_cpuid, newcpu = newp->v_cpuid;
644 647 uchar_t oldvec = oldp->v_vector, newvec = newp->v_vector;
645 648 int i, inum;
646 649
647 650 ASSERT(oldp->v_type != APIX_TYPE_IPI);
648 651
649 652 for (i = 0; i < count; i++) {
650 653 fromp = xv_vector(oldcpu, oldvec + i);
651 654 top = xv_vector(newcpu, newvec + i);
652 655 ASSERT(fromp != NULL && top != NULL);
653 656
654 657 /* copy over original one */
655 658 top->v_state = fromp->v_state;
656 659 top->v_type = fromp->v_type;
657 660 top->v_bound_cpuid = fromp->v_bound_cpuid;
658 661 top->v_inum = fromp->v_inum;
659 662 top->v_flags = fromp->v_flags;
660 663 top->v_intrmap_private = fromp->v_intrmap_private;
661 664
662 665 for (avp = fromp->v_autovect; avp != NULL; avp = avp->av_link) {
663 666 if (avp->av_vector == NULL)
664 667 continue;
665 668
666 669 apix_insert_av(top, avp->av_intr_id, avp->av_vector,
667 670 avp->av_intarg1, avp->av_intarg2, avp->av_ticksp,
668 671 avp->av_prilevel, avp->av_dip);
669 672
670 673 if (fromp->v_type == APIX_TYPE_FIXED &&
671 674 avp->av_dip != NULL) {
672 675 inum = GET_INTR_INUM(avp->av_intr_id);
673 676 apix_set_dev_map(top, avp->av_dip, inum);
674 677 }
675 678 }
676 679
677 680 if (DDI_INTR_IS_MSI_OR_MSIX(fromp->v_type) &&
678 681 fromp->v_devp != NULL)
679 682 apix_set_dev_map(top, fromp->v_devp->dv_dip,
680 683 fromp->v_devp->dv_inum);
681 684 }
682 685 }
683 686
684 687 static apix_vector_t *
685 688 apix_init_vector(processorid_t cpuid, uchar_t vector)
686 689 {
687 690 apix_impl_t *apixp = apixs[cpuid];
688 691 apix_vector_t *vecp = apixp->x_vectbl[vector];
689 692
690 693 ASSERT(IS_VECT_FREE(vecp));
691 694
692 695 if (vecp == NULL) {
693 696 vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP);
694 697 if (vecp == NULL) {
695 698 cmn_err(CE_WARN, "apix: no memory to allocate vector");
696 699 return (NULL);
697 700 }
698 701 apixp->x_vectbl[vector] = vecp;
699 702 }
700 703 vecp->v_state = APIX_STATE_ALLOCED;
701 704 vecp->v_cpuid = vecp->v_bound_cpuid = cpuid;
702 705 vecp->v_vector = vector;
703 706
704 707 return (vecp);
705 708 }
706 709
707 710 static void
708 711 apix_cleanup_vector(apix_vector_t *vecp)
709 712 {
710 713 ASSERT(vecp->v_share == 0);
711 714 vecp->v_bound_cpuid = IRQ_UNINIT;
712 715 vecp->v_state = APIX_STATE_FREED;
713 716 vecp->v_type = 0;
714 717 vecp->v_flags = 0;
715 718 vecp->v_busy = 0;
716 719 vecp->v_intrmap_private = NULL;
717 720 }
718 721
719 722 static void
720 723 apix_dprint_vector(apix_vector_t *vecp, dev_info_t *dip, int count)
721 724 {
722 725 #ifdef DEBUG
723 726 major_t major;
724 727 char *name, *drv_name;
725 728 int instance, len, t_len;
726 729 char mesg[1024] = "apix: ";
727 730
728 731 t_len = sizeof (mesg);
729 732 len = strlen(mesg);
730 733 if (dip != NULL) {
731 734 name = ddi_get_name(dip);
732 735 major = ddi_name_to_major(name);
733 736 drv_name = ddi_major_to_name(major);
734 737 instance = ddi_get_instance(dip);
735 738 (void) snprintf(mesg + len, t_len - len, "%s (%s) instance %d ",
736 739 name, drv_name, instance);
737 740 }
738 741 len = strlen(mesg);
739 742
740 743 switch (vecp->v_type) {
741 744 case APIX_TYPE_FIXED:
742 745 (void) snprintf(mesg + len, t_len - len, "irqno %d",
743 746 vecp->v_inum);
744 747 break;
745 748 case APIX_TYPE_MSI:
746 749 (void) snprintf(mesg + len, t_len - len,
747 750 "msi inum %d (count %d)", vecp->v_inum, count);
748 751 break;
749 752 case APIX_TYPE_MSIX:
750 753 (void) snprintf(mesg + len, t_len - len, "msi-x inum %d",
751 754 vecp->v_inum);
752 755 break;
753 756 default:
754 757 break;
755 758
756 759 }
757 760
758 761 APIC_VERBOSE(ALLOC, (CE_CONT, "%s allocated with vector 0x%x on "
759 762 "cpu %d\n", mesg, vecp->v_vector, vecp->v_cpuid));
760 763 #endif /* DEBUG */
761 764 }
762 765
763 766 /*
764 767 * Operations on avintr
765 768 */
766 769
767 770 #define INIT_AUTOVEC(p, intr_id, f, arg1, arg2, ticksp, ipl, dip) \
768 771 do { \
769 772 (p)->av_intr_id = intr_id; \
770 773 (p)->av_vector = f; \
771 774 (p)->av_intarg1 = arg1; \
772 775 (p)->av_intarg2 = arg2; \
773 776 (p)->av_ticksp = ticksp; \
774 777 (p)->av_prilevel = ipl; \
775 778 (p)->av_dip = dip; \
776 779 (p)->av_flags = 0; \
777 780 _NOTE(CONSTCOND)} while (0)
778 781
779 782 /*
780 783 * Insert an interrupt service routine into chain by its priority from
781 784 * high to low
782 785 */
783 786 static void
784 787 apix_insert_av(apix_vector_t *vecp, void *intr_id, avfunc f, caddr_t arg1,
785 788 caddr_t arg2, uint64_t *ticksp, int ipl, dev_info_t *dip)
786 789 {
787 790 struct autovec *p, *prep, *mem;
788 791
789 792 APIC_VERBOSE(INTR, (CE_CONT, "apix_insert_av: dip %p, vector 0x%x, "
790 793 "cpu %d\n", (void *)dip, vecp->v_vector, vecp->v_cpuid));
791 794
792 795 mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
793 796 INIT_AUTOVEC(mem, intr_id, f, arg1, arg2, ticksp, ipl, dip);
794 797 if (vecp->v_type == APIX_TYPE_FIXED && apic_level_intr[vecp->v_inum])
795 798 mem->av_flags |= AV_PENTRY_LEVEL;
796 799
797 800 vecp->v_share++;
798 801 vecp->v_pri = (ipl > vecp->v_pri) ? ipl : vecp->v_pri;
799 802 if (vecp->v_autovect == NULL) { /* Nothing on list - put it at head */
800 803 vecp->v_autovect = mem;
801 804 return;
802 805 }
803 806
804 807 if (DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) { /* MSI/X */
805 808 ASSERT(vecp->v_share == 1); /* No sharing for MSI/X */
806 809
807 810 INIT_AUTOVEC(vecp->v_autovect, intr_id, f, arg1, arg2, ticksp,
808 811 ipl, dip);
809 812 prep = vecp->v_autovect->av_link;
810 813 vecp->v_autovect->av_link = NULL;
811 814
812 815 /* Free the following autovect chain */
813 816 while (prep != NULL) {
814 817 ASSERT(prep->av_vector == NULL);
815 818
816 819 p = prep;
817 820 prep = prep->av_link;
818 821 kmem_free(p, sizeof (struct autovec));
819 822 }
820 823
821 824 kmem_free(mem, sizeof (struct autovec));
822 825 return;
823 826 }
824 827
825 828 /* find where it goes in list */
826 829 prep = NULL;
827 830 for (p = vecp->v_autovect; p != NULL; p = p->av_link) {
828 831 if (p->av_vector && p->av_prilevel <= ipl)
829 832 break;
830 833 prep = p;
831 834 }
832 835 if (prep != NULL) {
833 836 if (prep->av_vector == NULL) { /* freed struct available */
834 837 INIT_AUTOVEC(prep, intr_id, f, arg1, arg2,
835 838 ticksp, ipl, dip);
836 839 prep->av_flags = mem->av_flags;
837 840 kmem_free(mem, sizeof (struct autovec));
838 841 return;
839 842 }
840 843
841 844 mem->av_link = prep->av_link;
842 845 prep->av_link = mem;
843 846 } else {
844 847 /* insert new intpt at beginning of chain */
845 848 mem->av_link = vecp->v_autovect;
846 849 vecp->v_autovect = mem;
847 850 }
848 851 }
849 852
850 853 /*
851 854 * After having made a change to an autovector list, wait until we have
852 855 * seen specified cpu not executing an interrupt at that level--so we
853 856 * know our change has taken effect completely (no old state in registers,
854 857 * etc).
855 858 */
856 859 #define APIX_CPU_ENABLED(_cp) \
857 860 (quiesce_active == 0 && \
858 861 (((_cp)->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) == 0))
859 862
860 863 static void
861 864 apix_wait_till_seen(processorid_t cpuid, int ipl)
862 865 {
863 866 struct cpu *cp = cpu[cpuid];
864 867
865 868 if (cp == NULL || LOCAL_WITH_INTR_DISABLED(cpuid))
866 869 return;
867 870
868 871 /*
869 872 * Don't wait if the CPU is quiesced or offlined. This can happen
870 873 * when a CPU is running pause thread but hardware triggered an
871 874 * interrupt and the interrupt gets queued.
872 875 */
873 876 for (;;) {
874 877 if (!INTR_ACTIVE((volatile struct cpu *)cpu[cpuid], ipl) &&
875 878 (!APIX_CPU_ENABLED(cp) ||
876 879 !INTR_PENDING((volatile apix_impl_t *)apixs[cpuid], ipl)))
877 880 return;
878 881 }
879 882 }
880 883
881 884 static void
882 885 apix_remove_av(apix_vector_t *vecp, struct autovec *target)
883 886 {
884 887 int hi_pri = 0;
885 888 struct autovec *p;
886 889
887 890 if (target == NULL)
888 891 return;
889 892
890 893 APIC_VERBOSE(INTR, (CE_CONT, "apix_remove_av: dip %p, vector 0x%x, "
891 894 "cpu %d\n", (void *)target->av_dip, vecp->v_vector, vecp->v_cpuid));
892 895
893 896 for (p = vecp->v_autovect; p; p = p->av_link) {
894 897 if (p == target || p->av_vector == NULL)
895 898 continue;
896 899 hi_pri = (p->av_prilevel > hi_pri) ? p->av_prilevel : hi_pri;
897 900 }
898 901
899 902 vecp->v_share--;
900 903 vecp->v_pri = hi_pri;
901 904
902 905 /*
903 906 * This drops the handler from the chain, it can no longer be called.
904 907 * However, there is no guarantee that the handler is not currently
905 908 * still executing.
906 909 */
907 910 target->av_vector = NULL;
908 911 /*
909 912 * There is a race where we could be just about to pick up the ticksp
910 913 * pointer to increment it after returning from the service routine
911 914 * in av_dispatch_autovect. Rather than NULL it out let's just point
912 915 * it off to something safe so that any final tick update attempt
913 916 * won't fault.
914 917 */
915 918 target->av_ticksp = &dummy_tick;
916 919 apix_wait_till_seen(vecp->v_cpuid, target->av_prilevel);
917 920 }
918 921
919 922 static struct autovec *
920 923 apix_find_av(apix_vector_t *vecp, void *intr_id, avfunc f)
921 924 {
922 925 struct autovec *p;
923 926
924 927 for (p = vecp->v_autovect; p; p = p->av_link) {
925 928 if ((p->av_vector == f) && (p->av_intr_id == intr_id)) {
926 929 /* found the handler */
927 930 return (p);
928 931 }
929 932 }
930 933
931 934 return (NULL);
932 935 }
933 936
934 937 static apix_vector_t *
935 938 apix_find_vector_by_avintr(void *intr_id, avfunc f)
936 939 {
937 940 apix_vector_t *vecp;
938 941 processorid_t n;
939 942 uchar_t v;
940 943
941 944 for (n = 0; n < apic_nproc; n++) {
942 945 if (!apix_is_cpu_enabled(n))
943 946 continue;
944 947
945 948 for (v = APIX_AVINTR_MIN; v <= APIX_AVINTR_MIN; v++) {
946 949 vecp = xv_vector(n, v);
947 950 if (vecp == NULL ||
948 951 vecp->v_state <= APIX_STATE_OBSOLETED)
949 952 continue;
950 953
951 954 if (apix_find_av(vecp, intr_id, f) != NULL)
952 955 return (vecp);
953 956 }
954 957 }
955 958
956 959 return (NULL);
957 960 }
958 961
959 962 /*
960 963 * Add interrupt service routine.
961 964 *
962 965 * For legacy interrupts (HPET timer, ACPI SCI), the vector is actually
963 966 * IRQ no. A vector is then allocated. Otherwise, the vector is already
964 967 * allocated. The input argument virt_vect is virtual vector of format
965 968 * APIX_VIRTVEC_VECTOR(cpuid, vector).
966 969 *
967 970 * Return 1 on success, 0 on failure.
968 971 */
969 972 int
970 973 apix_add_avintr(void *intr_id, int ipl, avfunc xxintr, char *name,
971 974 int virt_vect, caddr_t arg1, caddr_t arg2, uint64_t *ticksp,
972 975 dev_info_t *dip)
973 976 {
974 977 int cpuid;
975 978 uchar_t v = (uchar_t)APIX_VIRTVEC_VECTOR(virt_vect);
976 979 apix_vector_t *vecp;
977 980
978 981 if (xxintr == NULL) {
979 982 cmn_err(CE_WARN, "Attempt to add null for %s "
980 983 "on vector 0x%x,0x%x", name,
981 984 APIX_VIRTVEC_CPU(virt_vect),
982 985 APIX_VIRTVEC_VECTOR(virt_vect));
983 986 return (0);
984 987 }
985 988
986 989 if (v >= APIX_IPI_MIN) /* IPIs */
987 990 return (apix_add_ipi(ipl, xxintr, name, v, arg1, arg2));
988 991
989 992 if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */
990 993 int irqno = virt_vect;
991 994 int inum = GET_INTR_INUM(intr_id);
992 995
993 996 /*
994 997 * Senarios include:
995 998 * a. add_avintr() is called before irqp initialized (legacy)
996 999 * b. irqp is initialized, vector is not allocated (fixed)
997 1000 * c. irqp is initialized, vector is allocated (fixed & shared)
998 1001 */
999 1002 if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
1000 1003 return (0);
1001 1004
1002 1005 cpuid = vecp->v_cpuid;
1003 1006 v = vecp->v_vector;
1004 1007 virt_vect = APIX_VIRTVECTOR(cpuid, v);
1005 1008 } else { /* got virtual vector */
1006 1009 cpuid = APIX_VIRTVEC_CPU(virt_vect);
1007 1010 vecp = xv_vector(cpuid, v);
1008 1011 ASSERT(vecp != NULL);
1009 1012 }
1010 1013
1011 1014 lock_set(&apix_lock);
1012 1015 if (vecp->v_state <= APIX_STATE_OBSOLETED) {
1013 1016 vecp = NULL;
1014 1017
1015 1018 /*
1016 1019 * Basically the allocated but not enabled interrupts
1017 1020 * will not get re-targeted. But MSIs in allocated state
1018 1021 * could be re-targeted due to group re-targeting.
1019 1022 */
1020 1023 if (intr_id != NULL && dip != NULL) {
1021 1024 ddi_intr_handle_impl_t *hdlp = intr_id;
1022 1025 vecp = apix_get_dev_map(dip, hdlp->ih_inum,
1023 1026 hdlp->ih_type);
1024 1027 ASSERT(vecp->v_state == APIX_STATE_ALLOCED);
1025 1028 }
1026 1029 if (vecp == NULL) {
1027 1030 lock_clear(&apix_lock);
1028 1031 cmn_err(CE_WARN, "Invalid interrupt 0x%x,0x%x "
1029 1032 " for %p to add", cpuid, v, intr_id);
1030 1033 return (0);
1031 1034 }
1032 1035 cpuid = vecp->v_cpuid;
1033 1036 virt_vect = APIX_VIRTVECTOR(cpuid, vecp->v_vector);
1034 1037 }
1035 1038
1036 1039 APIX_ENTER_CPU_LOCK(cpuid);
1037 1040 apix_insert_av(vecp, intr_id, xxintr, arg1, arg2, ticksp, ipl, dip);
1038 1041 APIX_LEAVE_CPU_LOCK(cpuid);
1039 1042
1040 1043 (void) apix_addspl(virt_vect, ipl, 0, 0);
1041 1044
1042 1045 lock_clear(&apix_lock);
1043 1046
1044 1047 return (1);
1045 1048 }
1046 1049
1047 1050 /*
1048 1051 * Remove avintr
1049 1052 *
1050 1053 * For fixed, if it's the last one of shared interrupts, free the vector.
1051 1054 * For msi/x, only disable the interrupt but not free the vector, which
1052 1055 * is freed by PSM_XXX_FREE_XXX.
1053 1056 */
1054 1057 void
1055 1058 apix_rem_avintr(void *intr_id, int ipl, avfunc xxintr, int virt_vect)
1056 1059 {
1057 1060 avfunc f;
1058 1061 apix_vector_t *vecp;
1059 1062 struct autovec *avp;
1060 1063 processorid_t cpuid;
1061 1064
1062 1065 if ((f = xxintr) == NULL)
1063 1066 return;
1064 1067
1065 1068 lock_set(&apix_lock);
1066 1069
1067 1070 if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */
1068 1071 vecp = apix_intx_get_vector(virt_vect);
1069 1072 virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1070 1073 } else /* got virtual vector */
1071 1074 vecp = xv_vector(APIX_VIRTVEC_CPU(virt_vect),
1072 1075 APIX_VIRTVEC_VECTOR(virt_vect));
1073 1076
1074 1077 if (vecp == NULL) {
1075 1078 lock_clear(&apix_lock);
1076 1079 cmn_err(CE_CONT, "Invalid interrupt 0x%x,0x%x to remove",
1077 1080 APIX_VIRTVEC_CPU(virt_vect),
1078 1081 APIX_VIRTVEC_VECTOR(virt_vect));
1079 1082 return;
1080 1083 }
1081 1084
1082 1085 if (vecp->v_state <= APIX_STATE_OBSOLETED ||
1083 1086 ((avp = apix_find_av(vecp, intr_id, f)) == NULL)) {
1084 1087 /*
1085 1088 * It's possible that the interrupt is rebound to a
1086 1089 * different cpu before rem_avintr() is called. Search
1087 1090 * through all vectors once it happens.
1088 1091 */
1089 1092 if ((vecp = apix_find_vector_by_avintr(intr_id, f))
1090 1093 == NULL) {
1091 1094 lock_clear(&apix_lock);
1092 1095 cmn_err(CE_CONT, "Unknown interrupt 0x%x,0x%x "
1093 1096 "for %p to remove", APIX_VIRTVEC_CPU(virt_vect),
1094 1097 APIX_VIRTVEC_VECTOR(virt_vect), intr_id);
1095 1098 return;
1096 1099 }
1097 1100 virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1098 1101 avp = apix_find_av(vecp, intr_id, f);
1099 1102 }
1100 1103 cpuid = vecp->v_cpuid;
1101 1104
1102 1105 /* disable interrupt */
1103 1106 (void) apix_delspl(virt_vect, ipl, 0, 0);
1104 1107
1105 1108 /* remove ISR entry */
1106 1109 APIX_ENTER_CPU_LOCK(cpuid);
1107 1110 apix_remove_av(vecp, avp);
1108 1111 APIX_LEAVE_CPU_LOCK(cpuid);
1109 1112
1110 1113 lock_clear(&apix_lock);
1111 1114 }
1112 1115
1113 1116 /*
1114 1117 * Device to vector mapping table
1115 1118 */
1116 1119
1117 1120 static void
1118 1121 apix_clear_dev_map(dev_info_t *dip, int inum, int type)
1119 1122 {
1120 1123 char *name;
1121 1124 major_t major;
1122 1125 apix_dev_vector_t *dvp, *prev = NULL;
1123 1126 int found = 0;
1124 1127
1125 1128 name = ddi_get_name(dip);
1126 1129 major = ddi_name_to_major(name);
1127 1130
1128 1131 mutex_enter(&apix_mutex);
1129 1132
1130 1133 for (dvp = apix_dev_vector[major]; dvp != NULL;
1131 1134 prev = dvp, dvp = dvp->dv_next) {
1132 1135 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1133 1136 dvp->dv_type == type) {
1134 1137 found++;
1135 1138 break;
1136 1139 }
1137 1140 }
1138 1141
1139 1142 if (!found) {
1140 1143 mutex_exit(&apix_mutex);
1141 1144 return;
1142 1145 }
1143 1146
1144 1147 if (prev != NULL)
1145 1148 prev->dv_next = dvp->dv_next;
1146 1149
1147 1150 if (apix_dev_vector[major] == dvp)
1148 1151 apix_dev_vector[major] = dvp->dv_next;
1149 1152
1150 1153 dvp->dv_vector->v_devp = NULL;
1151 1154
1152 1155 mutex_exit(&apix_mutex);
1153 1156
1154 1157 kmem_free(dvp, sizeof (apix_dev_vector_t));
1155 1158 }
1156 1159
1157 1160 void
1158 1161 apix_set_dev_map(apix_vector_t *vecp, dev_info_t *dip, int inum)
1159 1162 {
1160 1163 apix_dev_vector_t *dvp;
1161 1164 char *name;
1162 1165 major_t major;
1163 1166 uint32_t found = 0;
1164 1167
1165 1168 ASSERT(dip != NULL);
1166 1169 name = ddi_get_name(dip);
1167 1170 major = ddi_name_to_major(name);
1168 1171
1169 1172 mutex_enter(&apix_mutex);
1170 1173
1171 1174 for (dvp = apix_dev_vector[major]; dvp != NULL;
1172 1175 dvp = dvp->dv_next) {
1173 1176 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1174 1177 dvp->dv_type == vecp->v_type) {
1175 1178 found++;
1176 1179 break;
1177 1180 }
1178 1181 }
1179 1182
1180 1183 if (found == 0) { /* not found */
1181 1184 dvp = kmem_zalloc(sizeof (apix_dev_vector_t), KM_SLEEP);
1182 1185 dvp->dv_dip = dip;
1183 1186 dvp->dv_inum = inum;
1184 1187 dvp->dv_type = vecp->v_type;
1185 1188
1186 1189 dvp->dv_next = apix_dev_vector[major];
1187 1190 apix_dev_vector[major] = dvp;
1188 1191 }
1189 1192 dvp->dv_vector = vecp;
1190 1193 vecp->v_devp = dvp;
1191 1194
1192 1195 mutex_exit(&apix_mutex);
1193 1196
1194 1197 DDI_INTR_IMPLDBG((CE_CONT, "apix_set_dev_map: dip=0x%p "
1195 1198 "inum=0x%x vector=0x%x/0x%x\n",
1196 1199 (void *)dip, inum, vecp->v_cpuid, vecp->v_vector));
1197 1200 }
1198 1201
1199 1202 apix_vector_t *
1200 1203 apix_get_dev_map(dev_info_t *dip, int inum, int type)
1201 1204 {
1202 1205 char *name;
1203 1206 major_t major;
1204 1207 apix_dev_vector_t *dvp;
1205 1208 apix_vector_t *vecp;
1206 1209
1207 1210 name = ddi_get_name(dip);
1208 1211 if ((major = ddi_name_to_major(name)) == DDI_MAJOR_T_NONE)
1209 1212 return (NULL);
1210 1213
1211 1214 mutex_enter(&apix_mutex);
1212 1215 for (dvp = apix_dev_vector[major]; dvp != NULL;
1213 1216 dvp = dvp->dv_next) {
1214 1217 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1215 1218 dvp->dv_type == type) {
1216 1219 vecp = dvp->dv_vector;
1217 1220 mutex_exit(&apix_mutex);
1218 1221 return (vecp);
1219 1222 }
1220 1223 }
1221 1224 mutex_exit(&apix_mutex);
1222 1225
1223 1226 return (NULL);
1224 1227 }
1225 1228
1226 1229 /*
1227 1230 * Get minimum inum for specified device, used for MSI
1228 1231 */
1229 1232 int
1230 1233 apix_get_min_dev_inum(dev_info_t *dip, int type)
1231 1234 {
1232 1235 char *name;
1233 1236 major_t major;
1234 1237 apix_dev_vector_t *dvp;
1235 1238 int inum = -1;
1236 1239
1237 1240 name = ddi_get_name(dip);
1238 1241 major = ddi_name_to_major(name);
1239 1242
1240 1243 mutex_enter(&apix_mutex);
1241 1244 for (dvp = apix_dev_vector[major]; dvp != NULL;
1242 1245 dvp = dvp->dv_next) {
1243 1246 if (dvp->dv_dip == dip && dvp->dv_type == type) {
1244 1247 if (inum == -1)
1245 1248 inum = dvp->dv_inum;
1246 1249 else
1247 1250 inum = (dvp->dv_inum < inum) ?
1248 1251 dvp->dv_inum : inum;
1249 1252 }
1250 1253 }
1251 1254 mutex_exit(&apix_mutex);
1252 1255
1253 1256 return (inum);
1254 1257 }
1255 1258
1256 1259 int
1257 1260 apix_get_max_dev_inum(dev_info_t *dip, int type)
1258 1261 {
1259 1262 char *name;
1260 1263 major_t major;
1261 1264 apix_dev_vector_t *dvp;
1262 1265 int inum = -1;
1263 1266
1264 1267 name = ddi_get_name(dip);
1265 1268 major = ddi_name_to_major(name);
1266 1269
1267 1270 mutex_enter(&apix_mutex);
1268 1271 for (dvp = apix_dev_vector[major]; dvp != NULL;
1269 1272 dvp = dvp->dv_next) {
1270 1273 if (dvp->dv_dip == dip && dvp->dv_type == type) {
1271 1274 if (inum == -1)
1272 1275 inum = dvp->dv_inum;
1273 1276 else
1274 1277 inum = (dvp->dv_inum > inum) ?
1275 1278 dvp->dv_inum : inum;
1276 1279 }
1277 1280 }
1278 1281 mutex_exit(&apix_mutex);
1279 1282
1280 1283 return (inum);
1281 1284 }
1282 1285
1283 1286 /*
1284 1287 * Major to cpu binding, for INTR_ROUND_ROBIN_WITH_AFFINITY cpu
1285 1288 * binding policy
1286 1289 */
1287 1290
1288 1291 static uint32_t
1289 1292 apix_get_dev_binding(dev_info_t *dip)
1290 1293 {
1291 1294 major_t major;
1292 1295 char *name;
1293 1296 uint32_t cpu = IRQ_UNINIT;
1294 1297
1295 1298 name = ddi_get_name(dip);
1296 1299 major = ddi_name_to_major(name);
1297 1300 if (major < devcnt) {
1298 1301 mutex_enter(&apix_mutex);
1299 1302 cpu = apix_major_to_cpu[major];
1300 1303 mutex_exit(&apix_mutex);
1301 1304 }
1302 1305
1303 1306 return (cpu);
1304 1307 }
1305 1308
1306 1309 static void
1307 1310 apix_set_dev_binding(dev_info_t *dip, uint32_t cpu)
1308 1311 {
1309 1312 major_t major;
1310 1313 char *name;
1311 1314
1312 1315 /* setup major to cpu mapping */
1313 1316 name = ddi_get_name(dip);
1314 1317 major = ddi_name_to_major(name);
1315 1318 if (apix_major_to_cpu[major] == IRQ_UNINIT) {
1316 1319 mutex_enter(&apix_mutex);
1317 1320 apix_major_to_cpu[major] = cpu;
1318 1321 mutex_exit(&apix_mutex);
1319 1322 }
1320 1323 }
1321 1324
1322 1325 /*
1323 1326 * return the cpu to which this intr should be bound.
1324 1327 * Check properties or any other mechanism to see if user wants it
1325 1328 * bound to a specific CPU. If so, return the cpu id with high bit set.
1326 1329 * If not, use the policy to choose a cpu and return the id.
1327 1330 */
1328 1331 uint32_t
1329 1332 apix_bind_cpu(dev_info_t *dip)
1330 1333 {
1331 1334 int instance, instno, prop_len, bind_cpu, count;
1332 1335 uint_t i, rc;
1333 1336 major_t major;
1334 1337 char *name, *drv_name, *prop_val, *cptr;
1335 1338 char prop_name[32];
1336 1339
1337 1340 lock_set(&apix_lock);
1338 1341
1339 1342 if (apic_intr_policy == INTR_LOWEST_PRIORITY) {
1340 1343 cmn_err(CE_WARN, "apix: unsupported interrupt binding policy "
1341 1344 "LOWEST PRIORITY, use ROUND ROBIN instead");
1342 1345 apic_intr_policy = INTR_ROUND_ROBIN;
1343 1346 }
1344 1347
1345 1348 if (apic_nproc == 1) {
1346 1349 lock_clear(&apix_lock);
1347 1350 return (0);
1348 1351 }
1349 1352
1350 1353 drv_name = NULL;
1351 1354 rc = DDI_PROP_NOT_FOUND;
1352 1355 major = (major_t)-1;
1353 1356 if (dip != NULL) {
1354 1357 name = ddi_get_name(dip);
1355 1358 major = ddi_name_to_major(name);
1356 1359 drv_name = ddi_major_to_name(major);
1357 1360 instance = ddi_get_instance(dip);
1358 1361 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1359 1362 bind_cpu = apix_get_dev_binding(dip);
1360 1363 if (bind_cpu != IRQ_UNINIT) {
1361 1364 lock_clear(&apix_lock);
1362 1365 return (bind_cpu);
1363 1366 }
1364 1367 }
1365 1368 /*
1366 1369 * search for "drvname"_intpt_bind_cpus property first, the
1367 1370 * syntax of the property should be "a[,b,c,...]" where
1368 1371 * instance 0 binds to cpu a, instance 1 binds to cpu b,
1369 1372 * instance 3 binds to cpu c...
1370 1373 * ddi_getlongprop() will search /option first, then /
1371 1374 * if "drvname"_intpt_bind_cpus doesn't exist, then find
1372 1375 * intpt_bind_cpus property. The syntax is the same, and
1373 1376 * it applies to all the devices if its "drvname" specific
1374 1377 * property doesn't exist
1375 1378 */
1376 1379 (void) strcpy(prop_name, drv_name);
1377 1380 (void) strcat(prop_name, "_intpt_bind_cpus");
1378 1381 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1379 1382 (caddr_t)&prop_val, &prop_len);
1380 1383 if (rc != DDI_PROP_SUCCESS) {
1381 1384 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1382 1385 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1383 1386 }
1384 1387 }
1385 1388 if (rc == DDI_PROP_SUCCESS) {
1386 1389 for (i = count = 0; i < (prop_len - 1); i++)
1387 1390 if (prop_val[i] == ',')
1388 1391 count++;
1389 1392 if (prop_val[i-1] != ',')
1390 1393 count++;
1391 1394 /*
1392 1395 * if somehow the binding instances defined in the
1393 1396 * property are not enough for this instno., then
1394 1397 * reuse the pattern for the next instance until
1395 1398 * it reaches the requested instno
1396 1399 */
1397 1400 instno = instance % count;
1398 1401 i = 0;
1399 1402 cptr = prop_val;
1400 1403 while (i < instno)
1401 1404 if (*cptr++ == ',')
1402 1405 i++;
1403 1406 bind_cpu = stoi(&cptr);
1404 1407 kmem_free(prop_val, prop_len);
1405 1408 /* if specific cpu is bogus, then default to cpu 0 */
1406 1409 if (bind_cpu >= apic_nproc) {
1407 1410 cmn_err(CE_WARN, "apix: %s=%s: CPU %d not present",
1408 1411 prop_name, prop_val, bind_cpu);
1409 1412 bind_cpu = 0;
1410 1413 } else {
1411 1414 /* indicate that we are bound at user request */
1412 1415 bind_cpu |= IRQ_USER_BOUND;
1413 1416 }
1414 1417 /*
1415 1418 * no need to check apic_cpus[].aci_status, if specific cpu is
1416 1419 * not up, then post_cpu_start will handle it.
1417 1420 */
1418 1421 } else {
1419 1422 bind_cpu = apic_get_next_bind_cpu();
1420 1423 }
1421 1424
1422 1425 lock_clear(&apix_lock);
1423 1426
1424 1427 return ((uint32_t)bind_cpu);
1425 1428 }
1426 1429
1427 1430 static boolean_t
1428 1431 apix_is_cpu_enabled(processorid_t cpuid)
1429 1432 {
1430 1433 apic_cpus_info_t *cpu_infop;
1431 1434
1432 1435 cpu_infop = &apic_cpus[cpuid];
1433 1436
1434 1437 if ((cpu_infop->aci_status & APIC_CPU_INTR_ENABLE) == 0)
1435 1438 return (B_FALSE);
1436 1439
1437 1440 return (B_TRUE);
1438 1441 }
1439 1442
1440 1443 /*
1441 1444 * Must be called with apix_lock held. This function can be
1442 1445 * called from above lock level by apix_intr_redistribute().
1443 1446 *
1444 1447 * Arguments:
1445 1448 * vecp : Vector to be rebound
1446 1449 * tocpu : Target cpu. IRQ_UNINIT means target is vecp->v_cpuid.
1447 1450 * count : Number of continuous vectors
1448 1451 *
1449 1452 * Return new vector being bound to
1450 1453 */
1451 1454 apix_vector_t *
1452 1455 apix_rebind(apix_vector_t *vecp, processorid_t newcpu, int count)
1453 1456 {
1454 1457 apix_vector_t *newp, *oldp;
1455 1458 processorid_t oldcpu = vecp->v_cpuid;
1456 1459 uchar_t newvec, oldvec = vecp->v_vector;
1457 1460 int i;
1458 1461
1459 1462 ASSERT(LOCK_HELD(&apix_lock) && count > 0);
1460 1463
1461 1464 if (!apix_is_cpu_enabled(newcpu))
1462 1465 return (NULL);
1463 1466
1464 1467 if (vecp->v_cpuid == newcpu) /* rebind to the same cpu */
1465 1468 return (vecp);
1466 1469
1467 1470 APIX_ENTER_CPU_LOCK(oldcpu);
1468 1471 APIX_ENTER_CPU_LOCK(newcpu);
1469 1472
1470 1473 /* allocate vector */
1471 1474 if (count == 1)
1472 1475 newp = apix_alloc_vector_oncpu(newcpu, NULL, 0, vecp->v_type);
1473 1476 else {
1474 1477 ASSERT(vecp->v_type == APIX_TYPE_MSI);
1475 1478 newp = apix_alloc_nvectors_oncpu(newcpu, NULL, 0, count,
1476 1479 vecp->v_type);
1477 1480 }
1478 1481 if (newp == NULL) {
1479 1482 APIX_LEAVE_CPU_LOCK(newcpu);
1480 1483 APIX_LEAVE_CPU_LOCK(oldcpu);
1481 1484 return (NULL);
1482 1485 }
1483 1486
1484 1487 newvec = newp->v_vector;
1485 1488 apix_dup_vectors(vecp, newp, count);
1486 1489
1487 1490 APIX_LEAVE_CPU_LOCK(newcpu);
1488 1491 APIX_LEAVE_CPU_LOCK(oldcpu);
1489 1492
1490 1493 if (!DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) {
1491 1494 ASSERT(count == 1);
1492 1495 if (apix_intx_rebind(vecp->v_inum, newcpu, newvec) != 0) {
1493 1496 struct autovec *avp;
1494 1497 int inum;
1495 1498
1496 1499 /* undo duplication */
1497 1500 APIX_ENTER_CPU_LOCK(oldcpu);
1498 1501 APIX_ENTER_CPU_LOCK(newcpu);
1499 1502 for (avp = newp->v_autovect; avp != NULL;
1500 1503 avp = avp->av_link) {
1501 1504 if (avp->av_dip != NULL) {
1502 1505 inum = GET_INTR_INUM(avp->av_intr_id);
1503 1506 apix_set_dev_map(vecp, avp->av_dip,
1504 1507 inum);
1505 1508 }
1506 1509 apix_remove_av(newp, avp);
1507 1510 }
1508 1511 apix_cleanup_vector(newp);
1509 1512 APIX_LEAVE_CPU_LOCK(newcpu);
1510 1513 APIX_LEAVE_CPU_LOCK(oldcpu);
1511 1514 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed "
1512 1515 "interrupt 0x%x to cpu %d failed\n",
1513 1516 vecp->v_inum, newcpu));
1514 1517 return (NULL);
1515 1518 }
1516 1519
1517 1520 APIX_ENTER_CPU_LOCK(oldcpu);
1518 1521 (void) apix_obsolete_vector(vecp);
1519 1522 APIX_LEAVE_CPU_LOCK(oldcpu);
1520 1523 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed interrupt"
1521 1524 " 0x%x/0x%x to 0x%x/0x%x\n",
1522 1525 oldcpu, oldvec, newcpu, newvec));
1523 1526 return (newp);
1524 1527 }
1525 1528
1526 1529 for (i = 0; i < count; i++) {
1527 1530 oldp = xv_vector(oldcpu, oldvec + i);
1528 1531 newp = xv_vector(newcpu, newvec + i);
1529 1532
1530 1533 if (newp->v_share > 0) {
1531 1534 APIX_SET_REBIND_INFO(oldp, newp);
1532 1535
1533 1536 apix_enable_vector(newp);
1534 1537
1535 1538 APIX_CLR_REBIND_INFO();
1536 1539 }
1537 1540
1538 1541 APIX_ENTER_CPU_LOCK(oldcpu);
1539 1542 (void) apix_obsolete_vector(oldp);
1540 1543 APIX_LEAVE_CPU_LOCK(oldcpu);
1541 1544 }
1542 1545 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind vector 0x%x/0x%x "
1543 1546 "to 0x%x/0x%x, count=%d\n",
1544 1547 oldcpu, oldvec, newcpu, newvec, count));
1545 1548
1546 1549 return (xv_vector(newcpu, newvec));
1547 1550 }
1548 1551
1549 1552 /*
1550 1553 * Senarios include:
1551 1554 * a. add_avintr() is called before irqp initialized (legacy)
1552 1555 * b. irqp is initialized, vector is not allocated (fixed interrupts)
1553 1556 * c. irqp is initialized, vector is allocated (shared interrupts)
1554 1557 */
1555 1558 apix_vector_t *
1556 1559 apix_alloc_intx(dev_info_t *dip, int inum, int irqno)
1557 1560 {
1558 1561 apic_irq_t *irqp;
1559 1562 apix_vector_t *vecp;
1560 1563
1561 1564 /*
1562 1565 * Allocate IRQ. Caller is later responsible for the
1563 1566 * initialization
1564 1567 */
1565 1568 mutex_enter(&airq_mutex);
1566 1569 if ((irqp = apic_irq_table[irqno]) == NULL) {
1567 1570 /* allocate irq */
1568 1571 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1569 1572 irqp->airq_mps_intr_index = FREE_INDEX;
1570 1573 apic_irq_table[irqno] = irqp;
1571 1574 }
1572 1575 if (irqp->airq_mps_intr_index == FREE_INDEX) {
1573 1576 irqp->airq_mps_intr_index = DEFAULT_INDEX;
1574 1577 irqp->airq_cpu = IRQ_UNINIT;
1575 1578 irqp->airq_origirq = (uchar_t)irqno;
1576 1579 }
1577 1580
1578 1581 mutex_exit(&airq_mutex);
1579 1582
1580 1583 /*
1581 1584 * allocate vector
1582 1585 */
1583 1586 if (irqp->airq_cpu == IRQ_UNINIT) {
1584 1587 uint32_t bindcpu, cpuid;
1585 1588
1586 1589 /* select cpu by system policy */
1587 1590 bindcpu = apix_bind_cpu(dip);
1588 1591 cpuid = bindcpu & ~IRQ_USER_BOUND;
1589 1592
1590 1593 /* allocate vector */
1591 1594 APIX_ENTER_CPU_LOCK(cpuid);
1592 1595
1593 1596 if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum,
1594 1597 APIX_TYPE_FIXED)) == NULL) {
1595 1598 cmn_err(CE_WARN, "No interrupt vector for irq %x",
1596 1599 irqno);
1597 1600 APIX_LEAVE_CPU_LOCK(cpuid);
1598 1601 return (NULL);
1599 1602 }
1600 1603 vecp->v_inum = irqno;
1601 1604 vecp->v_flags |= APIX_VECT_MASKABLE;
1602 1605
1603 1606 apix_intx_set_vector(irqno, vecp->v_cpuid, vecp->v_vector);
1604 1607
1605 1608 APIX_LEAVE_CPU_LOCK(cpuid);
1606 1609 } else {
1607 1610 vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1608 1611 ASSERT(!IS_VECT_FREE(vecp));
1609 1612
1610 1613 if (dip != NULL)
1611 1614 apix_set_dev_map(vecp, dip, inum);
1612 1615 }
1613 1616
1614 1617 if ((dip != NULL) &&
1615 1618 (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1616 1619 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1617 1620 apix_set_dev_binding(dip, vecp->v_cpuid);
1618 1621
1619 1622 apix_dprint_vector(vecp, dip, 1);
1620 1623
1621 1624 return (vecp);
1622 1625 }
1623 1626
1624 1627 int
1625 1628 apix_alloc_msi(dev_info_t *dip, int inum, int count, int behavior)
1626 1629 {
1627 1630 int i, cap_ptr, rcount = count;
1628 1631 apix_vector_t *vecp;
1629 1632 processorid_t bindcpu, cpuid;
1630 1633 ushort_t msi_ctrl;
1631 1634 ddi_acc_handle_t handle;
1632 1635
1633 1636 DDI_INTR_IMPLDBG((CE_CONT, "apix_alloc_msi_vectors: dip=0x%p "
1634 1637 "inum=0x%x count=0x%x behavior=%d\n",
1635 1638 (void *)dip, inum, count, behavior));
1636 1639
1637 1640 if (count > 1) {
1638 1641 if (behavior == DDI_INTR_ALLOC_STRICT &&
1639 1642 apic_multi_msi_enable == 0)
1640 1643 return (0);
1641 1644 if (apic_multi_msi_enable == 0)
1642 1645 count = 1;
1643 1646 }
1644 1647
1645 1648 /* Check whether it supports per-vector masking */
1646 1649 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1647 1650 handle = i_ddi_get_pci_config_handle(dip);
1648 1651 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1649 1652
1650 1653 /* bind to cpu */
1651 1654 bindcpu = apix_bind_cpu(dip);
1652 1655 cpuid = bindcpu & ~IRQ_USER_BOUND;
1653 1656
1654 1657 /* if not ISP2, then round it down */
1655 1658 if (!ISP2(rcount))
1656 1659 rcount = 1 << (highbit(rcount) - 1);
1657 1660
1658 1661 APIX_ENTER_CPU_LOCK(cpuid);
1659 1662 for (vecp = NULL; rcount > 0; rcount >>= 1) {
1660 1663 vecp = apix_alloc_nvectors_oncpu(bindcpu, dip, inum, rcount,
1661 1664 APIX_TYPE_MSI);
1662 1665 if (vecp != NULL || behavior == DDI_INTR_ALLOC_STRICT)
1663 1666 break;
1664 1667 }
1665 1668 for (i = 0; vecp && i < rcount; i++)
1666 1669 xv_vector(vecp->v_cpuid, vecp->v_vector + i)->v_flags |=
1667 1670 (msi_ctrl & PCI_MSI_PVM_MASK) ? APIX_VECT_MASKABLE : 0;
1668 1671 APIX_LEAVE_CPU_LOCK(cpuid);
1669 1672 if (vecp == NULL) {
1670 1673 APIC_VERBOSE(INTR, (CE_CONT,
1671 1674 "apix_alloc_msi: no %d cont vectors found on cpu 0x%x\n",
1672 1675 count, bindcpu));
1673 1676 return (0);
1674 1677 }
1675 1678
1676 1679 /* major to cpu binding */
1677 1680 if ((apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1678 1681 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1679 1682 apix_set_dev_binding(dip, vecp->v_cpuid);
1680 1683
1681 1684 apix_dprint_vector(vecp, dip, rcount);
1682 1685
1683 1686 return (rcount);
1684 1687 }
1685 1688
1686 1689 int
1687 1690 apix_alloc_msix(dev_info_t *dip, int inum, int count, int behavior)
1688 1691 {
1689 1692 apix_vector_t *vecp;
1690 1693 processorid_t bindcpu, cpuid;
1691 1694 int i;
1692 1695
1693 1696 for (i = 0; i < count; i++) {
1694 1697 /* select cpu by system policy */
1695 1698 bindcpu = apix_bind_cpu(dip);
1696 1699 cpuid = bindcpu & ~IRQ_USER_BOUND;
1697 1700
1698 1701 /* allocate vector */
1699 1702 APIX_ENTER_CPU_LOCK(cpuid);
1700 1703 if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum + i,
1701 1704 APIX_TYPE_MSIX)) == NULL) {
1702 1705 APIX_LEAVE_CPU_LOCK(cpuid);
1703 1706 APIC_VERBOSE(INTR, (CE_CONT, "apix_alloc_msix: "
1704 1707 "allocate msix for device dip=%p, inum=%d on"
1705 1708 " cpu %d failed", (void *)dip, inum + i, bindcpu));
1706 1709 break;
1707 1710 }
1708 1711 vecp->v_flags |= APIX_VECT_MASKABLE;
1709 1712 APIX_LEAVE_CPU_LOCK(cpuid);
1710 1713
1711 1714 /* major to cpu mapping */
1712 1715 if ((i == 0) &&
1713 1716 (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1714 1717 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1715 1718 apix_set_dev_binding(dip, vecp->v_cpuid);
1716 1719
1717 1720 apix_dprint_vector(vecp, dip, 1);
1718 1721 }
1719 1722
1720 1723 if (i < count && behavior == DDI_INTR_ALLOC_STRICT) {
1721 1724 APIC_VERBOSE(INTR, (CE_WARN, "apix_alloc_msix: "
1722 1725 "strictly allocate %d vectors failed, got %d\n",
1723 1726 count, i));
1724 1727 apix_free_vectors(dip, inum, i, APIX_TYPE_MSIX);
1725 1728 i = 0;
1726 1729 }
1727 1730
1728 1731 return (i);
1729 1732 }
1730 1733
1731 1734 /*
1732 1735 * A rollback free for vectors allocated by apix_alloc_xxx().
1733 1736 */
1734 1737 void
1735 1738 apix_free_vectors(dev_info_t *dip, int inum, int count, int type)
1736 1739 {
1737 1740 int i, cpuid;
1738 1741 apix_vector_t *vecp;
1739 1742
1740 1743 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: dip: %p inum: %x "
1741 1744 "count: %x type: %x\n",
1742 1745 (void *)dip, inum, count, type));
1743 1746
1744 1747 lock_set(&apix_lock);
1745 1748
1746 1749 for (i = 0; i < count; i++, inum++) {
1747 1750 if ((vecp = apix_get_dev_map(dip, inum, type)) == NULL) {
1748 1751 lock_clear(&apix_lock);
1749 1752 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: "
1750 1753 "dip=0x%p inum=0x%x type=0x%x apix_find_intr() "
1751 1754 "failed\n", (void *)dip, inum, type));
1752 1755 continue;
1753 1756 }
1754 1757
1755 1758 APIX_ENTER_CPU_LOCK(vecp->v_cpuid);
1756 1759 cpuid = vecp->v_cpuid;
1757 1760
1758 1761 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: "
1759 1762 "dip=0x%p inum=0x%x type=0x%x vector 0x%x (share %d)\n",
1760 1763 (void *)dip, inum, type, vecp->v_vector, vecp->v_share));
1761 1764
1762 1765 /* tear down device interrupt to vector mapping */
1763 1766 apix_clear_dev_map(dip, inum, type);
1764 1767
1765 1768 if (vecp->v_type == APIX_TYPE_FIXED) {
1766 1769 if (vecp->v_share > 0) { /* share IRQ line */
1767 1770 APIX_LEAVE_CPU_LOCK(cpuid);
1768 1771 continue;
1769 1772 }
1770 1773
1771 1774 /* Free apic_irq_table entry */
1772 1775 apix_intx_free(vecp->v_inum);
1773 1776 }
1774 1777
1775 1778 /* free vector */
1776 1779 apix_cleanup_vector(vecp);
1777 1780
1778 1781 APIX_LEAVE_CPU_LOCK(cpuid);
1779 1782 }
1780 1783
1781 1784 lock_clear(&apix_lock);
1782 1785 }
1783 1786
1784 1787 /*
1785 1788 * Must be called with apix_lock held
1786 1789 */
1787 1790 apix_vector_t *
1788 1791 apix_setup_io_intr(apix_vector_t *vecp)
1789 1792 {
1790 1793 processorid_t bindcpu;
1791 1794 int ret;
1792 1795
1793 1796 ASSERT(LOCK_HELD(&apix_lock));
1794 1797
1795 1798 /*
1796 1799 * Interrupts are enabled on the CPU, programme IOAPIC RDT
1797 1800 * entry or MSI/X address/data to enable the interrupt.
1798 1801 */
1799 1802 if (apix_is_cpu_enabled(vecp->v_cpuid)) {
1800 1803 apix_enable_vector(vecp);
1801 1804 return (vecp);
1802 1805 }
1803 1806
1804 1807 /*
1805 1808 * CPU is not up or interrupts are disabled. Fall back to the
1806 1809 * first avialable CPU.
1807 1810 */
1808 1811 bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
1809 1812
1810 1813 if (vecp->v_type == APIX_TYPE_MSI)
1811 1814 return (apix_grp_set_cpu(vecp, bindcpu, &ret));
1812 1815
1813 1816 return (apix_set_cpu(vecp, bindcpu, &ret));
1814 1817 }
1815 1818
1816 1819 /*
1817 1820 * For interrupts which call add_avintr() before apic is initialized.
1818 1821 * ioapix_setup_intr() will
1819 1822 * - allocate vector
1820 1823 * - copy over ISR
1821 1824 */
|
↓ open down ↓ |
1783 lines elided |
↑ open up ↑ |
1822 1825 static void
1823 1826 ioapix_setup_intr(int irqno, iflag_t *flagp)
1824 1827 {
1825 1828 extern struct av_head autovect[];
1826 1829 apix_vector_t *vecp;
1827 1830 apic_irq_t *irqp;
1828 1831 uchar_t ioapicindex, ipin;
1829 1832 ulong_t iflag;
1830 1833 struct autovec *avp;
1831 1834
1832 - irqp = apic_irq_table[irqno];
1833 1835 ioapicindex = acpi_find_ioapic(irqno);
1834 1836 ASSERT(ioapicindex != 0xFF);
1835 1837 ipin = irqno - apic_io_vectbase[ioapicindex];
1836 1838
1837 - if ((irqp != NULL) && (irqp->airq_mps_intr_index == ACPI_INDEX)) {
1839 + mutex_enter(&airq_mutex);
1840 + irqp = apic_irq_table[irqno];
1841 +
1842 + /*
1843 + * The irq table entry should not exist unless the interrupts are shared.
1844 + * In that case, make sure it matches what we would initialize it to.
1845 + */
1846 + if (irqp != NULL) {
1847 + ASSERT(irqp->airq_mps_intr_index == ACPI_INDEX);
1838 1848 ASSERT(irqp->airq_intin_no == ipin &&
1839 1849 irqp->airq_ioapicindex == ioapicindex);
1840 1850 vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1841 1851 ASSERT(!IS_VECT_FREE(vecp));
1852 + mutex_exit(&airq_mutex);
1842 1853 } else {
1843 - vecp = apix_alloc_intx(NULL, 0, irqno);
1854 + irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1844 1855
1845 - irqp = apic_irq_table[irqno];
1856 + irqp->airq_cpu = IRQ_UNINIT;
1857 + irqp->airq_origirq = (uchar_t)irqno;
1846 1858 irqp->airq_mps_intr_index = ACPI_INDEX;
1847 1859 irqp->airq_ioapicindex = ioapicindex;
1848 1860 irqp->airq_intin_no = ipin;
1849 1861 irqp->airq_iflag = *flagp;
1850 1862 irqp->airq_share++;
1851 - apic_record_rdt_entry(irqp, irqno);
1863 +
1864 + apic_irq_table[irqno] = irqp;
1865 + mutex_exit(&airq_mutex);
1866 +
1867 + vecp = apix_alloc_intx(NULL, 0, irqno);
1852 1868 }
1853 1869
1854 1870 /* copy over autovect */
1855 1871 for (avp = autovect[irqno].avh_link; avp; avp = avp->av_link)
1856 1872 apix_insert_av(vecp, avp->av_intr_id, avp->av_vector,
1857 1873 avp->av_intarg1, avp->av_intarg2, avp->av_ticksp,
1858 1874 avp->av_prilevel, avp->av_dip);
1859 1875
1860 1876 /* Program I/O APIC */
1861 1877 iflag = intr_clear();
1862 1878 lock_set(&apix_lock);
1863 1879
1864 1880 (void) apix_setup_io_intr(vecp);
1865 1881
1866 1882 lock_clear(&apix_lock);
1867 1883 intr_restore(iflag);
1868 1884
1869 1885 APIC_VERBOSE_IOAPIC((CE_CONT, "apix: setup ioapic, irqno %x "
1870 1886 "(ioapic %x, ipin %x) is bound to cpu %x, vector %x\n",
1871 1887 irqno, ioapicindex, ipin, irqp->airq_cpu, irqp->airq_vector));
1872 1888 }
1873 1889
1874 1890 void
1875 1891 ioapix_init_intr(int mask_apic)
1876 1892 {
1877 1893 int ioapicindex;
1878 1894 int i, j;
1879 1895
1880 1896 /* mask interrupt vectors */
1881 1897 for (j = 0; j < apic_io_max && mask_apic; j++) {
1882 1898 int intin_max;
1883 1899
1884 1900 ioapicindex = j;
1885 1901 /* Bits 23-16 define the maximum redirection entries */
1886 1902 intin_max = (ioapic_read(ioapicindex, APIC_VERS_CMD) >> 16)
1887 1903 & 0xff;
1888 1904 for (i = 0; i <= intin_max; i++)
1889 1905 ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * i,
1890 1906 AV_MASK);
1891 1907 }
1892 1908
1893 1909 /*
1894 1910 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
1895 1911 */
1896 1912 if (apic_sci_vect > 0)
1897 1913 ioapix_setup_intr(apic_sci_vect, &apic_sci_flags);
1898 1914
1899 1915 /*
1900 1916 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
1901 1917 */
1902 1918 if (apic_hpet_vect > 0)
1903 1919 ioapix_setup_intr(apic_hpet_vect, &apic_hpet_flags);
1904 1920 }
|
↓ open down ↓ |
43 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX