Print this page
10924 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Peter Tribble <peter.tribble@gmail.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/io/apix/apix_utils.c
+++ new/usr/src/uts/i86pc/io/apix/apix_utils.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
|
↓ open down ↓ |
21 lines elided |
↑ open up ↑ |
22 22 /*
23 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25 /*
26 26 * Copyright (c) 2010, Intel Corporation.
27 27 * All rights reserved.
28 28 */
29 29 /*
30 30 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
31 31 * Copyright 2013 Pluribus Networks, Inc.
32 + * Copyright 2018 Joyent, Inc.
32 33 */
33 34
34 35 #include <sys/processor.h>
35 36 #include <sys/time.h>
36 37 #include <sys/psm.h>
37 38 #include <sys/smp_impldefs.h>
38 39 #include <sys/cram.h>
39 40 #include <sys/acpi/acpi.h>
40 41 #include <sys/acpica.h>
41 42 #include <sys/psm_common.h>
42 43 #include <sys/pit.h>
43 44 #include <sys/ddi.h>
44 45 #include <sys/sunddi.h>
45 46 #include <sys/ddi_impldefs.h>
46 47 #include <sys/pci.h>
47 48 #include <sys/promif.h>
48 49 #include <sys/x86_archext.h>
49 50 #include <sys/cpc_impl.h>
50 51 #include <sys/uadmin.h>
51 52 #include <sys/panic.h>
52 53 #include <sys/debug.h>
53 54 #include <sys/archsystm.h>
54 55 #include <sys/trap.h>
55 56 #include <sys/machsystm.h>
56 57 #include <sys/sysmacros.h>
57 58 #include <sys/cpuvar.h>
58 59 #include <sys/rm_platter.h>
|
↓ open down ↓ |
17 lines elided |
↑ open up ↑ |
59 60 #include <sys/privregs.h>
60 61 #include <sys/note.h>
61 62 #include <sys/pci_intr_lib.h>
62 63 #include <sys/spl.h>
63 64 #include <sys/clock.h>
64 65 #include <sys/dditypes.h>
65 66 #include <sys/sunddi.h>
66 67 #include <sys/x_call.h>
67 68 #include <sys/reboot.h>
68 69 #include <sys/apix.h>
70 +#include <sys/ht.h>
69 71
70 72 static int apix_get_avail_vector_oncpu(uint32_t, int, int);
71 73 static apix_vector_t *apix_init_vector(processorid_t, uchar_t);
72 74 static void apix_cleanup_vector(apix_vector_t *);
73 75 static void apix_insert_av(apix_vector_t *, void *, avfunc, caddr_t, caddr_t,
74 76 uint64_t *, int, dev_info_t *);
75 77 static void apix_remove_av(apix_vector_t *, struct autovec *);
76 78 static void apix_clear_dev_map(dev_info_t *, int, int);
77 79 static boolean_t apix_is_cpu_enabled(processorid_t);
78 80 static void apix_wait_till_seen(processorid_t, int);
79 81
80 82 #define GET_INTR_INUM(ihdlp) \
81 83 (((ihdlp) != NULL) ? ((ddi_intr_handle_impl_t *)(ihdlp))->ih_inum : 0)
82 84
83 85 apix_rebind_info_t apix_rebindinfo = {0, 0, 0, NULL, 0, NULL};
84 86
85 87 /*
86 88 * Allocate IPI
87 89 *
88 90 * Return vector number or 0 on error
89 91 */
90 92 uchar_t
91 93 apix_alloc_ipi(int ipl)
92 94 {
93 95 apix_vector_t *vecp;
94 96 uchar_t vector;
95 97 int cpun;
96 98 int nproc;
97 99
98 100 APIX_ENTER_CPU_LOCK(0);
99 101
100 102 vector = apix_get_avail_vector_oncpu(0, APIX_IPI_MIN, APIX_IPI_MAX);
101 103 if (vector == 0) {
102 104 APIX_LEAVE_CPU_LOCK(0);
103 105 cmn_err(CE_WARN, "apix: no available IPI\n");
104 106 apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
105 107 return (0);
106 108 }
107 109
108 110 nproc = max(apic_nproc, apic_max_nproc);
109 111 for (cpun = 0; cpun < nproc; cpun++) {
110 112 vecp = xv_vector(cpun, vector);
111 113 if (vecp == NULL) {
112 114 vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP);
113 115 if (vecp == NULL) {
114 116 cmn_err(CE_WARN, "apix: No memory for ipi");
115 117 goto fail;
116 118 }
117 119 xv_vector(cpun, vector) = vecp;
118 120 }
119 121 vecp->v_state = APIX_STATE_ALLOCED;
120 122 vecp->v_type = APIX_TYPE_IPI;
121 123 vecp->v_cpuid = vecp->v_bound_cpuid = cpun;
122 124 vecp->v_vector = vector;
123 125 vecp->v_pri = ipl;
124 126 }
125 127 APIX_LEAVE_CPU_LOCK(0);
126 128 return (vector);
127 129
128 130 fail:
129 131 while (--cpun >= 0)
130 132 apix_cleanup_vector(xv_vector(cpun, vector));
131 133 APIX_LEAVE_CPU_LOCK(0);
132 134 return (0);
133 135 }
134 136
135 137 /*
136 138 * Add IPI service routine
137 139 */
138 140 static int
139 141 apix_add_ipi(int ipl, avfunc xxintr, char *name, int vector,
140 142 caddr_t arg1, caddr_t arg2)
141 143 {
142 144 int cpun;
143 145 apix_vector_t *vecp;
144 146 int nproc;
145 147
146 148 ASSERT(vector >= APIX_IPI_MIN && vector <= APIX_IPI_MAX);
147 149
148 150 nproc = max(apic_nproc, apic_max_nproc);
149 151 for (cpun = 0; cpun < nproc; cpun++) {
150 152 APIX_ENTER_CPU_LOCK(cpun);
151 153 vecp = xv_vector(cpun, vector);
152 154 apix_insert_av(vecp, NULL, xxintr, arg1, arg2, NULL, ipl, NULL);
153 155 vecp->v_state = APIX_STATE_ENABLED;
154 156 APIX_LEAVE_CPU_LOCK(cpun);
155 157 }
156 158
157 159 APIC_VERBOSE(IPI, (CE_CONT, "apix: add ipi for %s, vector %x "
158 160 "ipl %x\n", name, vector, ipl));
159 161
160 162 return (1);
161 163 }
162 164
163 165 /*
164 166 * Find and return first free vector in range (start, end)
165 167 */
166 168 static int
167 169 apix_get_avail_vector_oncpu(uint32_t cpuid, int start, int end)
168 170 {
169 171 int i;
170 172 apix_impl_t *apixp = apixs[cpuid];
171 173
172 174 for (i = start; i <= end; i++) {
173 175 if (APIC_CHECK_RESERVE_VECTORS(i))
174 176 continue;
175 177 if (IS_VECT_FREE(apixp->x_vectbl[i]))
176 178 return (i);
177 179 }
178 180
179 181 return (0);
180 182 }
181 183
182 184 /*
183 185 * Allocate a vector on specified cpu
184 186 *
185 187 * Return NULL on error
186 188 */
187 189 static apix_vector_t *
188 190 apix_alloc_vector_oncpu(uint32_t cpuid, dev_info_t *dip, int inum, int type)
189 191 {
190 192 processorid_t tocpu = cpuid & ~IRQ_USER_BOUND;
191 193 apix_vector_t *vecp;
192 194 int vector;
193 195
194 196 ASSERT(APIX_CPU_LOCK_HELD(tocpu));
195 197
196 198 /* find free vector */
197 199 vector = apix_get_avail_vector_oncpu(tocpu, APIX_AVINTR_MIN,
198 200 APIX_AVINTR_MAX);
199 201 if (vector == 0)
200 202 return (NULL);
201 203
202 204 vecp = apix_init_vector(tocpu, vector);
203 205 vecp->v_type = (ushort_t)type;
204 206 vecp->v_inum = inum;
205 207 vecp->v_flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0;
206 208
207 209 if (dip != NULL)
208 210 apix_set_dev_map(vecp, dip, inum);
209 211
210 212 return (vecp);
211 213 }
212 214
213 215 /*
214 216 * Allocates "count" contiguous MSI vectors starting at the proper alignment.
215 217 * Caller needs to make sure that count has to be power of 2 and should not
216 218 * be < 1.
217 219 *
218 220 * Return first vector number
219 221 */
220 222 apix_vector_t *
221 223 apix_alloc_nvectors_oncpu(uint32_t cpuid, dev_info_t *dip, int inum,
222 224 int count, int type)
223 225 {
224 226 int i, msibits, start = 0, navail = 0;
225 227 apix_vector_t *vecp, *startp = NULL;
226 228 processorid_t tocpu = cpuid & ~IRQ_USER_BOUND;
227 229 uint_t flags;
228 230
229 231 ASSERT(APIX_CPU_LOCK_HELD(tocpu));
230 232
231 233 /*
232 234 * msibits is the no. of lower order message data bits for the
233 235 * allocated MSI vectors and is used to calculate the aligned
234 236 * starting vector
235 237 */
236 238 msibits = count - 1;
237 239
238 240 /* It has to be contiguous */
239 241 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
240 242 if (!IS_VECT_FREE(xv_vector(tocpu, i)))
241 243 continue;
242 244
243 245 /*
244 246 * starting vector has to be aligned accordingly for
245 247 * multiple MSIs
246 248 */
247 249 if (msibits)
248 250 i = (i + msibits) & ~msibits;
249 251
250 252 for (navail = 0, start = i; i <= APIX_AVINTR_MAX; i++) {
251 253 if (!IS_VECT_FREE(xv_vector(tocpu, i)))
252 254 break;
253 255 if (APIC_CHECK_RESERVE_VECTORS(i))
254 256 break;
255 257 if (++navail == count)
256 258 goto done;
257 259 }
258 260 }
259 261
260 262 return (NULL);
261 263
262 264 done:
263 265 flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0;
264 266
265 267 for (i = 0; i < count; i++) {
266 268 if ((vecp = apix_init_vector(tocpu, start + i)) == NULL)
267 269 goto fail;
268 270
269 271 vecp->v_type = (ushort_t)type;
270 272 vecp->v_inum = inum + i;
271 273 vecp->v_flags = flags;
272 274
273 275 if (dip != NULL)
274 276 apix_set_dev_map(vecp, dip, inum + i);
275 277
276 278 if (i == 0)
277 279 startp = vecp;
278 280 }
279 281
280 282 return (startp);
281 283
282 284 fail:
283 285 while (i-- > 0) { /* Free allocated vectors */
284 286 vecp = xv_vector(tocpu, start + i);
285 287 apix_clear_dev_map(dip, inum + i, type);
286 288 apix_cleanup_vector(vecp);
287 289 }
288 290 return (NULL);
289 291 }
290 292
291 293 #define APIX_WRITE_MSI_DATA(_hdl, _cap, _ctrl, _v)\
292 294 do {\
293 295 if ((_ctrl) & PCI_MSI_64BIT_MASK)\
294 296 pci_config_put16((_hdl), (_cap) + PCI_MSI_64BIT_DATA, (_v));\
295 297 else\
296 298 pci_config_put16((_hdl), (_cap) + PCI_MSI_32BIT_DATA, (_v));\
297 299 _NOTE(CONSTCOND)} while (0)
298 300
299 301 static void
300 302 apix_pci_msi_enable_vector(apix_vector_t *vecp, dev_info_t *dip, int type,
301 303 int inum, int count, uchar_t vector, int target_apic_id)
302 304 {
303 305 uint64_t msi_addr, msi_data;
304 306 ushort_t msi_ctrl;
305 307 int i, cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
306 308 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip);
307 309 msi_regs_t msi_regs;
308 310 void *intrmap_tbl[PCI_MSI_MAX_INTRS];
309 311
310 312 DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: dip=0x%p\n"
311 313 "\tdriver = %s, inum=0x%x vector=0x%x apicid=0x%x\n", (void *)dip,
312 314 ddi_driver_name(dip), inum, vector, target_apic_id));
313 315
314 316 ASSERT((handle != NULL) && (cap_ptr != 0));
315 317
316 318 msi_regs.mr_data = vector;
317 319 msi_regs.mr_addr = target_apic_id;
318 320
319 321 for (i = 0; i < count; i++)
320 322 intrmap_tbl[i] = xv_intrmap_private(vecp->v_cpuid, vector + i);
321 323 apic_vt_ops->apic_intrmap_alloc_entry(intrmap_tbl, dip, type,
322 324 count, 0xff);
323 325 for (i = 0; i < count; i++)
324 326 xv_intrmap_private(vecp->v_cpuid, vector + i) = intrmap_tbl[i];
325 327
326 328 apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
327 329 (void *)&msi_regs, type, count);
328 330 apic_vt_ops->apic_intrmap_record_msi(vecp->v_intrmap_private,
329 331 &msi_regs);
330 332
331 333 /* MSI Address */
332 334 msi_addr = msi_regs.mr_addr;
333 335
334 336 /* MSI Data: MSI is edge triggered according to spec */
335 337 msi_data = msi_regs.mr_data;
336 338
337 339 DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: addr=0x%lx "
338 340 "data=0x%lx\n", (long)msi_addr, (long)msi_data));
339 341
340 342 if (type == APIX_TYPE_MSI) {
341 343 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
342 344
343 345 /* Set the bits to inform how many MSIs are enabled */
344 346 msi_ctrl |= ((highbit(count) - 1) << PCI_MSI_MME_SHIFT);
345 347 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
346 348
347 349 if ((vecp->v_flags & APIX_VECT_MASKABLE) == 0)
348 350 APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl,
349 351 APIX_RESV_VECTOR);
350 352
351 353 pci_config_put32(handle,
352 354 cap_ptr + PCI_MSI_ADDR_OFFSET, msi_addr);
353 355 if (msi_ctrl & PCI_MSI_64BIT_MASK)
354 356 pci_config_put32(handle,
355 357 cap_ptr + PCI_MSI_ADDR_OFFSET + 4, msi_addr >> 32);
356 358
357 359 APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl, msi_data);
358 360 } else if (type == APIX_TYPE_MSIX) {
359 361 uintptr_t off;
360 362 ddi_intr_msix_t *msix_p = i_ddi_get_msix(dip);
361 363
362 364 /* Offset into the "inum"th entry in the MSI-X table */
363 365 off = (uintptr_t)msix_p->msix_tbl_addr +
364 366 (inum * PCI_MSIX_VECTOR_SIZE);
365 367
366 368 ddi_put32(msix_p->msix_tbl_hdl,
367 369 (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), msi_data);
368 370 ddi_put32(msix_p->msix_tbl_hdl,
369 371 (uint32_t *)(off + PCI_MSIX_LOWER_ADDR_OFFSET), msi_addr);
370 372 ddi_put32(msix_p->msix_tbl_hdl,
371 373 (uint32_t *)(off + PCI_MSIX_UPPER_ADDR_OFFSET),
372 374 msi_addr >> 32);
373 375 }
374 376 }
375 377
376 378 static void
377 379 apix_pci_msi_enable_mode(dev_info_t *dip, int type, int inum)
378 380 {
379 381 ushort_t msi_ctrl;
380 382 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
381 383 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip);
382 384
383 385 ASSERT((handle != NULL) && (cap_ptr != 0));
384 386
385 387 if (type == APIX_TYPE_MSI) {
386 388 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
387 389 if ((msi_ctrl & PCI_MSI_ENABLE_BIT))
388 390 return;
389 391
390 392 msi_ctrl |= PCI_MSI_ENABLE_BIT;
391 393 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
392 394
393 395 } else if (type == DDI_INTR_TYPE_MSIX) {
394 396 uintptr_t off;
395 397 uint32_t mask;
396 398 ddi_intr_msix_t *msix_p;
397 399
398 400 msix_p = i_ddi_get_msix(dip);
399 401
400 402 /* Offset into "inum"th entry in the MSI-X table & clear mask */
401 403 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
402 404 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
403 405
404 406 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
405 407
406 408 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask & ~1));
407 409
408 410 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
409 411
410 412 if (!(msi_ctrl & PCI_MSIX_ENABLE_BIT)) {
411 413 msi_ctrl |= PCI_MSIX_ENABLE_BIT;
412 414 pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
413 415 msi_ctrl);
414 416 }
415 417 }
416 418 }
417 419
418 420 /*
419 421 * Setup interrupt, pogramming IO-APIC or MSI/X address/data.
420 422 */
421 423 void
422 424 apix_enable_vector(apix_vector_t *vecp)
423 425 {
424 426 int tocpu = vecp->v_cpuid, type = vecp->v_type;
425 427 apic_cpus_info_t *cpu_infop;
426 428 ulong_t iflag;
427 429
428 430 ASSERT(tocpu < apic_nproc);
429 431
430 432 cpu_infop = &apic_cpus[tocpu];
431 433 if (vecp->v_flags & APIX_VECT_USER_BOUND)
432 434 cpu_infop->aci_bound++;
433 435 else
434 436 cpu_infop->aci_temp_bound++;
435 437
436 438 iflag = intr_clear();
437 439 lock_set(&apic_ioapic_lock);
438 440
439 441 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) { /* fixed */
440 442 apix_intx_enable(vecp->v_inum);
441 443 } else {
442 444 int inum = vecp->v_inum;
443 445 dev_info_t *dip = APIX_GET_DIP(vecp);
444 446 int count = i_ddi_intr_get_current_nintrs(dip);
445 447
446 448 if (type == APIX_TYPE_MSI) { /* MSI */
447 449 if (inum == apix_get_max_dev_inum(dip, type)) {
448 450 /* last one */
449 451 uchar_t start_inum = inum + 1 - count;
450 452 uchar_t start_vect = vecp->v_vector + 1 - count;
451 453 apix_vector_t *start_vecp =
452 454 xv_vector(vecp->v_cpuid, start_vect);
453 455
454 456 APIC_VERBOSE(INTR, (CE_CONT, "apix: call "
455 457 "apix_pci_msi_enable_vector\n"));
456 458 apix_pci_msi_enable_vector(start_vecp, dip,
457 459 type, start_inum, count, start_vect,
458 460 cpu_infop->aci_local_id);
459 461
460 462 APIC_VERBOSE(INTR, (CE_CONT, "apix: call "
461 463 "apix_pci_msi_enable_mode\n"));
462 464 apix_pci_msi_enable_mode(dip, type, inum);
463 465 }
464 466 } else { /* MSI-X */
465 467 apix_pci_msi_enable_vector(vecp, dip,
466 468 type, inum, 1, vecp->v_vector,
467 469 cpu_infop->aci_local_id);
468 470 apix_pci_msi_enable_mode(dip, type, inum);
469 471 }
470 472 }
471 473 vecp->v_state = APIX_STATE_ENABLED;
472 474 apic_redist_cpu_skip &= ~(1 << tocpu);
473 475
474 476 lock_clear(&apic_ioapic_lock);
475 477 intr_restore(iflag);
476 478 }
477 479
478 480 /*
479 481 * Disable the interrupt
480 482 */
481 483 void
482 484 apix_disable_vector(apix_vector_t *vecp)
483 485 {
484 486 struct autovec *avp = vecp->v_autovect;
485 487 ulong_t iflag;
486 488
487 489 ASSERT(avp != NULL);
488 490
489 491 iflag = intr_clear();
490 492 lock_set(&apic_ioapic_lock);
491 493
492 494 switch (vecp->v_type) {
493 495 case APIX_TYPE_MSI:
494 496 ASSERT(avp->av_vector != NULL && avp->av_dip != NULL);
495 497 /*
496 498 * Disable the MSI vector
497 499 * Make sure we only disable on the last
498 500 * of the multi-MSI support
499 501 */
500 502 if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) {
501 503 apic_pci_msi_disable_mode(avp->av_dip,
502 504 DDI_INTR_TYPE_MSI);
503 505 }
504 506 break;
505 507 case APIX_TYPE_MSIX:
506 508 ASSERT(avp->av_vector != NULL && avp->av_dip != NULL);
507 509 /*
508 510 * Disable the MSI-X vector
509 511 * needs to clear its mask and addr/data for each MSI-X
510 512 */
511 513 apic_pci_msi_unconfigure(avp->av_dip, DDI_INTR_TYPE_MSIX,
512 514 vecp->v_inum);
513 515 /*
514 516 * Make sure we only disable on the last MSI-X
515 517 */
516 518 if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) {
517 519 apic_pci_msi_disable_mode(avp->av_dip,
518 520 DDI_INTR_TYPE_MSIX);
519 521 }
520 522 break;
521 523 default:
522 524 apix_intx_disable(vecp->v_inum);
523 525 break;
524 526 }
525 527
526 528 if (!(apic_cpus[vecp->v_cpuid].aci_status & APIC_CPU_SUSPEND))
527 529 vecp->v_state = APIX_STATE_DISABLED;
528 530 apic_vt_ops->apic_intrmap_free_entry(&vecp->v_intrmap_private);
529 531 vecp->v_intrmap_private = NULL;
530 532
531 533 lock_clear(&apic_ioapic_lock);
532 534 intr_restore(iflag);
533 535 }
534 536
535 537 /*
536 538 * Mark vector as obsoleted or freed. The vector is marked
537 539 * obsoleted if there are pending requests on it. Otherwise,
538 540 * free the vector. The obsoleted vectors get freed after
539 541 * being serviced.
540 542 *
541 543 * Return 1 on being obosoleted and 0 on being freed.
542 544 */
543 545 #define INTR_BUSY(_avp)\
544 546 ((((volatile ushort_t)(_avp)->av_flags) &\
545 547 (AV_PENTRY_PEND | AV_PENTRY_ONPROC)) != 0)
546 548 #define LOCAL_WITH_INTR_DISABLED(_cpuid)\
547 549 ((_cpuid) == psm_get_cpu_id() && !interrupts_enabled())
548 550 static uint64_t dummy_tick;
549 551
550 552 int
551 553 apix_obsolete_vector(apix_vector_t *vecp)
552 554 {
553 555 struct autovec *avp = vecp->v_autovect;
554 556 int repeats, tries, ipl, busy = 0, cpuid = vecp->v_cpuid;
555 557 apix_impl_t *apixp = apixs[cpuid];
556 558
557 559 ASSERT(APIX_CPU_LOCK_HELD(cpuid));
558 560
559 561 for (avp = vecp->v_autovect; avp != NULL; avp = avp->av_link) {
560 562 if (avp->av_vector == NULL)
561 563 continue;
562 564
563 565 if (LOCAL_WITH_INTR_DISABLED(cpuid)) {
564 566 int bit, index, irr;
565 567
566 568 if (INTR_BUSY(avp)) {
567 569 busy++;
568 570 continue;
569 571 }
570 572
571 573 /* check IRR for pending interrupts */
572 574 index = vecp->v_vector / 32;
573 575 bit = vecp->v_vector % 32;
574 576 irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
575 577 if ((irr & (1 << bit)) != 0)
576 578 busy++;
577 579
578 580 if (!busy)
579 581 apix_remove_av(vecp, avp);
580 582
581 583 continue;
582 584 }
583 585
584 586 repeats = 0;
585 587 do {
586 588 repeats++;
587 589 for (tries = 0; tries < apic_max_reps_clear_pending;
588 590 tries++)
589 591 if (!INTR_BUSY(avp))
590 592 break;
591 593 } while (INTR_BUSY(avp) &&
592 594 (repeats < apic_max_reps_clear_pending));
593 595
594 596 if (INTR_BUSY(avp))
595 597 busy++;
596 598 else {
597 599 /*
598 600 * Interrupt is not in pending list or being serviced.
599 601 * However it might be cached in Local APIC's IRR
600 602 * register. It's impossible to check another CPU's
601 603 * IRR register. Then wait till lower levels finish
602 604 * running.
603 605 */
604 606 for (ipl = 1; ipl < MIN(LOCK_LEVEL, vecp->v_pri); ipl++)
605 607 apix_wait_till_seen(cpuid, ipl);
606 608 if (INTR_BUSY(avp))
607 609 busy++;
608 610 }
609 611
610 612 if (!busy)
611 613 apix_remove_av(vecp, avp);
612 614 }
613 615
614 616 if (busy) {
615 617 apix_vector_t *tp = apixp->x_obsoletes;
616 618
617 619 if (vecp->v_state == APIX_STATE_OBSOLETED)
618 620 return (1);
619 621
620 622 vecp->v_state = APIX_STATE_OBSOLETED;
621 623 vecp->v_next = NULL;
622 624 if (tp == NULL)
623 625 apixp->x_obsoletes = vecp;
624 626 else {
625 627 while (tp->v_next != NULL)
626 628 tp = tp->v_next;
627 629 tp->v_next = vecp;
628 630 }
629 631 return (1);
630 632 }
631 633
632 634 /* interrupt is not busy */
633 635 if (vecp->v_state == APIX_STATE_OBSOLETED) {
634 636 /* remove from obsoleted list */
635 637 apixp->x_obsoletes = vecp->v_next;
636 638 vecp->v_next = NULL;
637 639 }
638 640 apix_cleanup_vector(vecp);
639 641 return (0);
640 642 }
641 643
642 644 /*
643 645 * Duplicate number of continuous vectors to specified target vectors.
644 646 */
645 647 static void
646 648 apix_dup_vectors(apix_vector_t *oldp, apix_vector_t *newp, int count)
647 649 {
648 650 struct autovec *avp;
649 651 apix_vector_t *fromp, *top;
650 652 processorid_t oldcpu = oldp->v_cpuid, newcpu = newp->v_cpuid;
651 653 uchar_t oldvec = oldp->v_vector, newvec = newp->v_vector;
652 654 int i, inum;
653 655
654 656 ASSERT(oldp->v_type != APIX_TYPE_IPI);
655 657
656 658 for (i = 0; i < count; i++) {
657 659 fromp = xv_vector(oldcpu, oldvec + i);
658 660 top = xv_vector(newcpu, newvec + i);
659 661 ASSERT(fromp != NULL && top != NULL);
660 662
661 663 /* copy over original one */
662 664 top->v_state = fromp->v_state;
663 665 top->v_type = fromp->v_type;
664 666 top->v_bound_cpuid = fromp->v_bound_cpuid;
665 667 top->v_inum = fromp->v_inum;
666 668 top->v_flags = fromp->v_flags;
667 669 top->v_intrmap_private = fromp->v_intrmap_private;
668 670
669 671 for (avp = fromp->v_autovect; avp != NULL; avp = avp->av_link) {
670 672 if (avp->av_vector == NULL)
671 673 continue;
672 674
673 675 apix_insert_av(top, avp->av_intr_id, avp->av_vector,
674 676 avp->av_intarg1, avp->av_intarg2, avp->av_ticksp,
675 677 avp->av_prilevel, avp->av_dip);
676 678
677 679 if (fromp->v_type == APIX_TYPE_FIXED &&
678 680 avp->av_dip != NULL) {
679 681 inum = GET_INTR_INUM(avp->av_intr_id);
680 682 apix_set_dev_map(top, avp->av_dip, inum);
681 683 }
682 684 }
683 685
684 686 if (DDI_INTR_IS_MSI_OR_MSIX(fromp->v_type) &&
685 687 fromp->v_devp != NULL)
686 688 apix_set_dev_map(top, fromp->v_devp->dv_dip,
687 689 fromp->v_devp->dv_inum);
688 690 }
689 691 }
690 692
691 693 static apix_vector_t *
692 694 apix_init_vector(processorid_t cpuid, uchar_t vector)
693 695 {
694 696 apix_impl_t *apixp = apixs[cpuid];
695 697 apix_vector_t *vecp = apixp->x_vectbl[vector];
696 698
697 699 ASSERT(IS_VECT_FREE(vecp));
698 700
699 701 if (vecp == NULL) {
700 702 vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP);
701 703 if (vecp == NULL) {
702 704 cmn_err(CE_WARN, "apix: no memory to allocate vector");
703 705 return (NULL);
704 706 }
705 707 apixp->x_vectbl[vector] = vecp;
706 708 }
707 709 vecp->v_state = APIX_STATE_ALLOCED;
708 710 vecp->v_cpuid = vecp->v_bound_cpuid = cpuid;
709 711 vecp->v_vector = vector;
710 712
711 713 return (vecp);
712 714 }
713 715
714 716 static void
715 717 apix_cleanup_vector(apix_vector_t *vecp)
716 718 {
717 719 ASSERT(vecp->v_share == 0);
718 720 vecp->v_bound_cpuid = IRQ_UNINIT;
719 721 vecp->v_state = APIX_STATE_FREED;
720 722 vecp->v_type = 0;
721 723 vecp->v_flags = 0;
722 724 vecp->v_busy = 0;
723 725 vecp->v_intrmap_private = NULL;
724 726 }
725 727
726 728 static void
727 729 apix_dprint_vector(apix_vector_t *vecp, dev_info_t *dip, int count)
728 730 {
729 731 #ifdef DEBUG
730 732 major_t major;
731 733 char *name, *drv_name;
732 734 int instance, len, t_len;
733 735 char mesg[1024] = "apix: ";
734 736
735 737 t_len = sizeof (mesg);
736 738 len = strlen(mesg);
737 739 if (dip != NULL) {
738 740 name = ddi_get_name(dip);
739 741 major = ddi_name_to_major(name);
740 742 drv_name = ddi_major_to_name(major);
741 743 instance = ddi_get_instance(dip);
742 744 (void) snprintf(mesg + len, t_len - len, "%s (%s) instance %d ",
743 745 name, drv_name, instance);
744 746 }
745 747 len = strlen(mesg);
746 748
747 749 switch (vecp->v_type) {
748 750 case APIX_TYPE_FIXED:
749 751 (void) snprintf(mesg + len, t_len - len, "irqno %d",
750 752 vecp->v_inum);
751 753 break;
752 754 case APIX_TYPE_MSI:
753 755 (void) snprintf(mesg + len, t_len - len,
754 756 "msi inum %d (count %d)", vecp->v_inum, count);
755 757 break;
756 758 case APIX_TYPE_MSIX:
757 759 (void) snprintf(mesg + len, t_len - len, "msi-x inum %d",
758 760 vecp->v_inum);
759 761 break;
760 762 default:
761 763 break;
762 764
763 765 }
764 766
765 767 APIC_VERBOSE(ALLOC, (CE_CONT, "%s allocated with vector 0x%x on "
766 768 "cpu %d\n", mesg, vecp->v_vector, vecp->v_cpuid));
767 769 #endif /* DEBUG */
768 770 }
769 771
770 772 /*
771 773 * Operations on avintr
772 774 */
773 775
774 776 #define INIT_AUTOVEC(p, intr_id, f, arg1, arg2, ticksp, ipl, dip) \
775 777 do { \
776 778 (p)->av_intr_id = intr_id; \
777 779 (p)->av_vector = f; \
778 780 (p)->av_intarg1 = arg1; \
779 781 (p)->av_intarg2 = arg2; \
780 782 (p)->av_ticksp = ticksp; \
781 783 (p)->av_prilevel = ipl; \
782 784 (p)->av_dip = dip; \
783 785 (p)->av_flags = 0; \
784 786 _NOTE(CONSTCOND)} while (0)
785 787
786 788 /*
787 789 * Insert an interrupt service routine into chain by its priority from
788 790 * high to low
789 791 */
790 792 static void
791 793 apix_insert_av(apix_vector_t *vecp, void *intr_id, avfunc f, caddr_t arg1,
792 794 caddr_t arg2, uint64_t *ticksp, int ipl, dev_info_t *dip)
793 795 {
794 796 struct autovec *p, *prep, *mem;
795 797
|
↓ open down ↓ |
717 lines elided |
↑ open up ↑ |
796 798 APIC_VERBOSE(INTR, (CE_CONT, "apix_insert_av: dip %p, vector 0x%x, "
797 799 "cpu %d\n", (void *)dip, vecp->v_vector, vecp->v_cpuid));
798 800
799 801 mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP);
800 802 INIT_AUTOVEC(mem, intr_id, f, arg1, arg2, ticksp, ipl, dip);
801 803 if (vecp->v_type == APIX_TYPE_FIXED && apic_level_intr[vecp->v_inum])
802 804 mem->av_flags |= AV_PENTRY_LEVEL;
803 805
804 806 vecp->v_share++;
805 807 vecp->v_pri = (ipl > vecp->v_pri) ? ipl : vecp->v_pri;
808 +
809 + ht_intr_alloc_pil(vecp->v_pri);
810 +
806 811 if (vecp->v_autovect == NULL) { /* Nothing on list - put it at head */
807 812 vecp->v_autovect = mem;
808 813 return;
809 814 }
810 815
811 816 if (DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) { /* MSI/X */
812 817 ASSERT(vecp->v_share == 1); /* No sharing for MSI/X */
813 818
814 819 INIT_AUTOVEC(vecp->v_autovect, intr_id, f, arg1, arg2, ticksp,
815 820 ipl, dip);
816 821 prep = vecp->v_autovect->av_link;
817 822 vecp->v_autovect->av_link = NULL;
818 823
819 824 /* Free the following autovect chain */
820 825 while (prep != NULL) {
821 826 ASSERT(prep->av_vector == NULL);
822 827
823 828 p = prep;
824 829 prep = prep->av_link;
825 830 kmem_free(p, sizeof (struct autovec));
826 831 }
827 832
828 833 kmem_free(mem, sizeof (struct autovec));
829 834 return;
830 835 }
831 836
832 837 /* find where it goes in list */
833 838 prep = NULL;
834 839 for (p = vecp->v_autovect; p != NULL; p = p->av_link) {
835 840 if (p->av_vector && p->av_prilevel <= ipl)
836 841 break;
837 842 prep = p;
838 843 }
839 844 if (prep != NULL) {
840 845 if (prep->av_vector == NULL) { /* freed struct available */
841 846 INIT_AUTOVEC(prep, intr_id, f, arg1, arg2,
842 847 ticksp, ipl, dip);
843 848 prep->av_flags = mem->av_flags;
844 849 kmem_free(mem, sizeof (struct autovec));
845 850 return;
846 851 }
847 852
848 853 mem->av_link = prep->av_link;
849 854 prep->av_link = mem;
850 855 } else {
851 856 /* insert new intpt at beginning of chain */
852 857 mem->av_link = vecp->v_autovect;
853 858 vecp->v_autovect = mem;
854 859 }
855 860 }
856 861
857 862 /*
858 863 * After having made a change to an autovector list, wait until we have
859 864 * seen specified cpu not executing an interrupt at that level--so we
860 865 * know our change has taken effect completely (no old state in registers,
861 866 * etc).
862 867 */
863 868 #define APIX_CPU_ENABLED(_cp) \
864 869 (quiesce_active == 0 && \
865 870 (((_cp)->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) == 0))
866 871
867 872 static void
868 873 apix_wait_till_seen(processorid_t cpuid, int ipl)
869 874 {
870 875 struct cpu *cp = cpu[cpuid];
871 876
872 877 if (cp == NULL || LOCAL_WITH_INTR_DISABLED(cpuid))
873 878 return;
874 879
875 880 /*
876 881 * Don't wait if the CPU is quiesced or offlined. This can happen
877 882 * when a CPU is running pause thread but hardware triggered an
878 883 * interrupt and the interrupt gets queued.
879 884 */
880 885 for (;;) {
881 886 if (!INTR_ACTIVE((volatile struct cpu *)cpu[cpuid], ipl) &&
882 887 (!APIX_CPU_ENABLED(cp) ||
883 888 !INTR_PENDING((volatile apix_impl_t *)apixs[cpuid], ipl)))
884 889 return;
885 890 }
886 891 }
887 892
888 893 static void
889 894 apix_remove_av(apix_vector_t *vecp, struct autovec *target)
890 895 {
891 896 int hi_pri = 0;
892 897 struct autovec *p;
893 898
894 899 if (target == NULL)
895 900 return;
896 901
897 902 APIC_VERBOSE(INTR, (CE_CONT, "apix_remove_av: dip %p, vector 0x%x, "
898 903 "cpu %d\n", (void *)target->av_dip, vecp->v_vector, vecp->v_cpuid));
899 904
900 905 for (p = vecp->v_autovect; p; p = p->av_link) {
901 906 if (p == target || p->av_vector == NULL)
902 907 continue;
903 908 hi_pri = (p->av_prilevel > hi_pri) ? p->av_prilevel : hi_pri;
904 909 }
905 910
906 911 vecp->v_share--;
907 912 vecp->v_pri = hi_pri;
908 913
909 914 /*
910 915 * This drops the handler from the chain, it can no longer be called.
911 916 * However, there is no guarantee that the handler is not currently
912 917 * still executing.
913 918 */
914 919 target->av_vector = NULL;
915 920 /*
916 921 * There is a race where we could be just about to pick up the ticksp
917 922 * pointer to increment it after returning from the service routine
918 923 * in av_dispatch_autovect. Rather than NULL it out let's just point
919 924 * it off to something safe so that any final tick update attempt
920 925 * won't fault.
921 926 */
922 927 target->av_ticksp = &dummy_tick;
923 928 apix_wait_till_seen(vecp->v_cpuid, target->av_prilevel);
924 929 }
925 930
926 931 static struct autovec *
927 932 apix_find_av(apix_vector_t *vecp, void *intr_id, avfunc f)
928 933 {
929 934 struct autovec *p;
930 935
931 936 for (p = vecp->v_autovect; p; p = p->av_link) {
932 937 if ((p->av_vector == f) && (p->av_intr_id == intr_id)) {
933 938 /* found the handler */
934 939 return (p);
935 940 }
936 941 }
937 942
938 943 return (NULL);
939 944 }
940 945
941 946 static apix_vector_t *
942 947 apix_find_vector_by_avintr(void *intr_id, avfunc f)
943 948 {
944 949 apix_vector_t *vecp;
945 950 processorid_t n;
946 951 uchar_t v;
947 952
948 953 for (n = 0; n < apic_nproc; n++) {
949 954 if (!apix_is_cpu_enabled(n))
950 955 continue;
951 956
952 957 for (v = APIX_AVINTR_MIN; v <= APIX_AVINTR_MIN; v++) {
953 958 vecp = xv_vector(n, v);
954 959 if (vecp == NULL ||
955 960 vecp->v_state <= APIX_STATE_OBSOLETED)
956 961 continue;
957 962
958 963 if (apix_find_av(vecp, intr_id, f) != NULL)
959 964 return (vecp);
960 965 }
961 966 }
962 967
963 968 return (NULL);
964 969 }
965 970
966 971 /*
967 972 * Add interrupt service routine.
968 973 *
969 974 * For legacy interrupts (HPET timer, ACPI SCI), the vector is actually
970 975 * IRQ no. A vector is then allocated. Otherwise, the vector is already
971 976 * allocated. The input argument virt_vect is virtual vector of format
972 977 * APIX_VIRTVEC_VECTOR(cpuid, vector).
973 978 *
974 979 * Return 1 on success, 0 on failure.
975 980 */
976 981 int
977 982 apix_add_avintr(void *intr_id, int ipl, avfunc xxintr, char *name,
978 983 int virt_vect, caddr_t arg1, caddr_t arg2, uint64_t *ticksp,
979 984 dev_info_t *dip)
980 985 {
981 986 int cpuid;
982 987 uchar_t v = (uchar_t)APIX_VIRTVEC_VECTOR(virt_vect);
983 988 apix_vector_t *vecp;
984 989
985 990 if (xxintr == NULL) {
986 991 cmn_err(CE_WARN, "Attempt to add null for %s "
987 992 "on vector 0x%x,0x%x", name,
988 993 APIX_VIRTVEC_CPU(virt_vect),
989 994 APIX_VIRTVEC_VECTOR(virt_vect));
990 995 return (0);
991 996 }
992 997
993 998 if (v >= APIX_IPI_MIN) /* IPIs */
994 999 return (apix_add_ipi(ipl, xxintr, name, v, arg1, arg2));
995 1000
996 1001 if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */
997 1002 int irqno = virt_vect;
998 1003 int inum = GET_INTR_INUM(intr_id);
999 1004
1000 1005 /*
1001 1006 * Senarios include:
1002 1007 * a. add_avintr() is called before irqp initialized (legacy)
1003 1008 * b. irqp is initialized, vector is not allocated (fixed)
1004 1009 * c. irqp is initialized, vector is allocated (fixed & shared)
1005 1010 */
1006 1011 if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
1007 1012 return (0);
1008 1013
1009 1014 cpuid = vecp->v_cpuid;
1010 1015 v = vecp->v_vector;
1011 1016 virt_vect = APIX_VIRTVECTOR(cpuid, v);
1012 1017 } else { /* got virtual vector */
1013 1018 cpuid = APIX_VIRTVEC_CPU(virt_vect);
1014 1019 vecp = xv_vector(cpuid, v);
1015 1020 ASSERT(vecp != NULL);
1016 1021 }
1017 1022
1018 1023 lock_set(&apix_lock);
1019 1024 if (vecp->v_state <= APIX_STATE_OBSOLETED) {
1020 1025 vecp = NULL;
1021 1026
1022 1027 /*
1023 1028 * Basically the allocated but not enabled interrupts
1024 1029 * will not get re-targeted. But MSIs in allocated state
1025 1030 * could be re-targeted due to group re-targeting.
1026 1031 */
1027 1032 if (intr_id != NULL && dip != NULL) {
1028 1033 ddi_intr_handle_impl_t *hdlp = intr_id;
1029 1034 vecp = apix_get_dev_map(dip, hdlp->ih_inum,
1030 1035 hdlp->ih_type);
1031 1036 ASSERT(vecp->v_state == APIX_STATE_ALLOCED);
1032 1037 }
1033 1038 if (vecp == NULL) {
1034 1039 lock_clear(&apix_lock);
1035 1040 cmn_err(CE_WARN, "Invalid interrupt 0x%x,0x%x "
1036 1041 " for %p to add", cpuid, v, intr_id);
1037 1042 return (0);
1038 1043 }
1039 1044 cpuid = vecp->v_cpuid;
1040 1045 virt_vect = APIX_VIRTVECTOR(cpuid, vecp->v_vector);
1041 1046 }
1042 1047
1043 1048 APIX_ENTER_CPU_LOCK(cpuid);
1044 1049 apix_insert_av(vecp, intr_id, xxintr, arg1, arg2, ticksp, ipl, dip);
1045 1050 APIX_LEAVE_CPU_LOCK(cpuid);
1046 1051
1047 1052 (void) apix_addspl(virt_vect, ipl, 0, 0);
1048 1053
1049 1054 lock_clear(&apix_lock);
1050 1055
1051 1056 return (1);
1052 1057 }
1053 1058
1054 1059 /*
1055 1060 * Remove avintr
1056 1061 *
1057 1062 * For fixed, if it's the last one of shared interrupts, free the vector.
1058 1063 * For msi/x, only disable the interrupt but not free the vector, which
1059 1064 * is freed by PSM_XXX_FREE_XXX.
1060 1065 */
1061 1066 void
1062 1067 apix_rem_avintr(void *intr_id, int ipl, avfunc xxintr, int virt_vect)
1063 1068 {
1064 1069 avfunc f;
1065 1070 apix_vector_t *vecp;
1066 1071 struct autovec *avp;
1067 1072 processorid_t cpuid;
1068 1073
1069 1074 if ((f = xxintr) == NULL)
1070 1075 return;
1071 1076
1072 1077 lock_set(&apix_lock);
1073 1078
1074 1079 if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */
1075 1080 vecp = apix_intx_get_vector(virt_vect);
1076 1081 virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1077 1082 } else /* got virtual vector */
1078 1083 vecp = xv_vector(APIX_VIRTVEC_CPU(virt_vect),
1079 1084 APIX_VIRTVEC_VECTOR(virt_vect));
1080 1085
1081 1086 if (vecp == NULL) {
1082 1087 lock_clear(&apix_lock);
1083 1088 cmn_err(CE_CONT, "Invalid interrupt 0x%x,0x%x to remove",
1084 1089 APIX_VIRTVEC_CPU(virt_vect),
1085 1090 APIX_VIRTVEC_VECTOR(virt_vect));
1086 1091 return;
1087 1092 }
1088 1093
1089 1094 if (vecp->v_state <= APIX_STATE_OBSOLETED ||
1090 1095 ((avp = apix_find_av(vecp, intr_id, f)) == NULL)) {
1091 1096 /*
1092 1097 * It's possible that the interrupt is rebound to a
1093 1098 * different cpu before rem_avintr() is called. Search
1094 1099 * through all vectors once it happens.
1095 1100 */
1096 1101 if ((vecp = apix_find_vector_by_avintr(intr_id, f))
1097 1102 == NULL) {
1098 1103 lock_clear(&apix_lock);
1099 1104 cmn_err(CE_CONT, "Unknown interrupt 0x%x,0x%x "
1100 1105 "for %p to remove", APIX_VIRTVEC_CPU(virt_vect),
1101 1106 APIX_VIRTVEC_VECTOR(virt_vect), intr_id);
1102 1107 return;
1103 1108 }
1104 1109 virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1105 1110 avp = apix_find_av(vecp, intr_id, f);
1106 1111 }
1107 1112 cpuid = vecp->v_cpuid;
1108 1113
1109 1114 /* disable interrupt */
1110 1115 (void) apix_delspl(virt_vect, ipl, 0, 0);
1111 1116
1112 1117 /* remove ISR entry */
1113 1118 APIX_ENTER_CPU_LOCK(cpuid);
1114 1119 apix_remove_av(vecp, avp);
1115 1120 APIX_LEAVE_CPU_LOCK(cpuid);
1116 1121
1117 1122 lock_clear(&apix_lock);
1118 1123 }
1119 1124
1120 1125 /*
1121 1126 * Device to vector mapping table
1122 1127 */
1123 1128
1124 1129 static void
1125 1130 apix_clear_dev_map(dev_info_t *dip, int inum, int type)
1126 1131 {
1127 1132 char *name;
1128 1133 major_t major;
1129 1134 apix_dev_vector_t *dvp, *prev = NULL;
1130 1135 int found = 0;
1131 1136
1132 1137 name = ddi_get_name(dip);
1133 1138 major = ddi_name_to_major(name);
1134 1139
1135 1140 mutex_enter(&apix_mutex);
1136 1141
1137 1142 for (dvp = apix_dev_vector[major]; dvp != NULL;
1138 1143 prev = dvp, dvp = dvp->dv_next) {
1139 1144 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1140 1145 dvp->dv_type == type) {
1141 1146 found++;
1142 1147 break;
1143 1148 }
1144 1149 }
1145 1150
1146 1151 if (!found) {
1147 1152 mutex_exit(&apix_mutex);
1148 1153 return;
1149 1154 }
1150 1155
1151 1156 if (prev != NULL)
1152 1157 prev->dv_next = dvp->dv_next;
1153 1158
1154 1159 if (apix_dev_vector[major] == dvp)
1155 1160 apix_dev_vector[major] = dvp->dv_next;
1156 1161
1157 1162 dvp->dv_vector->v_devp = NULL;
1158 1163
1159 1164 mutex_exit(&apix_mutex);
1160 1165
1161 1166 kmem_free(dvp, sizeof (apix_dev_vector_t));
1162 1167 }
1163 1168
1164 1169 void
1165 1170 apix_set_dev_map(apix_vector_t *vecp, dev_info_t *dip, int inum)
1166 1171 {
1167 1172 apix_dev_vector_t *dvp;
1168 1173 char *name;
1169 1174 major_t major;
1170 1175 uint32_t found = 0;
1171 1176
1172 1177 ASSERT(dip != NULL);
1173 1178 name = ddi_get_name(dip);
1174 1179 major = ddi_name_to_major(name);
1175 1180
1176 1181 mutex_enter(&apix_mutex);
1177 1182
1178 1183 for (dvp = apix_dev_vector[major]; dvp != NULL;
1179 1184 dvp = dvp->dv_next) {
1180 1185 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1181 1186 dvp->dv_type == vecp->v_type) {
1182 1187 found++;
1183 1188 break;
1184 1189 }
1185 1190 }
1186 1191
1187 1192 if (found == 0) { /* not found */
1188 1193 dvp = kmem_zalloc(sizeof (apix_dev_vector_t), KM_SLEEP);
1189 1194 dvp->dv_dip = dip;
1190 1195 dvp->dv_inum = inum;
1191 1196 dvp->dv_type = vecp->v_type;
1192 1197
1193 1198 dvp->dv_next = apix_dev_vector[major];
1194 1199 apix_dev_vector[major] = dvp;
1195 1200 }
1196 1201 dvp->dv_vector = vecp;
1197 1202 vecp->v_devp = dvp;
1198 1203
1199 1204 mutex_exit(&apix_mutex);
1200 1205
1201 1206 DDI_INTR_IMPLDBG((CE_CONT, "apix_set_dev_map: dip=0x%p "
1202 1207 "inum=0x%x vector=0x%x/0x%x\n",
1203 1208 (void *)dip, inum, vecp->v_cpuid, vecp->v_vector));
1204 1209 }
1205 1210
1206 1211 apix_vector_t *
1207 1212 apix_get_dev_map(dev_info_t *dip, int inum, int type)
1208 1213 {
1209 1214 char *name;
1210 1215 major_t major;
1211 1216 apix_dev_vector_t *dvp;
1212 1217 apix_vector_t *vecp;
1213 1218
1214 1219 name = ddi_get_name(dip);
1215 1220 if ((major = ddi_name_to_major(name)) == DDI_MAJOR_T_NONE)
1216 1221 return (NULL);
1217 1222
1218 1223 mutex_enter(&apix_mutex);
1219 1224 for (dvp = apix_dev_vector[major]; dvp != NULL;
1220 1225 dvp = dvp->dv_next) {
1221 1226 if (dvp->dv_dip == dip && dvp->dv_inum == inum &&
1222 1227 dvp->dv_type == type) {
1223 1228 vecp = dvp->dv_vector;
1224 1229 mutex_exit(&apix_mutex);
1225 1230 return (vecp);
1226 1231 }
1227 1232 }
1228 1233 mutex_exit(&apix_mutex);
1229 1234
1230 1235 return (NULL);
1231 1236 }
1232 1237
1233 1238 /*
1234 1239 * Get minimum inum for specified device, used for MSI
1235 1240 */
1236 1241 int
1237 1242 apix_get_min_dev_inum(dev_info_t *dip, int type)
1238 1243 {
1239 1244 char *name;
1240 1245 major_t major;
1241 1246 apix_dev_vector_t *dvp;
1242 1247 int inum = -1;
1243 1248
1244 1249 name = ddi_get_name(dip);
1245 1250 major = ddi_name_to_major(name);
1246 1251
1247 1252 mutex_enter(&apix_mutex);
1248 1253 for (dvp = apix_dev_vector[major]; dvp != NULL;
1249 1254 dvp = dvp->dv_next) {
1250 1255 if (dvp->dv_dip == dip && dvp->dv_type == type) {
1251 1256 if (inum == -1)
1252 1257 inum = dvp->dv_inum;
1253 1258 else
1254 1259 inum = (dvp->dv_inum < inum) ?
1255 1260 dvp->dv_inum : inum;
1256 1261 }
1257 1262 }
1258 1263 mutex_exit(&apix_mutex);
1259 1264
1260 1265 return (inum);
1261 1266 }
1262 1267
1263 1268 int
1264 1269 apix_get_max_dev_inum(dev_info_t *dip, int type)
1265 1270 {
1266 1271 char *name;
1267 1272 major_t major;
1268 1273 apix_dev_vector_t *dvp;
1269 1274 int inum = -1;
1270 1275
1271 1276 name = ddi_get_name(dip);
1272 1277 major = ddi_name_to_major(name);
1273 1278
1274 1279 mutex_enter(&apix_mutex);
1275 1280 for (dvp = apix_dev_vector[major]; dvp != NULL;
1276 1281 dvp = dvp->dv_next) {
1277 1282 if (dvp->dv_dip == dip && dvp->dv_type == type) {
1278 1283 if (inum == -1)
1279 1284 inum = dvp->dv_inum;
1280 1285 else
1281 1286 inum = (dvp->dv_inum > inum) ?
1282 1287 dvp->dv_inum : inum;
1283 1288 }
1284 1289 }
1285 1290 mutex_exit(&apix_mutex);
1286 1291
1287 1292 return (inum);
1288 1293 }
1289 1294
1290 1295 /*
1291 1296 * Major to cpu binding, for INTR_ROUND_ROBIN_WITH_AFFINITY cpu
1292 1297 * binding policy
1293 1298 */
1294 1299
1295 1300 static uint32_t
1296 1301 apix_get_dev_binding(dev_info_t *dip)
1297 1302 {
1298 1303 major_t major;
1299 1304 char *name;
1300 1305 uint32_t cpu = IRQ_UNINIT;
1301 1306
1302 1307 name = ddi_get_name(dip);
1303 1308 major = ddi_name_to_major(name);
1304 1309 if (major < devcnt) {
1305 1310 mutex_enter(&apix_mutex);
1306 1311 cpu = apix_major_to_cpu[major];
1307 1312 mutex_exit(&apix_mutex);
1308 1313 }
1309 1314
1310 1315 return (cpu);
1311 1316 }
1312 1317
1313 1318 static void
1314 1319 apix_set_dev_binding(dev_info_t *dip, uint32_t cpu)
1315 1320 {
1316 1321 major_t major;
1317 1322 char *name;
1318 1323
1319 1324 /* setup major to cpu mapping */
1320 1325 name = ddi_get_name(dip);
1321 1326 major = ddi_name_to_major(name);
1322 1327 if (apix_major_to_cpu[major] == IRQ_UNINIT) {
1323 1328 mutex_enter(&apix_mutex);
1324 1329 apix_major_to_cpu[major] = cpu;
1325 1330 mutex_exit(&apix_mutex);
1326 1331 }
1327 1332 }
1328 1333
1329 1334 /*
1330 1335 * return the cpu to which this intr should be bound.
1331 1336 * Check properties or any other mechanism to see if user wants it
1332 1337 * bound to a specific CPU. If so, return the cpu id with high bit set.
1333 1338 * If not, use the policy to choose a cpu and return the id.
1334 1339 */
1335 1340 uint32_t
1336 1341 apix_bind_cpu(dev_info_t *dip)
1337 1342 {
1338 1343 int instance, instno, prop_len, bind_cpu, count;
1339 1344 uint_t i, rc;
1340 1345 major_t major;
1341 1346 char *name, *drv_name, *prop_val, *cptr;
1342 1347 char prop_name[32];
1343 1348
1344 1349 lock_set(&apix_lock);
1345 1350
1346 1351 if (apic_intr_policy == INTR_LOWEST_PRIORITY) {
1347 1352 cmn_err(CE_WARN, "apix: unsupported interrupt binding policy "
1348 1353 "LOWEST PRIORITY, use ROUND ROBIN instead");
1349 1354 apic_intr_policy = INTR_ROUND_ROBIN;
1350 1355 }
1351 1356
1352 1357 if (apic_nproc == 1) {
1353 1358 lock_clear(&apix_lock);
1354 1359 return (0);
1355 1360 }
1356 1361
1357 1362 drv_name = NULL;
1358 1363 rc = DDI_PROP_NOT_FOUND;
1359 1364 major = (major_t)-1;
1360 1365 if (dip != NULL) {
1361 1366 name = ddi_get_name(dip);
1362 1367 major = ddi_name_to_major(name);
1363 1368 drv_name = ddi_major_to_name(major);
1364 1369 instance = ddi_get_instance(dip);
1365 1370 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
1366 1371 bind_cpu = apix_get_dev_binding(dip);
1367 1372 if (bind_cpu != IRQ_UNINIT) {
1368 1373 lock_clear(&apix_lock);
1369 1374 return (bind_cpu);
1370 1375 }
1371 1376 }
1372 1377 /*
1373 1378 * search for "drvname"_intpt_bind_cpus property first, the
1374 1379 * syntax of the property should be "a[,b,c,...]" where
1375 1380 * instance 0 binds to cpu a, instance 1 binds to cpu b,
1376 1381 * instance 3 binds to cpu c...
1377 1382 * ddi_getlongprop() will search /option first, then /
1378 1383 * if "drvname"_intpt_bind_cpus doesn't exist, then find
1379 1384 * intpt_bind_cpus property. The syntax is the same, and
1380 1385 * it applies to all the devices if its "drvname" specific
1381 1386 * property doesn't exist
1382 1387 */
1383 1388 (void) strcpy(prop_name, drv_name);
1384 1389 (void) strcat(prop_name, "_intpt_bind_cpus");
1385 1390 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
1386 1391 (caddr_t)&prop_val, &prop_len);
1387 1392 if (rc != DDI_PROP_SUCCESS) {
1388 1393 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
1389 1394 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
1390 1395 }
1391 1396 }
1392 1397 if (rc == DDI_PROP_SUCCESS) {
1393 1398 for (i = count = 0; i < (prop_len - 1); i++)
1394 1399 if (prop_val[i] == ',')
1395 1400 count++;
1396 1401 if (prop_val[i-1] != ',')
1397 1402 count++;
1398 1403 /*
1399 1404 * if somehow the binding instances defined in the
1400 1405 * property are not enough for this instno., then
1401 1406 * reuse the pattern for the next instance until
1402 1407 * it reaches the requested instno
1403 1408 */
1404 1409 instno = instance % count;
1405 1410 i = 0;
1406 1411 cptr = prop_val;
1407 1412 while (i < instno)
1408 1413 if (*cptr++ == ',')
1409 1414 i++;
1410 1415 bind_cpu = stoi(&cptr);
1411 1416 kmem_free(prop_val, prop_len);
1412 1417 /* if specific cpu is bogus, then default to cpu 0 */
1413 1418 if (bind_cpu >= apic_nproc) {
1414 1419 cmn_err(CE_WARN, "apix: %s=%s: CPU %d not present",
1415 1420 prop_name, prop_val, bind_cpu);
1416 1421 bind_cpu = 0;
1417 1422 } else {
1418 1423 /* indicate that we are bound at user request */
1419 1424 bind_cpu |= IRQ_USER_BOUND;
1420 1425 }
1421 1426 /*
1422 1427 * no need to check apic_cpus[].aci_status, if specific cpu is
1423 1428 * not up, then post_cpu_start will handle it.
1424 1429 */
1425 1430 } else {
1426 1431 bind_cpu = apic_get_next_bind_cpu();
1427 1432 }
1428 1433
1429 1434 lock_clear(&apix_lock);
1430 1435
1431 1436 return ((uint32_t)bind_cpu);
1432 1437 }
1433 1438
1434 1439 static boolean_t
1435 1440 apix_is_cpu_enabled(processorid_t cpuid)
1436 1441 {
1437 1442 apic_cpus_info_t *cpu_infop;
1438 1443
1439 1444 cpu_infop = &apic_cpus[cpuid];
1440 1445
1441 1446 if ((cpu_infop->aci_status & APIC_CPU_INTR_ENABLE) == 0)
1442 1447 return (B_FALSE);
1443 1448
1444 1449 return (B_TRUE);
1445 1450 }
1446 1451
1447 1452 /*
1448 1453 * Must be called with apix_lock held. This function can be
1449 1454 * called from above lock level by apix_intr_redistribute().
1450 1455 *
1451 1456 * Arguments:
1452 1457 * vecp : Vector to be rebound
1453 1458 * tocpu : Target cpu. IRQ_UNINIT means target is vecp->v_cpuid.
1454 1459 * count : Number of continuous vectors
1455 1460 *
1456 1461 * Return new vector being bound to
1457 1462 */
1458 1463 apix_vector_t *
1459 1464 apix_rebind(apix_vector_t *vecp, processorid_t newcpu, int count)
1460 1465 {
|
↓ open down ↓ |
645 lines elided |
↑ open up ↑ |
1461 1466 apix_vector_t *newp, *oldp;
1462 1467 processorid_t oldcpu = vecp->v_cpuid;
1463 1468 uchar_t newvec, oldvec = vecp->v_vector;
1464 1469 int i;
1465 1470
1466 1471 ASSERT(LOCK_HELD(&apix_lock) && count > 0);
1467 1472
1468 1473 if (!apix_is_cpu_enabled(newcpu))
1469 1474 return (NULL);
1470 1475
1471 - if (vecp->v_cpuid == newcpu) /* rebind to the same cpu */
1476 + if (vecp->v_cpuid == newcpu) /* rebind to the same cpu */
1472 1477 return (vecp);
1473 1478
1474 1479 APIX_ENTER_CPU_LOCK(oldcpu);
1475 1480 APIX_ENTER_CPU_LOCK(newcpu);
1476 1481
1477 1482 /* allocate vector */
1478 1483 if (count == 1)
1479 1484 newp = apix_alloc_vector_oncpu(newcpu, NULL, 0, vecp->v_type);
1480 1485 else {
1481 1486 ASSERT(vecp->v_type == APIX_TYPE_MSI);
1482 1487 newp = apix_alloc_nvectors_oncpu(newcpu, NULL, 0, count,
1483 1488 vecp->v_type);
1484 1489 }
1485 1490 if (newp == NULL) {
1486 1491 APIX_LEAVE_CPU_LOCK(newcpu);
1487 1492 APIX_LEAVE_CPU_LOCK(oldcpu);
1488 1493 return (NULL);
1489 1494 }
1490 1495
1491 1496 newvec = newp->v_vector;
1492 1497 apix_dup_vectors(vecp, newp, count);
1493 1498
1494 1499 APIX_LEAVE_CPU_LOCK(newcpu);
1495 1500 APIX_LEAVE_CPU_LOCK(oldcpu);
1496 1501
1497 1502 if (!DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) {
1498 1503 ASSERT(count == 1);
1499 1504 if (apix_intx_rebind(vecp->v_inum, newcpu, newvec) != 0) {
1500 1505 struct autovec *avp;
1501 1506 int inum;
1502 1507
1503 1508 /* undo duplication */
1504 1509 APIX_ENTER_CPU_LOCK(oldcpu);
1505 1510 APIX_ENTER_CPU_LOCK(newcpu);
1506 1511 for (avp = newp->v_autovect; avp != NULL;
1507 1512 avp = avp->av_link) {
1508 1513 if (avp->av_dip != NULL) {
1509 1514 inum = GET_INTR_INUM(avp->av_intr_id);
1510 1515 apix_set_dev_map(vecp, avp->av_dip,
1511 1516 inum);
1512 1517 }
1513 1518 apix_remove_av(newp, avp);
1514 1519 }
1515 1520 apix_cleanup_vector(newp);
1516 1521 APIX_LEAVE_CPU_LOCK(newcpu);
1517 1522 APIX_LEAVE_CPU_LOCK(oldcpu);
1518 1523 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed "
1519 1524 "interrupt 0x%x to cpu %d failed\n",
1520 1525 vecp->v_inum, newcpu));
1521 1526 return (NULL);
1522 1527 }
1523 1528
1524 1529 APIX_ENTER_CPU_LOCK(oldcpu);
1525 1530 (void) apix_obsolete_vector(vecp);
1526 1531 APIX_LEAVE_CPU_LOCK(oldcpu);
1527 1532 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed interrupt"
1528 1533 " 0x%x/0x%x to 0x%x/0x%x\n",
1529 1534 oldcpu, oldvec, newcpu, newvec));
1530 1535 return (newp);
1531 1536 }
1532 1537
1533 1538 for (i = 0; i < count; i++) {
1534 1539 oldp = xv_vector(oldcpu, oldvec + i);
1535 1540 newp = xv_vector(newcpu, newvec + i);
1536 1541
1537 1542 if (newp->v_share > 0) {
1538 1543 APIX_SET_REBIND_INFO(oldp, newp);
1539 1544
1540 1545 apix_enable_vector(newp);
1541 1546
1542 1547 APIX_CLR_REBIND_INFO();
1543 1548 }
1544 1549
1545 1550 APIX_ENTER_CPU_LOCK(oldcpu);
1546 1551 (void) apix_obsolete_vector(oldp);
1547 1552 APIX_LEAVE_CPU_LOCK(oldcpu);
1548 1553 }
1549 1554 APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind vector 0x%x/0x%x "
1550 1555 "to 0x%x/0x%x, count=%d\n",
1551 1556 oldcpu, oldvec, newcpu, newvec, count));
1552 1557
1553 1558 return (xv_vector(newcpu, newvec));
1554 1559 }
1555 1560
1556 1561 /*
1557 1562 * Senarios include:
1558 1563 * a. add_avintr() is called before irqp initialized (legacy)
1559 1564 * b. irqp is initialized, vector is not allocated (fixed interrupts)
1560 1565 * c. irqp is initialized, vector is allocated (shared interrupts)
1561 1566 */
1562 1567 apix_vector_t *
1563 1568 apix_alloc_intx(dev_info_t *dip, int inum, int irqno)
1564 1569 {
1565 1570 apic_irq_t *irqp;
1566 1571 apix_vector_t *vecp;
1567 1572
1568 1573 /*
1569 1574 * Allocate IRQ. Caller is later responsible for the
1570 1575 * initialization
1571 1576 */
1572 1577 mutex_enter(&airq_mutex);
1573 1578 if ((irqp = apic_irq_table[irqno]) == NULL) {
1574 1579 /* allocate irq */
1575 1580 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1576 1581 irqp->airq_mps_intr_index = FREE_INDEX;
1577 1582 apic_irq_table[irqno] = irqp;
1578 1583 }
1579 1584 if (irqp->airq_mps_intr_index == FREE_INDEX) {
1580 1585 irqp->airq_mps_intr_index = DEFAULT_INDEX;
1581 1586 irqp->airq_cpu = IRQ_UNINIT;
1582 1587 irqp->airq_origirq = (uchar_t)irqno;
1583 1588 }
1584 1589
1585 1590 mutex_exit(&airq_mutex);
1586 1591
1587 1592 /*
1588 1593 * allocate vector
1589 1594 */
1590 1595 if (irqp->airq_cpu == IRQ_UNINIT) {
1591 1596 uint32_t bindcpu, cpuid;
1592 1597
1593 1598 /* select cpu by system policy */
1594 1599 bindcpu = apix_bind_cpu(dip);
1595 1600 cpuid = bindcpu & ~IRQ_USER_BOUND;
1596 1601
1597 1602 /* allocate vector */
1598 1603 APIX_ENTER_CPU_LOCK(cpuid);
1599 1604
1600 1605 if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum,
1601 1606 APIX_TYPE_FIXED)) == NULL) {
1602 1607 cmn_err(CE_WARN, "No interrupt vector for irq %x",
1603 1608 irqno);
1604 1609 APIX_LEAVE_CPU_LOCK(cpuid);
1605 1610 return (NULL);
1606 1611 }
1607 1612 vecp->v_inum = irqno;
1608 1613 vecp->v_flags |= APIX_VECT_MASKABLE;
1609 1614
1610 1615 apix_intx_set_vector(irqno, vecp->v_cpuid, vecp->v_vector);
1611 1616
1612 1617 APIX_LEAVE_CPU_LOCK(cpuid);
1613 1618 } else {
1614 1619 vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1615 1620 ASSERT(!IS_VECT_FREE(vecp));
1616 1621
1617 1622 if (dip != NULL)
1618 1623 apix_set_dev_map(vecp, dip, inum);
1619 1624 }
1620 1625
1621 1626 if ((dip != NULL) &&
1622 1627 (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1623 1628 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1624 1629 apix_set_dev_binding(dip, vecp->v_cpuid);
1625 1630
1626 1631 apix_dprint_vector(vecp, dip, 1);
1627 1632
1628 1633 return (vecp);
1629 1634 }
1630 1635
1631 1636 int
1632 1637 apix_alloc_msi(dev_info_t *dip, int inum, int count, int behavior)
1633 1638 {
1634 1639 int i, cap_ptr, rcount = count;
1635 1640 apix_vector_t *vecp;
1636 1641 processorid_t bindcpu, cpuid;
1637 1642 ushort_t msi_ctrl;
1638 1643 ddi_acc_handle_t handle;
1639 1644
1640 1645 DDI_INTR_IMPLDBG((CE_CONT, "apix_alloc_msi_vectors: dip=0x%p "
1641 1646 "inum=0x%x count=0x%x behavior=%d\n",
1642 1647 (void *)dip, inum, count, behavior));
1643 1648
1644 1649 if (count > 1) {
1645 1650 if (behavior == DDI_INTR_ALLOC_STRICT &&
1646 1651 apic_multi_msi_enable == 0)
1647 1652 return (0);
1648 1653 if (apic_multi_msi_enable == 0)
1649 1654 count = 1;
1650 1655 }
1651 1656
1652 1657 /* Check whether it supports per-vector masking */
1653 1658 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1654 1659 handle = i_ddi_get_pci_config_handle(dip);
1655 1660 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1656 1661
1657 1662 /* bind to cpu */
1658 1663 bindcpu = apix_bind_cpu(dip);
1659 1664 cpuid = bindcpu & ~IRQ_USER_BOUND;
1660 1665
1661 1666 /* if not ISP2, then round it down */
1662 1667 if (!ISP2(rcount))
1663 1668 rcount = 1 << (highbit(rcount) - 1);
1664 1669
1665 1670 APIX_ENTER_CPU_LOCK(cpuid);
1666 1671 for (vecp = NULL; rcount > 0; rcount >>= 1) {
1667 1672 vecp = apix_alloc_nvectors_oncpu(bindcpu, dip, inum, rcount,
1668 1673 APIX_TYPE_MSI);
1669 1674 if (vecp != NULL || behavior == DDI_INTR_ALLOC_STRICT)
1670 1675 break;
1671 1676 }
1672 1677 for (i = 0; vecp && i < rcount; i++)
1673 1678 xv_vector(vecp->v_cpuid, vecp->v_vector + i)->v_flags |=
1674 1679 (msi_ctrl & PCI_MSI_PVM_MASK) ? APIX_VECT_MASKABLE : 0;
1675 1680 APIX_LEAVE_CPU_LOCK(cpuid);
1676 1681 if (vecp == NULL) {
1677 1682 APIC_VERBOSE(INTR, (CE_CONT,
1678 1683 "apix_alloc_msi: no %d cont vectors found on cpu 0x%x\n",
1679 1684 count, bindcpu));
1680 1685 return (0);
1681 1686 }
1682 1687
1683 1688 /* major to cpu binding */
1684 1689 if ((apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1685 1690 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1686 1691 apix_set_dev_binding(dip, vecp->v_cpuid);
1687 1692
1688 1693 apix_dprint_vector(vecp, dip, rcount);
1689 1694
1690 1695 return (rcount);
1691 1696 }
1692 1697
1693 1698 int
1694 1699 apix_alloc_msix(dev_info_t *dip, int inum, int count, int behavior)
1695 1700 {
1696 1701 apix_vector_t *vecp;
1697 1702 processorid_t bindcpu, cpuid;
1698 1703 int i;
1699 1704
1700 1705 for (i = 0; i < count; i++) {
1701 1706 /* select cpu by system policy */
1702 1707 bindcpu = apix_bind_cpu(dip);
1703 1708 cpuid = bindcpu & ~IRQ_USER_BOUND;
1704 1709
1705 1710 /* allocate vector */
1706 1711 APIX_ENTER_CPU_LOCK(cpuid);
1707 1712 if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum + i,
1708 1713 APIX_TYPE_MSIX)) == NULL) {
1709 1714 APIX_LEAVE_CPU_LOCK(cpuid);
1710 1715 APIC_VERBOSE(INTR, (CE_CONT, "apix_alloc_msix: "
1711 1716 "allocate msix for device dip=%p, inum=%d on"
1712 1717 " cpu %d failed", (void *)dip, inum + i, bindcpu));
1713 1718 break;
1714 1719 }
1715 1720 vecp->v_flags |= APIX_VECT_MASKABLE;
1716 1721 APIX_LEAVE_CPU_LOCK(cpuid);
1717 1722
1718 1723 /* major to cpu mapping */
1719 1724 if ((i == 0) &&
1720 1725 (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) &&
1721 1726 ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0))
1722 1727 apix_set_dev_binding(dip, vecp->v_cpuid);
1723 1728
1724 1729 apix_dprint_vector(vecp, dip, 1);
1725 1730 }
1726 1731
1727 1732 if (i < count && behavior == DDI_INTR_ALLOC_STRICT) {
1728 1733 APIC_VERBOSE(INTR, (CE_WARN, "apix_alloc_msix: "
1729 1734 "strictly allocate %d vectors failed, got %d\n",
1730 1735 count, i));
1731 1736 apix_free_vectors(dip, inum, i, APIX_TYPE_MSIX);
1732 1737 i = 0;
1733 1738 }
1734 1739
1735 1740 return (i);
1736 1741 }
1737 1742
1738 1743 /*
1739 1744 * A rollback free for vectors allocated by apix_alloc_xxx().
1740 1745 */
1741 1746 void
1742 1747 apix_free_vectors(dev_info_t *dip, int inum, int count, int type)
1743 1748 {
1744 1749 int i, cpuid;
1745 1750 apix_vector_t *vecp;
1746 1751
1747 1752 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: dip: %p inum: %x "
1748 1753 "count: %x type: %x\n",
1749 1754 (void *)dip, inum, count, type));
1750 1755
1751 1756 lock_set(&apix_lock);
1752 1757
1753 1758 for (i = 0; i < count; i++, inum++) {
1754 1759 if ((vecp = apix_get_dev_map(dip, inum, type)) == NULL) {
1755 1760 lock_clear(&apix_lock);
1756 1761 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: "
1757 1762 "dip=0x%p inum=0x%x type=0x%x apix_find_intr() "
1758 1763 "failed\n", (void *)dip, inum, type));
1759 1764 continue;
1760 1765 }
1761 1766
1762 1767 APIX_ENTER_CPU_LOCK(vecp->v_cpuid);
1763 1768 cpuid = vecp->v_cpuid;
1764 1769
1765 1770 DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: "
1766 1771 "dip=0x%p inum=0x%x type=0x%x vector 0x%x (share %d)\n",
1767 1772 (void *)dip, inum, type, vecp->v_vector, vecp->v_share));
1768 1773
1769 1774 /* tear down device interrupt to vector mapping */
1770 1775 apix_clear_dev_map(dip, inum, type);
1771 1776
1772 1777 if (vecp->v_type == APIX_TYPE_FIXED) {
1773 1778 if (vecp->v_share > 0) { /* share IRQ line */
1774 1779 APIX_LEAVE_CPU_LOCK(cpuid);
1775 1780 continue;
1776 1781 }
1777 1782
1778 1783 /* Free apic_irq_table entry */
1779 1784 apix_intx_free(vecp->v_inum);
1780 1785 }
1781 1786
1782 1787 /* free vector */
1783 1788 apix_cleanup_vector(vecp);
1784 1789
1785 1790 APIX_LEAVE_CPU_LOCK(cpuid);
1786 1791 }
1787 1792
1788 1793 lock_clear(&apix_lock);
1789 1794 }
1790 1795
1791 1796 /*
1792 1797 * Must be called with apix_lock held
1793 1798 */
1794 1799 apix_vector_t *
1795 1800 apix_setup_io_intr(apix_vector_t *vecp)
1796 1801 {
1797 1802 processorid_t bindcpu;
1798 1803 int ret;
1799 1804
1800 1805 ASSERT(LOCK_HELD(&apix_lock));
1801 1806
1802 1807 /*
1803 1808 * Interrupts are enabled on the CPU, programme IOAPIC RDT
1804 1809 * entry or MSI/X address/data to enable the interrupt.
1805 1810 */
1806 1811 if (apix_is_cpu_enabled(vecp->v_cpuid)) {
1807 1812 apix_enable_vector(vecp);
1808 1813 return (vecp);
1809 1814 }
1810 1815
1811 1816 /*
1812 1817 * CPU is not up or interrupts are disabled. Fall back to the
1813 1818 * first avialable CPU.
1814 1819 */
1815 1820 bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
1816 1821
1817 1822 if (vecp->v_type == APIX_TYPE_MSI)
1818 1823 return (apix_grp_set_cpu(vecp, bindcpu, &ret));
1819 1824
1820 1825 return (apix_set_cpu(vecp, bindcpu, &ret));
1821 1826 }
1822 1827
1823 1828 /*
1824 1829 * For interrupts which call add_avintr() before apic is initialized.
1825 1830 * ioapix_setup_intr() will
1826 1831 * - allocate vector
1827 1832 * - copy over ISR
1828 1833 */
1829 1834 static void
1830 1835 ioapix_setup_intr(int irqno, iflag_t *flagp)
1831 1836 {
1832 1837 extern struct av_head autovect[];
1833 1838 apix_vector_t *vecp;
1834 1839 apic_irq_t *irqp;
1835 1840 uchar_t ioapicindex, ipin;
1836 1841 ulong_t iflag;
1837 1842 struct autovec *avp;
1838 1843
1839 1844 ioapicindex = acpi_find_ioapic(irqno);
1840 1845 ASSERT(ioapicindex != 0xFF);
1841 1846 ipin = irqno - apic_io_vectbase[ioapicindex];
1842 1847
1843 1848 mutex_enter(&airq_mutex);
1844 1849 irqp = apic_irq_table[irqno];
1845 1850
1846 1851 /*
1847 1852 * The irq table entry shouldn't exist unless the interrupts are shared.
1848 1853 * In that case, make sure it matches what we would initialize it to.
1849 1854 */
1850 1855 if (irqp != NULL) {
1851 1856 ASSERT(irqp->airq_mps_intr_index == ACPI_INDEX);
1852 1857 ASSERT(irqp->airq_intin_no == ipin &&
1853 1858 irqp->airq_ioapicindex == ioapicindex);
1854 1859 vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1855 1860 ASSERT(!IS_VECT_FREE(vecp));
1856 1861 mutex_exit(&airq_mutex);
1857 1862 } else {
1858 1863 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
1859 1864
1860 1865 irqp->airq_cpu = IRQ_UNINIT;
1861 1866 irqp->airq_origirq = (uchar_t)irqno;
1862 1867 irqp->airq_mps_intr_index = ACPI_INDEX;
1863 1868 irqp->airq_ioapicindex = ioapicindex;
1864 1869 irqp->airq_intin_no = ipin;
1865 1870 irqp->airq_iflag = *flagp;
1866 1871 irqp->airq_share++;
1867 1872
1868 1873 apic_irq_table[irqno] = irqp;
1869 1874 mutex_exit(&airq_mutex);
1870 1875
1871 1876 vecp = apix_alloc_intx(NULL, 0, irqno);
1872 1877 }
1873 1878
1874 1879 /* copy over autovect */
1875 1880 for (avp = autovect[irqno].avh_link; avp; avp = avp->av_link)
1876 1881 apix_insert_av(vecp, avp->av_intr_id, avp->av_vector,
1877 1882 avp->av_intarg1, avp->av_intarg2, avp->av_ticksp,
1878 1883 avp->av_prilevel, avp->av_dip);
1879 1884
1880 1885 /* Program I/O APIC */
1881 1886 iflag = intr_clear();
1882 1887 lock_set(&apix_lock);
1883 1888
1884 1889 (void) apix_setup_io_intr(vecp);
1885 1890
1886 1891 lock_clear(&apix_lock);
1887 1892 intr_restore(iflag);
1888 1893
1889 1894 APIC_VERBOSE_IOAPIC((CE_CONT, "apix: setup ioapic, irqno %x "
1890 1895 "(ioapic %x, ipin %x) is bound to cpu %x, vector %x\n",
1891 1896 irqno, ioapicindex, ipin, irqp->airq_cpu, irqp->airq_vector));
1892 1897 }
1893 1898
1894 1899 void
1895 1900 ioapix_init_intr(int mask_apic)
1896 1901 {
1897 1902 int ioapicindex;
1898 1903 int i, j;
1899 1904
1900 1905 /* mask interrupt vectors */
1901 1906 for (j = 0; j < apic_io_max && mask_apic; j++) {
1902 1907 int intin_max;
1903 1908
1904 1909 ioapicindex = j;
1905 1910 /* Bits 23-16 define the maximum redirection entries */
1906 1911 intin_max = (ioapic_read(ioapicindex, APIC_VERS_CMD) >> 16)
1907 1912 & 0xff;
1908 1913 for (i = 0; i <= intin_max; i++)
1909 1914 ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * i,
1910 1915 AV_MASK);
1911 1916 }
1912 1917
1913 1918 /*
1914 1919 * Hack alert: deal with ACPI SCI interrupt chicken/egg here
1915 1920 */
1916 1921 if (apic_sci_vect > 0)
1917 1922 ioapix_setup_intr(apic_sci_vect, &apic_sci_flags);
1918 1923
1919 1924 /*
1920 1925 * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
1921 1926 */
1922 1927 if (apic_hpet_vect > 0)
1923 1928 ioapix_setup_intr(apic_hpet_vect, &apic_hpet_flags);
1924 1929 }
|
↓ open down ↓ |
443 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX