1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/sysmacros.h> 27 #include <sys/stack.h> 28 #include <sys/cpuvar.h> 29 #include <sys/ivintr.h> 30 #include <sys/intreg.h> 31 #include <sys/membar.h> 32 #include <sys/kmem.h> 33 #include <sys/intr.h> 34 #include <sys/sunddi.h> 35 #include <sys/sunndi.h> 36 #include <sys/cmn_err.h> 37 #include <sys/privregs.h> 38 #include <sys/systm.h> 39 #include <sys/archsystm.h> 40 #include <sys/machsystm.h> 41 #include <sys/x_call.h> 42 #include <vm/seg_kp.h> 43 #include <sys/debug.h> 44 #include <sys/cyclic.h> 45 #include <sys/kdi_impl.h> 46 #include <sys/ddi_timer.h> 47 48 #include <sys/cpu_sgnblk_defs.h> 49 50 /* Global locks which protect the interrupt distribution lists */ 51 static kmutex_t intr_dist_lock; 52 static kmutex_t intr_dist_cpu_lock; 53 54 /* Head of the interrupt distribution lists */ 55 static struct intr_dist *intr_dist_head = NULL; 56 static struct intr_dist *intr_dist_whead = NULL; 57 58 static uint64_t siron_inum[DDI_IPL_10]; /* software interrupt numbers */ 59 uint64_t *siron_cpu_inum = NULL; 60 uint64_t siron_poke_cpu_inum; 61 static int siron_cpu_setup(cpu_setup_t, int, void *); 62 extern uint_t softlevel1(); 63 64 static uint64_t siron1_inum; /* backward compatibility */ 65 uint64_t poke_cpu_inum; 66 uint_t poke_cpu_intr(caddr_t arg1, caddr_t arg2); 67 uint_t siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2); 68 69 /* 70 * Variable to enable/disable printing a message when an invalid vecintr 71 * is received. 72 */ 73 uint_t ignore_invalid_vecintr = 0; 74 75 /* 76 * Note:- 77 * siron_pending was originally created to prevent a resource over consumption 78 * bug in setsoftint(exhaustion of interrupt pool free list). 79 * It's original intention is obsolete with the use of iv_pending in 80 * setsoftint. However, siron_pending stayed around, acting as a second 81 * gatekeeper preventing soft interrupts from being queued. In this capacity, 82 * it can lead to hangs on MP systems, where due to global visibility issues 83 * it can end up set while iv_pending is reset, preventing soft interrupts from 84 * ever being processed. In addition to its gatekeeper role, init_intr also 85 * uses it to flag the situation where siron() was called before siron_inum has 86 * been defined. 87 * 88 * siron() does not need an extra gatekeeper; any cpu that wishes should be 89 * allowed to queue a soft interrupt. It is softint()'s job to ensure 90 * correct handling of the queues. Therefore, siron_pending has been 91 * stripped of its gatekeeper task, retaining only its intr_init job, where 92 * it indicates that there is a pending need to call siron(). 93 */ 94 static int siron_pending[DDI_IPL_10]; /* software interrupt pending flags */ 95 static int siron1_pending; /* backward compatibility */ 96 97 int intr_policy = INTR_WEIGHTED_DIST; /* interrupt distribution policy */ 98 int intr_dist_debug = 0; 99 int32_t intr_dist_weight_max = 1; 100 int32_t intr_dist_weight_maxmax = 1000; 101 int intr_dist_weight_maxfactor = 2; 102 #define INTR_DEBUG(args) if (intr_dist_debug) cmn_err args 103 104 /* 105 * intr_init() - Interrupt initialization 106 * Initialize the system's interrupt vector table. 107 */ 108 void 109 intr_init(cpu_t *cp) 110 { 111 int i; 112 extern uint_t softlevel1(); 113 114 init_ivintr(); 115 REGISTER_BBUS_INTR(); 116 117 /* 118 * Register these software interrupts for ddi timer. 119 * Software interrupts up to the level 10 are supported. 120 */ 121 for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) { 122 siron_inum[i-1] = add_softintr(i, (softintrfunc)timer_softintr, 123 (caddr_t)(uintptr_t)(i), SOFTINT_ST); 124 } 125 126 siron1_inum = add_softintr(PIL_1, softlevel1, 0, SOFTINT_ST); 127 poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0, SOFTINT_MT); 128 siron_poke_cpu_inum = add_softintr(PIL_13, 129 siron_poke_cpu_intr, 0, SOFTINT_MT); 130 cp->cpu_m.poke_cpu_outstanding = B_FALSE; 131 132 mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL); 133 mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL); 134 135 /* 136 * A soft interrupt may have been requested prior to the initialization 137 * of soft interrupts. Soft interrupts can't be dispatched until after 138 * init_intr(), so we have to wait until now before we can dispatch the 139 * pending soft interrupt (if any). 140 */ 141 for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) { 142 if (siron_pending[i-1]) { 143 siron_pending[i-1] = 0; 144 sir_on(i); 145 } 146 } 147 if (siron1_pending) { 148 siron1_pending = 0; 149 siron(); 150 } 151 } 152 153 /* 154 * poke_cpu_intr - fall through when poke_cpu calls 155 */ 156 /* ARGSUSED */ 157 uint_t 158 poke_cpu_intr(caddr_t arg1, caddr_t arg2) 159 { 160 CPU->cpu_m.poke_cpu_outstanding = B_FALSE; 161 membar_stld_stst(); 162 return (1); 163 } 164 165 /* 166 * Trigger software interrupts dedicated to ddi timer. 167 */ 168 void 169 sir_on(int level) 170 { 171 ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10); 172 if (siron_inum[level-1]) 173 setsoftint(siron_inum[level-1]); 174 else 175 siron_pending[level-1] = 1; 176 } 177 178 /* 179 * kmdb uses siron (and thus setsoftint) while the world is stopped in order to 180 * inform its driver component that there's work to be done. We need to keep 181 * DTrace from instrumenting kmdb's siron and setsoftint. We duplicate siron, 182 * giving kmdb's version a kdi_ prefix to keep DTrace at bay. The 183 * implementation of setsoftint is complicated enough that we don't want to 184 * duplicate it, but at the same time we don't want to preclude tracing either. 185 * The meat of setsoftint() therefore goes into kdi_setsoftint, with 186 * setsoftint() implemented as a wrapper. This allows tracing, while still 187 * providing a way for kmdb to sneak in unmolested. 188 */ 189 void 190 kdi_siron(void) 191 { 192 if (siron1_inum != 0) 193 kdi_setsoftint(siron1_inum); 194 else 195 siron1_pending = 1; 196 } 197 198 void 199 setsoftint(uint64_t inum) 200 { 201 kdi_setsoftint(inum); 202 } 203 204 /* 205 * Generates softlevel1 interrupt on current CPU if it 206 * is not pending already. 207 */ 208 void 209 siron(void) 210 { 211 uint64_t inum; 212 213 if (siron1_inum != 0) { 214 /* 215 * Once siron_cpu_inum has been allocated, we can 216 * use per-CPU siron inum. 217 */ 218 if (siron_cpu_inum && siron_cpu_inum[CPU->cpu_id] != 0) 219 inum = siron_cpu_inum[CPU->cpu_id]; 220 else 221 inum = siron1_inum; 222 223 setsoftint(inum); 224 } else 225 siron1_pending = 1; 226 } 227 228 229 static void 230 siron_init(void) 231 { 232 /* 233 * We just allocate memory for per-cpu siron right now. Rest of 234 * the work is done when CPU is configured. 235 */ 236 siron_cpu_inum = kmem_zalloc(sizeof (uint64_t) * NCPU, KM_SLEEP); 237 } 238 239 /* 240 * This routine creates per-CPU siron inum for CPUs which are 241 * configured during boot. 242 */ 243 void 244 siron_mp_init() 245 { 246 cpu_t *c; 247 248 /* 249 * Get the memory for per-CPU siron inums 250 */ 251 siron_init(); 252 253 mutex_enter(&cpu_lock); 254 c = cpu_list; 255 do { 256 (void) siron_cpu_setup(CPU_CONFIG, c->cpu_id, NULL); 257 } while ((c = c->cpu_next) != cpu_list); 258 259 register_cpu_setup_func(siron_cpu_setup, NULL); 260 mutex_exit(&cpu_lock); 261 } 262 263 /* 264 * siron_poke_cpu_intr - cross-call handler. 265 */ 266 /* ARGSUSED */ 267 uint_t 268 siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2) 269 { 270 /* generate level1 softint */ 271 siron(); 272 return (1); 273 } 274 275 /* 276 * This routine generates a cross-call on target CPU(s). 277 */ 278 void 279 siron_poke_cpu(cpuset_t poke) 280 { 281 int cpuid = CPU->cpu_id; 282 283 if (CPU_IN_SET(poke, cpuid)) { 284 siron(); 285 CPUSET_DEL(poke, cpuid); 286 if (CPUSET_ISNULL(poke)) 287 return; 288 } 289 290 xt_some(poke, setsoftint_tl1, siron_poke_cpu_inum, 0); 291 } 292 293 /* 294 * This callback function allows us to create per-CPU siron inum. 295 */ 296 /* ARGSUSED */ 297 static int 298 siron_cpu_setup(cpu_setup_t what, int id, void *arg) 299 { 300 cpu_t *cp = cpu[id]; 301 302 ASSERT(MUTEX_HELD(&cpu_lock)); 303 ASSERT(cp != NULL); 304 305 switch (what) { 306 case CPU_CONFIG: 307 siron_cpu_inum[cp->cpu_id] = add_softintr(PIL_1, 308 (softintrfunc)softlevel1, 0, SOFTINT_ST); 309 break; 310 case CPU_UNCONFIG: 311 (void) rem_softintr(siron_cpu_inum[cp->cpu_id]); 312 siron_cpu_inum[cp->cpu_id] = 0; 313 break; 314 default: 315 break; 316 } 317 318 return (0); 319 } 320 321 /* 322 * no_ivintr() 323 * called by setvecint_tl1() through sys_trap() 324 * vector interrupt received but not valid or not 325 * registered in intr_vec_table 326 * considered as a spurious mondo interrupt 327 */ 328 /* ARGSUSED */ 329 void 330 no_ivintr(struct regs *rp, int inum, int pil) 331 { 332 if (!ignore_invalid_vecintr) 333 cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x", 334 inum, pil); 335 336 #ifdef DEBUG_VEC_INTR 337 prom_enter_mon(); 338 #endif /* DEBUG_VEC_INTR */ 339 } 340 341 void 342 intr_dequeue_req(uint_t pil, uint64_t inum) 343 { 344 intr_vec_t *iv, *next, *prev; 345 struct machcpu *mcpu; 346 uint32_t clr; 347 processorid_t cpu_id; 348 extern uint_t getpstate(void); 349 350 ASSERT((getpstate() & PSTATE_IE) == 0); 351 352 mcpu = &CPU->cpu_m; 353 cpu_id = CPU->cpu_id; 354 355 iv = (intr_vec_t *)inum; 356 prev = NULL; 357 next = mcpu->intr_head[pil]; 358 359 /* Find a matching entry in the list */ 360 while (next != NULL) { 361 if (next == iv) 362 break; 363 prev = next; 364 next = IV_GET_PIL_NEXT(next, cpu_id); 365 } 366 367 if (next != NULL) { 368 intr_vec_t *next_iv = IV_GET_PIL_NEXT(next, cpu_id); 369 370 /* Remove entry from list */ 371 if (prev != NULL) 372 IV_SET_PIL_NEXT(prev, cpu_id, next_iv); /* non-head */ 373 else 374 mcpu->intr_head[pil] = next_iv; /* head */ 375 376 if (next_iv == NULL) 377 mcpu->intr_tail[pil] = prev; /* tail */ 378 } 379 380 /* Clear pending interrupts at this level if the list is empty */ 381 if (mcpu->intr_head[pil] == NULL) { 382 clr = 1 << pil; 383 if (pil == PIL_14) 384 clr |= (TICK_INT_MASK | STICK_INT_MASK); 385 wr_clr_softint(clr); 386 } 387 } 388 389 390 /* 391 * Send a directed interrupt of specified interrupt number id to a cpu. 392 */ 393 void 394 send_dirint( 395 int cpuix, /* cpu to be interrupted */ 396 int intr_id) /* interrupt number id */ 397 { 398 xt_one(cpuix, setsoftint_tl1, intr_id, 0); 399 } 400 401 /* 402 * Take the specified CPU out of participation in interrupts. 403 * Called by p_online(2) when a processor is being taken off-line. 404 * This allows interrupt threads being handled on the processor to 405 * complete before the processor is idled. 406 */ 407 int 408 cpu_disable_intr(struct cpu *cp) 409 { 410 ASSERT(MUTEX_HELD(&cpu_lock)); 411 412 /* 413 * Turn off the CPU_ENABLE flag before calling the redistribution 414 * function, since it checks for this in the cpu flags. 415 */ 416 cp->cpu_flags &= ~CPU_ENABLE; 417 418 intr_redist_all_cpus(); 419 420 return (0); 421 } 422 423 /* 424 * Allow the specified CPU to participate in interrupts. 425 * Called by p_online(2) if a processor could not be taken off-line 426 * because of bound threads, in order to resume processing interrupts. 427 * Also called after starting a processor. 428 */ 429 void 430 cpu_enable_intr(struct cpu *cp) 431 { 432 ASSERT(MUTEX_HELD(&cpu_lock)); 433 434 cp->cpu_flags |= CPU_ENABLE; 435 436 intr_redist_all_cpus(); 437 } 438 439 /* 440 * Add function to callback list for intr_redist_all_cpus. We keep two lists, 441 * one for weighted callbacks and one for normal callbacks. Weighted callbacks 442 * are issued to redirect interrupts of a specified weight, from heavy to 443 * light. This allows all the interrupts of a given weight to be redistributed 444 * for all weighted nexus drivers prior to those of less weight. 445 */ 446 static void 447 intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg) 448 { 449 struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP); 450 struct intr_dist *iptr; 451 struct intr_dist **pptr; 452 453 ASSERT(func); 454 new->func = func; 455 new->arg = arg; 456 new->next = NULL; 457 458 /* Add to tail so that redistribution occurs in original order. */ 459 mutex_enter(&intr_dist_lock); 460 for (iptr = *phead, pptr = phead; iptr != NULL; 461 pptr = &iptr->next, iptr = iptr->next) { 462 /* check for problems as we locate the tail */ 463 if ((iptr->func == func) && (iptr->arg == arg)) { 464 cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate"); 465 /*NOTREACHED*/ 466 } 467 } 468 *pptr = new; 469 470 mutex_exit(&intr_dist_lock); 471 } 472 473 void 474 intr_dist_add(void (*func)(void *), void *arg) 475 { 476 intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg); 477 } 478 479 void 480 intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 481 { 482 intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg); 483 } 484 485 /* 486 * Search for the interrupt distribution structure with the specified 487 * mondo vec reg in the interrupt distribution list. If a match is found, 488 * then delete the entry from the list. The caller is responsible for 489 * modifying the mondo vector registers. 490 */ 491 static void 492 intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg) 493 { 494 struct intr_dist *iptr; 495 struct intr_dist **vect; 496 497 mutex_enter(&intr_dist_lock); 498 for (iptr = *headp, vect = headp; 499 iptr != NULL; vect = &iptr->next, iptr = iptr->next) { 500 if ((iptr->func == func) && (iptr->arg == arg)) { 501 *vect = iptr->next; 502 kmem_free(iptr, sizeof (struct intr_dist)); 503 mutex_exit(&intr_dist_lock); 504 return; 505 } 506 } 507 508 if (!panicstr) 509 cmn_err(CE_PANIC, "intr_dist_rem_list: not found"); 510 mutex_exit(&intr_dist_lock); 511 } 512 513 void 514 intr_dist_rem(void (*func)(void *), void *arg) 515 { 516 intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg); 517 } 518 519 void 520 intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 521 { 522 intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg); 523 } 524 525 /* 526 * Initiate interrupt redistribution. Redistribution improves the isolation 527 * associated with interrupt weights by ordering operations from heavy weight 528 * to light weight. When a CPUs orientation changes relative to interrupts, 529 * there is *always* a redistribution to accommodate this change (call to 530 * intr_redist_all_cpus()). As devices (not CPUs) attach/detach it is possible 531 * that a redistribution could improve the quality of an initialization. For 532 * example, if you are not using a NIC it may not be attached with s10 (devfs). 533 * If you then configure the NIC (ifconfig), this may cause the NIC to attach 534 * and plumb interrupts. The CPU assignment for the NIC's interrupts is 535 * occurring late, so optimal "isolation" relative to weight is not occurring. 536 * The same applies to detach, although in this case doing the redistribution 537 * might improve "spread" for medium weight devices since the "isolation" of 538 * a higher weight device may no longer be present. 539 * 540 * NB: We should provide a utility to trigger redistribution (ala "intradm -r"). 541 * 542 * NB: There is risk associated with automatically triggering execution of the 543 * redistribution code at arbitrary times. The risk comes from the fact that 544 * there is a lot of low-level hardware interaction associated with a 545 * redistribution. At some point we may want this code to perform automatic 546 * redistribution (redistribution thread; trigger timeout when add/remove 547 * weight delta is large enough, and call cv_signal from timeout - causing 548 * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too 549 * risky at this time. 550 */ 551 void 552 i_ddi_intr_redist_all_cpus() 553 { 554 mutex_enter(&cpu_lock); 555 INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n")); 556 intr_redist_all_cpus(); 557 mutex_exit(&cpu_lock); 558 } 559 560 /* 561 * Redistribute all interrupts 562 * 563 * This function redistributes all interrupting devices, running the 564 * parent callback functions for each node. 565 */ 566 void 567 intr_redist_all_cpus(void) 568 { 569 struct cpu *cp; 570 struct intr_dist *iptr; 571 int32_t weight, max_weight; 572 573 ASSERT(MUTEX_HELD(&cpu_lock)); 574 mutex_enter(&intr_dist_lock); 575 576 /* 577 * zero cpu_intr_weight on all cpus - it is safe to traverse 578 * cpu_list since we hold cpu_lock. 579 */ 580 cp = cpu_list; 581 do { 582 cp->cpu_intr_weight = 0; 583 } while ((cp = cp->cpu_next) != cpu_list); 584 585 /* 586 * Assume that this redistribution may encounter a device weight 587 * via driver.conf tuning of "ddi-intr-weight" that is at most 588 * intr_dist_weight_maxfactor times larger. 589 */ 590 max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor; 591 if (max_weight > intr_dist_weight_maxmax) 592 max_weight = intr_dist_weight_maxmax; 593 intr_dist_weight_max = 1; 594 595 INTR_DEBUG((CE_CONT, "intr_dist: " 596 "intr_redist_all_cpus: %d-0\n", max_weight)); 597 598 /* 599 * Redistribute weighted, from heavy to light. The callback that 600 * specifies a weight equal to weight_max should redirect all 601 * interrupts of weight weight_max or greater [weight_max, inf.). 602 * Interrupts of lesser weight should be processed on the call with 603 * the matching weight. This allows all the heaver weight interrupts 604 * on all weighted busses (multiple pci busses) to be redirected prior 605 * to any lesser weight interrupts. 606 */ 607 for (weight = max_weight; weight >= 0; weight--) 608 for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next) 609 ((void (*)(void *, int32_t, int32_t))iptr->func) 610 (iptr->arg, max_weight, weight); 611 612 /* redistribute normal (non-weighted) interrupts */ 613 for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next) 614 ((void (*)(void *))iptr->func)(iptr->arg); 615 mutex_exit(&intr_dist_lock); 616 } 617 618 void 619 intr_redist_all_cpus_shutdown(void) 620 { 621 intr_policy = INTR_CURRENT_CPU; 622 intr_redist_all_cpus(); 623 } 624 625 /* 626 * Determine what CPU to target, based on interrupt policy. 627 * 628 * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and 629 * advance through interrupt enabled cpus (round-robin). 630 * 631 * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest 632 * cpu_intr_weight, round robin when all equal. 633 * 634 * Weighted interrupt distribution provides two things: "spread" of weight 635 * (associated with algorithm itself) and "isolation" (associated with a 636 * particular device weight). A redistribution is what provides optimal 637 * "isolation" of heavy weight interrupts, optimal "spread" of weight 638 * (relative to what came before) is always occurring. 639 * 640 * An interrupt weight is a subjective number that represents the 641 * percentage of a CPU required to service a device's interrupts: the 642 * default weight is 0% (however the algorithm still maintains 643 * round-robin), a network interface controller (NIC) may have a large 644 * weight (35%). Interrupt weight only has meaning relative to the 645 * interrupt weight of other devices: a CPU can be weighted more than 646 * 100%, and a single device might consume more than 100% of a CPU. 647 * 648 * A coarse interrupt weight can be defined by the parent nexus driver 649 * based on bus specific information, like pci class codes. A nexus 650 * driver that supports device interrupt weighting for its children 651 * should call intr_dist_cpuid_add/rem_device_weight(), which adds 652 * and removes the weight of a device from the CPU that an interrupt 653 * is directed at. The quality of initialization improves when the 654 * device interrupt weights more accuracy reflect actual run-time weights, 655 * and as the assignments are ordered from is heavy to light. 656 * 657 * The implementation also supports interrupt weight being specified in 658 * driver.conf files via the property "ddi-intr-weight", which takes 659 * precedence over the nexus supplied weight. This support is added to 660 * permit possible tweaking in the product in response to customer 661 * problems. This is not a formal or committed interface. 662 * 663 * While a weighted approach chooses the CPU providing the best spread 664 * given past weights, less than optimal isolation can result in cases 665 * where heavy weight devices show up last. The nexus driver's interrupt 666 * redistribution logic should use intr_dist_add/rem_weighted so that 667 * interrupts can be redistributed heavy first for optimal isolation. 668 */ 669 uint32_t 670 intr_dist_cpuid(void) 671 { 672 static struct cpu *curr_cpu; 673 struct cpu *start_cpu; 674 struct cpu *new_cpu; 675 struct cpu *cp; 676 int cpuid = -1; 677 678 /* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */ 679 mutex_enter(&intr_dist_cpu_lock); 680 681 switch (intr_policy) { 682 case INTR_CURRENT_CPU: 683 cpuid = CPU->cpu_id; 684 break; 685 686 case INTR_BOOT_CPU: 687 panic("INTR_BOOT_CPU no longer supported."); 688 /*NOTREACHED*/ 689 690 case INTR_FLAT_DIST: 691 case INTR_WEIGHTED_DIST: 692 default: 693 /* 694 * Ensure that curr_cpu is valid - cpu_next will be NULL if 695 * the cpu has been deleted (cpu structs are never freed). 696 */ 697 if (curr_cpu == NULL || curr_cpu->cpu_next == NULL) 698 curr_cpu = CPU; 699 700 /* 701 * Advance to online CPU after curr_cpu (round-robin). For 702 * INTR_WEIGHTED_DIST we choose the cpu with the lightest 703 * weight. For a nexus that does not support weight the 704 * default weight of zero is used. We degrade to round-robin 705 * behavior among equal weightes. The default weight is zero 706 * and round-robin behavior continues. 707 * 708 * Disable preemption while traversing cpu_next_onln to 709 * ensure the list does not change. This works because 710 * modifiers of this list and other lists in a struct cpu 711 * call pause_cpus() before making changes. 712 */ 713 kpreempt_disable(); 714 cp = start_cpu = curr_cpu->cpu_next_onln; 715 new_cpu = NULL; 716 do { 717 /* Skip CPUs with interrupts disabled */ 718 if ((cp->cpu_flags & CPU_ENABLE) == 0) 719 continue; 720 721 if (intr_policy == INTR_FLAT_DIST) { 722 /* select CPU */ 723 new_cpu = cp; 724 break; 725 } else if ((new_cpu == NULL) || 726 (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) { 727 /* Choose if lighter weight */ 728 new_cpu = cp; 729 } 730 } while ((cp = cp->cpu_next_onln) != start_cpu); 731 ASSERT(new_cpu); 732 cpuid = new_cpu->cpu_id; 733 734 INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: " 735 "targeted\n", cpuid, new_cpu->cpu_intr_weight)); 736 737 /* update static pointer for next round-robin */ 738 curr_cpu = new_cpu; 739 kpreempt_enable(); 740 break; 741 } 742 mutex_exit(&intr_dist_cpu_lock); 743 return (cpuid); 744 } 745 746 /* 747 * Add or remove the the weight of a device from a CPUs interrupt weight. 748 * 749 * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for 750 * their children to improve the overall quality of interrupt initialization. 751 * 752 * If a nexues shares the CPU returned by a single intr_dist_cpuid() call 753 * among multiple devices (sharing ino) then the nexus should call 754 * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices 755 * that share must specify the same cpuid. 756 * 757 * If a nexus driver is unable to determine the cpu at remove_intr time 758 * for some of its interrupts, then it should not call add_device_weight - 759 * intr_dist_cpuid will still provide round-robin. 760 * 761 * An established device weight (from dev_info node) takes precedence over 762 * the weight passed in. If a device weight is not already established 763 * then the passed in nexus weight is established. 764 */ 765 void 766 intr_dist_cpuid_add_device_weight(uint32_t cpuid, 767 dev_info_t *dip, int32_t nweight) 768 { 769 int32_t eweight; 770 771 /* 772 * For non-weighted policy everything has weight of zero (and we get 773 * round-robin distribution from intr_dist_cpuid). 774 * NB: intr_policy is limited to this file. A weighted nexus driver is 775 * calls this rouitne even if intr_policy has been patched to 776 * INTR_FLAG_DIST. 777 */ 778 ASSERT(dip); 779 if (intr_policy != INTR_WEIGHTED_DIST) 780 return; 781 782 eweight = i_ddi_get_intr_weight(dip); 783 INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for " 784 "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, 785 nweight, eweight, ddi_driver_name(ddi_get_parent(dip)), 786 ddi_get_instance(ddi_get_parent(dip)), 787 ddi_driver_name(dip), ddi_get_instance(dip))); 788 789 /* if no establish weight, establish nexus weight */ 790 if (eweight < 0) { 791 if (nweight > 0) 792 (void) i_ddi_set_intr_weight(dip, nweight); 793 else 794 nweight = 0; 795 } else 796 nweight = eweight; /* use established weight */ 797 798 /* Establish exclusion for cpu_intr_weight manipulation */ 799 mutex_enter(&intr_dist_cpu_lock); 800 cpu[cpuid]->cpu_intr_weight += nweight; 801 802 /* update intr_dist_weight_max */ 803 if (nweight > intr_dist_weight_max) 804 intr_dist_weight_max = nweight; 805 mutex_exit(&intr_dist_cpu_lock); 806 } 807 808 void 809 intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip) 810 { 811 struct cpu *cp; 812 int32_t weight; 813 814 ASSERT(dip); 815 if (intr_policy != INTR_WEIGHTED_DIST) 816 return; 817 818 /* remove weight of device from cpu */ 819 weight = i_ddi_get_intr_weight(dip); 820 if (weight < 0) 821 weight = 0; 822 INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d for " 823 "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight, 824 ddi_driver_name(ddi_get_parent(dip)), 825 ddi_get_instance(ddi_get_parent(dip)), 826 ddi_driver_name(dip), ddi_get_instance(dip))); 827 828 /* Establish exclusion for cpu_intr_weight manipulation */ 829 mutex_enter(&intr_dist_cpu_lock); 830 cp = cpu[cpuid]; 831 cp->cpu_intr_weight -= weight; 832 if (cp->cpu_intr_weight < 0) 833 cp->cpu_intr_weight = 0; /* sanity */ 834 mutex_exit(&intr_dist_cpu_lock); 835 } 836 837 ulong_t 838 create_softint(uint_t pil, uint_t (*func)(caddr_t, caddr_t), caddr_t arg1) 839 { 840 uint64_t inum; 841 842 inum = add_softintr(pil, func, arg1, SOFTINT_MT); 843 return ((ulong_t)inum); 844 } 845 846 void 847 invoke_softint(processorid_t cpuid, ulong_t hdl) 848 { 849 uint64_t inum = hdl; 850 851 if (cpuid == CPU->cpu_id) 852 setsoftint(inum); 853 else 854 xt_one(cpuid, setsoftint_tl1, inum, 0); 855 } 856 857 void 858 remove_softint(ulong_t hdl) 859 { 860 uint64_t inum = hdl; 861 862 (void) rem_softintr(inum); 863 } 864 865 void 866 sync_softint(cpuset_t set) 867 { 868 xt_sync(set); 869 }