1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  27  */
  28 
  29 #include <sys/sysmacros.h>
  30 #include <sys/stack.h>
  31 #include <sys/cpuvar.h>
  32 #include <sys/ivintr.h>
  33 #include <sys/intreg.h>
  34 #include <sys/membar.h>
  35 #include <sys/kmem.h>
  36 #include <sys/intr.h>
  37 #include <sys/sunddi.h>
  38 #include <sys/sunndi.h>
  39 #include <sys/cmn_err.h>
  40 #include <sys/privregs.h>
  41 #include <sys/systm.h>
  42 #include <sys/archsystm.h>
  43 #include <sys/machsystm.h>
  44 #include <sys/x_call.h>
  45 #include <vm/seg_kp.h>
  46 #include <sys/debug.h>
  47 #include <sys/cyclic.h>
  48 #include <sys/kdi_impl.h>
  49 #include <sys/ddi_timer.h>
  50 
  51 #include <sys/cpu_sgnblk_defs.h>
  52 
  53 /* Global locks which protect the interrupt distribution lists */
  54 static kmutex_t intr_dist_lock;
  55 static kmutex_t intr_dist_cpu_lock;
  56 
  57 /* Head of the interrupt distribution lists */
  58 static struct intr_dist *intr_dist_head = NULL;
  59 static struct intr_dist *intr_dist_whead = NULL;
  60 
  61 static uint64_t siron_inum[DDI_IPL_10]; /* software interrupt numbers */
  62 uint64_t *siron_cpu_inum = NULL;
  63 uint64_t siron_poke_cpu_inum;
  64 static int siron_cpu_setup(cpu_setup_t, int, void *);
  65 extern uint_t softlevel1();
  66 
  67 static uint64_t siron1_inum; /* backward compatibility */
  68 uint64_t poke_cpu_inum;
  69 uint_t poke_cpu_intr(caddr_t arg1, caddr_t arg2);
  70 uint_t siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2);
  71 
  72 /*
  73  * Variable to enable/disable printing a message when an invalid vecintr
  74  * is received.
  75  */
  76 uint_t ignore_invalid_vecintr = 0;
  77 
  78 /*
  79  * Note:-
  80  * siron_pending was originally created to prevent a resource over consumption
  81  * bug in setsoftint(exhaustion of interrupt pool free list).
  82  * It's original intention is obsolete with the use of iv_pending in
  83  * setsoftint. However, siron_pending stayed around, acting as a second
  84  * gatekeeper preventing soft interrupts from being queued. In this capacity,
  85  * it can lead to hangs on MP systems, where due to global visibility issues
  86  * it can end up set while iv_pending is reset, preventing soft interrupts from
  87  * ever being processed. In addition to its gatekeeper role, init_intr also
  88  * uses it to flag the situation where siron() was called before siron_inum has
  89  * been defined.
  90  *
  91  * siron() does not need an extra gatekeeper; any cpu that wishes should be
  92  * allowed to queue a soft interrupt. It is softint()'s job to ensure
  93  * correct handling of the queues. Therefore, siron_pending has been
  94  * stripped of its gatekeeper task, retaining only its intr_init job, where
  95  * it indicates that there is a pending need to call siron().
  96  */
  97 static int siron_pending[DDI_IPL_10]; /* software interrupt pending flags */
  98 static int siron1_pending; /* backward compatibility */
  99 
 100 int intr_policy = INTR_WEIGHTED_DIST;   /* interrupt distribution policy */
 101 int intr_dist_debug = 0;
 102 int32_t intr_dist_weight_max = 1;
 103 int32_t intr_dist_weight_maxmax = 1000;
 104 int intr_dist_weight_maxfactor = 2;
 105 #define INTR_DEBUG(args) if (intr_dist_debug) cmn_err args
 106 
 107 /*
 108  * intr_init() - Interrupt initialization
 109  *      Initialize the system's interrupt vector table.
 110  */
 111 void
 112 intr_init(cpu_t *cp)
 113 {
 114         int i;
 115         extern uint_t softlevel1();
 116 
 117         init_ivintr();
 118         REGISTER_BBUS_INTR();
 119 
 120         /*
 121          * Register these software interrupts for ddi timer.
 122          * Software interrupts up to the level 10 are supported.
 123          */
 124         for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
 125                 siron_inum[i - 1] = add_softintr(i,
 126                     (softintrfunc)ddi_periodic_softintr,
 127                     (caddr_t)(uintptr_t)(i), SOFTINT_ST);
 128         }
 129 
 130         siron1_inum = add_softintr(PIL_1, softlevel1, 0, SOFTINT_ST);
 131         poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0, SOFTINT_MT);
 132         siron_poke_cpu_inum = add_softintr(PIL_13,
 133             siron_poke_cpu_intr, 0, SOFTINT_MT);
 134         cp->cpu_m.poke_cpu_outstanding = B_FALSE;
 135 
 136         mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL);
 137         mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL);
 138 
 139         /*
 140          * A soft interrupt may have been requested prior to the initialization
 141          * of soft interrupts.  Soft interrupts can't be dispatched until after
 142          * init_intr(), so we have to wait until now before we can dispatch the
 143          * pending soft interrupt (if any).
 144          */
 145         for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
 146                 if (siron_pending[i-1]) {
 147                         siron_pending[i-1] = 0;
 148                         sir_on(i);
 149                 }
 150         }
 151         if (siron1_pending) {
 152                 siron1_pending = 0;
 153                 siron();
 154         }
 155 }
 156 
 157 /*
 158  * poke_cpu_intr - fall through when poke_cpu calls
 159  */
 160 /* ARGSUSED */
 161 uint_t
 162 poke_cpu_intr(caddr_t arg1, caddr_t arg2)
 163 {
 164         CPU->cpu_m.poke_cpu_outstanding = B_FALSE;
 165         membar_stld_stst();
 166         return (1);
 167 }
 168 
 169 /*
 170  * Trigger software interrupts dedicated to ddi timer.
 171  */
 172 void
 173 sir_on(int level)
 174 {
 175         ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10);
 176         if (siron_inum[level-1])
 177                 setsoftint(siron_inum[level-1]);
 178         else
 179                 siron_pending[level-1] = 1;
 180 }
 181 
 182 /*
 183  * kmdb uses siron (and thus setsoftint) while the world is stopped in order to
 184  * inform its driver component that there's work to be done.  We need to keep
 185  * DTrace from instrumenting kmdb's siron and setsoftint.  We duplicate siron,
 186  * giving kmdb's version a kdi_ prefix to keep DTrace at bay.  The
 187  * implementation of setsoftint is complicated enough that we don't want to
 188  * duplicate it, but at the same time we don't want to preclude tracing either.
 189  * The meat of setsoftint() therefore goes into kdi_setsoftint, with
 190  * setsoftint() implemented as a wrapper.  This allows tracing, while still
 191  * providing a way for kmdb to sneak in unmolested.
 192  */
 193 void
 194 kdi_siron(void)
 195 {
 196         if (siron1_inum != 0)
 197                 kdi_setsoftint(siron1_inum);
 198         else
 199                 siron1_pending = 1;
 200 }
 201 
 202 void
 203 setsoftint(uint64_t inum)
 204 {
 205         kdi_setsoftint(inum);
 206 }
 207 
 208 /*
 209  * Generates softlevel1 interrupt on current CPU if it
 210  * is not pending already.
 211  */
 212 void
 213 siron(void)
 214 {
 215         uint64_t inum;
 216 
 217         if (siron1_inum != 0) {
 218                 /*
 219                  * Once siron_cpu_inum has been allocated, we can
 220                  * use per-CPU siron inum.
 221                  */
 222                 if (siron_cpu_inum && siron_cpu_inum[CPU->cpu_id] != 0)
 223                         inum = siron_cpu_inum[CPU->cpu_id];
 224                 else
 225                         inum = siron1_inum;
 226 
 227                 setsoftint(inum);
 228         } else
 229                 siron1_pending = 1;
 230 }
 231 
 232 
 233 static void
 234 siron_init(void)
 235 {
 236         /*
 237          * We just allocate memory for per-cpu siron right now. Rest of
 238          * the work is done when CPU is configured.
 239          */
 240         siron_cpu_inum = kmem_zalloc(sizeof (uint64_t) * NCPU, KM_SLEEP);
 241 }
 242 
 243 /*
 244  * This routine creates per-CPU siron inum for CPUs which are
 245  * configured during boot.
 246  */
 247 void
 248 siron_mp_init()
 249 {
 250         cpu_t *c;
 251 
 252         /*
 253          * Get the memory for per-CPU siron inums
 254          */
 255         siron_init();
 256 
 257         mutex_enter(&cpu_lock);
 258         c = cpu_list;
 259         do {
 260                 (void) siron_cpu_setup(CPU_CONFIG, c->cpu_id, NULL);
 261         } while ((c = c->cpu_next) != cpu_list);
 262 
 263         register_cpu_setup_func(siron_cpu_setup, NULL);
 264         mutex_exit(&cpu_lock);
 265 }
 266 
 267 /*
 268  * siron_poke_cpu_intr - cross-call handler.
 269  */
 270 /* ARGSUSED */
 271 uint_t
 272 siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2)
 273 {
 274         /* generate level1 softint */
 275         siron();
 276         return (1);
 277 }
 278 
 279 /*
 280  * This routine generates a cross-call on target CPU(s).
 281  */
 282 void
 283 siron_poke_cpu(cpuset_t poke)
 284 {
 285         int cpuid = CPU->cpu_id;
 286 
 287         if (CPU_IN_SET(poke, cpuid)) {
 288                 siron();
 289                 CPUSET_DEL(poke, cpuid);
 290                 if (CPUSET_ISNULL(poke))
 291                         return;
 292         }
 293 
 294         xt_some(poke, setsoftint_tl1, siron_poke_cpu_inum, 0);
 295 }
 296 
 297 /*
 298  * This callback function allows us to create per-CPU siron inum.
 299  */
 300 /* ARGSUSED */
 301 static int
 302 siron_cpu_setup(cpu_setup_t what, int id, void *arg)
 303 {
 304         cpu_t *cp = cpu[id];
 305 
 306         ASSERT(MUTEX_HELD(&cpu_lock));
 307         ASSERT(cp != NULL);
 308 
 309         switch (what) {
 310         case CPU_CONFIG:
 311                 siron_cpu_inum[cp->cpu_id] = add_softintr(PIL_1,
 312                     (softintrfunc)softlevel1, 0, SOFTINT_ST);
 313                 break;
 314         case CPU_UNCONFIG:
 315                 (void) rem_softintr(siron_cpu_inum[cp->cpu_id]);
 316                 siron_cpu_inum[cp->cpu_id] = 0;
 317                 break;
 318         default:
 319                 break;
 320         }
 321 
 322         return (0);
 323 }
 324 
 325 /*
 326  * no_ivintr()
 327  *      called by setvecint_tl1() through sys_trap()
 328  *      vector interrupt received but not valid or not
 329  *      registered in intr_vec_table
 330  *      considered as a spurious mondo interrupt
 331  */
 332 /* ARGSUSED */
 333 void
 334 no_ivintr(struct regs *rp, int inum, int pil)
 335 {
 336         if (!ignore_invalid_vecintr)
 337                 cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x",
 338                     inum, pil);
 339 
 340 #ifdef DEBUG_VEC_INTR
 341         prom_enter_mon();
 342 #endif /* DEBUG_VEC_INTR */
 343 }
 344 
 345 void
 346 intr_dequeue_req(uint_t pil, uint64_t inum)
 347 {
 348         intr_vec_t      *iv, *next, *prev;
 349         struct machcpu  *mcpu;
 350         uint32_t        clr;
 351         processorid_t   cpu_id;
 352         extern uint_t   getpstate(void);
 353 
 354         ASSERT((getpstate() & PSTATE_IE) == 0);
 355 
 356         mcpu = &CPU->cpu_m;
 357         cpu_id = CPU->cpu_id;
 358 
 359         iv = (intr_vec_t *)inum;
 360         prev = NULL;
 361         next = mcpu->intr_head[pil];
 362 
 363         /* Find a matching entry in the list */
 364         while (next != NULL) {
 365                 if (next == iv)
 366                         break;
 367                 prev = next;
 368                 next = IV_GET_PIL_NEXT(next, cpu_id);
 369         }
 370 
 371         if (next != NULL) {
 372                 intr_vec_t      *next_iv = IV_GET_PIL_NEXT(next, cpu_id);
 373 
 374                 /* Remove entry from list */
 375                 if (prev != NULL)
 376                         IV_SET_PIL_NEXT(prev, cpu_id, next_iv); /* non-head */
 377                 else
 378                         mcpu->intr_head[pil] = next_iv; /* head */
 379 
 380                 if (next_iv == NULL)
 381                         mcpu->intr_tail[pil] = prev; /* tail */
 382         }
 383 
 384         /* Clear pending interrupts at this level if the list is empty */
 385         if (mcpu->intr_head[pil] == NULL) {
 386                 clr = 1 << pil;
 387                 if (pil == PIL_14)
 388                         clr |= (TICK_INT_MASK | STICK_INT_MASK);
 389                 wr_clr_softint(clr);
 390         }
 391 }
 392 
 393 
 394 /*
 395  * Send a directed interrupt of specified interrupt number id to a cpu.
 396  */
 397 void
 398 send_dirint(
 399         int cpuix,              /* cpu to be interrupted */
 400         int intr_id)            /* interrupt number id */
 401 {
 402         xt_one(cpuix, setsoftint_tl1, intr_id, 0);
 403 }
 404 
 405 /*
 406  * Take the specified CPU out of participation in interrupts.
 407  *      Called by p_online(2) when a processor is being taken off-line.
 408  *      This allows interrupt threads being handled on the processor to
 409  *      complete before the processor is idled.
 410  */
 411 int
 412 cpu_disable_intr(struct cpu *cp)
 413 {
 414         ASSERT(MUTEX_HELD(&cpu_lock));
 415 
 416         /*
 417          * Turn off the CPU_ENABLE flag before calling the redistribution
 418          * function, since it checks for this in the cpu flags.
 419          */
 420         cp->cpu_flags &= ~CPU_ENABLE;
 421 
 422         intr_redist_all_cpus();
 423 
 424         return (0);
 425 }
 426 
 427 /*
 428  * Allow the specified CPU to participate in interrupts.
 429  *      Called by p_online(2) if a processor could not be taken off-line
 430  *      because of bound threads, in order to resume processing interrupts.
 431  *      Also called after starting a processor.
 432  */
 433 void
 434 cpu_enable_intr(struct cpu *cp)
 435 {
 436         ASSERT(MUTEX_HELD(&cpu_lock));
 437 
 438         cp->cpu_flags |= CPU_ENABLE;
 439 
 440         intr_redist_all_cpus();
 441 }
 442 
 443 /*
 444  * Add function to callback list for intr_redist_all_cpus.  We keep two lists,
 445  * one for weighted callbacks and one for normal callbacks. Weighted callbacks
 446  * are issued to redirect interrupts of a specified weight, from heavy to
 447  * light.  This allows all the interrupts of a given weight to be redistributed
 448  * for all weighted nexus drivers prior to those of less weight.
 449  */
 450 static void
 451 intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg)
 452 {
 453         struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP);
 454         struct intr_dist *iptr;
 455         struct intr_dist **pptr;
 456 
 457         ASSERT(func);
 458         new->func = func;
 459         new->arg = arg;
 460         new->next = NULL;
 461 
 462         /* Add to tail so that redistribution occurs in original order. */
 463         mutex_enter(&intr_dist_lock);
 464         for (iptr = *phead, pptr = phead; iptr != NULL;
 465             pptr = &iptr->next, iptr = iptr->next) {
 466                 /* check for problems as we locate the tail */
 467                 if ((iptr->func == func) && (iptr->arg == arg)) {
 468                         cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate");
 469                         /*NOTREACHED*/
 470                 }
 471         }
 472         *pptr = new;
 473 
 474         mutex_exit(&intr_dist_lock);
 475 }
 476 
 477 void
 478 intr_dist_add(void (*func)(void *), void *arg)
 479 {
 480         intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg);
 481 }
 482 
 483 void
 484 intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
 485 {
 486         intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg);
 487 }
 488 
 489 /*
 490  * Search for the interrupt distribution structure with the specified
 491  * mondo vec reg in the interrupt distribution list. If a match is found,
 492  * then delete the entry from the list. The caller is responsible for
 493  * modifying the mondo vector registers.
 494  */
 495 static void
 496 intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg)
 497 {
 498         struct intr_dist *iptr;
 499         struct intr_dist **vect;
 500 
 501         mutex_enter(&intr_dist_lock);
 502         for (iptr = *headp, vect = headp;
 503             iptr != NULL; vect = &iptr->next, iptr = iptr->next) {
 504                 if ((iptr->func == func) && (iptr->arg == arg)) {
 505                         *vect = iptr->next;
 506                         kmem_free(iptr, sizeof (struct intr_dist));
 507                         mutex_exit(&intr_dist_lock);
 508                         return;
 509                 }
 510         }
 511 
 512         if (!panicstr)
 513                 cmn_err(CE_PANIC, "intr_dist_rem_list: not found");
 514         mutex_exit(&intr_dist_lock);
 515 }
 516 
 517 void
 518 intr_dist_rem(void (*func)(void *), void *arg)
 519 {
 520         intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg);
 521 }
 522 
 523 void
 524 intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
 525 {
 526         intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg);
 527 }
 528 
 529 /*
 530  * Initiate interrupt redistribution.  Redistribution improves the isolation
 531  * associated with interrupt weights by ordering operations from heavy weight
 532  * to light weight.  When a CPUs orientation changes relative to interrupts,
 533  * there is *always* a redistribution to accommodate this change (call to
 534  * intr_redist_all_cpus()).  As devices (not CPUs) attach/detach it is possible
 535  * that a redistribution could improve the quality of an initialization. For
 536  * example, if you are not using a NIC it may not be attached with s10 (devfs).
 537  * If you then configure the NIC (ifconfig), this may cause the NIC to attach
 538  * and plumb interrupts.  The CPU assignment for the NIC's interrupts is
 539  * occurring late, so optimal "isolation" relative to weight is not occurring.
 540  * The same applies to detach, although in this case doing the redistribution
 541  * might improve "spread" for medium weight devices since the "isolation" of
 542  * a higher weight device may no longer be present.
 543  *
 544  * NB: We should provide a utility to trigger redistribution (ala "intradm -r").
 545  *
 546  * NB: There is risk associated with automatically triggering execution of the
 547  * redistribution code at arbitrary times. The risk comes from the fact that
 548  * there is a lot of low-level hardware interaction associated with a
 549  * redistribution.  At some point we may want this code to perform automatic
 550  * redistribution (redistribution thread; trigger timeout when add/remove
 551  * weight delta is large enough, and call cv_signal from timeout - causing
 552  * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too
 553  * risky at this time.
 554  */
 555 void
 556 i_ddi_intr_redist_all_cpus()
 557 {
 558         mutex_enter(&cpu_lock);
 559         INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n"));
 560         intr_redist_all_cpus();
 561         mutex_exit(&cpu_lock);
 562 }
 563 
 564 /*
 565  * Redistribute all interrupts
 566  *
 567  * This function redistributes all interrupting devices, running the
 568  * parent callback functions for each node.
 569  */
 570 void
 571 intr_redist_all_cpus(void)
 572 {
 573         struct cpu *cp;
 574         struct intr_dist *iptr;
 575         int32_t weight, max_weight;
 576 
 577         ASSERT(MUTEX_HELD(&cpu_lock));
 578         mutex_enter(&intr_dist_lock);
 579 
 580         /*
 581          * zero cpu_intr_weight on all cpus - it is safe to traverse
 582          * cpu_list since we hold cpu_lock.
 583          */
 584         cp = cpu_list;
 585         do {
 586                 cp->cpu_intr_weight = 0;
 587         } while ((cp = cp->cpu_next) != cpu_list);
 588 
 589         /*
 590          * Assume that this redistribution may encounter a device weight
 591          * via driver.conf tuning of "ddi-intr-weight" that is at most
 592          * intr_dist_weight_maxfactor times larger.
 593          */
 594         max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor;
 595         if (max_weight > intr_dist_weight_maxmax)
 596                 max_weight = intr_dist_weight_maxmax;
 597         intr_dist_weight_max = 1;
 598 
 599         INTR_DEBUG((CE_CONT, "intr_dist: "
 600             "intr_redist_all_cpus: %d-0\n", max_weight));
 601 
 602         /*
 603          * Redistribute weighted, from heavy to light.  The callback that
 604          * specifies a weight equal to weight_max should redirect all
 605          * interrupts of weight weight_max or greater [weight_max, inf.).
 606          * Interrupts of lesser weight should be processed on the call with
 607          * the matching weight. This allows all the heaver weight interrupts
 608          * on all weighted busses (multiple pci busses) to be redirected prior
 609          * to any lesser weight interrupts.
 610          */
 611         for (weight = max_weight; weight >= 0; weight--)
 612                 for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next)
 613                         ((void (*)(void *, int32_t, int32_t))iptr->func)
 614                             (iptr->arg, max_weight, weight);
 615 
 616         /* redistribute normal (non-weighted) interrupts */
 617         for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next)
 618                 ((void (*)(void *))iptr->func)(iptr->arg);
 619         mutex_exit(&intr_dist_lock);
 620 }
 621 
 622 void
 623 intr_redist_all_cpus_shutdown(void)
 624 {
 625         intr_policy = INTR_CURRENT_CPU;
 626         intr_redist_all_cpus();
 627 }
 628 
 629 /*
 630  * Determine what CPU to target, based on interrupt policy.
 631  *
 632  * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and
 633  *      advance through interrupt enabled cpus (round-robin).
 634  *
 635  * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest
 636  *      cpu_intr_weight, round robin when all equal.
 637  *
 638  *      Weighted interrupt distribution provides two things: "spread" of weight
 639  *      (associated with algorithm itself) and "isolation" (associated with a
 640  *      particular device weight). A redistribution is what provides optimal
 641  *      "isolation" of heavy weight interrupts, optimal "spread" of weight
 642  *      (relative to what came before) is always occurring.
 643  *
 644  *      An interrupt weight is a subjective number that represents the
 645  *      percentage of a CPU required to service a device's interrupts: the
 646  *      default weight is 0% (however the algorithm still maintains
 647  *      round-robin), a network interface controller (NIC) may have a large
 648  *      weight (35%). Interrupt weight only has meaning relative to the
 649  *      interrupt weight of other devices: a CPU can be weighted more than
 650  *      100%, and a single device might consume more than 100% of a CPU.
 651  *
 652  *      A coarse interrupt weight can be defined by the parent nexus driver
 653  *      based on bus specific information, like pci class codes. A nexus
 654  *      driver that supports device interrupt weighting for its children
 655  *      should call intr_dist_cpuid_add/rem_device_weight(), which adds
 656  *      and removes the weight of a device from the CPU that an interrupt
 657  *      is directed at.  The quality of initialization improves when the
 658  *      device interrupt weights more accuracy reflect actual run-time weights,
 659  *      and as the assignments are ordered from is heavy to light.
 660  *
 661  *      The implementation also supports interrupt weight being specified in
 662  *      driver.conf files via the property "ddi-intr-weight", which takes
 663  *      precedence over the nexus supplied weight.  This support is added to
 664  *      permit possible tweaking in the product in response to customer
 665  *      problems. This is not a formal or committed interface.
 666  *
 667  *      While a weighted approach chooses the CPU providing the best spread
 668  *      given past weights, less than optimal isolation can result in cases
 669  *      where heavy weight devices show up last. The nexus driver's interrupt
 670  *      redistribution logic should use intr_dist_add/rem_weighted so that
 671  *      interrupts can be redistributed heavy first for optimal isolation.
 672  */
 673 uint32_t
 674 intr_dist_cpuid(void)
 675 {
 676         static struct cpu       *curr_cpu;
 677         struct cpu              *start_cpu;
 678         struct cpu              *new_cpu;
 679         struct cpu              *cp;
 680         int                     cpuid = -1;
 681 
 682         /* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */
 683         mutex_enter(&intr_dist_cpu_lock);
 684 
 685         switch (intr_policy) {
 686         case INTR_CURRENT_CPU:
 687                 cpuid = CPU->cpu_id;
 688                 break;
 689 
 690         case INTR_BOOT_CPU:
 691                 panic("INTR_BOOT_CPU no longer supported.");
 692                 /*NOTREACHED*/
 693 
 694         case INTR_FLAT_DIST:
 695         case INTR_WEIGHTED_DIST:
 696         default:
 697                 /*
 698                  * Ensure that curr_cpu is valid - cpu_next will be NULL if
 699                  * the cpu has been deleted (cpu structs are never freed).
 700                  */
 701                 if (curr_cpu == NULL || curr_cpu->cpu_next == NULL)
 702                         curr_cpu = CPU;
 703 
 704                 /*
 705                  * Advance to online CPU after curr_cpu (round-robin). For
 706                  * INTR_WEIGHTED_DIST we choose the cpu with the lightest
 707                  * weight.  For a nexus that does not support weight the
 708                  * default weight of zero is used. We degrade to round-robin
 709                  * behavior among equal weightes.  The default weight is zero
 710                  * and round-robin behavior continues.
 711                  *
 712                  * Disable preemption while traversing cpu_next_onln to
 713                  * ensure the list does not change.  This works because
 714                  * modifiers of this list and other lists in a struct cpu
 715                  * call pause_cpus() before making changes.
 716                  */
 717                 kpreempt_disable();
 718                 cp = start_cpu = curr_cpu->cpu_next_onln;
 719                 new_cpu = NULL;
 720                 do {
 721                         /* Skip CPUs with interrupts disabled */
 722                         if ((cp->cpu_flags & CPU_ENABLE) == 0)
 723                                 continue;
 724 
 725                         if (intr_policy == INTR_FLAT_DIST) {
 726                                 /* select CPU */
 727                                 new_cpu = cp;
 728                                 break;
 729                         } else if ((new_cpu == NULL) ||
 730                             (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) {
 731                                 /* Choose if lighter weight */
 732                                 new_cpu = cp;
 733                         }
 734                 } while ((cp = cp->cpu_next_onln) != start_cpu);
 735                 ASSERT(new_cpu);
 736                 cpuid = new_cpu->cpu_id;
 737 
 738                 INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: "
 739                     "targeted\n", cpuid, new_cpu->cpu_intr_weight));
 740 
 741                 /* update static pointer for next round-robin */
 742                 curr_cpu = new_cpu;
 743                 kpreempt_enable();
 744                 break;
 745         }
 746         mutex_exit(&intr_dist_cpu_lock);
 747         return (cpuid);
 748 }
 749 
 750 /*
 751  * Add or remove the the weight of a device from a CPUs interrupt weight.
 752  *
 753  * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for
 754  * their children to improve the overall quality of interrupt initialization.
 755  *
 756  * If a nexues shares the CPU returned by a single intr_dist_cpuid() call
 757  * among multiple devices (sharing ino) then the nexus should call
 758  * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices
 759  * that share must specify the same cpuid.
 760  *
 761  * If a nexus driver is unable to determine the cpu at remove_intr time
 762  * for some of its interrupts, then it should not call add_device_weight -
 763  * intr_dist_cpuid will still provide round-robin.
 764  *
 765  * An established device weight (from dev_info node) takes precedence over
 766  * the weight passed in.  If a device weight is not already established
 767  * then the passed in nexus weight is established.
 768  */
 769 void
 770 intr_dist_cpuid_add_device_weight(uint32_t cpuid,
 771     dev_info_t *dip, int32_t nweight)
 772 {
 773         int32_t         eweight;
 774 
 775         /*
 776          * For non-weighted policy everything has weight of zero (and we get
 777          * round-robin distribution from intr_dist_cpuid).
 778          * NB: intr_policy is limited to this file. A weighted nexus driver is
 779          * calls this rouitne even if intr_policy has been patched to
 780          * INTR_FLAG_DIST.
 781          */
 782         ASSERT(dip);
 783         if (intr_policy != INTR_WEIGHTED_DIST)
 784                 return;
 785 
 786         eweight = i_ddi_get_intr_weight(dip);
 787         INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for "
 788             "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight,
 789             nweight, eweight, ddi_driver_name(ddi_get_parent(dip)),
 790             ddi_get_instance(ddi_get_parent(dip)),
 791             ddi_driver_name(dip), ddi_get_instance(dip)));
 792 
 793         /* if no establish weight, establish nexus weight */
 794         if (eweight < 0) {
 795                 if (nweight > 0)
 796                         (void) i_ddi_set_intr_weight(dip, nweight);
 797                 else
 798                         nweight = 0;
 799         } else
 800                 nweight = eweight;      /* use established weight */
 801 
 802         /* Establish exclusion for cpu_intr_weight manipulation */
 803         mutex_enter(&intr_dist_cpu_lock);
 804         cpu[cpuid]->cpu_intr_weight += nweight;
 805 
 806         /* update intr_dist_weight_max */
 807         if (nweight > intr_dist_weight_max)
 808                 intr_dist_weight_max = nweight;
 809         mutex_exit(&intr_dist_cpu_lock);
 810 }
 811 
 812 void
 813 intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip)
 814 {
 815         struct cpu      *cp;
 816         int32_t         weight;
 817 
 818         ASSERT(dip);
 819         if (intr_policy != INTR_WEIGHTED_DIST)
 820                 return;
 821 
 822         /* remove weight of device from cpu */
 823         weight = i_ddi_get_intr_weight(dip);
 824         if (weight < 0)
 825                 weight = 0;
 826         INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d    for "
 827             "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight,
 828             ddi_driver_name(ddi_get_parent(dip)),
 829             ddi_get_instance(ddi_get_parent(dip)),
 830             ddi_driver_name(dip), ddi_get_instance(dip)));
 831 
 832         /* Establish exclusion for cpu_intr_weight manipulation */
 833         mutex_enter(&intr_dist_cpu_lock);
 834         cp = cpu[cpuid];
 835         cp->cpu_intr_weight -= weight;
 836         if (cp->cpu_intr_weight < 0)
 837                 cp->cpu_intr_weight = 0;     /* sanity */
 838         mutex_exit(&intr_dist_cpu_lock);
 839 }
 840 
 841 ulong_t
 842 create_softint(uint_t pil, uint_t (*func)(caddr_t, caddr_t), caddr_t arg1)
 843 {
 844         uint64_t inum;
 845 
 846         inum = add_softintr(pil, func, arg1, SOFTINT_MT);
 847         return ((ulong_t)inum);
 848 }
 849 
 850 void
 851 invoke_softint(processorid_t cpuid, ulong_t hdl)
 852 {
 853         uint64_t inum = hdl;
 854 
 855         if (cpuid == CPU->cpu_id)
 856                 setsoftint(inum);
 857         else
 858                 xt_one(cpuid, setsoftint_tl1, inum, 0);
 859 }
 860 
 861 void
 862 remove_softint(ulong_t hdl)
 863 {
 864         uint64_t inum = hdl;
 865 
 866         (void) rem_softintr(inum);
 867 }
 868 
 869 void
 870 sync_softint(cpuset_t set)
 871 {
 872         xt_sync(set);
 873 }