1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2018 Joyent, Inc.  All rights reserverd.
  25  */
  26 
  27 /*
  28  * To understand the present state of interrupt handling on i86pc, we must
  29  * first consider the history of interrupt controllers and our way of handling
  30  * interrupts.
  31  *
  32  * History of Interrupt Controllers on i86pc
  33  * -----------------------------------------
  34  *
  35  *    Intel 8259 and 8259A
  36  *
  37  * The first interrupt controller that attained widespread use on i86pc was
  38  * the Intel 8259(A) Programmable Interrupt Controller that first saw use with
  39  * the 8086. It took up to 8 interrupt sources and combined them into one
  40  * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
  41  * With the switch to the 8259A, level mode interrupts became possible. For a
  42  * long time on i86pc the 8259A was the only way to handle interrupts and it
  43  * had its own set of quirks. The 8259A and its corresponding interval timer
  44  * the 8254 are programmed using outb and inb instructions.
  45  *
  46  *    Intel Advanced Programmable Interrupt Controller (APIC)
  47  *
  48  * Starting around the time of the introduction of the P6 family
  49  * microarchitecture (i686) Intel introduced a new interrupt controller.
  50  * Instead of having the series of slaved 8259A devices, Intel opted to outfit
  51  * each processor with a Local APIC (lapic) and to outfit the system with at
  52  * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
  53  * initially communicated over a dedicated bus, but this has since been
  54  * replaced. Each physical core and even hyperthread currently contains its
  55  * own local apic, which is not shared. There are a few exceptions for
  56  * hyperthreads, but that does not usually concern us.
  57  *
  58  * Instead of talking directly to 8259 for status, sending End Of Interrupt
  59  * (EOI), etc. a microprocessor now communicates directly to the lapic. This
  60  * also allows for each microprocessor to be able to have independent controls.
  61  * The programming method is different from the 8259. Consumers map the lapic
  62  * registers into uncacheable memory to read and manipulate the state.
  63  *
  64  * The number of addressable interrupt vectors was increased to 256. However
  65  * vectors 0-31 are reserved for the processor exception handling, leaving the
  66  * remaining vectors for general use. In addition to hardware generated
  67  * interrupts, the lapic provides a way for generating inter-processor
  68  * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
  69  *
  70  * AMD ended up implementing the Intel APIC architecture in lieu of their work
  71  * with Cyrix.
  72  *
  73  *    Intel x2apic
  74  *
  75  * The x2apic is an extension to the lapic which started showing up around the
  76  * same time as the Sandy Bridge chipsets. It provides a new programming mode
  77  * as well as new features. The goal of the x2apic is to solve a few problems
  78  * with the previous generation of lapic and the x2apic is backwards compatible
  79  * with the previous programming and model. The only downsides to using the
  80  * backwards compatibility is that you are not able to take advantage of the new
  81  * x2apic features.
  82  *
  83  *    o The APIC ID is increased from an 8-bit value to a 32-bit value. This
  84  *    increases the maximum number of addressable physical processors beyond
  85  *    256. This new ID is assembled in a similar manner as the information that
  86  *    is obtainable by the extended cpuid topology leaves.
  87  *
  88  *    o A new means of generating IPIs was introduced.
  89  *
  90  *    o Instead of memory mapping the registers, the x2apic only allows for
  91  *    programming it through a series of wrmsrs. This has important semantic
  92  *    side effects. Recall that the registers were previously all mapped to
  93  *    uncachable memory which meant that all operations to the local apic were
  94  *    serializing instructions. With the switch to using wrmsrs this has been
  95  *    relaxed and these operations can no longer be assumed to be serializing
  96  *    instructions.
  97  *
  98  * Note for the rest of this we are only going to concern ourselves with the
  99  * apic and x2apic which practically all of i86pc has been using now for
 100  * quite some time.
 101  *
 102  * Interrupt Priority Levels
 103  * -------------------------
 104  *
 105  * On i86pc systems there are a total of fifteen interrupt priority levels
 106  * (ipls) which range from 1-15. Level 0 is for normal processing and
 107  * non-interrupt processing. To manipulate these values the family of spl
 108  * functions (which date back to UNIX on the PDP-11) are used. Specifically,
 109  * splr() to raise the priority level and splx() to lower it. One should not
 110  * generally call setspl() directly.
 111  *
 112  * Both i86pc and the supported SPARC platforms honor the same conventions for
 113  * the meaning behind these IPLs. The most important IPL is the platform's
 114  * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
 115  * sleep on any synchronization object. The only allowed synchronization
 116  * primitive is a mutex that has been specifically initialized to be a spin
 117  * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
 118  * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
 119  * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
 120  *
 121  * Each interrupt that is registered in the system fires at a specific IPL.
 122  * Generally most interrupts fire below LOCK_LEVEL.
 123  *
 124  * PSM Drivers
 125  * -----------
 126  *
 127  * We currently have three sets of PSM (platform specific module) drivers
 128  * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
 129  * driver that interacts with the 8259A and 8254. In general, it is not used
 130  * anymore given the prevalence of the apic.
 131  *
 132  * The system prefers to use the apix driver over the pcplusmp driver. The apix
 133  * driver requires HW support for an x2apic. If there is no x2apic HW, apix
 134  * will not be used. In general we prefer using the apix driver over the
 135  * pcplusmp driver because it gives us much more flexibility with respect to
 136  * interrupts. In the apix driver each local apic has its own independent set
 137  * of  interrupts, whereas the pcplusmp driver only has a single global set of
 138  * interrupts. This is why pcplusmp only supports a finite number of interrupts
 139  * per IPL -- generally 16, often less. The apix driver supports using either
 140  * the x2apic or the local apic programing modes. The programming mode does not
 141  * change the number of interrupts available, just the number of processors
 142  * that we can address. For the apix driver, the x2apic mode is enabled if the
 143  * system supports interrupt re-mapping, otherwise the module manages the
 144  * x2apic in local mode.
 145  *
 146  * When there is no x2apic present, we default back to the pcplusmp PSM driver.
 147  * In general, this is not problematic unless you have more than 256
 148  * processors in the machine or you do not have enough interrupts available.
 149  *
 150  * Controlling Interrupt Generation on i86pc
 151  * -----------------------------------------
 152  *
 153  * There are two different ways to manipulate which interrupts will be
 154  * generated on i86pc. Each offers different degrees of control.
 155  *
 156  * The first is through the flags register (eflags and rflags on i386 and amd64
 157  * respectively). The IF bit determines whether or not interrupts are enabled
 158  * or disabled. This is manipulated in one of several ways. The most common way
 159  * is through the cli and sti instructions. These clear the IF flag and set it,
 160  * respectively, for the current processor. The other common way is through the
 161  * use of the intr_clear and intr_restore functions.
 162  *
 163  * Assuming interrupts are not blocked by the IF flag, then the second form is
 164  * through the Processor-Priority Register (PPR). The PPR is used to determine
 165  * whether or not a pending interrupt should be delivered. If the ipl of the
 166  * new interrupt is higher than the current value in the PPR, then the lapic
 167  * will either deliver it immediately (if interrupts are not in progress) or it
 168  * will deliver it once the current interrupt processing has issued an EOI. The
 169  * highest unmasked interrupt will be the one delivered.
 170  *
 171  * The PPR register is based upon the max of the following two registers in the
 172  * lapic, the TPR register (also known as CR8 on amd64) that can be used to
 173  * mask interrupt levels, and the current vector. Because the pcplusmp module
 174  * always sets TPR appropriately early in the do_interrupt path, we can usually
 175  * just think that the PPR is the TPR. The pcplusmp module also issues an EOI
 176  * once it has set the TPR, so higher priority interrupts can come in while
 177  * we're servicing a lower priority interrupt.
 178  *
 179  * Handling Interrupts
 180  * -------------------
 181  *
 182  * Interrupts can be broken down into three categories based on priority and
 183  * source:
 184  *
 185  *   o High level interrupts
 186  *   o Low level hardware interrupts
 187  *   o Low level software interrupts
 188  *
 189  *   High Level Interrupts
 190  *
 191  * High level interrupts encompasses both hardware-sourced and software-sourced
 192  * interrupts. Examples of high level hardware interrupts include the serial
 193  * console. High level software-sourced interrupts are still delivered through
 194  * the local apic through IPIs. This is primarily cross calls.
 195  *
 196  * When a high level interrupt comes in, we will raise the SPL and then pin the
 197  * current lwp to the processor. We will use its lwp, but our own interrupt
 198  * stack and process the high level interrupt in-situ. These handlers are
 199  * designed to be very short in nature and cannot go to sleep, only block on a
 200  * spin lock. If the interrupt has a lot of work to do, it must generate a
 201  * low-priority software interrupt that will be processed later.
 202  *
 203  *   Low level hardware interrupts
 204  *
 205  * Low level hardware interrupts start off like their high-level cousins. The
 206  * current CPU contains a number of kernel threads (kthread_t) that can be used
 207  * to process low level interrupts. These are shared between both low level
 208  * hardware and software interrupts. Note that while we run with our
 209  * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
 210  * synchronization object. If we hit one and need to sleep, then the scheduler
 211  * will instead create the rest of what we need.
 212  *
 213  *   Low level software interrupts
 214  *
 215  * Low level software interrupts are handled in a similar way as hardware
 216  * interrupts, but the notification vector is different. Each CPU has a bitmask
 217  * of pending software interrupts. We can notify a CPU to process software
 218  * interrupts through a specific trap vector as well as through several
 219  * checks that are performed throughout the code. These checks will look at
 220  * processing software interrupts as we lower our spl.
 221  *
 222  * We attempt to process the highest pending software interrupt that we can
 223  * which is greater than our current IPL. If none currently exist, then we move
 224  * on. We process a software interrupt in a similar fashion to a hardware
 225  * interrupt.
 226  *
 227  * Traditional Interrupt Flow
 228  * --------------------------
 229  *
 230  * The following diagram tracks the flow of the traditional uppc and pcplusmp
 231  * interrupt handlers. The apix driver has its own version of do_interrupt().
 232  * We come into the interrupt handler with all interrupts masked by the IF
 233  * flag. This is because we set up the handler using an interrupt-gate, which
 234  * is defined architecturally to have cleared the IF flag for us.
 235  *
 236  * +--------------+    +----------------+    +-----------+
 237  * | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
 238  * +--------------+    +----------------+    +-----------+
 239  *                       |      |     |
 240  *                       |      |     |
 241  *              low-level|      |     | softint
 242  *                HW int |      |     +---------------------------------------+
 243  * +--------------+      |      |                                             |
 244  * | intr_thread_ |<-----+      | hi-level int                                |
 245  * | prolog()     |             |    +----------+                             |
 246  * +--------------+             +--->| hilevel_ |      Not on intr stack      |
 247  *       |                           | intr_    |-----------------+           |
 248  *       |                           | prolog() |                 |           |
 249  * +------------+                    +----------+                 |           |
 250  * | switch_sp_ |                        | On intr                v           |
 251  * | and_call() |                        | Stack          +------------+      |
 252  * +------------+                        |                | switch_sp_ |      |
 253  *       |                               v                | and_call() |      |
 254  *       v                             +-----------+      +------------+      |
 255  * +-----------+                       | dispatch_ |             |            |
 256  * | dispatch_ |   +-------------------| hilevel() |<------------+            |
 257  * | hardint() |   |                   +-----------+                          |
 258  * +-----------+   |                                                          |
 259  *       |         v                                                          |
 260  *       |     +-----+  +----------------------+  +-----+  hi-level           |
 261  *       +---->| sti |->| av_dispatch_autovect |->| cli |---------+           |
 262  *             +-----+  +----------------------+  +-----+         |           |
 263  *                                |                |              |           |
 264  *                                v                |              |           |
 265  *                         +----------+            |              |           |
 266  *                         | for each |            |              |           |
 267  *                         | handler  |            |              |           |
 268  *                         |  *intr() |            |              v           |
 269  * +--------------+        +----------+            |      +----------------+  |
 270  * | intr_thread_ |                      low-level |      | hilevel_intr_  |  |
 271  * | epilog()     |<-------------------------------+      | epilog()       |  |
 272  * +--------------+                                       +----------------+  |
 273  *   |       |                                                   |            |
 274  *   |       +----------------------v      v---------------------+            |
 275  *   |                           +------------+                               |
 276  *   |   +---------------------->| *setlvlx() |                               |
 277  *   |   |                       +------------+                               |
 278  *   |   |                              |                                     |
 279  *   |   |                              v                                     |
 280  *   |   |      +--------+     +------------------+      +-------------+      |
 281  *   |   |      | return |<----| softint pending? |----->| dosoftint() |<-----+
 282  *   |   |      +--------+  no +------------------+ yes  +-------------+
 283  *   |   |           ^                                      |     |
 284  *   |   |           |  softint pil too low                 |     |
 285  *   |   |           +--------------------------------------+     |
 286  *   |   |                                                        v
 287  *   |   |    +-----------+      +------------+          +-----------+
 288  *   |   |    | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
 289  *   |   |    | softint() |      | and_call() |          +-----------+
 290  *   |   |    +-----------+      +------------+
 291  *   |   |        |
 292  *   |   |        v
 293  *   |   |      +-----+  +----------------------+  +-----+  +------------+
 294  *   |   |      | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
 295  *   |   |      +-----+  +----------------------+  +-----+  | epilog()   |
 296  *   |   |                                                  +------------+
 297  *   |   |                                                    |     |
 298  *   |   +----------------------------------------------------+     |
 299  *   v                                                              |
 300  * +-----------+                                                    |
 301  * | interrupt |                                                    |
 302  * | thread    |<---------------------------------------------------+
 303  * | blocked   |
 304  * +-----------+
 305  *      |
 306  *      v
 307  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
 308  *  | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
 309  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
 310  *
 311  *    Calls made on Interrupt Stacks and Epilogue routines
 312  *
 313  * We use the switch_sp_and_call() assembly routine to switch our sp to the
 314  * interrupt stacks and then call the appropriate dispatch function. In the
 315  * case of interrupts which may block, softints and hardints, we always ensure
 316  * that we are still on the interrupt thread when we call the epilog routine.
 317  * This is not just important, it's necessary. If the interrupt thread blocked,
 318  * we won't return from our switch_sp_and_call() function and instead we'll go
 319  * through and set ourselves up to swtch() directly.
 320  *
 321  * New Interrupt Flow
 322  * ------------------
 323  *
 324  * The apix module has its own interrupt path. This is done for various
 325  * reasons. The first is that rather than having global interrupt vectors, we
 326  * now have per-cpu vectors.
 327  *
 328  * The other substantial change is that the apix design does not use the TPR to
 329  * mask interrupts below the current level. In fact, except for one special
 330  * case, it does not use the TPR at all. Instead, it only uses the IF flag
 331  * (cli/sti) to either block all interrupts or allow any interrupts to come in.
 332  * The design is such that when interrupts are allowed to come in, if we are
 333  * currently servicing a higher priority interupt, the new interrupt is treated
 334  * as pending and serviced later. Specifically, in the pcplusmp module's
 335  * apic_intr_enter() the code masks interrupts at or below the current
 336  * IPL using the TPR before sending EOI, whereas the apix module's
 337  * apix_intr_enter() simply sends EOI.
 338  *
 339  * The one special case where the apix code uses the TPR is when it calls
 340  * through the apic_reg_ops function pointer apic_write_task_reg in
 341  * apix_init_intr() to initially mask all levels and then finally to enable all
 342  * levels.
 343  *
 344  * Recall that we come into the interrupt handler with all interrupts masked
 345  * by the IF flag. This is because we set up the handler using an
 346  * interrupt-gate which is defined architecturally to have cleared the IF flag
 347  * for us.
 348  *
 349  * +--------------+    +---------------------+
 350  * | _interrupt() |--->| apix_do_interrupt() |
 351  * +--------------+    +---------------------+
 352  *                               |
 353  *                hard int? +----+--------+ softint?
 354  *                          |             | (but no low-level looping)
 355  *                   +-----------+        |
 356  *                   | *setlvl() |        |
 357  * +---------+       +-----------+        +----------------------------------+
 358  * |apix_add_|    check IPL |                                                |
 359  * |pending_ |<-------------+------+----------------------+                  |
 360  * |hardint()|        low-level int|          hi-level int|                  |
 361  * +---------+                     v                      v                  |
 362  *     | check IPL       +-----------------+     +---------------+           |
 363  *  +--+-----+           | apix_intr_      |     | apix_hilevel_ |           |
 364  *  |        |           | thread_prolog() |     | intr_prolog() |           |
 365  *  |      return        +-----------------+     +---------------+           |
 366  *  |                         |                    | On intr                 |
 367  *  |                   +------------+             | stack?  +------------+  |
 368  *  |                   | switch_sp_ |             +---------| switch_sp_ |  |
 369  *  |                   | and_call() |             |         | and_call() |  |
 370  *  |                   +------------+             |         +------------+  |
 371  *  |                         |                    |          |              |
 372  *  |                   +----------------+     +----------------+            |
 373  *  |                   | apix_dispatch_ |     | apix_dispatch_ |            |
 374  *  |                   | lowlevel()     |     | hilevel()      |            |
 375  *  |                   +----------------+     +----------------+            |
 376  *  |                                |             |                         |
 377  *  |                                v             v                         |
 378  *  |                       +-------------------------+                      |
 379  *  |                       |apix_dispatch_by_vector()|----+                 |
 380  *  |                       +-------------------------+    |                 |
 381  *  |               !XC_HI_PIL|         |         |        |                 |
 382  *  |                       +---+   +-------+   +---+      |                 |
 383  *  |                       |sti|   |*intr()|   |cli|      |                 |
 384  *  |                       +---+   +-------+   +---+      |  hi-level?      |
 385  *  |                          +---------------------------+----+            |
 386  *  |                          v                low-level?      v            |
 387  *  |                  +----------------+               +----------------+   |
 388  *  |                  | apix_intr_     |               | apix_hilevel_  |   |
 389  *  |                  | thread_epilog()|               | intr_epilog()  |   |
 390  *  |                  +----------------+               +----------------+   |
 391  *  |                          |                                |            |
 392  *  |        v-----------------+--------------------------------+            |
 393  *  |  +------------+                                                        |
 394  *  |  | *setlvlx() |   +----------------------------------------------------+
 395  *  |  +------------+   |
 396  *  |      |            |            +--------------------------------+ low
 397  *  v      v     v------+            v                                | level
 398  * +------------------+      +------------------+      +-----------+  | pending?
 399  * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_  |--+
 400  * | hilevel()        |      | hardint()        |      | softint() |  |
 401  * +------------------+      +------------------+      +-----------+    return
 402  *     |                       |                         |
 403  *     | while pending         | while pending           | while pending
 404  *     | hi-level              | low-level               | softint
 405  *     |                       |                         |
 406  *  +---------------+        +-----------------+       +-----------------+
 407  *  | apix_hilevel_ |        | apix_intr_      |       | apix_do_        |
 408  *  | intr_prolog() |        | thread_prolog() |       | softint_prolog()|
 409  *  +---------------+        +-----------------+       +-----------------+
 410  *     | On intr                       |                      |
 411  *     | stack? +------------+    +------------+        +------------+
 412  *     +--------| switch_sp_ |    | switch_sp_ |        | switch_sp_ |
 413  *     |        | and_call() |    | and_call() |        | and_call() |
 414  *     |        +------------+    +------------+        +------------+
 415  *     |           |                   |                      |
 416  *  +------------------+   +------------------+   +------------------------+
 417  *  | apix_dispatch_   |   | apix_dispatch_   |   | apix_dispatch_softint()|
 418  *  | pending_hilevel()|   | pending_hardint()|   +------------------------+
 419  *  +------------------+   +------------------+      |    |      |      |
 420  *    |         |           |         |              |    |      |      |
 421  *    | +----------------+  | +----------------+     |    |      |      |
 422  *    | | apix_hilevel_  |  | | apix_intr_     |     |    |      |      |
 423  *    | | intr_epilog()  |  | | thread_epilog()|     |    |      |      |
 424  *    | +----------------+  | +----------------+     |    |      |      |
 425  *    |         |           |       |                |    |      |      |
 426  *    |   +------------+    |  +----------+   +------+    |      |      |
 427  *    |   | *setlvlx() |    |  |*setlvlx()|   |           |      |      |
 428  *    |   +------------+    |  +----------+   |   +----------+   |   +---------+
 429  *    |                     |               +---+ |av_       | +---+ |apix_do_ |
 430  * +---------------------------------+      |sti| |dispatch_ | |cli| |softint_ |
 431  * | apix_dispatch_pending_autovect()|      +---+ |softvect()| +---+ |epilog() |
 432  * +---------------------------------+            +----------+       +---------+
 433  *  |!XC_HI_PIL  |       |         |                    |
 434  * +---+  +-------+    +---+  +----------+          +-------+
 435  * |sti|  |*intr()|    |cli|  |apix_post_|          |*intr()|
 436  * +---+  +-------+    +---+  |hardint() |          +-------+
 437  *                            +----------+
 438  */
 439 
 440 #include <sys/cpuvar.h>
 441 #include <sys/cpu_event.h>
 442 #include <sys/regset.h>
 443 #include <sys/psw.h>
 444 #include <sys/types.h>
 445 #include <sys/thread.h>
 446 #include <sys/systm.h>
 447 #include <sys/segments.h>
 448 #include <sys/pcb.h>
 449 #include <sys/trap.h>
 450 #include <sys/ftrace.h>
 451 #include <sys/traptrace.h>
 452 #include <sys/clock.h>
 453 #include <sys/panic.h>
 454 #include <sys/disp.h>
 455 #include <vm/seg_kp.h>
 456 #include <sys/stack.h>
 457 #include <sys/sysmacros.h>
 458 #include <sys/cmn_err.h>
 459 #include <sys/kstat.h>
 460 #include <sys/smp_impldefs.h>
 461 #include <sys/pool_pset.h>
 462 #include <sys/zone.h>
 463 #include <sys/bitmap.h>
 464 #include <sys/archsystm.h>
 465 #include <sys/machsystm.h>
 466 #include <sys/ontrap.h>
 467 #include <sys/x86_archext.h>
 468 #include <sys/promif.h>
 469 #include <sys/ht.h>
 470 #include <vm/hat_i86.h>
 471 #if defined(__xpv)
 472 #include <sys/hypervisor.h>
 473 #endif
 474 
 475 /* If these fail, then the padding numbers in machcpuvar.h are wrong. */
 476 #if !defined(__xpv)
 477 #define MCOFF(member)   \
 478         (offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, member))
 479 CTASSERT(MCOFF(mcpu_pad) == MACHCPU_SIZE);
 480 CTASSERT(MCOFF(mcpu_pad2) == MMU_PAGESIZE);
 481 CTASSERT((MCOFF(mcpu_kpti) & 0xF) == 0);
 482 CTASSERT(((sizeof (struct kpti_frame)) & 0xF) == 0);
 483 CTASSERT((offsetof(struct kpti_frame, kf_tr_rsp) & 0xF) == 0);
 484 CTASSERT(MCOFF(mcpu_pad3) < 2 * MMU_PAGESIZE);
 485 #endif
 486 
 487 #if defined(__xpv) && defined(DEBUG)
 488 
 489 /*
 490  * This panic message is intended as an aid to interrupt debugging.
 491  *
 492  * The associated assertion tests the condition of enabling
 493  * events when events are already enabled.  The implication
 494  * being that whatever code the programmer thought was
 495  * protected by having events disabled until the second
 496  * enable happened really wasn't protected at all ..
 497  */
 498 
 499 int stistipanic = 1;    /* controls the debug panic check */
 500 const char *stistimsg = "stisti";
 501 ulong_t laststi[NCPU];
 502 
 503 /*
 504  * This variable tracks the last place events were disabled on each cpu
 505  * it assists in debugging when asserts that interrupts are enabled trip.
 506  */
 507 ulong_t lastcli[NCPU];
 508 
 509 #endif
 510 
 511 void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
 512 
 513 void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt;
 514 uintptr_t (*get_intr_handler)(int, short) = NULL;
 515 
 516 /*
 517  * Set cpu's base SPL level to the highest active interrupt level
 518  */
 519 void
 520 set_base_spl(void)
 521 {
 522         struct cpu *cpu = CPU;
 523         uint16_t active = (uint16_t)cpu->cpu_intr_actv;
 524 
 525         cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
 526 }
 527 
 528 /*
 529  * Do all the work necessary to set up the cpu and thread structures
 530  * to dispatch a high-level interrupt.
 531  *
 532  * Returns 0 if we're -not- already on the high-level interrupt stack,
 533  * (and *must* switch to it), non-zero if we are already on that stack.
 534  *
 535  * Called with interrupts masked.
 536  * The 'pil' is already set to the appropriate level for rp->r_trapno.
 537  */
 538 static int
 539 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
 540 {
 541         struct machcpu *mcpu = &cpu->cpu_m;
 542         uint_t mask;
 543         hrtime_t intrtime;
 544         hrtime_t now = tsc_read();
 545 
 546         ASSERT(pil > LOCK_LEVEL);
 547 
 548         if (pil == CBE_HIGH_PIL) {
 549                 cpu->cpu_profile_pil = oldpil;
 550                 if (USERMODE(rp->r_cs)) {
 551                         cpu->cpu_profile_pc = 0;
 552                         cpu->cpu_profile_upc = rp->r_pc;
 553                         cpu->cpu_cpcprofile_pc = 0;
 554                         cpu->cpu_cpcprofile_upc = rp->r_pc;
 555                 } else {
 556                         cpu->cpu_profile_pc = rp->r_pc;
 557                         cpu->cpu_profile_upc = 0;
 558                         cpu->cpu_cpcprofile_pc = rp->r_pc;
 559                         cpu->cpu_cpcprofile_upc = 0;
 560                 }
 561         }
 562 
 563         mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
 564         if (mask != 0) {
 565                 int nestpil;
 566 
 567                 /*
 568                  * We have interrupted another high-level interrupt.
 569                  * Load starting timestamp, compute interval, update
 570                  * cumulative counter.
 571                  */
 572                 nestpil = bsrw_insn((uint16_t)mask);
 573                 ASSERT(nestpil < pil);
 574                 intrtime = now -
 575                     mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
 576                 mcpu->intrstat[nestpil][0] += intrtime;
 577                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 578                 /*
 579                  * Another high-level interrupt is active below this one, so
 580                  * there is no need to check for an interrupt thread.  That
 581                  * will be done by the lowest priority high-level interrupt
 582                  * active.
 583                  */
 584         } else {
 585                 kthread_t *t = cpu->cpu_thread;
 586 
 587                 /*
 588                  * See if we are interrupting a low-level interrupt thread.
 589                  * If so, account for its time slice only if its time stamp
 590                  * is non-zero.
 591                  */
 592                 if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
 593                         intrtime = now - t->t_intr_start;
 594                         mcpu->intrstat[t->t_pil][0] += intrtime;
 595                         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 596                         t->t_intr_start = 0;
 597                 }
 598         }
 599 
 600         ht_begin_intr(pil);
 601 
 602         /*
 603          * Store starting timestamp in CPU structure for this PIL.
 604          */
 605         mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
 606 
 607         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
 608 
 609         if (pil == 15) {
 610                 /*
 611                  * To support reentrant level 15 interrupts, we maintain a
 612                  * recursion count in the top half of cpu_intr_actv.  Only
 613                  * when this count hits zero do we clear the PIL 15 bit from
 614                  * the lower half of cpu_intr_actv.
 615                  */
 616                 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
 617                 (*refcntp)++;
 618         }
 619 
 620         mask = cpu->cpu_intr_actv;
 621 
 622         cpu->cpu_intr_actv |= (1 << pil);
 623 
 624         return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
 625 }
 626 
 627 /*
 628  * Does most of the work of returning from a high level interrupt.
 629  *
 630  * Returns 0 if there are no more high level interrupts (in which
 631  * case we must switch back to the interrupted thread stack) or
 632  * non-zero if there are more (in which case we should stay on it).
 633  *
 634  * Called with interrupts masked
 635  */
 636 static int
 637 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
 638 {
 639         struct machcpu *mcpu = &cpu->cpu_m;
 640         uint_t mask;
 641         hrtime_t intrtime;
 642         hrtime_t now = tsc_read();
 643 
 644         ASSERT(mcpu->mcpu_pri == pil);
 645 
 646         cpu->cpu_stats.sys.intr[pil - 1]++;
 647 
 648         ASSERT(cpu->cpu_intr_actv & (1 << pil));
 649 
 650         if (pil == 15) {
 651                 /*
 652                  * To support reentrant level 15 interrupts, we maintain a
 653                  * recursion count in the top half of cpu_intr_actv.  Only
 654                  * when this count hits zero do we clear the PIL 15 bit from
 655                  * the lower half of cpu_intr_actv.
 656                  */
 657                 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
 658 
 659                 ASSERT(*refcntp > 0);
 660 
 661                 if (--(*refcntp) == 0)
 662                         cpu->cpu_intr_actv &= ~(1 << pil);
 663         } else {
 664                 cpu->cpu_intr_actv &= ~(1 << pil);
 665         }
 666 
 667         ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
 668 
 669         intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
 670         mcpu->intrstat[pil][0] += intrtime;
 671         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 672 
 673         /*
 674          * Check for lower-pil nested high-level interrupt beneath
 675          * current one.  If so, place a starting timestamp in its
 676          * pil_high_start entry.
 677          */
 678         mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
 679         if (mask != 0) {
 680                 int nestpil;
 681 
 682                 /*
 683                  * find PIL of nested interrupt
 684                  */
 685                 nestpil = bsrw_insn((uint16_t)mask);
 686                 ASSERT(nestpil < pil);
 687                 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
 688                 /*
 689                  * (Another high-level interrupt is active below this one,
 690                  * so there is no need to check for an interrupt
 691                  * thread.  That will be done by the lowest priority
 692                  * high-level interrupt active.)
 693                  */
 694         } else {
 695                 /*
 696                  * Check to see if there is a low-level interrupt active.
 697                  * If so, place a starting timestamp in the thread
 698                  * structure.
 699                  */
 700                 kthread_t *t = cpu->cpu_thread;
 701 
 702                 if (t->t_flag & T_INTR_THREAD)
 703                         t->t_intr_start = now;
 704         }
 705 
 706         ht_end_intr();
 707 
 708         mcpu->mcpu_pri = oldpil;
 709         (void) (*setlvlx)(oldpil, vecnum);
 710 
 711         return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
 712 }
 713 
 714 /*
 715  * Set up the cpu, thread and interrupt thread structures for
 716  * executing an interrupt thread.  The new stack pointer of the
 717  * interrupt thread (which *must* be switched to) is returned.
 718  */
 719 static caddr_t
 720 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
 721 {
 722         struct machcpu *mcpu = &cpu->cpu_m;
 723         kthread_t *t, *volatile it;
 724         hrtime_t now = tsc_read();
 725 
 726         ASSERT(pil > 0);
 727         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
 728         cpu->cpu_intr_actv |= (1 << pil);
 729 
 730         /*
 731          * Get set to run an interrupt thread.
 732          * There should always be an interrupt thread, since we
 733          * allocate one for each level on each CPU.
 734          *
 735          * t_intr_start could be zero due to cpu_intr_swtch_enter.
 736          */
 737         t = cpu->cpu_thread;
 738         if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
 739                 hrtime_t intrtime = now - t->t_intr_start;
 740                 mcpu->intrstat[t->t_pil][0] += intrtime;
 741                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 742                 t->t_intr_start = 0;
 743         }
 744 
 745         ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
 746 
 747         t->t_sp = (uintptr_t)stackptr;       /* mark stack in curthread for resume */
 748 
 749         /*
 750          * unlink the interrupt thread off the cpu
 751          *
 752          * Note that the code in kcpc_overflow_intr -relies- on the
 753          * ordering of events here - in particular that t->t_lwp of
 754          * the interrupt thread is set to the pinned thread *before*
 755          * curthread is changed.
 756          */
 757         it = cpu->cpu_intr_thread;
 758         cpu->cpu_intr_thread = it->t_link;
 759         it->t_intr = t;
 760         it->t_lwp = t->t_lwp;
 761 
 762         /*
 763          * (threads on the interrupt thread free list could have state
 764          * preset to TS_ONPROC, but it helps in debugging if
 765          * they're TS_FREE.)
 766          */
 767         it->t_state = TS_ONPROC;
 768 
 769         cpu->cpu_thread = it;                /* new curthread on this cpu */
 770         ht_begin_intr(pil);
 771 
 772         it->t_pil = (uchar_t)pil;
 773         it->t_pri = intr_pri + (pri_t)pil;
 774         it->t_intr_start = now;
 775 
 776         return (it->t_stk);
 777 }
 778 
 779 
 780 #ifdef DEBUG
 781 int intr_thread_cnt;
 782 #endif
 783 
 784 /*
 785  * Called with interrupts disabled
 786  */
 787 static void
 788 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
 789 {
 790         struct machcpu *mcpu = &cpu->cpu_m;
 791         kthread_t *t;
 792         kthread_t *it = cpu->cpu_thread;     /* curthread */
 793         uint_t pil, basespl;
 794         hrtime_t intrtime;
 795         hrtime_t now = tsc_read();
 796 
 797         pil = it->t_pil;
 798         cpu->cpu_stats.sys.intr[pil - 1]++;
 799 
 800         ASSERT(it->t_intr_start != 0);
 801         intrtime = now - it->t_intr_start;
 802         mcpu->intrstat[pil][0] += intrtime;
 803         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 804 
 805         ASSERT(cpu->cpu_intr_actv & (1 << pil));
 806         cpu->cpu_intr_actv &= ~(1 << pil);
 807 
 808         /*
 809          * If there is still an interrupted thread underneath this one
 810          * then the interrupt was never blocked and the return is
 811          * fairly simple.  Otherwise it isn't.
 812          */
 813         if ((t = it->t_intr) == NULL) {
 814                 /*
 815                  * The interrupted thread is no longer pinned underneath
 816                  * the interrupt thread.  This means the interrupt must
 817                  * have blocked, and the interrupted thread has been
 818                  * unpinned, and has probably been running around the
 819                  * system for a while.
 820                  *
 821                  * Since there is no longer a thread under this one, put
 822                  * this interrupt thread back on the CPU's free list and
 823                  * resume the idle thread which will dispatch the next
 824                  * thread to run.
 825                  */
 826 #ifdef DEBUG
 827                 intr_thread_cnt++;
 828 #endif
 829                 cpu->cpu_stats.sys.intrblk++;
 830                 /*
 831                  * Set CPU's base SPL based on active interrupts bitmask
 832                  */
 833                 set_base_spl();
 834                 basespl = cpu->cpu_base_spl;
 835                 mcpu->mcpu_pri = basespl;
 836                 (*setlvlx)(basespl, vec);
 837                 (void) splhigh();
 838                 sti();
 839                 it->t_state = TS_FREE;
 840                 /*
 841                  * Return interrupt thread to pool
 842                  */
 843                 it->t_link = cpu->cpu_intr_thread;
 844                 cpu->cpu_intr_thread = it;
 845                 swtch();
 846                 panic("intr_thread_epilog: swtch returned");
 847                 /*NOTREACHED*/
 848         }
 849 
 850         /*
 851          * Return interrupt thread to the pool
 852          */
 853         it->t_link = cpu->cpu_intr_thread;
 854         cpu->cpu_intr_thread = it;
 855         it->t_state = TS_FREE;
 856 
 857         basespl = cpu->cpu_base_spl;
 858         pil = MAX(oldpil, basespl);
 859         mcpu->mcpu_pri = pil;
 860         (*setlvlx)(pil, vec);
 861         t->t_intr_start = now;
 862         ht_end_intr();
 863         cpu->cpu_thread = t;
 864 }
 865 
 866 /*
 867  * intr_get_time() is a resource for interrupt handlers to determine how
 868  * much time has been spent handling the current interrupt. Such a function
 869  * is needed because higher level interrupts can arrive during the
 870  * processing of an interrupt.  intr_get_time() only returns time spent in the
 871  * current interrupt handler.
 872  *
 873  * The caller must be calling from an interrupt handler running at a pil
 874  * below or at lock level. Timings are not provided for high-level
 875  * interrupts.
 876  *
 877  * The first time intr_get_time() is called while handling an interrupt,
 878  * it returns the time since the interrupt handler was invoked. Subsequent
 879  * calls will return the time since the prior call to intr_get_time(). Time
 880  * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
 881  *
 882  * Theory Of Intrstat[][]:
 883  *
 884  * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
 885  * uint64_ts per pil.
 886  *
 887  * intrstat[pil][0] is a cumulative count of the number of ticks spent
 888  * handling all interrupts at the specified pil on this CPU. It is
 889  * exported via kstats to the user.
 890  *
 891  * intrstat[pil][1] is always a count of ticks less than or equal to the
 892  * value in [0]. The difference between [1] and [0] is the value returned
 893  * by a call to intr_get_time(). At the start of interrupt processing,
 894  * [0] and [1] will be equal (or nearly so). As the interrupt consumes
 895  * time, [0] will increase, but [1] will remain the same. A call to
 896  * intr_get_time() will return the difference, then update [1] to be the
 897  * same as [0]. Future calls will return the time since the last call.
 898  * Finally, when the interrupt completes, [1] is updated to the same as [0].
 899  *
 900  * Implementation:
 901  *
 902  * intr_get_time() works much like a higher level interrupt arriving. It
 903  * "checkpoints" the timing information by incrementing intrstat[pil][0]
 904  * to include elapsed running time, and by setting t_intr_start to rdtsc.
 905  * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
 906  * and updates intrstat[pil][1] to be the same as the new value of
 907  * intrstat[pil][0].
 908  *
 909  * In the normal handling of interrupts, after an interrupt handler returns
 910  * and the code in intr_thread() updates intrstat[pil][0], it then sets
 911  * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
 912  * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
 913  * is 0.
 914  *
 915  * Whenever interrupts arrive on a CPU which is handling a lower pil
 916  * interrupt, they update the lower pil's [0] to show time spent in the
 917  * handler that they've interrupted. This results in a growing discrepancy
 918  * between [0] and [1], which is returned the next time intr_get_time() is
 919  * called. Time spent in the higher-pil interrupt will not be returned in
 920  * the next intr_get_time() call from the original interrupt, because
 921  * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
 922  */
 923 uint64_t
 924 intr_get_time(void)
 925 {
 926         struct cpu *cpu;
 927         struct machcpu *mcpu;
 928         kthread_t *t;
 929         uint64_t time, delta, ret;
 930         uint_t pil;
 931 
 932         cli();
 933         cpu = CPU;
 934         mcpu = &cpu->cpu_m;
 935         t = cpu->cpu_thread;
 936         pil = t->t_pil;
 937         ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
 938         ASSERT(t->t_flag & T_INTR_THREAD);
 939         ASSERT(pil != 0);
 940         ASSERT(t->t_intr_start != 0);
 941 
 942         time = tsc_read();
 943         delta = time - t->t_intr_start;
 944         t->t_intr_start = time;
 945 
 946         time = mcpu->intrstat[pil][0] + delta;
 947         ret = time - mcpu->intrstat[pil][1];
 948         mcpu->intrstat[pil][0] = time;
 949         mcpu->intrstat[pil][1] = time;
 950         cpu->cpu_intracct[cpu->cpu_mstate] += delta;
 951 
 952         sti();
 953         return (ret);
 954 }
 955 
 956 static caddr_t
 957 dosoftint_prolog(
 958         struct cpu *cpu,
 959         caddr_t stackptr,
 960         uint32_t st_pending,
 961         uint_t oldpil)
 962 {
 963         kthread_t *t, *volatile it;
 964         struct machcpu *mcpu = &cpu->cpu_m;
 965         uint_t pil;
 966         hrtime_t now;
 967 
 968 top:
 969         ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
 970 
 971         pil = bsrw_insn((uint16_t)st_pending);
 972         if (pil <= oldpil || pil <= cpu->cpu_base_spl)
 973                 return (0);
 974 
 975         /*
 976          * XX64 Sigh.
 977          *
 978          * This is a transliteration of the i386 assembler code for
 979          * soft interrupts.  One question is "why does this need
 980          * to be atomic?"  One possible race is -other- processors
 981          * posting soft interrupts to us in set_pending() i.e. the
 982          * CPU might get preempted just after the address computation,
 983          * but just before the atomic transaction, so another CPU would
 984          * actually set the original CPU's st_pending bit.  However,
 985          * it looks like it would be simpler to disable preemption there.
 986          * Are there other races for which preemption control doesn't work?
 987          *
 988          * The i386 assembler version -also- checks to see if the bit
 989          * being cleared was actually set; if it wasn't, it rechecks
 990          * for more.  This seems a bit strange, as the only code that
 991          * ever clears the bit is -this- code running with interrupts
 992          * disabled on -this- CPU.  This code would probably be cheaper:
 993          *
 994          * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
 995          *   ~(1 << pil));
 996          *
 997          * and t->t_preempt--/++ around set_pending() even cheaper,
 998          * but at this point, correctness is critical, so we slavishly
 999          * emulate the i386 port.
1000          */
1001         if (atomic_btr32((uint32_t *)
1002             &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
1003                 st_pending = mcpu->mcpu_softinfo.st_pending;
1004                 goto top;
1005         }
1006 
1007         mcpu->mcpu_pri = pil;
1008         (*setspl)(pil);
1009 
1010         now = tsc_read();
1011 
1012         /*
1013          * Get set to run interrupt thread.
1014          * There should always be an interrupt thread since we
1015          * allocate one for each level on the CPU.
1016          */
1017         it = cpu->cpu_intr_thread;
1018         cpu->cpu_intr_thread = it->t_link;
1019 
1020         /* t_intr_start could be zero due to cpu_intr_swtch_enter. */
1021         t = cpu->cpu_thread;
1022         if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
1023                 hrtime_t intrtime = now - t->t_intr_start;
1024                 mcpu->intrstat[pil][0] += intrtime;
1025                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1026                 t->t_intr_start = 0;
1027         }
1028 
1029         /*
1030          * Note that the code in kcpc_overflow_intr -relies- on the
1031          * ordering of events here - in particular that t->t_lwp of
1032          * the interrupt thread is set to the pinned thread *before*
1033          * curthread is changed.
1034          */
1035         it->t_lwp = t->t_lwp;
1036         it->t_state = TS_ONPROC;
1037 
1038         /*
1039          * Push interrupted thread onto list from new thread.
1040          * Set the new thread as the current one.
1041          * Set interrupted thread's T_SP because if it is the idle thread,
1042          * resume() may use that stack between threads.
1043          */
1044 
1045         ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
1046         t->t_sp = (uintptr_t)stackptr;
1047 
1048         it->t_intr = t;
1049         cpu->cpu_thread = it;
1050         ht_begin_intr(pil);
1051 
1052         /*
1053          * Set bit for this pil in CPU's interrupt active bitmask.
1054          */
1055         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1056         cpu->cpu_intr_actv |= (1 << pil);
1057 
1058         /*
1059          * Initialize thread priority level from intr_pri
1060          */
1061         it->t_pil = (uchar_t)pil;
1062         it->t_pri = (pri_t)pil + intr_pri;
1063         it->t_intr_start = now;
1064 
1065         return (it->t_stk);
1066 }
1067 
1068 static void
1069 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
1070 {
1071         struct machcpu *mcpu = &cpu->cpu_m;
1072         kthread_t *t, *it;
1073         uint_t pil, basespl;
1074         hrtime_t intrtime;
1075         hrtime_t now = tsc_read();
1076 
1077         it = cpu->cpu_thread;
1078         pil = it->t_pil;
1079 
1080         cpu->cpu_stats.sys.intr[pil - 1]++;
1081 
1082         ASSERT(cpu->cpu_intr_actv & (1 << pil));
1083         cpu->cpu_intr_actv &= ~(1 << pil);
1084         intrtime = now - it->t_intr_start;
1085         mcpu->intrstat[pil][0] += intrtime;
1086         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1087 
1088         /*
1089          * If there is still an interrupted thread underneath this one
1090          * then the interrupt was never blocked and the return is
1091          * fairly simple.  Otherwise it isn't.
1092          */
1093         if ((t = it->t_intr) == NULL) {
1094                 /*
1095                  * Put thread back on the interrupt thread list.
1096                  * This was an interrupt thread, so set CPU's base SPL.
1097                  */
1098                 set_base_spl();
1099                 it->t_state = TS_FREE;
1100                 it->t_link = cpu->cpu_intr_thread;
1101                 cpu->cpu_intr_thread = it;
1102                 (void) splhigh();
1103                 sti();
1104                 swtch();
1105                 /*NOTREACHED*/
1106                 panic("dosoftint_epilog: swtch returned");
1107         }
1108         it->t_link = cpu->cpu_intr_thread;
1109         cpu->cpu_intr_thread = it;
1110         it->t_state = TS_FREE;
1111         ht_end_intr();
1112         cpu->cpu_thread = t;
1113 
1114         if (t->t_flag & T_INTR_THREAD)
1115                 t->t_intr_start = now;
1116         basespl = cpu->cpu_base_spl;
1117         pil = MAX(oldpil, basespl);
1118         mcpu->mcpu_pri = pil;
1119         (*setspl)(pil);
1120 }
1121 
1122 
1123 /*
1124  * Make the interrupted thread 'to' be runnable.
1125  *
1126  * Since t->t_sp has already been saved, t->t_pc is all
1127  * that needs to be set in this function.
1128  *
1129  * Returns the interrupt level of the interrupt thread.
1130  */
1131 int
1132 intr_passivate(
1133         kthread_t *it,          /* interrupt thread */
1134         kthread_t *t)           /* interrupted thread */
1135 {
1136         extern void _sys_rtt();
1137 
1138         ASSERT(it->t_flag & T_INTR_THREAD);
1139         ASSERT(SA(t->t_sp) == t->t_sp);
1140 
1141         t->t_pc = (uintptr_t)_sys_rtt;
1142         return (it->t_pil);
1143 }
1144 
1145 /*
1146  * Create interrupt kstats for this CPU.
1147  */
1148 void
1149 cpu_create_intrstat(cpu_t *cp)
1150 {
1151         int             i;
1152         kstat_t         *intr_ksp;
1153         kstat_named_t   *knp;
1154         char            name[KSTAT_STRLEN];
1155         zoneid_t        zoneid;
1156 
1157         ASSERT(MUTEX_HELD(&cpu_lock));
1158 
1159         if (pool_pset_enabled())
1160                 zoneid = GLOBAL_ZONEID;
1161         else
1162                 zoneid = ALL_ZONES;
1163 
1164         intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
1165             KSTAT_TYPE_NAMED, PIL_MAX * 2, 0, zoneid);
1166 
1167         /*
1168          * Initialize each PIL's named kstat
1169          */
1170         if (intr_ksp != NULL) {
1171                 intr_ksp->ks_update = cpu_kstat_intrstat_update;
1172                 knp = (kstat_named_t *)intr_ksp->ks_data;
1173                 intr_ksp->ks_private = cp;
1174                 for (i = 0; i < PIL_MAX; i++) {
1175                         (void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
1176                             i + 1);
1177                         kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
1178                         (void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
1179                             i + 1);
1180                         kstat_named_init(&knp[(i * 2) + 1], name,
1181                             KSTAT_DATA_UINT64);
1182                 }
1183                 kstat_install(intr_ksp);
1184         }
1185 }
1186 
1187 /*
1188  * Delete interrupt kstats for this CPU.
1189  */
1190 void
1191 cpu_delete_intrstat(cpu_t *cp)
1192 {
1193         kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
1194 }
1195 
1196 /*
1197  * Convert interrupt statistics from CPU ticks to nanoseconds and
1198  * update kstat.
1199  */
1200 int
1201 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
1202 {
1203         kstat_named_t   *knp = ksp->ks_data;
1204         cpu_t           *cpup = (cpu_t *)ksp->ks_private;
1205         int             i;
1206         hrtime_t        hrt;
1207 
1208         if (rw == KSTAT_WRITE)
1209                 return (EACCES);
1210 
1211         for (i = 0; i < PIL_MAX; i++) {
1212                 hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
1213                 scalehrtimef(&hrt);
1214                 knp[i * 2].value.ui64 = (uint64_t)hrt;
1215                 knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
1216         }
1217 
1218         return (0);
1219 }
1220 
1221 /*
1222  * An interrupt thread is ending a time slice, so compute the interval it
1223  * ran for and update the statistic for its PIL.
1224  */
1225 void
1226 cpu_intr_swtch_enter(kthread_id_t t)
1227 {
1228         uint64_t        interval;
1229         uint64_t        start;
1230         cpu_t           *cpu;
1231 
1232         ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1233         ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1234 
1235         /*
1236          * We could be here with a zero timestamp. This could happen if:
1237          * an interrupt thread which no longer has a pinned thread underneath
1238          * it (i.e. it blocked at some point in its past) has finished running
1239          * its handler. intr_thread() updated the interrupt statistic for its
1240          * PIL and zeroed its timestamp. Since there was no pinned thread to
1241          * return to, swtch() gets called and we end up here.
1242          *
1243          * Note that we use atomic ops below (atomic_cas_64 and
1244          * atomic_add_64), which we don't use in the functions above,
1245          * because we're not called with interrupts blocked, but the
1246          * epilog/prolog functions are.
1247          */
1248         if (t->t_intr_start) {
1249                 do {
1250                         start = t->t_intr_start;
1251                         interval = tsc_read() - start;
1252                 } while (atomic_cas_64(&t->t_intr_start, start, 0) != start);
1253                 cpu = CPU;
1254                 cpu->cpu_m.intrstat[t->t_pil][0] += interval;
1255 
1256                 atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
1257                     interval);
1258         } else
1259                 ASSERT(t->t_intr == NULL);
1260 }
1261 
1262 /*
1263  * An interrupt thread is returning from swtch(). Place a starting timestamp
1264  * in its thread structure.
1265  */
1266 void
1267 cpu_intr_swtch_exit(kthread_id_t t)
1268 {
1269         uint64_t ts;
1270 
1271         ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1272         ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1273 
1274         do {
1275                 ts = t->t_intr_start;
1276         } while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts);
1277 }
1278 
1279 /*
1280  * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
1281  */
1282 /*ARGSUSED*/
1283 static void
1284 dispatch_hilevel(uint_t vector, uint_t arg2)
1285 {
1286         sti();
1287         av_dispatch_autovect(vector);
1288         cli();
1289 }
1290 
1291 /*
1292  * Dispatch a soft interrupt
1293  */
1294 /*ARGSUSED*/
1295 static void
1296 dispatch_softint(uint_t oldpil, uint_t arg2)
1297 {
1298         struct cpu *cpu = CPU;
1299 
1300         sti();
1301         av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
1302         cli();
1303 
1304         /*
1305          * Must run softint_epilog() on the interrupt thread stack, since
1306          * there may not be a return from it if the interrupt thread blocked.
1307          */
1308         dosoftint_epilog(cpu, oldpil);
1309 }
1310 
1311 /*
1312  * Dispatch a normal interrupt
1313  */
1314 static void
1315 dispatch_hardint(uint_t vector, uint_t oldipl)
1316 {
1317         struct cpu *cpu = CPU;
1318 
1319         sti();
1320         av_dispatch_autovect(vector);
1321         cli();
1322 
1323         /*
1324          * Must run intr_thread_epilog() on the interrupt thread stack, since
1325          * there may not be a return from it if the interrupt thread blocked.
1326          */
1327         intr_thread_epilog(cpu, vector, oldipl);
1328 }
1329 
1330 /*
1331  * Deliver any softints the current interrupt priority allows.
1332  * Called with interrupts disabled.
1333  */
1334 void
1335 dosoftint(struct regs *regs)
1336 {
1337         struct cpu *cpu = CPU;
1338         int oldipl;
1339         caddr_t newsp;
1340 
1341         while (cpu->cpu_softinfo.st_pending) {
1342                 oldipl = cpu->cpu_pri;
1343                 newsp = dosoftint_prolog(cpu, (caddr_t)regs,
1344                     cpu->cpu_softinfo.st_pending, oldipl);
1345                 /*
1346                  * If returned stack pointer is NULL, priority is too high
1347                  * to run any of the pending softints now.
1348                  * Break out and they will be run later.
1349                  */
1350                 if (newsp == NULL)
1351                         break;
1352                 switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
1353         }
1354 }
1355 
1356 /*
1357  * Interrupt service routine, called with interrupts disabled.
1358  */
1359 /*ARGSUSED*/
1360 void
1361 do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
1362 {
1363         struct cpu *cpu = CPU;
1364         int newipl, oldipl = cpu->cpu_pri;
1365         uint_t vector;
1366         caddr_t newsp;
1367 
1368 #ifdef TRAPTRACE
1369         ttp->ttr_marker = TT_INTERRUPT;
1370         ttp->ttr_ipl = 0xff;
1371         ttp->ttr_pri = oldipl;
1372         ttp->ttr_spl = cpu->cpu_base_spl;
1373         ttp->ttr_vector = 0xff;
1374 #endif  /* TRAPTRACE */
1375 
1376         cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR);
1377 
1378         ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp;
1379 
1380         /*
1381          * If it's a softint go do it now.
1382          */
1383         if (rp->r_trapno == T_SOFTINT) {
1384                 dosoftint(rp);
1385                 ASSERT(!interrupts_enabled());
1386                 return;
1387         }
1388 
1389         /*
1390          * Raise the interrupt priority.
1391          */
1392         newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
1393 #ifdef TRAPTRACE
1394         ttp->ttr_ipl = newipl;
1395 #endif  /* TRAPTRACE */
1396 
1397         /*
1398          * Bail if it is a spurious interrupt
1399          */
1400         if (newipl == -1)
1401                 return;
1402         cpu->cpu_pri = newipl;
1403         vector = rp->r_trapno;
1404 #ifdef TRAPTRACE
1405         ttp->ttr_vector = vector;
1406 #endif  /* TRAPTRACE */
1407         if (newipl > LOCK_LEVEL) {
1408                 /*
1409                  * High priority interrupts run on this cpu's interrupt stack.
1410                  */
1411                 if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
1412                         newsp = cpu->cpu_intr_stack;
1413                         switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
1414                 } else { /* already on the interrupt stack */
1415                         dispatch_hilevel(vector, 0);
1416                 }
1417                 (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
1418         } else {
1419                 /*
1420                  * Run this interrupt in a separate thread.
1421                  */
1422                 newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
1423                 switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
1424         }
1425 
1426 #if !defined(__xpv)
1427         /*
1428          * Deliver any pending soft interrupts.
1429          */
1430         if (cpu->cpu_softinfo.st_pending)
1431                 dosoftint(rp);
1432 #endif  /* !__xpv */
1433 }
1434 
1435 
1436 /*
1437  * Common tasks always done by _sys_rtt, called with interrupts disabled.
1438  * Returns 1 if returning to userland, 0 if returning to system mode.
1439  */
1440 int
1441 sys_rtt_common(struct regs *rp)
1442 {
1443         kthread_t *tp;
1444         extern void mutex_exit_critical_start();
1445         extern long mutex_exit_critical_size;
1446         extern void mutex_owner_running_critical_start();
1447         extern long mutex_owner_running_critical_size;
1448 
1449 loop:
1450 
1451         /*
1452          * Check if returning to user
1453          */
1454         tp = CPU->cpu_thread;
1455         if (USERMODE(rp->r_cs)) {
1456                 pcb_t *pcb;
1457 
1458                 /*
1459                  * Check if AST pending.
1460                  */
1461                 if (tp->t_astflag) {
1462                         /*
1463                          * Let trap() handle the AST
1464                          */
1465                         sti();
1466                         rp->r_trapno = T_AST;
1467                         trap(rp, (caddr_t)0, CPU->cpu_id);
1468                         cli();
1469                         goto loop;
1470                 }
1471 
1472                 pcb = &tp->t_lwp->lwp_pcb;
1473 
1474                 /*
1475                  * Check to see if we need to initialize the FPU for this
1476                  * thread. This should be an uncommon occurrence, but may happen
1477                  * in the case where the system creates an lwp through an
1478                  * abnormal path such as the agent lwp. Make sure that we still
1479                  * happen to have the FPU in a good state.
1480                  */
1481                 if ((pcb->pcb_fpu.fpu_flags & FPU_EN) == 0) {
1482                         kpreempt_disable();
1483                         fp_seed();
1484                         kpreempt_enable();
1485                         PCB_SET_UPDATE_FPU(pcb);
1486                 }
1487 
1488                 /*
1489                  * We are done if segment registers do not need updating.
1490                  */
1491                 if (!PCB_NEED_UPDATE(pcb))
1492                         return (1);
1493 
1494                 if (PCB_NEED_UPDATE_SEGS(pcb) && update_sregs(rp, tp->t_lwp)) {
1495                         /*
1496                          * 1 or more of the selectors is bad.
1497                          * Deliver a SIGSEGV.
1498                          */
1499                         proc_t *p = ttoproc(tp);
1500 
1501                         sti();
1502                         mutex_enter(&p->p_lock);
1503                         tp->t_lwp->lwp_cursig = SIGSEGV;
1504                         mutex_exit(&p->p_lock);
1505                         psig();
1506                         tp->t_sig_check = 1;
1507                         cli();
1508                 }
1509                 PCB_CLEAR_UPDATE_SEGS(pcb);
1510 
1511                 if (PCB_NEED_UPDATE_FPU(pcb)) {
1512                         fprestore_ctxt(&pcb->pcb_fpu);
1513                 }
1514                 PCB_CLEAR_UPDATE_FPU(pcb);
1515 
1516                 ASSERT0(PCB_NEED_UPDATE(pcb));
1517 
1518                 return (1);
1519         }
1520 
1521 #if !defined(__xpv)
1522         /*
1523          * Assert that we're not trying to return into the syscall return
1524          * trampolines. Things will go baaaaad if we try to do that.
1525          *
1526          * Note that none of these run with interrupts on, so this should
1527          * never happen (even in the sysexit case the STI doesn't take effect
1528          * until after sysexit finishes).
1529          */
1530         extern void tr_sysc_ret_start();
1531         extern void tr_sysc_ret_end();
1532         ASSERT(!(rp->r_pc >= (uintptr_t)tr_sysc_ret_start &&
1533             rp->r_pc <= (uintptr_t)tr_sysc_ret_end));
1534 #endif
1535 
1536         /*
1537          * Here if we are returning to supervisor mode.
1538          * Check for a kernel preemption request.
1539          */
1540         if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1541 
1542                 /*
1543                  * Do nothing if already in kpreempt
1544                  */
1545                 if (!tp->t_preempt_lk) {
1546                         tp->t_preempt_lk = 1;
1547                         sti();
1548                         kpreempt(1); /* asynchronous kpreempt call */
1549                         cli();
1550                         tp->t_preempt_lk = 0;
1551                 }
1552         }
1553 
1554         /*
1555          * If we interrupted the mutex_exit() critical region we must
1556          * reset the PC back to the beginning to prevent missed wakeups
1557          * See the comments in mutex_exit() for details.
1558          */
1559         if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1560             mutex_exit_critical_size) {
1561                 rp->r_pc = (greg_t)mutex_exit_critical_start;
1562         }
1563 
1564         /*
1565          * If we interrupted the mutex_owner_running() critical region we
1566          * must reset the PC back to the beginning to prevent dereferencing
1567          * of a freed thread pointer. See the comments in mutex_owner_running
1568          * for details.
1569          */
1570         if ((uintptr_t)rp->r_pc -
1571             (uintptr_t)mutex_owner_running_critical_start <
1572             mutex_owner_running_critical_size) {
1573                 rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1574         }
1575 
1576         return (0);
1577 }
1578 
1579 void
1580 send_dirint(int cpuid, int int_level)
1581 {
1582         (*send_dirintf)(cpuid, int_level);
1583 }
1584 
1585 #define IS_FAKE_SOFTINT(flag, newpri)           \
1586         (((flag) & PS_IE) &&                                \
1587             (((*get_pending_spl)() > (newpri)) ||    \
1588             bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1589 
1590 /*
1591  * do_splx routine, takes new ipl to set
1592  * returns the old ipl.
1593  * We are careful not to set priority lower than CPU->cpu_base_pri,
1594  * even though it seems we're raising the priority, it could be set
1595  * higher at any time by an interrupt routine, so we must block interrupts
1596  * and look at CPU->cpu_base_pri
1597  */
1598 int
1599 do_splx(int newpri)
1600 {
1601         ulong_t flag;
1602         cpu_t   *cpu;
1603         int     curpri, basepri;
1604 
1605         flag = intr_clear();
1606         cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1607         curpri = cpu->cpu_m.mcpu_pri;
1608         basepri = cpu->cpu_base_spl;
1609         if (newpri < basepri)
1610                 newpri = basepri;
1611         cpu->cpu_m.mcpu_pri = newpri;
1612         (*setspl)(newpri);
1613         /*
1614          * If we are going to reenable interrupts see if new priority level
1615          * allows pending softint delivery.
1616          */
1617         if (IS_FAKE_SOFTINT(flag, newpri))
1618                 fakesoftint();
1619         ASSERT(!interrupts_enabled());
1620         intr_restore(flag);
1621         return (curpri);
1622 }
1623 
1624 /*
1625  * Common spl raise routine, takes new ipl to set
1626  * returns the old ipl, will not lower ipl.
1627  */
1628 int
1629 splr(int newpri)
1630 {
1631         ulong_t flag;
1632         cpu_t   *cpu;
1633         int     curpri, basepri;
1634 
1635         flag = intr_clear();
1636         cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1637         curpri = cpu->cpu_m.mcpu_pri;
1638         /*
1639          * Only do something if new priority is larger
1640          */
1641         if (newpri > curpri) {
1642                 basepri = cpu->cpu_base_spl;
1643                 if (newpri < basepri)
1644                         newpri = basepri;
1645                 cpu->cpu_m.mcpu_pri = newpri;
1646                 (*setspl)(newpri);
1647                 /*
1648                  * See if new priority level allows pending softint delivery
1649                  */
1650                 if (IS_FAKE_SOFTINT(flag, newpri))
1651                         fakesoftint();
1652         }
1653         intr_restore(flag);
1654         return (curpri);
1655 }
1656 
1657 int
1658 getpil(void)
1659 {
1660         return (CPU->cpu_m.mcpu_pri);
1661 }
1662 
1663 int
1664 spl_xcall(void)
1665 {
1666         return (splr(ipltospl(XCALL_PIL)));
1667 }
1668 
1669 int
1670 interrupts_enabled(void)
1671 {
1672         ulong_t flag;
1673 
1674         flag = getflags();
1675         return ((flag & PS_IE) == PS_IE);
1676 }
1677 
1678 #ifdef DEBUG
1679 void
1680 assert_ints_enabled(void)
1681 {
1682         ASSERT(!interrupts_unleashed || interrupts_enabled());
1683 }
1684 #endif  /* DEBUG */