1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2018 Joyent, Inc. All rights reserverd.
25 */
26
27 /*
28 * To understand the present state of interrupt handling on i86pc, we must
29 * first consider the history of interrupt controllers and our way of handling
30 * interrupts.
31 *
32 * History of Interrupt Controllers on i86pc
33 * -----------------------------------------
34 *
35 * Intel 8259 and 8259A
36 *
37 * The first interrupt controller that attained widespread use on i86pc was
38 * the Intel 8259(A) Programmable Interrupt Controller that first saw use with
39 * the 8086. It took up to 8 interrupt sources and combined them into one
40 * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
41 * With the switch to the 8259A, level mode interrupts became possible. For a
42 * long time on i86pc the 8259A was the only way to handle interrupts and it
43 * had its own set of quirks. The 8259A and its corresponding interval timer
44 * the 8254 are programmed using outb and inb instructions.
45 *
46 * Intel Advanced Programmable Interrupt Controller (APIC)
47 *
48 * Starting around the time of the introduction of the P6 family
49 * microarchitecture (i686) Intel introduced a new interrupt controller.
50 * Instead of having the series of slaved 8259A devices, Intel opted to outfit
51 * each processor with a Local APIC (lapic) and to outfit the system with at
52 * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
53 * initially communicated over a dedicated bus, but this has since been
54 * replaced. Each physical core and even hyperthread currently contains its
55 * own local apic, which is not shared. There are a few exceptions for
56 * hyperthreads, but that does not usually concern us.
57 *
58 * Instead of talking directly to 8259 for status, sending End Of Interrupt
59 * (EOI), etc. a microprocessor now communicates directly to the lapic. This
60 * also allows for each microprocessor to be able to have independent controls.
61 * The programming method is different from the 8259. Consumers map the lapic
62 * registers into uncacheable memory to read and manipulate the state.
63 *
64 * The number of addressable interrupt vectors was increased to 256. However
65 * vectors 0-31 are reserved for the processor exception handling, leaving the
66 * remaining vectors for general use. In addition to hardware generated
67 * interrupts, the lapic provides a way for generating inter-processor
68 * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
69 *
70 * AMD ended up implementing the Intel APIC architecture in lieu of their work
71 * with Cyrix.
72 *
73 * Intel x2apic
74 *
75 * The x2apic is an extension to the lapic which started showing up around the
76 * same time as the Sandy Bridge chipsets. It provides a new programming mode
77 * as well as new features. The goal of the x2apic is to solve a few problems
78 * with the previous generation of lapic and the x2apic is backwards compatible
79 * with the previous programming and model. The only downsides to using the
80 * backwards compatibility is that you are not able to take advantage of the new
81 * x2apic features.
82 *
83 * o The APIC ID is increased from an 8-bit value to a 32-bit value. This
84 * increases the maximum number of addressable physical processors beyond
85 * 256. This new ID is assembled in a similar manner as the information that
86 * is obtainable by the extended cpuid topology leaves.
87 *
88 * o A new means of generating IPIs was introduced.
89 *
90 * o Instead of memory mapping the registers, the x2apic only allows for
91 * programming it through a series of wrmsrs. This has important semantic
92 * side effects. Recall that the registers were previously all mapped to
93 * uncachable memory which meant that all operations to the local apic were
94 * serializing instructions. With the switch to using wrmsrs this has been
95 * relaxed and these operations can no longer be assumed to be serializing
96 * instructions.
97 *
98 * Note for the rest of this we are only going to concern ourselves with the
99 * apic and x2apic which practically all of i86pc has been using now for
100 * quite some time.
101 *
102 * Interrupt Priority Levels
103 * -------------------------
104 *
105 * On i86pc systems there are a total of fifteen interrupt priority levels
106 * (ipls) which range from 1-15. Level 0 is for normal processing and
107 * non-interrupt processing. To manipulate these values the family of spl
108 * functions (which date back to UNIX on the PDP-11) are used. Specifically,
109 * splr() to raise the priority level and splx() to lower it. One should not
110 * generally call setspl() directly.
111 *
112 * Both i86pc and the supported SPARC platforms honor the same conventions for
113 * the meaning behind these IPLs. The most important IPL is the platform's
114 * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
115 * sleep on any synchronization object. The only allowed synchronization
116 * primitive is a mutex that has been specifically initialized to be a spin
117 * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
118 * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
119 * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
120 *
121 * Each interrupt that is registered in the system fires at a specific IPL.
122 * Generally most interrupts fire below LOCK_LEVEL.
123 *
124 * PSM Drivers
125 * -----------
126 *
127 * We currently have three sets of PSM (platform specific module) drivers
128 * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
129 * driver that interacts with the 8259A and 8254. In general, it is not used
130 * anymore given the prevalence of the apic.
131 *
132 * The system prefers to use the apix driver over the pcplusmp driver. The apix
133 * driver requires HW support for an x2apic. If there is no x2apic HW, apix
134 * will not be used. In general we prefer using the apix driver over the
135 * pcplusmp driver because it gives us much more flexibility with respect to
136 * interrupts. In the apix driver each local apic has its own independent set
137 * of interrupts, whereas the pcplusmp driver only has a single global set of
138 * interrupts. This is why pcplusmp only supports a finite number of interrupts
139 * per IPL -- generally 16, often less. The apix driver supports using either
140 * the x2apic or the local apic programing modes. The programming mode does not
141 * change the number of interrupts available, just the number of processors
142 * that we can address. For the apix driver, the x2apic mode is enabled if the
143 * system supports interrupt re-mapping, otherwise the module manages the
144 * x2apic in local mode.
145 *
146 * When there is no x2apic present, we default back to the pcplusmp PSM driver.
147 * In general, this is not problematic unless you have more than 256
148 * processors in the machine or you do not have enough interrupts available.
149 *
150 * Controlling Interrupt Generation on i86pc
151 * -----------------------------------------
152 *
153 * There are two different ways to manipulate which interrupts will be
154 * generated on i86pc. Each offers different degrees of control.
155 *
156 * The first is through the flags register (eflags and rflags on i386 and amd64
157 * respectively). The IF bit determines whether or not interrupts are enabled
158 * or disabled. This is manipulated in one of several ways. The most common way
159 * is through the cli and sti instructions. These clear the IF flag and set it,
160 * respectively, for the current processor. The other common way is through the
161 * use of the intr_clear and intr_restore functions.
162 *
163 * Assuming interrupts are not blocked by the IF flag, then the second form is
164 * through the Processor-Priority Register (PPR). The PPR is used to determine
165 * whether or not a pending interrupt should be delivered. If the ipl of the
166 * new interrupt is higher than the current value in the PPR, then the lapic
167 * will either deliver it immediately (if interrupts are not in progress) or it
168 * will deliver it once the current interrupt processing has issued an EOI. The
169 * highest unmasked interrupt will be the one delivered.
170 *
171 * The PPR register is based upon the max of the following two registers in the
172 * lapic, the TPR register (also known as CR8 on amd64) that can be used to
173 * mask interrupt levels, and the current vector. Because the pcplusmp module
174 * always sets TPR appropriately early in the do_interrupt path, we can usually
175 * just think that the PPR is the TPR. The pcplusmp module also issues an EOI
176 * once it has set the TPR, so higher priority interrupts can come in while
177 * we're servicing a lower priority interrupt.
178 *
179 * Handling Interrupts
180 * -------------------
181 *
182 * Interrupts can be broken down into three categories based on priority and
183 * source:
184 *
185 * o High level interrupts
186 * o Low level hardware interrupts
187 * o Low level software interrupts
188 *
189 * High Level Interrupts
190 *
191 * High level interrupts encompasses both hardware-sourced and software-sourced
192 * interrupts. Examples of high level hardware interrupts include the serial
193 * console. High level software-sourced interrupts are still delivered through
194 * the local apic through IPIs. This is primarily cross calls.
195 *
196 * When a high level interrupt comes in, we will raise the SPL and then pin the
197 * current lwp to the processor. We will use its lwp, but our own interrupt
198 * stack and process the high level interrupt in-situ. These handlers are
199 * designed to be very short in nature and cannot go to sleep, only block on a
200 * spin lock. If the interrupt has a lot of work to do, it must generate a
201 * low-priority software interrupt that will be processed later.
202 *
203 * Low level hardware interrupts
204 *
205 * Low level hardware interrupts start off like their high-level cousins. The
206 * current CPU contains a number of kernel threads (kthread_t) that can be used
207 * to process low level interrupts. These are shared between both low level
208 * hardware and software interrupts. Note that while we run with our
209 * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
210 * synchronization object. If we hit one and need to sleep, then the scheduler
211 * will instead create the rest of what we need.
212 *
213 * Low level software interrupts
214 *
215 * Low level software interrupts are handled in a similar way as hardware
216 * interrupts, but the notification vector is different. Each CPU has a bitmask
217 * of pending software interrupts. We can notify a CPU to process software
218 * interrupts through a specific trap vector as well as through several
219 * checks that are performed throughout the code. These checks will look at
220 * processing software interrupts as we lower our spl.
221 *
222 * We attempt to process the highest pending software interrupt that we can
223 * which is greater than our current IPL. If none currently exist, then we move
224 * on. We process a software interrupt in a similar fashion to a hardware
225 * interrupt.
226 *
227 * Traditional Interrupt Flow
228 * --------------------------
229 *
230 * The following diagram tracks the flow of the traditional uppc and pcplusmp
231 * interrupt handlers. The apix driver has its own version of do_interrupt().
232 * We come into the interrupt handler with all interrupts masked by the IF
233 * flag. This is because we set up the handler using an interrupt-gate, which
234 * is defined architecturally to have cleared the IF flag for us.
235 *
236 * +--------------+ +----------------+ +-----------+
237 * | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
238 * +--------------+ +----------------+ +-----------+
239 * | | |
240 * | | |
241 * low-level| | | softint
242 * HW int | | +---------------------------------------+
243 * +--------------+ | | |
244 * | intr_thread_ |<-----+ | hi-level int |
245 * | prolog() | | +----------+ |
246 * +--------------+ +--->| hilevel_ | Not on intr stack |
247 * | | intr_ |-----------------+ |
248 * | | prolog() | | |
249 * +------------+ +----------+ | |
250 * | switch_sp_ | | On intr v |
251 * | and_call() | | Stack +------------+ |
252 * +------------+ | | switch_sp_ | |
253 * | v | and_call() | |
254 * v +-----------+ +------------+ |
255 * +-----------+ | dispatch_ | | |
256 * | dispatch_ | +-------------------| hilevel() |<------------+ |
257 * | hardint() | | +-----------+ |
258 * +-----------+ | |
259 * | v |
260 * | +-----+ +----------------------+ +-----+ hi-level |
261 * +---->| sti |->| av_dispatch_autovect |->| cli |---------+ |
262 * +-----+ +----------------------+ +-----+ | |
263 * | | | |
264 * v | | |
265 * +----------+ | | |
266 * | for each | | | |
267 * | handler | | | |
268 * | *intr() | | v |
269 * +--------------+ +----------+ | +----------------+ |
270 * | intr_thread_ | low-level | | hilevel_intr_ | |
271 * | epilog() |<-------------------------------+ | epilog() | |
272 * +--------------+ +----------------+ |
273 * | | | |
274 * | +----------------------v v---------------------+ |
275 * | +------------+ |
276 * | +---------------------->| *setlvlx() | |
277 * | | +------------+ |
278 * | | | |
279 * | | v |
280 * | | +--------+ +------------------+ +-------------+ |
281 * | | | return |<----| softint pending? |----->| dosoftint() |<-----+
282 * | | +--------+ no +------------------+ yes +-------------+
283 * | | ^ | |
284 * | | | softint pil too low | |
285 * | | +--------------------------------------+ |
286 * | | v
287 * | | +-----------+ +------------+ +-----------+
288 * | | | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
289 * | | | softint() | | and_call() | +-----------+
290 * | | +-----------+ +------------+
291 * | | |
292 * | | v
293 * | | +-----+ +----------------------+ +-----+ +------------+
294 * | | | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
295 * | | +-----+ +----------------------+ +-----+ | epilog() |
296 * | | +------------+
297 * | | | |
298 * | +----------------------------------------------------+ |
299 * v |
300 * +-----------+ |
301 * | interrupt | |
302 * | thread |<---------------------------------------------------+
303 * | blocked |
304 * +-----------+
305 * |
306 * v
307 * +----------------+ +------------+ +-----------+ +-------+ +---------+
308 * | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
309 * +----------------+ +------------+ +-----------+ +-------+ +---------+
310 *
311 * Calls made on Interrupt Stacks and Epilogue routines
312 *
313 * We use the switch_sp_and_call() assembly routine to switch our sp to the
314 * interrupt stacks and then call the appropriate dispatch function. In the
315 * case of interrupts which may block, softints and hardints, we always ensure
316 * that we are still on the interrupt thread when we call the epilog routine.
317 * This is not just important, it's necessary. If the interrupt thread blocked,
318 * we won't return from our switch_sp_and_call() function and instead we'll go
319 * through and set ourselves up to swtch() directly.
320 *
321 * New Interrupt Flow
322 * ------------------
323 *
324 * The apix module has its own interrupt path. This is done for various
325 * reasons. The first is that rather than having global interrupt vectors, we
326 * now have per-cpu vectors.
327 *
328 * The other substantial change is that the apix design does not use the TPR to
329 * mask interrupts below the current level. In fact, except for one special
330 * case, it does not use the TPR at all. Instead, it only uses the IF flag
331 * (cli/sti) to either block all interrupts or allow any interrupts to come in.
332 * The design is such that when interrupts are allowed to come in, if we are
333 * currently servicing a higher priority interupt, the new interrupt is treated
334 * as pending and serviced later. Specifically, in the pcplusmp module's
335 * apic_intr_enter() the code masks interrupts at or below the current
336 * IPL using the TPR before sending EOI, whereas the apix module's
337 * apix_intr_enter() simply sends EOI.
338 *
339 * The one special case where the apix code uses the TPR is when it calls
340 * through the apic_reg_ops function pointer apic_write_task_reg in
341 * apix_init_intr() to initially mask all levels and then finally to enable all
342 * levels.
343 *
344 * Recall that we come into the interrupt handler with all interrupts masked
345 * by the IF flag. This is because we set up the handler using an
346 * interrupt-gate which is defined architecturally to have cleared the IF flag
347 * for us.
348 *
349 * +--------------+ +---------------------+
350 * | _interrupt() |--->| apix_do_interrupt() |
351 * +--------------+ +---------------------+
352 * |
353 * hard int? +----+--------+ softint?
354 * | | (but no low-level looping)
355 * +-----------+ |
356 * | *setlvl() | |
357 * +---------+ +-----------+ +----------------------------------+
358 * |apix_add_| check IPL | |
359 * |pending_ |<-------------+------+----------------------+ |
360 * |hardint()| low-level int| hi-level int| |
361 * +---------+ v v |
362 * | check IPL +-----------------+ +---------------+ |
363 * +--+-----+ | apix_intr_ | | apix_hilevel_ | |
364 * | | | thread_prolog() | | intr_prolog() | |
365 * | return +-----------------+ +---------------+ |
366 * | | | On intr |
367 * | +------------+ | stack? +------------+ |
368 * | | switch_sp_ | +---------| switch_sp_ | |
369 * | | and_call() | | | and_call() | |
370 * | +------------+ | +------------+ |
371 * | | | | |
372 * | +----------------+ +----------------+ |
373 * | | apix_dispatch_ | | apix_dispatch_ | |
374 * | | lowlevel() | | hilevel() | |
375 * | +----------------+ +----------------+ |
376 * | | | |
377 * | v v |
378 * | +-------------------------+ |
379 * | |apix_dispatch_by_vector()|----+ |
380 * | +-------------------------+ | |
381 * | !XC_HI_PIL| | | | |
382 * | +---+ +-------+ +---+ | |
383 * | |sti| |*intr()| |cli| | |
384 * | +---+ +-------+ +---+ | hi-level? |
385 * | +---------------------------+----+ |
386 * | v low-level? v |
387 * | +----------------+ +----------------+ |
388 * | | apix_intr_ | | apix_hilevel_ | |
389 * | | thread_epilog()| | intr_epilog() | |
390 * | +----------------+ +----------------+ |
391 * | | | |
392 * | v-----------------+--------------------------------+ |
393 * | +------------+ |
394 * | | *setlvlx() | +----------------------------------------------------+
395 * | +------------+ |
396 * | | | +--------------------------------+ low
397 * v v v------+ v | level
398 * +------------------+ +------------------+ +-----------+ | pending?
399 * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_ |--+
400 * | hilevel() | | hardint() | | softint() | |
401 * +------------------+ +------------------+ +-----------+ return
402 * | | |
403 * | while pending | while pending | while pending
404 * | hi-level | low-level | softint
405 * | | |
406 * +---------------+ +-----------------+ +-----------------+
407 * | apix_hilevel_ | | apix_intr_ | | apix_do_ |
408 * | intr_prolog() | | thread_prolog() | | softint_prolog()|
409 * +---------------+ +-----------------+ +-----------------+
410 * | On intr | |
411 * | stack? +------------+ +------------+ +------------+
412 * +--------| switch_sp_ | | switch_sp_ | | switch_sp_ |
413 * | | and_call() | | and_call() | | and_call() |
414 * | +------------+ +------------+ +------------+
415 * | | | |
416 * +------------------+ +------------------+ +------------------------+
417 * | apix_dispatch_ | | apix_dispatch_ | | apix_dispatch_softint()|
418 * | pending_hilevel()| | pending_hardint()| +------------------------+
419 * +------------------+ +------------------+ | | | |
420 * | | | | | | | |
421 * | +----------------+ | +----------------+ | | | |
422 * | | apix_hilevel_ | | | apix_intr_ | | | | |
423 * | | intr_epilog() | | | thread_epilog()| | | | |
424 * | +----------------+ | +----------------+ | | | |
425 * | | | | | | | |
426 * | +------------+ | +----------+ +------+ | | |
427 * | | *setlvlx() | | |*setlvlx()| | | | |
428 * | +------------+ | +----------+ | +----------+ | +---------+
429 * | | +---+ |av_ | +---+ |apix_do_ |
430 * +---------------------------------+ |sti| |dispatch_ | |cli| |softint_ |
431 * | apix_dispatch_pending_autovect()| +---+ |softvect()| +---+ |epilog() |
432 * +---------------------------------+ +----------+ +---------+
433 * |!XC_HI_PIL | | | |
434 * +---+ +-------+ +---+ +----------+ +-------+
435 * |sti| |*intr()| |cli| |apix_post_| |*intr()|
436 * +---+ +-------+ +---+ |hardint() | +-------+
437 * +----------+
438 */
439
440 #include <sys/cpuvar.h>
441 #include <sys/cpu_event.h>
442 #include <sys/regset.h>
443 #include <sys/psw.h>
444 #include <sys/types.h>
445 #include <sys/thread.h>
446 #include <sys/systm.h>
447 #include <sys/segments.h>
448 #include <sys/pcb.h>
449 #include <sys/trap.h>
450 #include <sys/ftrace.h>
451 #include <sys/traptrace.h>
452 #include <sys/clock.h>
453 #include <sys/panic.h>
454 #include <sys/disp.h>
455 #include <vm/seg_kp.h>
456 #include <sys/stack.h>
457 #include <sys/sysmacros.h>
458 #include <sys/cmn_err.h>
459 #include <sys/kstat.h>
460 #include <sys/smp_impldefs.h>
461 #include <sys/pool_pset.h>
462 #include <sys/zone.h>
463 #include <sys/bitmap.h>
464 #include <sys/archsystm.h>
465 #include <sys/machsystm.h>
466 #include <sys/ontrap.h>
467 #include <sys/x86_archext.h>
468 #include <sys/promif.h>
469 #include <sys/ht.h>
470 #include <vm/hat_i86.h>
471 #if defined(__xpv)
472 #include <sys/hypervisor.h>
473 #endif
474
475 /* If these fail, then the padding numbers in machcpuvar.h are wrong. */
476 #if !defined(__xpv)
477 #define MCOFF(member) \
478 (offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, member))
479 CTASSERT(MCOFF(mcpu_pad) == MACHCPU_SIZE);
480 CTASSERT(MCOFF(mcpu_pad2) == MMU_PAGESIZE);
481 CTASSERT((MCOFF(mcpu_kpti) & 0xF) == 0);
482 CTASSERT(((sizeof (struct kpti_frame)) & 0xF) == 0);
483 CTASSERT((offsetof(struct kpti_frame, kf_tr_rsp) & 0xF) == 0);
484 CTASSERT(MCOFF(mcpu_pad3) < 2 * MMU_PAGESIZE);
485 #endif
486
487 #if defined(__xpv) && defined(DEBUG)
488
489 /*
490 * This panic message is intended as an aid to interrupt debugging.
491 *
492 * The associated assertion tests the condition of enabling
493 * events when events are already enabled. The implication
494 * being that whatever code the programmer thought was
495 * protected by having events disabled until the second
496 * enable happened really wasn't protected at all ..
497 */
498
499 int stistipanic = 1; /* controls the debug panic check */
500 const char *stistimsg = "stisti";
501 ulong_t laststi[NCPU];
502
503 /*
504 * This variable tracks the last place events were disabled on each cpu
505 * it assists in debugging when asserts that interrupts are enabled trip.
506 */
507 ulong_t lastcli[NCPU];
508
509 #endif
510
511 void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
512
513 void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt;
514 uintptr_t (*get_intr_handler)(int, short) = NULL;
515
516 /*
517 * Set cpu's base SPL level to the highest active interrupt level
518 */
519 void
520 set_base_spl(void)
521 {
522 struct cpu *cpu = CPU;
523 uint16_t active = (uint16_t)cpu->cpu_intr_actv;
524
525 cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
526 }
527
528 /*
529 * Do all the work necessary to set up the cpu and thread structures
530 * to dispatch a high-level interrupt.
531 *
532 * Returns 0 if we're -not- already on the high-level interrupt stack,
533 * (and *must* switch to it), non-zero if we are already on that stack.
534 *
535 * Called with interrupts masked.
536 * The 'pil' is already set to the appropriate level for rp->r_trapno.
537 */
538 static int
539 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
540 {
541 struct machcpu *mcpu = &cpu->cpu_m;
542 uint_t mask;
543 hrtime_t intrtime;
544 hrtime_t now = tsc_read();
545
546 ASSERT(pil > LOCK_LEVEL);
547
548 if (pil == CBE_HIGH_PIL) {
549 cpu->cpu_profile_pil = oldpil;
550 if (USERMODE(rp->r_cs)) {
551 cpu->cpu_profile_pc = 0;
552 cpu->cpu_profile_upc = rp->r_pc;
553 cpu->cpu_cpcprofile_pc = 0;
554 cpu->cpu_cpcprofile_upc = rp->r_pc;
555 } else {
556 cpu->cpu_profile_pc = rp->r_pc;
557 cpu->cpu_profile_upc = 0;
558 cpu->cpu_cpcprofile_pc = rp->r_pc;
559 cpu->cpu_cpcprofile_upc = 0;
560 }
561 }
562
563 mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
564 if (mask != 0) {
565 int nestpil;
566
567 /*
568 * We have interrupted another high-level interrupt.
569 * Load starting timestamp, compute interval, update
570 * cumulative counter.
571 */
572 nestpil = bsrw_insn((uint16_t)mask);
573 ASSERT(nestpil < pil);
574 intrtime = now -
575 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
576 mcpu->intrstat[nestpil][0] += intrtime;
577 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
578 /*
579 * Another high-level interrupt is active below this one, so
580 * there is no need to check for an interrupt thread. That
581 * will be done by the lowest priority high-level interrupt
582 * active.
583 */
584 } else {
585 kthread_t *t = cpu->cpu_thread;
586
587 /*
588 * See if we are interrupting a low-level interrupt thread.
589 * If so, account for its time slice only if its time stamp
590 * is non-zero.
591 */
592 if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
593 intrtime = now - t->t_intr_start;
594 mcpu->intrstat[t->t_pil][0] += intrtime;
595 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
596 t->t_intr_start = 0;
597 }
598 }
599
600 ht_begin_intr(pil);
601
602 /*
603 * Store starting timestamp in CPU structure for this PIL.
604 */
605 mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
606
607 ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
608
609 if (pil == 15) {
610 /*
611 * To support reentrant level 15 interrupts, we maintain a
612 * recursion count in the top half of cpu_intr_actv. Only
613 * when this count hits zero do we clear the PIL 15 bit from
614 * the lower half of cpu_intr_actv.
615 */
616 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
617 (*refcntp)++;
618 }
619
620 mask = cpu->cpu_intr_actv;
621
622 cpu->cpu_intr_actv |= (1 << pil);
623
624 return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
625 }
626
627 /*
628 * Does most of the work of returning from a high level interrupt.
629 *
630 * Returns 0 if there are no more high level interrupts (in which
631 * case we must switch back to the interrupted thread stack) or
632 * non-zero if there are more (in which case we should stay on it).
633 *
634 * Called with interrupts masked
635 */
636 static int
637 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
638 {
639 struct machcpu *mcpu = &cpu->cpu_m;
640 uint_t mask;
641 hrtime_t intrtime;
642 hrtime_t now = tsc_read();
643
644 ASSERT(mcpu->mcpu_pri == pil);
645
646 cpu->cpu_stats.sys.intr[pil - 1]++;
647
648 ASSERT(cpu->cpu_intr_actv & (1 << pil));
649
650 if (pil == 15) {
651 /*
652 * To support reentrant level 15 interrupts, we maintain a
653 * recursion count in the top half of cpu_intr_actv. Only
654 * when this count hits zero do we clear the PIL 15 bit from
655 * the lower half of cpu_intr_actv.
656 */
657 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
658
659 ASSERT(*refcntp > 0);
660
661 if (--(*refcntp) == 0)
662 cpu->cpu_intr_actv &= ~(1 << pil);
663 } else {
664 cpu->cpu_intr_actv &= ~(1 << pil);
665 }
666
667 ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
668
669 intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
670 mcpu->intrstat[pil][0] += intrtime;
671 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
672
673 /*
674 * Check for lower-pil nested high-level interrupt beneath
675 * current one. If so, place a starting timestamp in its
676 * pil_high_start entry.
677 */
678 mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
679 if (mask != 0) {
680 int nestpil;
681
682 /*
683 * find PIL of nested interrupt
684 */
685 nestpil = bsrw_insn((uint16_t)mask);
686 ASSERT(nestpil < pil);
687 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
688 /*
689 * (Another high-level interrupt is active below this one,
690 * so there is no need to check for an interrupt
691 * thread. That will be done by the lowest priority
692 * high-level interrupt active.)
693 */
694 } else {
695 /*
696 * Check to see if there is a low-level interrupt active.
697 * If so, place a starting timestamp in the thread
698 * structure.
699 */
700 kthread_t *t = cpu->cpu_thread;
701
702 if (t->t_flag & T_INTR_THREAD)
703 t->t_intr_start = now;
704 }
705
706 ht_end_intr();
707
708 mcpu->mcpu_pri = oldpil;
709 (void) (*setlvlx)(oldpil, vecnum);
710
711 return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
712 }
713
714 /*
715 * Set up the cpu, thread and interrupt thread structures for
716 * executing an interrupt thread. The new stack pointer of the
717 * interrupt thread (which *must* be switched to) is returned.
718 */
719 static caddr_t
720 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
721 {
722 struct machcpu *mcpu = &cpu->cpu_m;
723 kthread_t *t, *volatile it;
724 hrtime_t now = tsc_read();
725
726 ASSERT(pil > 0);
727 ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
728 cpu->cpu_intr_actv |= (1 << pil);
729
730 /*
731 * Get set to run an interrupt thread.
732 * There should always be an interrupt thread, since we
733 * allocate one for each level on each CPU.
734 *
735 * t_intr_start could be zero due to cpu_intr_swtch_enter.
736 */
737 t = cpu->cpu_thread;
738 if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
739 hrtime_t intrtime = now - t->t_intr_start;
740 mcpu->intrstat[t->t_pil][0] += intrtime;
741 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
742 t->t_intr_start = 0;
743 }
744
745 ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
746
747 t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */
748
749 /*
750 * unlink the interrupt thread off the cpu
751 *
752 * Note that the code in kcpc_overflow_intr -relies- on the
753 * ordering of events here - in particular that t->t_lwp of
754 * the interrupt thread is set to the pinned thread *before*
755 * curthread is changed.
756 */
757 it = cpu->cpu_intr_thread;
758 cpu->cpu_intr_thread = it->t_link;
759 it->t_intr = t;
760 it->t_lwp = t->t_lwp;
761
762 /*
763 * (threads on the interrupt thread free list could have state
764 * preset to TS_ONPROC, but it helps in debugging if
765 * they're TS_FREE.)
766 */
767 it->t_state = TS_ONPROC;
768
769 cpu->cpu_thread = it; /* new curthread on this cpu */
770 ht_begin_intr(pil);
771
772 it->t_pil = (uchar_t)pil;
773 it->t_pri = intr_pri + (pri_t)pil;
774 it->t_intr_start = now;
775
776 return (it->t_stk);
777 }
778
779
780 #ifdef DEBUG
781 int intr_thread_cnt;
782 #endif
783
784 /*
785 * Called with interrupts disabled
786 */
787 static void
788 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
789 {
790 struct machcpu *mcpu = &cpu->cpu_m;
791 kthread_t *t;
792 kthread_t *it = cpu->cpu_thread; /* curthread */
793 uint_t pil, basespl;
794 hrtime_t intrtime;
795 hrtime_t now = tsc_read();
796
797 pil = it->t_pil;
798 cpu->cpu_stats.sys.intr[pil - 1]++;
799
800 ASSERT(it->t_intr_start != 0);
801 intrtime = now - it->t_intr_start;
802 mcpu->intrstat[pil][0] += intrtime;
803 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
804
805 ASSERT(cpu->cpu_intr_actv & (1 << pil));
806 cpu->cpu_intr_actv &= ~(1 << pil);
807
808 /*
809 * If there is still an interrupted thread underneath this one
810 * then the interrupt was never blocked and the return is
811 * fairly simple. Otherwise it isn't.
812 */
813 if ((t = it->t_intr) == NULL) {
814 /*
815 * The interrupted thread is no longer pinned underneath
816 * the interrupt thread. This means the interrupt must
817 * have blocked, and the interrupted thread has been
818 * unpinned, and has probably been running around the
819 * system for a while.
820 *
821 * Since there is no longer a thread under this one, put
822 * this interrupt thread back on the CPU's free list and
823 * resume the idle thread which will dispatch the next
824 * thread to run.
825 */
826 #ifdef DEBUG
827 intr_thread_cnt++;
828 #endif
829 cpu->cpu_stats.sys.intrblk++;
830 /*
831 * Set CPU's base SPL based on active interrupts bitmask
832 */
833 set_base_spl();
834 basespl = cpu->cpu_base_spl;
835 mcpu->mcpu_pri = basespl;
836 (*setlvlx)(basespl, vec);
837 (void) splhigh();
838 sti();
839 it->t_state = TS_FREE;
840 /*
841 * Return interrupt thread to pool
842 */
843 it->t_link = cpu->cpu_intr_thread;
844 cpu->cpu_intr_thread = it;
845 swtch();
846 panic("intr_thread_epilog: swtch returned");
847 /*NOTREACHED*/
848 }
849
850 /*
851 * Return interrupt thread to the pool
852 */
853 it->t_link = cpu->cpu_intr_thread;
854 cpu->cpu_intr_thread = it;
855 it->t_state = TS_FREE;
856
857 basespl = cpu->cpu_base_spl;
858 pil = MAX(oldpil, basespl);
859 mcpu->mcpu_pri = pil;
860 (*setlvlx)(pil, vec);
861 t->t_intr_start = now;
862 ht_end_intr();
863 cpu->cpu_thread = t;
864 }
865
866 /*
867 * intr_get_time() is a resource for interrupt handlers to determine how
868 * much time has been spent handling the current interrupt. Such a function
869 * is needed because higher level interrupts can arrive during the
870 * processing of an interrupt. intr_get_time() only returns time spent in the
871 * current interrupt handler.
872 *
873 * The caller must be calling from an interrupt handler running at a pil
874 * below or at lock level. Timings are not provided for high-level
875 * interrupts.
876 *
877 * The first time intr_get_time() is called while handling an interrupt,
878 * it returns the time since the interrupt handler was invoked. Subsequent
879 * calls will return the time since the prior call to intr_get_time(). Time
880 * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
881 *
882 * Theory Of Intrstat[][]:
883 *
884 * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
885 * uint64_ts per pil.
886 *
887 * intrstat[pil][0] is a cumulative count of the number of ticks spent
888 * handling all interrupts at the specified pil on this CPU. It is
889 * exported via kstats to the user.
890 *
891 * intrstat[pil][1] is always a count of ticks less than or equal to the
892 * value in [0]. The difference between [1] and [0] is the value returned
893 * by a call to intr_get_time(). At the start of interrupt processing,
894 * [0] and [1] will be equal (or nearly so). As the interrupt consumes
895 * time, [0] will increase, but [1] will remain the same. A call to
896 * intr_get_time() will return the difference, then update [1] to be the
897 * same as [0]. Future calls will return the time since the last call.
898 * Finally, when the interrupt completes, [1] is updated to the same as [0].
899 *
900 * Implementation:
901 *
902 * intr_get_time() works much like a higher level interrupt arriving. It
903 * "checkpoints" the timing information by incrementing intrstat[pil][0]
904 * to include elapsed running time, and by setting t_intr_start to rdtsc.
905 * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
906 * and updates intrstat[pil][1] to be the same as the new value of
907 * intrstat[pil][0].
908 *
909 * In the normal handling of interrupts, after an interrupt handler returns
910 * and the code in intr_thread() updates intrstat[pil][0], it then sets
911 * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
912 * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
913 * is 0.
914 *
915 * Whenever interrupts arrive on a CPU which is handling a lower pil
916 * interrupt, they update the lower pil's [0] to show time spent in the
917 * handler that they've interrupted. This results in a growing discrepancy
918 * between [0] and [1], which is returned the next time intr_get_time() is
919 * called. Time spent in the higher-pil interrupt will not be returned in
920 * the next intr_get_time() call from the original interrupt, because
921 * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
922 */
923 uint64_t
924 intr_get_time(void)
925 {
926 struct cpu *cpu;
927 struct machcpu *mcpu;
928 kthread_t *t;
929 uint64_t time, delta, ret;
930 uint_t pil;
931
932 cli();
933 cpu = CPU;
934 mcpu = &cpu->cpu_m;
935 t = cpu->cpu_thread;
936 pil = t->t_pil;
937 ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
938 ASSERT(t->t_flag & T_INTR_THREAD);
939 ASSERT(pil != 0);
940 ASSERT(t->t_intr_start != 0);
941
942 time = tsc_read();
943 delta = time - t->t_intr_start;
944 t->t_intr_start = time;
945
946 time = mcpu->intrstat[pil][0] + delta;
947 ret = time - mcpu->intrstat[pil][1];
948 mcpu->intrstat[pil][0] = time;
949 mcpu->intrstat[pil][1] = time;
950 cpu->cpu_intracct[cpu->cpu_mstate] += delta;
951
952 sti();
953 return (ret);
954 }
955
956 static caddr_t
957 dosoftint_prolog(
958 struct cpu *cpu,
959 caddr_t stackptr,
960 uint32_t st_pending,
961 uint_t oldpil)
962 {
963 kthread_t *t, *volatile it;
964 struct machcpu *mcpu = &cpu->cpu_m;
965 uint_t pil;
966 hrtime_t now;
967
968 top:
969 ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
970
971 pil = bsrw_insn((uint16_t)st_pending);
972 if (pil <= oldpil || pil <= cpu->cpu_base_spl)
973 return (0);
974
975 /*
976 * XX64 Sigh.
977 *
978 * This is a transliteration of the i386 assembler code for
979 * soft interrupts. One question is "why does this need
980 * to be atomic?" One possible race is -other- processors
981 * posting soft interrupts to us in set_pending() i.e. the
982 * CPU might get preempted just after the address computation,
983 * but just before the atomic transaction, so another CPU would
984 * actually set the original CPU's st_pending bit. However,
985 * it looks like it would be simpler to disable preemption there.
986 * Are there other races for which preemption control doesn't work?
987 *
988 * The i386 assembler version -also- checks to see if the bit
989 * being cleared was actually set; if it wasn't, it rechecks
990 * for more. This seems a bit strange, as the only code that
991 * ever clears the bit is -this- code running with interrupts
992 * disabled on -this- CPU. This code would probably be cheaper:
993 *
994 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
995 * ~(1 << pil));
996 *
997 * and t->t_preempt--/++ around set_pending() even cheaper,
998 * but at this point, correctness is critical, so we slavishly
999 * emulate the i386 port.
1000 */
1001 if (atomic_btr32((uint32_t *)
1002 &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
1003 st_pending = mcpu->mcpu_softinfo.st_pending;
1004 goto top;
1005 }
1006
1007 mcpu->mcpu_pri = pil;
1008 (*setspl)(pil);
1009
1010 now = tsc_read();
1011
1012 /*
1013 * Get set to run interrupt thread.
1014 * There should always be an interrupt thread since we
1015 * allocate one for each level on the CPU.
1016 */
1017 it = cpu->cpu_intr_thread;
1018 cpu->cpu_intr_thread = it->t_link;
1019
1020 /* t_intr_start could be zero due to cpu_intr_swtch_enter. */
1021 t = cpu->cpu_thread;
1022 if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
1023 hrtime_t intrtime = now - t->t_intr_start;
1024 mcpu->intrstat[pil][0] += intrtime;
1025 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1026 t->t_intr_start = 0;
1027 }
1028
1029 /*
1030 * Note that the code in kcpc_overflow_intr -relies- on the
1031 * ordering of events here - in particular that t->t_lwp of
1032 * the interrupt thread is set to the pinned thread *before*
1033 * curthread is changed.
1034 */
1035 it->t_lwp = t->t_lwp;
1036 it->t_state = TS_ONPROC;
1037
1038 /*
1039 * Push interrupted thread onto list from new thread.
1040 * Set the new thread as the current one.
1041 * Set interrupted thread's T_SP because if it is the idle thread,
1042 * resume() may use that stack between threads.
1043 */
1044
1045 ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
1046 t->t_sp = (uintptr_t)stackptr;
1047
1048 it->t_intr = t;
1049 cpu->cpu_thread = it;
1050 ht_begin_intr(pil);
1051
1052 /*
1053 * Set bit for this pil in CPU's interrupt active bitmask.
1054 */
1055 ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1056 cpu->cpu_intr_actv |= (1 << pil);
1057
1058 /*
1059 * Initialize thread priority level from intr_pri
1060 */
1061 it->t_pil = (uchar_t)pil;
1062 it->t_pri = (pri_t)pil + intr_pri;
1063 it->t_intr_start = now;
1064
1065 return (it->t_stk);
1066 }
1067
1068 static void
1069 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
1070 {
1071 struct machcpu *mcpu = &cpu->cpu_m;
1072 kthread_t *t, *it;
1073 uint_t pil, basespl;
1074 hrtime_t intrtime;
1075 hrtime_t now = tsc_read();
1076
1077 it = cpu->cpu_thread;
1078 pil = it->t_pil;
1079
1080 cpu->cpu_stats.sys.intr[pil - 1]++;
1081
1082 ASSERT(cpu->cpu_intr_actv & (1 << pil));
1083 cpu->cpu_intr_actv &= ~(1 << pil);
1084 intrtime = now - it->t_intr_start;
1085 mcpu->intrstat[pil][0] += intrtime;
1086 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1087
1088 /*
1089 * If there is still an interrupted thread underneath this one
1090 * then the interrupt was never blocked and the return is
1091 * fairly simple. Otherwise it isn't.
1092 */
1093 if ((t = it->t_intr) == NULL) {
1094 /*
1095 * Put thread back on the interrupt thread list.
1096 * This was an interrupt thread, so set CPU's base SPL.
1097 */
1098 set_base_spl();
1099 it->t_state = TS_FREE;
1100 it->t_link = cpu->cpu_intr_thread;
1101 cpu->cpu_intr_thread = it;
1102 (void) splhigh();
1103 sti();
1104 swtch();
1105 /*NOTREACHED*/
1106 panic("dosoftint_epilog: swtch returned");
1107 }
1108 it->t_link = cpu->cpu_intr_thread;
1109 cpu->cpu_intr_thread = it;
1110 it->t_state = TS_FREE;
1111 ht_end_intr();
1112 cpu->cpu_thread = t;
1113
1114 if (t->t_flag & T_INTR_THREAD)
1115 t->t_intr_start = now;
1116 basespl = cpu->cpu_base_spl;
1117 pil = MAX(oldpil, basespl);
1118 mcpu->mcpu_pri = pil;
1119 (*setspl)(pil);
1120 }
1121
1122
1123 /*
1124 * Make the interrupted thread 'to' be runnable.
1125 *
1126 * Since t->t_sp has already been saved, t->t_pc is all
1127 * that needs to be set in this function.
1128 *
1129 * Returns the interrupt level of the interrupt thread.
1130 */
1131 int
1132 intr_passivate(
1133 kthread_t *it, /* interrupt thread */
1134 kthread_t *t) /* interrupted thread */
1135 {
1136 extern void _sys_rtt();
1137
1138 ASSERT(it->t_flag & T_INTR_THREAD);
1139 ASSERT(SA(t->t_sp) == t->t_sp);
1140
1141 t->t_pc = (uintptr_t)_sys_rtt;
1142 return (it->t_pil);
1143 }
1144
1145 /*
1146 * Create interrupt kstats for this CPU.
1147 */
1148 void
1149 cpu_create_intrstat(cpu_t *cp)
1150 {
1151 int i;
1152 kstat_t *intr_ksp;
1153 kstat_named_t *knp;
1154 char name[KSTAT_STRLEN];
1155 zoneid_t zoneid;
1156
1157 ASSERT(MUTEX_HELD(&cpu_lock));
1158
1159 if (pool_pset_enabled())
1160 zoneid = GLOBAL_ZONEID;
1161 else
1162 zoneid = ALL_ZONES;
1163
1164 intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
1165 KSTAT_TYPE_NAMED, PIL_MAX * 2, 0, zoneid);
1166
1167 /*
1168 * Initialize each PIL's named kstat
1169 */
1170 if (intr_ksp != NULL) {
1171 intr_ksp->ks_update = cpu_kstat_intrstat_update;
1172 knp = (kstat_named_t *)intr_ksp->ks_data;
1173 intr_ksp->ks_private = cp;
1174 for (i = 0; i < PIL_MAX; i++) {
1175 (void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
1176 i + 1);
1177 kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
1178 (void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
1179 i + 1);
1180 kstat_named_init(&knp[(i * 2) + 1], name,
1181 KSTAT_DATA_UINT64);
1182 }
1183 kstat_install(intr_ksp);
1184 }
1185 }
1186
1187 /*
1188 * Delete interrupt kstats for this CPU.
1189 */
1190 void
1191 cpu_delete_intrstat(cpu_t *cp)
1192 {
1193 kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
1194 }
1195
1196 /*
1197 * Convert interrupt statistics from CPU ticks to nanoseconds and
1198 * update kstat.
1199 */
1200 int
1201 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
1202 {
1203 kstat_named_t *knp = ksp->ks_data;
1204 cpu_t *cpup = (cpu_t *)ksp->ks_private;
1205 int i;
1206 hrtime_t hrt;
1207
1208 if (rw == KSTAT_WRITE)
1209 return (EACCES);
1210
1211 for (i = 0; i < PIL_MAX; i++) {
1212 hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
1213 scalehrtimef(&hrt);
1214 knp[i * 2].value.ui64 = (uint64_t)hrt;
1215 knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
1216 }
1217
1218 return (0);
1219 }
1220
1221 /*
1222 * An interrupt thread is ending a time slice, so compute the interval it
1223 * ran for and update the statistic for its PIL.
1224 */
1225 void
1226 cpu_intr_swtch_enter(kthread_id_t t)
1227 {
1228 uint64_t interval;
1229 uint64_t start;
1230 cpu_t *cpu;
1231
1232 ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1233 ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1234
1235 /*
1236 * We could be here with a zero timestamp. This could happen if:
1237 * an interrupt thread which no longer has a pinned thread underneath
1238 * it (i.e. it blocked at some point in its past) has finished running
1239 * its handler. intr_thread() updated the interrupt statistic for its
1240 * PIL and zeroed its timestamp. Since there was no pinned thread to
1241 * return to, swtch() gets called and we end up here.
1242 *
1243 * Note that we use atomic ops below (atomic_cas_64 and
1244 * atomic_add_64), which we don't use in the functions above,
1245 * because we're not called with interrupts blocked, but the
1246 * epilog/prolog functions are.
1247 */
1248 if (t->t_intr_start) {
1249 do {
1250 start = t->t_intr_start;
1251 interval = tsc_read() - start;
1252 } while (atomic_cas_64(&t->t_intr_start, start, 0) != start);
1253 cpu = CPU;
1254 cpu->cpu_m.intrstat[t->t_pil][0] += interval;
1255
1256 atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
1257 interval);
1258 } else
1259 ASSERT(t->t_intr == NULL);
1260 }
1261
1262 /*
1263 * An interrupt thread is returning from swtch(). Place a starting timestamp
1264 * in its thread structure.
1265 */
1266 void
1267 cpu_intr_swtch_exit(kthread_id_t t)
1268 {
1269 uint64_t ts;
1270
1271 ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1272 ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1273
1274 do {
1275 ts = t->t_intr_start;
1276 } while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts);
1277 }
1278
1279 /*
1280 * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
1281 */
1282 /*ARGSUSED*/
1283 static void
1284 dispatch_hilevel(uint_t vector, uint_t arg2)
1285 {
1286 sti();
1287 av_dispatch_autovect(vector);
1288 cli();
1289 }
1290
1291 /*
1292 * Dispatch a soft interrupt
1293 */
1294 /*ARGSUSED*/
1295 static void
1296 dispatch_softint(uint_t oldpil, uint_t arg2)
1297 {
1298 struct cpu *cpu = CPU;
1299
1300 sti();
1301 av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
1302 cli();
1303
1304 /*
1305 * Must run softint_epilog() on the interrupt thread stack, since
1306 * there may not be a return from it if the interrupt thread blocked.
1307 */
1308 dosoftint_epilog(cpu, oldpil);
1309 }
1310
1311 /*
1312 * Dispatch a normal interrupt
1313 */
1314 static void
1315 dispatch_hardint(uint_t vector, uint_t oldipl)
1316 {
1317 struct cpu *cpu = CPU;
1318
1319 sti();
1320 av_dispatch_autovect(vector);
1321 cli();
1322
1323 /*
1324 * Must run intr_thread_epilog() on the interrupt thread stack, since
1325 * there may not be a return from it if the interrupt thread blocked.
1326 */
1327 intr_thread_epilog(cpu, vector, oldipl);
1328 }
1329
1330 /*
1331 * Deliver any softints the current interrupt priority allows.
1332 * Called with interrupts disabled.
1333 */
1334 void
1335 dosoftint(struct regs *regs)
1336 {
1337 struct cpu *cpu = CPU;
1338 int oldipl;
1339 caddr_t newsp;
1340
1341 while (cpu->cpu_softinfo.st_pending) {
1342 oldipl = cpu->cpu_pri;
1343 newsp = dosoftint_prolog(cpu, (caddr_t)regs,
1344 cpu->cpu_softinfo.st_pending, oldipl);
1345 /*
1346 * If returned stack pointer is NULL, priority is too high
1347 * to run any of the pending softints now.
1348 * Break out and they will be run later.
1349 */
1350 if (newsp == NULL)
1351 break;
1352 switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
1353 }
1354 }
1355
1356 /*
1357 * Interrupt service routine, called with interrupts disabled.
1358 */
1359 /*ARGSUSED*/
1360 void
1361 do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
1362 {
1363 struct cpu *cpu = CPU;
1364 int newipl, oldipl = cpu->cpu_pri;
1365 uint_t vector;
1366 caddr_t newsp;
1367
1368 #ifdef TRAPTRACE
1369 ttp->ttr_marker = TT_INTERRUPT;
1370 ttp->ttr_ipl = 0xff;
1371 ttp->ttr_pri = oldipl;
1372 ttp->ttr_spl = cpu->cpu_base_spl;
1373 ttp->ttr_vector = 0xff;
1374 #endif /* TRAPTRACE */
1375
1376 cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR);
1377
1378 ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp;
1379
1380 /*
1381 * If it's a softint go do it now.
1382 */
1383 if (rp->r_trapno == T_SOFTINT) {
1384 dosoftint(rp);
1385 ASSERT(!interrupts_enabled());
1386 return;
1387 }
1388
1389 /*
1390 * Raise the interrupt priority.
1391 */
1392 newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
1393 #ifdef TRAPTRACE
1394 ttp->ttr_ipl = newipl;
1395 #endif /* TRAPTRACE */
1396
1397 /*
1398 * Bail if it is a spurious interrupt
1399 */
1400 if (newipl == -1)
1401 return;
1402 cpu->cpu_pri = newipl;
1403 vector = rp->r_trapno;
1404 #ifdef TRAPTRACE
1405 ttp->ttr_vector = vector;
1406 #endif /* TRAPTRACE */
1407 if (newipl > LOCK_LEVEL) {
1408 /*
1409 * High priority interrupts run on this cpu's interrupt stack.
1410 */
1411 if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
1412 newsp = cpu->cpu_intr_stack;
1413 switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
1414 } else { /* already on the interrupt stack */
1415 dispatch_hilevel(vector, 0);
1416 }
1417 (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
1418 } else {
1419 /*
1420 * Run this interrupt in a separate thread.
1421 */
1422 newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
1423 switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
1424 }
1425
1426 #if !defined(__xpv)
1427 /*
1428 * Deliver any pending soft interrupts.
1429 */
1430 if (cpu->cpu_softinfo.st_pending)
1431 dosoftint(rp);
1432 #endif /* !__xpv */
1433 }
1434
1435
1436 /*
1437 * Common tasks always done by _sys_rtt, called with interrupts disabled.
1438 * Returns 1 if returning to userland, 0 if returning to system mode.
1439 */
1440 int
1441 sys_rtt_common(struct regs *rp)
1442 {
1443 kthread_t *tp;
1444 extern void mutex_exit_critical_start();
1445 extern long mutex_exit_critical_size;
1446 extern void mutex_owner_running_critical_start();
1447 extern long mutex_owner_running_critical_size;
1448
1449 loop:
1450
1451 /*
1452 * Check if returning to user
1453 */
1454 tp = CPU->cpu_thread;
1455 if (USERMODE(rp->r_cs)) {
1456 pcb_t *pcb;
1457
1458 /*
1459 * Check if AST pending.
1460 */
1461 if (tp->t_astflag) {
1462 /*
1463 * Let trap() handle the AST
1464 */
1465 sti();
1466 rp->r_trapno = T_AST;
1467 trap(rp, (caddr_t)0, CPU->cpu_id);
1468 cli();
1469 goto loop;
1470 }
1471
1472 pcb = &tp->t_lwp->lwp_pcb;
1473
1474 /*
1475 * Check to see if we need to initialize the FPU for this
1476 * thread. This should be an uncommon occurrence, but may happen
1477 * in the case where the system creates an lwp through an
1478 * abnormal path such as the agent lwp. Make sure that we still
1479 * happen to have the FPU in a good state.
1480 */
1481 if ((pcb->pcb_fpu.fpu_flags & FPU_EN) == 0) {
1482 kpreempt_disable();
1483 fp_seed();
1484 kpreempt_enable();
1485 PCB_SET_UPDATE_FPU(pcb);
1486 }
1487
1488 /*
1489 * We are done if segment registers do not need updating.
1490 */
1491 if (!PCB_NEED_UPDATE(pcb))
1492 return (1);
1493
1494 if (PCB_NEED_UPDATE_SEGS(pcb) && update_sregs(rp, tp->t_lwp)) {
1495 /*
1496 * 1 or more of the selectors is bad.
1497 * Deliver a SIGSEGV.
1498 */
1499 proc_t *p = ttoproc(tp);
1500
1501 sti();
1502 mutex_enter(&p->p_lock);
1503 tp->t_lwp->lwp_cursig = SIGSEGV;
1504 mutex_exit(&p->p_lock);
1505 psig();
1506 tp->t_sig_check = 1;
1507 cli();
1508 }
1509 PCB_CLEAR_UPDATE_SEGS(pcb);
1510
1511 if (PCB_NEED_UPDATE_FPU(pcb)) {
1512 fprestore_ctxt(&pcb->pcb_fpu);
1513 }
1514 PCB_CLEAR_UPDATE_FPU(pcb);
1515
1516 ASSERT0(PCB_NEED_UPDATE(pcb));
1517
1518 return (1);
1519 }
1520
1521 #if !defined(__xpv)
1522 /*
1523 * Assert that we're not trying to return into the syscall return
1524 * trampolines. Things will go baaaaad if we try to do that.
1525 *
1526 * Note that none of these run with interrupts on, so this should
1527 * never happen (even in the sysexit case the STI doesn't take effect
1528 * until after sysexit finishes).
1529 */
1530 extern void tr_sysc_ret_start();
1531 extern void tr_sysc_ret_end();
1532 ASSERT(!(rp->r_pc >= (uintptr_t)tr_sysc_ret_start &&
1533 rp->r_pc <= (uintptr_t)tr_sysc_ret_end));
1534 #endif
1535
1536 /*
1537 * Here if we are returning to supervisor mode.
1538 * Check for a kernel preemption request.
1539 */
1540 if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1541
1542 /*
1543 * Do nothing if already in kpreempt
1544 */
1545 if (!tp->t_preempt_lk) {
1546 tp->t_preempt_lk = 1;
1547 sti();
1548 kpreempt(1); /* asynchronous kpreempt call */
1549 cli();
1550 tp->t_preempt_lk = 0;
1551 }
1552 }
1553
1554 /*
1555 * If we interrupted the mutex_exit() critical region we must
1556 * reset the PC back to the beginning to prevent missed wakeups
1557 * See the comments in mutex_exit() for details.
1558 */
1559 if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1560 mutex_exit_critical_size) {
1561 rp->r_pc = (greg_t)mutex_exit_critical_start;
1562 }
1563
1564 /*
1565 * If we interrupted the mutex_owner_running() critical region we
1566 * must reset the PC back to the beginning to prevent dereferencing
1567 * of a freed thread pointer. See the comments in mutex_owner_running
1568 * for details.
1569 */
1570 if ((uintptr_t)rp->r_pc -
1571 (uintptr_t)mutex_owner_running_critical_start <
1572 mutex_owner_running_critical_size) {
1573 rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1574 }
1575
1576 return (0);
1577 }
1578
1579 void
1580 send_dirint(int cpuid, int int_level)
1581 {
1582 (*send_dirintf)(cpuid, int_level);
1583 }
1584
1585 #define IS_FAKE_SOFTINT(flag, newpri) \
1586 (((flag) & PS_IE) && \
1587 (((*get_pending_spl)() > (newpri)) || \
1588 bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1589
1590 /*
1591 * do_splx routine, takes new ipl to set
1592 * returns the old ipl.
1593 * We are careful not to set priority lower than CPU->cpu_base_pri,
1594 * even though it seems we're raising the priority, it could be set
1595 * higher at any time by an interrupt routine, so we must block interrupts
1596 * and look at CPU->cpu_base_pri
1597 */
1598 int
1599 do_splx(int newpri)
1600 {
1601 ulong_t flag;
1602 cpu_t *cpu;
1603 int curpri, basepri;
1604
1605 flag = intr_clear();
1606 cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1607 curpri = cpu->cpu_m.mcpu_pri;
1608 basepri = cpu->cpu_base_spl;
1609 if (newpri < basepri)
1610 newpri = basepri;
1611 cpu->cpu_m.mcpu_pri = newpri;
1612 (*setspl)(newpri);
1613 /*
1614 * If we are going to reenable interrupts see if new priority level
1615 * allows pending softint delivery.
1616 */
1617 if (IS_FAKE_SOFTINT(flag, newpri))
1618 fakesoftint();
1619 ASSERT(!interrupts_enabled());
1620 intr_restore(flag);
1621 return (curpri);
1622 }
1623
1624 /*
1625 * Common spl raise routine, takes new ipl to set
1626 * returns the old ipl, will not lower ipl.
1627 */
1628 int
1629 splr(int newpri)
1630 {
1631 ulong_t flag;
1632 cpu_t *cpu;
1633 int curpri, basepri;
1634
1635 flag = intr_clear();
1636 cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1637 curpri = cpu->cpu_m.mcpu_pri;
1638 /*
1639 * Only do something if new priority is larger
1640 */
1641 if (newpri > curpri) {
1642 basepri = cpu->cpu_base_spl;
1643 if (newpri < basepri)
1644 newpri = basepri;
1645 cpu->cpu_m.mcpu_pri = newpri;
1646 (*setspl)(newpri);
1647 /*
1648 * See if new priority level allows pending softint delivery
1649 */
1650 if (IS_FAKE_SOFTINT(flag, newpri))
1651 fakesoftint();
1652 }
1653 intr_restore(flag);
1654 return (curpri);
1655 }
1656
1657 int
1658 getpil(void)
1659 {
1660 return (CPU->cpu_m.mcpu_pri);
1661 }
1662
1663 int
1664 spl_xcall(void)
1665 {
1666 return (splr(ipltospl(XCALL_PIL)));
1667 }
1668
1669 int
1670 interrupts_enabled(void)
1671 {
1672 ulong_t flag;
1673
1674 flag = getflags();
1675 return ((flag & PS_IE) == PS_IE);
1676 }
1677
1678 #ifdef DEBUG
1679 void
1680 assert_ints_enabled(void)
1681 {
1682 ASSERT(!interrupts_unleashed || interrupts_enabled());
1683 }
1684 #endif /* DEBUG */