1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include "assym.h"
26
27 #include <sys/cmn_err.h>
28 #include <sys/ftrace.h>
29 #include <sys/asm_linkage.h>
30 #include <sys/machthread.h>
31 #include <sys/machcpuvar.h>
32 #include <sys/intreg.h>
33 #include <sys/ivintr.h>
34
35 #ifdef TRAPTRACE
36 #include <sys/traptrace.h>
37 #endif /* TRAPTRACE */
38
39
40 /*
41 * (TT 0x40..0x4F, TL>0) Interrupt Level N Handler (N == 1..15)
42 * Register passed from LEVEL_INTERRUPT(level)
43 * %g4 - interrupt request level
44 */
45 ENTRY_NP(pil_interrupt)
46 !
47 ! Register usage
48 ! %g1 - cpu
49 ! %g2 - pointer to intr_vec_t (iv)
50 ! %g4 - pil
51 ! %g3, %g5, %g6, %g7 - temps
52 !
53 ! Grab the first or list head intr_vec_t off the intr_head[pil]
54 ! and panic immediately if list head is NULL. Otherwise, update
55 ! intr_head[pil] to next intr_vec_t on the list and clear softint
56 ! %clear_softint, if next intr_vec_t is NULL.
57 !
58 CPU_ADDR(%g1, %g5) ! %g1 = cpu
59 !
60 ALTENTRY(pil_interrupt_common)
61 sll %g4, CPTRSHIFT, %g5 ! %g5 = offset to the pil entry
62 add %g1, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head
63 add %g6, %g5, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil]
64 ldn [%g6], %g2 ! %g2 = cpu->m_cpu.intr_head[pil]
65 brnz,pt %g2, 0f ! check list head (iv) is NULL
66 nop
67 ba ptl1_panic ! panic, list head (iv) is NULL
68 mov PTL1_BAD_INTR_VEC, %g1
69 0:
70 lduh [%g2 + IV_FLAGS], %g7 ! %g7 = iv->iv_flags
71 and %g7, IV_SOFTINT_MT, %g3 ! %g3 = iv->iv_flags & IV_SOFTINT_MT
72 brz,pt %g3, 1f ! check for multi target softint
73 add %g2, IV_PIL_NEXT, %g7 ! g7% = &iv->iv_pil_next
74 ld [%g1 + CPU_ID], %g3 ! for multi target softint, use cpuid
75 sll %g3, CPTRSHIFT, %g3 ! convert cpuid to offset address
76 add %g7, %g3, %g7 ! %g5 = &iv->iv_xpil_next[cpuid]
77 1:
78 ldn [%g7], %g3 ! %g3 = next intr_vec_t
79 brnz,pn %g3, 2f ! branch if next intr_vec_t non NULL
80 stn %g3, [%g6] ! update cpu->m_cpu.intr_head[pil]
81 add %g1, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail
82 stn %g0, [%g5 + %g6] ! clear cpu->m_cpu.intr_tail[pil]
83 mov 1, %g5 ! %g5 = 1
84 sll %g5, %g4, %g5 ! %g5 = 1 << pil
85 wr %g5, CLEAR_SOFTINT ! clear interrupt on this pil
86 2:
87 #ifdef TRAPTRACE
88 TRACE_PTR(%g5, %g6)
89 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
90 rdpr %tt, %g6
91 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt
92 rdpr %tpc, %g6
93 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc
94 rdpr %tstate, %g6
95 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
96 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp
97 stna %g2, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = first intr_vec
98 stna %g3, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = next intr_vec
99 GET_TRACE_TICK(%g6, %g3)
100 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick
101 sll %g4, CPTRSHIFT, %g3
102 add %g1, INTR_HEAD, %g6
103 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_head[pil]
104 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil]
105 add %g1, INTR_TAIL, %g6
106 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_tail[pil]
107 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil]
108 stna %g4, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil
109 TRACE_NEXT(%g5, %g6, %g3)
110 #endif /* TRAPTRACE */
111 !
112 ! clear the iv_pending flag for this interrupt request
113 !
114 lduh [%g2 + IV_FLAGS], %g3 ! %g3 = iv->iv_flags
115 andn %g3, IV_SOFTINT_PEND, %g3 ! %g3 = !(iv->iv_flags & PEND)
116 sth %g3, [%g2 + IV_FLAGS] ! clear IV_SOFTINT_PEND flag
117 stn %g0, [%g7] ! clear iv->iv_pil_next or
118 ! iv->iv_pil_xnext
119
120 !
121 ! Prepare for sys_trap()
122 !
123 ! Registers passed to sys_trap()
124 ! %g1 - interrupt handler at TL==0
125 ! %g2 - pointer to current intr_vec_t (iv),
126 ! job queue for intr_thread or current_thread
127 ! %g3 - pil
128 ! %g4 - initial pil for handler
129 !
130 ! figure which handler to run and which %pil it starts at
131 ! intr_thread starts at DISP_LEVEL to prevent preemption
132 ! current_thread starts at PIL_MAX to protect cpu_intr_actv
133 !
134 mov %g4, %g3 ! %g3 = %g4, pil
135 cmp %g4, LOCK_LEVEL
136 bg,a,pt %xcc, 3f ! branch if pil > LOCK_LEVEL
137 mov PIL_MAX, %g4 ! %g4 = PIL_MAX (15)
138 sethi %hi(intr_thread), %g1 ! %g1 = intr_thread
139 mov DISP_LEVEL, %g4 ! %g4 = DISP_LEVEL (11)
140 ba,pt %xcc, sys_trap
141 or %g1, %lo(intr_thread), %g1
142 3:
143 sethi %hi(current_thread), %g1 ! %g1 = current_thread
144 ba,pt %xcc, sys_trap
145 or %g1, %lo(current_thread), %g1
146 SET_SIZE(pil_interrupt_common)
147 SET_SIZE(pil_interrupt)
148
149
150 _spurious:
151 .asciz "!interrupt 0x%x at level %d not serviced"
152
153 /*
154 * SERVE_INTR_PRE is called once, just before the first invocation
155 * of SERVE_INTR.
156 *
157 * Registers on entry:
158 *
159 * iv_p, cpu, regs: may be out-registers
160 * ls1, ls2: local scratch registers
161 * os1, os2, os3: scratch registers, may be out
162 */
163
164 #define SERVE_INTR_PRE(iv_p, cpu, ls1, ls2, os1, os2, os3, regs) \
165 mov iv_p, ls1; \
166 mov iv_p, ls2; \
167 SERVE_INTR_TRACE(iv_p, os1, os2, os3, regs);
168
169 /*
170 * SERVE_INTR is called immediately after either SERVE_INTR_PRE or
171 * SERVE_INTR_NEXT, without intervening code. No register values
172 * may be modified.
173 *
174 * After calling SERVE_INTR, the caller must check if os3 is set. If
175 * so, there is another interrupt to process. The caller must call
176 * SERVE_INTR_NEXT, immediately followed by SERVE_INTR.
177 *
178 * Before calling SERVE_INTR_NEXT, the caller may perform accounting
179 * and other actions which need to occur after invocation of an interrupt
180 * handler. However, the values of ls1 and os3 *must* be preserved and
181 * passed unmodified into SERVE_INTR_NEXT.
182 *
183 * Registers on return from SERVE_INTR:
184 *
185 * ls1 - the pil just processed
186 * ls2 - the pointer to intr_vec_t (iv) just processed
187 * os3 - if set, another interrupt needs to be processed
188 * cpu, ls1, os3 - must be preserved if os3 is set
189 */
190
191 #define SERVE_INTR(os5, cpu, ls1, ls2, os1, os2, os3, os4) \
192 ldn [ls1 + IV_HANDLER], os2; \
193 ldn [ls1 + IV_ARG1], %o0; \
194 ldn [ls1 + IV_ARG2], %o1; \
195 call os2; \
196 lduh [ls1 + IV_PIL], ls1; \
197 brnz,pt %o0, 2f; \
198 mov CE_WARN, %o0; \
199 set _spurious, %o1; \
200 mov ls2, %o2; \
201 call cmn_err; \
202 rdpr %pil, %o3; \
203 2: ldn [THREAD_REG + T_CPU], cpu; \
204 sll ls1, 3, os1; \
205 add os1, CPU_STATS_SYS_INTR - 8, os2; \
206 ldx [cpu + os2], os3; \
207 inc os3; \
208 stx os3, [cpu + os2]; \
209 sll ls1, CPTRSHIFT, os2; \
210 add cpu, INTR_HEAD, os1; \
211 add os1, os2, os1; \
212 ldn [os1], os3;
213
214 /*
215 * Registers on entry:
216 *
217 * cpu - cpu pointer (clobbered, set to cpu upon completion)
218 * ls1, os3 - preserved from prior call to SERVE_INTR
219 * ls2 - local scratch reg (not preserved)
220 * os1, os2, os4, os5 - scratch reg, can be out (not preserved)
221 */
222 #define SERVE_INTR_NEXT(os5, cpu, ls1, ls2, os1, os2, os3, os4) \
223 sll ls1, CPTRSHIFT, os4; \
224 add cpu, INTR_HEAD, os1; \
225 rdpr %pstate, ls2; \
226 wrpr ls2, PSTATE_IE, %pstate; \
227 lduh [os3 + IV_FLAGS], os2; \
228 and os2, IV_SOFTINT_MT, os2; \
229 brz,pt os2, 4f; \
230 add os3, IV_PIL_NEXT, os2; \
231 ld [cpu + CPU_ID], os5; \
232 sll os5, CPTRSHIFT, os5; \
233 add os2, os5, os2; \
234 4: ldn [os2], os5; \
235 brnz,pn os5, 5f; \
236 stn os5, [os1 + os4]; \
237 add cpu, INTR_TAIL, os1; \
238 stn %g0, [os1 + os4]; \
239 mov 1, os1; \
240 sll os1, ls1, os1; \
241 wr os1, CLEAR_SOFTINT; \
242 5: lduh [os3 + IV_FLAGS], ls1; \
243 andn ls1, IV_SOFTINT_PEND, ls1; \
244 sth ls1, [os3 + IV_FLAGS]; \
245 stn %g0, [os2]; \
246 wrpr %g0, ls2, %pstate; \
247 mov os3, ls1; \
248 mov os3, ls2; \
249 SERVE_INTR_TRACE2(os5, os1, os2, os3, os4);
250
251 #ifdef TRAPTRACE
252 /*
253 * inum - not modified, _spurious depends on it.
254 */
255 #define SERVE_INTR_TRACE(inum, os1, os2, os3, os4) \
256 rdpr %pstate, os3; \
257 andn os3, PSTATE_IE | PSTATE_AM, os2; \
258 wrpr %g0, os2, %pstate; \
259 TRACE_PTR(os1, os2); \
260 ldn [os4 + PC_OFF], os2; \
261 stna os2, [os1 + TRAP_ENT_TPC]%asi; \
262 ldx [os4 + TSTATE_OFF], os2; \
263 stxa os2, [os1 + TRAP_ENT_TSTATE]%asi; \
264 mov os3, os4; \
265 GET_TRACE_TICK(os2, os3); \
266 stxa os2, [os1 + TRAP_ENT_TICK]%asi; \
267 TRACE_SAVE_TL_GL_REGS(os1, os2); \
268 set TT_SERVE_INTR, os2; \
269 rdpr %pil, os3; \
270 or os2, os3, os2; \
271 stha os2, [os1 + TRAP_ENT_TT]%asi; \
272 stna %sp, [os1 + TRAP_ENT_SP]%asi; \
273 stna inum, [os1 + TRAP_ENT_TR]%asi; \
274 stna %g0, [os1 + TRAP_ENT_F1]%asi; \
275 stna %g0, [os1 + TRAP_ENT_F2]%asi; \
276 stna %g0, [os1 + TRAP_ENT_F3]%asi; \
277 stna %g0, [os1 + TRAP_ENT_F4]%asi; \
278 TRACE_NEXT(os1, os2, os3); \
279 wrpr %g0, os4, %pstate
280 #else /* TRAPTRACE */
281 #define SERVE_INTR_TRACE(inum, os1, os2, os3, os4)
282 #endif /* TRAPTRACE */
283
284 #ifdef TRAPTRACE
285 /*
286 * inum - not modified, _spurious depends on it.
287 */
288 #define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4) \
289 rdpr %pstate, os3; \
290 andn os3, PSTATE_IE | PSTATE_AM, os2; \
291 wrpr %g0, os2, %pstate; \
292 TRACE_PTR(os1, os2); \
293 stna %g0, [os1 + TRAP_ENT_TPC]%asi; \
294 stxa %g0, [os1 + TRAP_ENT_TSTATE]%asi; \
295 mov os3, os4; \
296 GET_TRACE_TICK(os2, os3); \
297 stxa os2, [os1 + TRAP_ENT_TICK]%asi; \
298 TRACE_SAVE_TL_GL_REGS(os1, os2); \
299 set TT_SERVE_INTR, os2; \
300 rdpr %pil, os3; \
301 or os2, os3, os2; \
302 stha os2, [os1 + TRAP_ENT_TT]%asi; \
303 stna %sp, [os1 + TRAP_ENT_SP]%asi; \
304 stna inum, [os1 + TRAP_ENT_TR]%asi; \
305 stna %g0, [os1 + TRAP_ENT_F1]%asi; \
306 stna %g0, [os1 + TRAP_ENT_F2]%asi; \
307 stna %g0, [os1 + TRAP_ENT_F3]%asi; \
308 stna %g0, [os1 + TRAP_ENT_F4]%asi; \
309 TRACE_NEXT(os1, os2, os3); \
310 wrpr %g0, os4, %pstate
311 #else /* TRAPTRACE */
312 #define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4)
313 #endif /* TRAPTRACE */
314
315 #define INTRCNT_LIMIT 16
316
317 /*
318 * Handle an interrupt in a new thread.
319 * Entry:
320 * %o0 = pointer to regs structure
321 * %o1 = pointer to current intr_vec_t (iv) to be processed
322 * %o2 = pil
323 * %sp = on current thread's kernel stack
324 * %o7 = return linkage to trap code
325 * %g7 = current thread
326 * %pstate = normal globals, interrupts enabled,
327 * privileged, fp disabled
328 * %pil = DISP_LEVEL
329 *
330 * Register Usage
331 * %l0 = return linkage
332 * %l1 = pil
333 * %l2 - %l3 = scratch
334 * %l4 - %l7 = reserved for sys_trap
335 * %o2 = cpu
336 * %o3 = intr thread
337 * %o0 = scratch
338 * %o4 - %o5 = scratch
339 */
340 ENTRY_NP(intr_thread)
341 mov %o7, %l0
342 mov %o2, %l1
343 !
344 ! See if we are interrupting another interrupt thread.
345 !
346 lduh [THREAD_REG + T_FLAGS], %o3
347 andcc %o3, T_INTR_THREAD, %g0
348 bz,pt %xcc, 1f
349 ldn [THREAD_REG + T_CPU], %o2 ! delay - load CPU pointer
350
351 ! We have interrupted an interrupt thread. Take a timestamp,
352 ! compute its interval, and update its cumulative counter.
353 add THREAD_REG, T_INTR_START, %o5
354 0:
355 ldx [%o5], %o3
356 brz,pn %o3, 1f
357 ! We came in on top of an interrupt thread that had no timestamp.
358 ! This could happen if, for instance, an interrupt thread which had
359 ! previously blocked is being set up to run again in resume(), but
360 ! resume() hasn't yet stored a timestamp for it. Or, it could be in
361 ! swtch() after its slice has been accounted for.
362 ! Only account for the time slice if the starting timestamp is non-zero.
363 RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
364 sub %o4, %o3, %o4 ! o4 has interval
365
366 ! A high-level interrupt in current_thread() interrupting here
367 ! will account for the interrupted thread's time slice, but
368 ! only if t_intr_start is non-zero. Since this code is going to account
369 ! for the time slice, we want to "atomically" load the thread's
370 ! starting timestamp, calculate the interval with %tick, and zero
371 ! its starting timestamp.
372 ! To do this, we do a casx on the t_intr_start field, and store 0 to it.
373 ! If it has changed since we loaded it above, we need to re-compute the
374 ! interval, since a changed t_intr_start implies current_thread placed
375 ! a new, later timestamp there after running a high-level interrupt,
376 ! and the %tick val in %o4 had become stale.
377 mov %g0, %l2
378 casx [%o5], %o3, %l2
379
380 ! If %l2 == %o3, our casx was successful. If not, the starting timestamp
381 ! changed between loading it (after label 0b) and computing the
382 ! interval above.
383 cmp %l2, %o3
384 bne,pn %xcc, 0b
385
386 ! Check for Energy Star mode
387 lduh [%o2 + CPU_DIVISOR], %l2 ! delay -- %l2 = clock divisor
388 cmp %l2, 1
389 bg,a,pn %xcc, 2f
390 mulx %o4, %l2, %o4 ! multiply interval by clock divisor iff > 1
391 2:
392 ! We now know that a valid interval for the interrupted interrupt
393 ! thread is in %o4. Update its cumulative counter.
394 ldub [THREAD_REG + T_PIL], %l3 ! load PIL
395 sllx %l3, 4, %l3 ! convert PIL index to byte offset
396 add %l3, CPU_MCPU, %l3 ! CPU_INTRSTAT is too big for use
397 add %l3, MCPU_INTRSTAT, %l3 ! as const, add offsets separately
398 ldx [%o2 + %l3], %o5 ! old counter in o5
399 add %o5, %o4, %o5 ! new counter in o5
400 stx %o5, [%o2 + %l3] ! store new counter
401
402 ! Also update intracct[]
403 lduh [%o2 + CPU_MSTATE], %l3
404 sllx %l3, 3, %l3
405 add %l3, CPU_INTRACCT, %l3
406 add %l3, %o2, %l3
407 0:
408 ldx [%l3], %o5
409 add %o5, %o4, %o3
410 casx [%l3], %o5, %o3
411 cmp %o5, %o3
412 bne,pn %xcc, 0b
413 nop
414
415 1:
416 !
417 ! Get set to run interrupt thread.
418 ! There should always be an interrupt thread since we allocate one
419 ! for each level on the CPU.
420 !
421 ! Note that the code in kcpc_overflow_intr -relies- on the ordering
422 ! of events here -- in particular that t->t_lwp of the interrupt thread
423 ! is set to the pinned thread *before* curthread is changed.
424 !
425 ldn [%o2 + CPU_INTR_THREAD], %o3 ! interrupt thread pool
426 ldn [%o3 + T_LINK], %o4 ! unlink thread from CPU's list
427 stn %o4, [%o2 + CPU_INTR_THREAD]
428 !
429 ! Set bit for this level in CPU's active interrupt bitmask.
430 !
431 ld [%o2 + CPU_INTR_ACTV], %o5
432 mov 1, %o4
433 sll %o4, %l1, %o4
434 #ifdef DEBUG
435 !
436 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
437 !
438 andcc %o5, %o4, %g0
439 bz,pt %xcc, 0f
440 nop
441 ! Do not call panic if a panic is already in progress.
442 sethi %hi(panic_quiesce), %l2
443 ld [%l2 + %lo(panic_quiesce)], %l2
444 brnz,pn %l2, 0f
445 nop
446 sethi %hi(intr_thread_actv_bit_set), %o0
447 call panic
448 or %o0, %lo(intr_thread_actv_bit_set), %o0
449 0:
450 #endif /* DEBUG */
451 or %o5, %o4, %o5
452 st %o5, [%o2 + CPU_INTR_ACTV]
453 !
454 ! Consider the new thread part of the same LWP so that
455 ! window overflow code can find the PCB.
456 !
457 ldn [THREAD_REG + T_LWP], %o4
458 stn %o4, [%o3 + T_LWP]
459 !
460 ! Threads on the interrupt thread free list could have state already
461 ! set to TS_ONPROC, but it helps in debugging if they're TS_FREE
462 ! Could eliminate the next two instructions with a little work.
463 !
464 mov TS_ONPROC, %o4
465 st %o4, [%o3 + T_STATE]
466 !
467 ! Push interrupted thread onto list from new thread.
468 ! Set the new thread as the current one.
469 ! Set interrupted thread's T_SP because if it is the idle thread,
470 ! resume may use that stack between threads.
471 !
472 stn %o7, [THREAD_REG + T_PC] ! mark pc for resume
473 stn %sp, [THREAD_REG + T_SP] ! mark stack for resume
474 stn THREAD_REG, [%o3 + T_INTR] ! push old thread
475 stn %o3, [%o2 + CPU_THREAD] ! set new thread
476 mov %o3, THREAD_REG ! set global curthread register
477 ldn [%o3 + T_STACK], %o4 ! interrupt stack pointer
478 sub %o4, STACK_BIAS, %sp
479 !
480 ! Initialize thread priority level from intr_pri
481 !
482 sethi %hi(intr_pri), %o4
483 ldsh [%o4 + %lo(intr_pri)], %o4 ! grab base interrupt priority
484 add %l1, %o4, %o4 ! convert level to dispatch priority
485 sth %o4, [THREAD_REG + T_PRI]
486 stub %l1, [THREAD_REG + T_PIL] ! save pil for intr_passivate
487
488 ! Store starting timestamp in thread structure.
489 add THREAD_REG, T_INTR_START, %o3
490 1:
491 ldx [%o3], %o5
492 RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
493 casx [%o3], %o5, %o4
494 cmp %o4, %o5
495 ! If a high-level interrupt occurred while we were attempting to store
496 ! the timestamp, try again.
497 bne,pn %xcc, 1b
498 nop
499
500 wrpr %g0, %l1, %pil ! lower %pil to new level
501 !
502 ! Fast event tracing.
503 !
504 ld [%o2 + CPU_FTRACE_STATE], %o4 ! %o2 = curthread->t_cpu
505 btst FTRACE_ENABLED, %o4
506 be,pt %icc, 1f ! skip if ftrace disabled
507 mov %l1, %o5
508 !
509 ! Tracing is enabled - write the trace entry.
510 !
511 save %sp, -SA(MINFRAME), %sp
512 set ftrace_intr_thread_format_str, %o0
513 mov %i0, %o1
514 mov %i1, %o2
515 mov %i5, %o3
516 call ftrace_3
517 ldn [%i0 + PC_OFF], %o4
518 restore
519 1:
520 !
521 ! call the handler
522 !
523 SERVE_INTR_PRE(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
524 !
525 ! %o0 and %o1 are now available as scratch registers.
526 !
527 0:
528 SERVE_INTR(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
529 !
530 ! If %o3 is set, we must call serve_intr_next, and both %l1 and %o3
531 ! must be preserved. %l1 holds our pil, %l3 holds our inum.
532 !
533 ! Note: %l1 is the pil level we're processing, but we may have a
534 ! higher effective pil because a higher-level interrupt may have
535 ! blocked.
536 !
537 wrpr %g0, DISP_LEVEL, %pil
538 !
539 ! Take timestamp, compute interval, update cumulative counter.
540 !
541 add THREAD_REG, T_INTR_START, %o5
542 1:
543 ldx [%o5], %o0
544 #ifdef DEBUG
545 brnz %o0, 9f
546 nop
547 ! Do not call panic if a panic is already in progress.
548 sethi %hi(panic_quiesce), %o1
549 ld [%o1 + %lo(panic_quiesce)], %o1
550 brnz,pn %o1, 9f
551 nop
552 sethi %hi(intr_thread_t_intr_start_zero), %o0
553 call panic
554 or %o0, %lo(intr_thread_t_intr_start_zero), %o0
555 9:
556 #endif /* DEBUG */
557 RD_CLOCK_TICK(%o1,%l2,%l3,__LINE__)
558 sub %o1, %o0, %l2 ! l2 has interval
559 !
560 ! The general outline of what the code here does is:
561 ! 1. load t_intr_start, %tick, and calculate the delta
562 ! 2. replace t_intr_start with %tick (if %o3 is set) or 0.
563 !
564 ! The problem is that a high-level interrupt could arrive at any time.
565 ! It will account for (%tick - t_intr_start) for us when it starts,
566 ! unless we have set t_intr_start to zero, and then set t_intr_start
567 ! to a new %tick when it finishes. To account for this, our first step
568 ! is to load t_intr_start and the last is to use casx to store the new
569 ! t_intr_start. This guarantees atomicity in reading t_intr_start,
570 ! reading %tick, and updating t_intr_start.
571 !
572 movrz %o3, %g0, %o1
573 casx [%o5], %o0, %o1
574 cmp %o0, %o1
575 bne,pn %xcc, 1b
576 !
577 ! Check for Energy Star mode
578 !
579 lduh [%o2 + CPU_DIVISOR], %o0 ! delay -- %o0 = clock divisor
580 cmp %o0, 1
581 bg,a,pn %xcc, 2f
582 mulx %l2, %o0, %l2 ! multiply interval by clock divisor iff > 1
583 2:
584 !
585 ! Update cpu_intrstat. If o3 is set then we will be processing another
586 ! interrupt. Above we have set t_intr_start to %tick, not 0. This
587 ! means a high-level interrupt can arrive and update the same stats
588 ! we're updating. Need to use casx.
589 !
590 sllx %l1, 4, %o1 ! delay - PIL as byte offset
591 add %o1, CPU_MCPU, %o1 ! CPU_INTRSTAT const too big
592 add %o1, MCPU_INTRSTAT, %o1 ! add parts separately
593 add %o1, %o2, %o1
594 1:
595 ldx [%o1], %o5 ! old counter in o5
596 add %o5, %l2, %o0 ! new counter in o0
597 stx %o0, [%o1 + 8] ! store into intrstat[pil][1]
598 casx [%o1], %o5, %o0 ! and into intrstat[pil][0]
599 cmp %o5, %o0
600 bne,pn %xcc, 1b
601 nop
602
603 ! Also update intracct[]
604 lduh [%o2 + CPU_MSTATE], %o1
605 sllx %o1, 3, %o1
606 add %o1, CPU_INTRACCT, %o1
607 add %o1, %o2, %o1
608 1:
609 ldx [%o1], %o5
610 add %o5, %l2, %o0
611 casx [%o1], %o5, %o0
612 cmp %o5, %o0
613 bne,pn %xcc, 1b
614 nop
615
616 !
617 ! Don't keep a pinned process pinned indefinitely. Bump cpu_intrcnt
618 ! for each interrupt handler we invoke. If we hit INTRCNT_LIMIT, then
619 ! we've crossed the threshold and we should unpin the pinned threads
620 ! by preempt()ing ourselves, which will bubble up the t_intr chain
621 ! until hitting the non-interrupt thread, which will then in turn
622 ! preempt itself allowing the interrupt processing to resume. Finally,
623 ! the scheduler takes over and picks the next thread to run.
624 !
625 ! If our CPU is quiesced, we cannot preempt because the idle thread
626 ! won't ever re-enter the scheduler, and the interrupt will be forever
627 ! blocked.
628 !
629 ! If t_intr is NULL, we're not pinning anyone, so we use a simpler
630 ! algorithm. Just check for cpu_kprunrun, and if set then preempt.
631 ! This insures we enter the scheduler if a higher-priority thread
632 ! has become runnable.
633 !
634 lduh [%o2 + CPU_FLAGS], %o5 ! don't preempt if quiesced
635 andcc %o5, CPU_QUIESCED, %g0
636 bnz,pn %xcc, 1f
637
638 ldn [THREAD_REG + T_INTR], %o5 ! pinning anything?
639 brz,pn %o5, 3f ! if not, don't inc intrcnt
640
641 ldub [%o2 + CPU_INTRCNT], %o5 ! delay - %o5 = cpu_intrcnt
642 inc %o5
643 cmp %o5, INTRCNT_LIMIT ! have we hit the limit?
644 bl,a,pt %xcc, 1f ! no preempt if < INTRCNT_LIMIT
645 stub %o5, [%o2 + CPU_INTRCNT] ! delay annul - inc CPU_INTRCNT
646 bg,pn %xcc, 2f ! don't inc stats again
647 !
648 ! We've reached the limit. Set cpu_intrcnt and cpu_kprunrun, and do
649 ! CPU_STATS_ADDQ(cp, sys, intrunpin, 1). Then call preempt.
650 !
651 mov 1, %o4 ! delay
652 stub %o4, [%o2 + CPU_KPRUNRUN]
653 ldx [%o2 + CPU_STATS_SYS_INTRUNPIN], %o4
654 inc %o4
655 stx %o4, [%o2 + CPU_STATS_SYS_INTRUNPIN]
656 ba 2f
657 stub %o5, [%o2 + CPU_INTRCNT] ! delay
658 3:
659 ! Code for t_intr == NULL
660 ldub [%o2 + CPU_KPRUNRUN], %o5
661 brz,pt %o5, 1f ! don't preempt unless kprunrun
662 2:
663 ! Time to call preempt
664 mov %o2, %l3 ! delay - save %o2
665 call preempt
666 mov %o3, %l2 ! delay - save %o3.
667 mov %l3, %o2 ! restore %o2
668 mov %l2, %o3 ! restore %o3
669 wrpr %g0, DISP_LEVEL, %pil ! up from cpu_base_spl
670 1:
671 !
672 ! Do we need to call serve_intr_next and do this again?
673 !
674 brz,a,pt %o3, 0f
675 ld [%o2 + CPU_INTR_ACTV], %o5 ! delay annulled
676 !
677 ! Restore %pil before calling serve_intr() again. We must check
678 ! CPU_BASE_SPL and set %pil to max(our-pil, CPU_BASE_SPL)
679 !
680 ld [%o2 + CPU_BASE_SPL], %o4
681 cmp %o4, %l1
682 movl %xcc, %l1, %o4
683 wrpr %g0, %o4, %pil
684 SERVE_INTR_NEXT(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
685 ba 0b ! compute new stats
686 nop
687 0:
688 !
689 ! Clear bit for this level in CPU's interrupt active bitmask.
690 !
691 mov 1, %o4
692 sll %o4, %l1, %o4
693 #ifdef DEBUG
694 !
695 ! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
696 !
697 andcc %o4, %o5, %g0
698 bnz,pt %xcc, 0f
699 nop
700 ! Do not call panic if a panic is already in progress.
701 sethi %hi(panic_quiesce), %l2
702 ld [%l2 + %lo(panic_quiesce)], %l2
703 brnz,pn %l2, 0f
704 nop
705 sethi %hi(intr_thread_actv_bit_not_set), %o0
706 call panic
707 or %o0, %lo(intr_thread_actv_bit_not_set), %o0
708 0:
709 #endif /* DEBUG */
710 andn %o5, %o4, %o5
711 st %o5, [%o2 + CPU_INTR_ACTV]
712 !
713 ! If there is still an interrupted thread underneath this one,
714 ! then the interrupt was never blocked and the return is fairly
715 ! simple. Otherwise jump to intr_thread_exit.
716 !
717 ldn [THREAD_REG + T_INTR], %o4 ! pinned thread
718 brz,pn %o4, intr_thread_exit ! branch if none
719 nop
720 !
721 ! link the thread back onto the interrupt thread pool
722 !
723 ldn [%o2 + CPU_INTR_THREAD], %o3
724 stn %o3, [THREAD_REG + T_LINK]
725 stn THREAD_REG, [%o2 + CPU_INTR_THREAD]
726 !
727 ! set the thread state to free so kernel debuggers don't see it
728 !
729 mov TS_FREE, %o5
730 st %o5, [THREAD_REG + T_STATE]
731 !
732 ! Switch back to the interrupted thread and return
733 !
734 stn %o4, [%o2 + CPU_THREAD]
735 membar #StoreLoad ! sync with mutex_exit()
736 mov %o4, THREAD_REG
737
738 ! If we pinned an interrupt thread, store its starting timestamp.
739 lduh [THREAD_REG + T_FLAGS], %o5
740 andcc %o5, T_INTR_THREAD, %g0
741 bz,pt %xcc, 1f
742 ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp
743
744 add THREAD_REG, T_INTR_START, %o3 ! o3 has &curthread->t_intr_star
745 0:
746 ldx [%o3], %o4 ! o4 = t_intr_start before
747 RD_CLOCK_TICK(%o5,%l2,%l3,__LINE__)
748 casx [%o3], %o4, %o5 ! put o5 in ts if o4 == ts after
749 cmp %o4, %o5
750 ! If a high-level interrupt occurred while we were attempting to store
751 ! the timestamp, try again.
752 bne,pn %xcc, 0b
753 ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp
754 1:
755 ! If the thread being restarted isn't pinning anyone, and no interrupts
756 ! are pending, zero out cpu_intrcnt
757 ldn [THREAD_REG + T_INTR], %o4
758 brnz,pn %o4, 2f
759 rd SOFTINT, %o4 ! delay
760 set SOFTINT_MASK, %o5
761 andcc %o4, %o5, %g0
762 bz,a,pt %xcc, 2f
763 stub %g0, [%o2 + CPU_INTRCNT] ! delay annul
764 2:
765 jmp %l0 + 8
766 nop
767 SET_SIZE(intr_thread)
768 /* Not Reached */
769
770 !
771 ! An interrupt returned on what was once (and still might be)
772 ! an interrupt thread stack, but the interrupted process is no longer
773 ! there. This means the interrupt must have blocked.
774 !
775 ! There is no longer a thread under this one, so put this thread back
776 ! on the CPU's free list and resume the idle thread which will dispatch
777 ! the next thread to run.
778 !
779 ! All traps below DISP_LEVEL are disabled here, but the mondo interrupt
780 ! is enabled.
781 !
782 ENTRY_NP(intr_thread_exit)
783 #ifdef TRAPTRACE
784 rdpr %pstate, %l2
785 andn %l2, PSTATE_IE | PSTATE_AM, %o4
786 wrpr %g0, %o4, %pstate ! cpu to known state
787 TRACE_PTR(%o4, %o5)
788 GET_TRACE_TICK(%o5, %o0)
789 stxa %o5, [%o4 + TRAP_ENT_TICK]%asi
790 TRACE_SAVE_TL_GL_REGS(%o4, %o5)
791 set TT_INTR_EXIT, %o5
792 stha %o5, [%o4 + TRAP_ENT_TT]%asi
793 stna %g0, [%o4 + TRAP_ENT_TPC]%asi
794 stxa %g0, [%o4 + TRAP_ENT_TSTATE]%asi
795 stna %sp, [%o4 + TRAP_ENT_SP]%asi
796 stna THREAD_REG, [%o4 + TRAP_ENT_TR]%asi
797 ld [%o2 + CPU_BASE_SPL], %o5
798 stna %o5, [%o4 + TRAP_ENT_F1]%asi
799 stna %g0, [%o4 + TRAP_ENT_F2]%asi
800 stna %g0, [%o4 + TRAP_ENT_F3]%asi
801 stna %g0, [%o4 + TRAP_ENT_F4]%asi
802 TRACE_NEXT(%o4, %o5, %o0)
803 wrpr %g0, %l2, %pstate
804 #endif /* TRAPTRACE */
805 ! cpu_stats.sys.intrblk++
806 ldx [%o2 + CPU_STATS_SYS_INTRBLK], %o4
807 inc %o4
808 stx %o4, [%o2 + CPU_STATS_SYS_INTRBLK]
809 !
810 ! Put thread back on the interrupt thread list.
811 !
812
813 !
814 ! Set the CPU's base SPL level.
815 !
816 #ifdef DEBUG
817 !
818 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
819 !
820 ld [%o2 + CPU_INTR_ACTV], %o5
821 mov 1, %o4
822 sll %o4, %l1, %o4
823 and %o5, %o4, %o4
824 brz,pt %o4, 0f
825 nop
826 ! Do not call panic if a panic is already in progress.
827 sethi %hi(panic_quiesce), %l2
828 ld [%l2 + %lo(panic_quiesce)], %l2
829 brnz,pn %l2, 0f
830 nop
831 sethi %hi(intr_thread_exit_actv_bit_set), %o0
832 call panic
833 or %o0, %lo(intr_thread_exit_actv_bit_set), %o0
834 0:
835 #endif /* DEBUG */
836 call _intr_set_spl ! set CPU's base SPL level
837 ld [%o2 + CPU_INTR_ACTV], %o5 ! delay - load active mask
838 !
839 ! set the thread state to free so kernel debuggers don't see it
840 !
841 mov TS_FREE, %o4
842 st %o4, [THREAD_REG + T_STATE]
843 !
844 ! Put thread on either the interrupt pool or the free pool and
845 ! call swtch() to resume another thread.
846 !
847 ldn [%o2 + CPU_INTR_THREAD], %o5 ! get list pointer
848 stn %o5, [THREAD_REG + T_LINK]
849 call swtch ! switch to best thread
850 stn THREAD_REG, [%o2 + CPU_INTR_THREAD] ! delay - put thread on list
851 ba,a,pt %xcc, . ! swtch() shouldn't return
852 SET_SIZE(intr_thread_exit)
853
854 .global ftrace_intr_thread_format_str
855 ftrace_intr_thread_format_str:
856 .asciz "intr_thread(): regs=0x%lx, int=0x%lx, pil=0x%lx"
857 #ifdef DEBUG
858 intr_thread_actv_bit_set:
859 .asciz "intr_thread(): cpu_intr_actv bit already set for PIL"
860 intr_thread_actv_bit_not_set:
861 .asciz "intr_thread(): cpu_intr_actv bit not set for PIL"
862 intr_thread_exit_actv_bit_set:
863 .asciz "intr_thread_exit(): cpu_intr_actv bit erroneously set for PIL"
864 intr_thread_t_intr_start_zero:
865 .asciz "intr_thread(): t_intr_start zero upon handler return"
866 #endif /* DEBUG */
867
868 /*
869 * Handle an interrupt in the current thread
870 * Entry:
871 * %o0 = pointer to regs structure
872 * %o1 = pointer to current intr_vec_t (iv) to be processed
873 * %o2 = pil
874 * %sp = on current thread's kernel stack
875 * %o7 = return linkage to trap code
876 * %g7 = current thread
877 * %pstate = normal globals, interrupts enabled,
878 * privileged, fp disabled
879 * %pil = PIL_MAX
880 *
881 * Register Usage
882 * %l0 = return linkage
883 * %l1 = old stack
884 * %l2 - %l3 = scratch
885 * %l4 - %l7 = reserved for sys_trap
886 * %o3 = cpu
887 * %o0 = scratch
888 * %o4 - %o5 = scratch
889 */
890 ENTRY_NP(current_thread)
891
892 mov %o7, %l0
893 ldn [THREAD_REG + T_CPU], %o3
894
895 ldn [THREAD_REG + T_ONFAULT], %l2
896 brz,pt %l2, no_onfault ! branch if no onfault label set
897 nop
898 stn %g0, [THREAD_REG + T_ONFAULT]! clear onfault label
899 ldn [THREAD_REG + T_LOFAULT], %l3
900 stn %g0, [THREAD_REG + T_LOFAULT]! clear lofault data
901
902 sub %o2, LOCK_LEVEL + 1, %o5
903 sll %o5, CPTRSHIFT, %o5
904 add %o5, CPU_OFD, %o4 ! %o4 has on_fault data offset
905 stn %l2, [%o3 + %o4] ! save onfault label for pil %o2
906 add %o5, CPU_LFD, %o4 ! %o4 has lofault data offset
907 stn %l3, [%o3 + %o4] ! save lofault data for pil %o2
908
909 no_onfault:
910 ldn [THREAD_REG + T_ONTRAP], %l2
911 brz,pt %l2, 6f ! branch if no on_trap protection
912 nop
913 stn %g0, [THREAD_REG + T_ONTRAP]! clear on_trap protection
914 sub %o2, LOCK_LEVEL + 1, %o5
915 sll %o5, CPTRSHIFT, %o5
916 add %o5, CPU_OTD, %o4 ! %o4 has on_trap data offset
917 stn %l2, [%o3 + %o4] ! save on_trap label for pil %o2
918
919 !
920 ! Set bit for this level in CPU's active interrupt bitmask.
921 !
922 6: ld [%o3 + CPU_INTR_ACTV], %o5 ! o5 has cpu_intr_actv b4 chng
923 mov 1, %o4
924 sll %o4, %o2, %o4 ! construct mask for level
925 #ifdef DEBUG
926 !
927 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
928 !
929 andcc %o5, %o4, %g0
930 bz,pt %xcc, 0f
931 nop
932 ! Do not call panic if a panic is already in progress.
933 sethi %hi(panic_quiesce), %l2
934 ld [%l2 + %lo(panic_quiesce)], %l2
935 brnz,pn %l2, 0f
936 nop
937 sethi %hi(current_thread_actv_bit_set), %o0
938 call panic
939 or %o0, %lo(current_thread_actv_bit_set), %o0
940 0:
941 #endif /* DEBUG */
942 or %o5, %o4, %o4
943 !
944 ! See if we are interrupting another high-level interrupt.
945 !
946 srl %o5, LOCK_LEVEL + 1, %o5 ! only look at high-level bits
947 brz,pt %o5, 1f
948 st %o4, [%o3 + CPU_INTR_ACTV] ! delay - store active mask
949 !
950 ! We have interrupted another high-level interrupt. Find its PIL,
951 ! compute the interval it ran for, and update its cumulative counter.
952 !
953 ! Register usage:
954
955 ! o2 = PIL of this interrupt
956 ! o5 = high PIL bits of INTR_ACTV (not including this PIL)
957 ! l1 = bitmask used to find other active high-level PIL
958 ! o4 = index of bit set in l1
959 ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
960 ! interrupted high-level interrupt.
961 ! Create mask for cpu_intr_actv. Begin by looking for bits set
962 ! at one level below the current PIL. Since %o5 contains the active
963 ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
964 ! at bit (current_pil - (LOCK_LEVEL + 2)).
965 sub %o2, LOCK_LEVEL + 2, %o4
966 mov 1, %l1
967 sll %l1, %o4, %l1
968 2:
969 #ifdef DEBUG
970 ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
971 brnz,pt %l1, 9f
972 nop
973
974 ! Don't panic if a panic is already in progress.
975 sethi %hi(panic_quiesce), %l3
976 ld [%l3 + %lo(panic_quiesce)], %l3
977 brnz,pn %l3, 9f
978 nop
979 sethi %hi(current_thread_nested_PIL_not_found), %o0
980 call panic
981 or %o0, %lo(current_thread_nested_PIL_not_found), %o0
982 9:
983 #endif /* DEBUG */
984 andcc %l1, %o5, %g0 ! test mask against high-level bits of
985 bnz %xcc, 3f ! cpu_intr_actv
986 nop
987 srl %l1, 1, %l1 ! No match. Try next lower PIL.
988 ba,pt %xcc, 2b
989 sub %o4, 1, %o4 ! delay - decrement PIL
990 3:
991 sll %o4, 3, %o4 ! index to byte offset
992 add %o4, CPU_MCPU, %l1 ! CPU_PIL_HIGH_START is too large
993 add %l1, MCPU_PIL_HIGH_START, %l1
994 ldx [%o3 + %l1], %l3 ! load starting timestamp
995 #ifdef DEBUG
996 brnz,pt %l3, 9f
997 nop
998 ! Don't panic if a panic is already in progress.
999 sethi %hi(panic_quiesce), %l1
1000 ld [%l1 + %lo(panic_quiesce)], %l1
1001 brnz,pn %l1, 9f
1002 nop
1003 srl %o4, 3, %o1 ! Find interrupted PIL for panic
1004 add %o1, LOCK_LEVEL + 1, %o1
1005 sethi %hi(current_thread_nested_pil_zero), %o0
1006 call panic
1007 or %o0, %lo(current_thread_nested_pil_zero), %o0
1008 9:
1009 #endif /* DEBUG */
1010 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%l1, %l2)
1011 sub %l1, %l3, %l3 ! interval in %l3
1012 !
1013 ! Check for Energy Star mode
1014 !
1015 lduh [%o3 + CPU_DIVISOR], %l1 ! %l1 = clock divisor
1016 cmp %l1, 1
1017 bg,a,pn %xcc, 2f
1018 mulx %l3, %l1, %l3 ! multiply interval by clock divisor iff > 1
1019 2:
1020 !
1021 ! We need to find the CPU offset of the cumulative counter. We start
1022 ! with %o4, which has (PIL - (LOCK_LEVEL + 1)) * 8. We need PIL * 16,
1023 ! so we shift left 1, then add (LOCK_LEVEL + 1) * 16, which is
1024 ! CPU_INTRSTAT_LOW_PIL_OFFSET.
1025 !
1026 sll %o4, 1, %o4
1027 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large
1028 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately
1029 add %o4, CPU_INTRSTAT_LOW_PIL_OFFSET, %o4
1030 ldx [%o3 + %o4], %l1 ! old counter in l1
1031 add %l1, %l3, %l1 ! new counter in l1
1032 stx %l1, [%o3 + %o4] ! store new counter
1033
1034 ! Also update intracct[]
1035 lduh [%o3 + CPU_MSTATE], %o4
1036 sllx %o4, 3, %o4
1037 add %o4, CPU_INTRACCT, %o4
1038 ldx [%o3 + %o4], %l1
1039 add %l1, %l3, %l1
1040 ! Another high-level interrupt is active below this one, so
1041 ! there is no need to check for an interrupt thread. That will be
1042 ! done by the lowest priority high-level interrupt active.
1043 ba,pt %xcc, 5f
1044 stx %l1, [%o3 + %o4] ! delay - store new counter
1045 1:
1046 ! If we haven't interrupted another high-level interrupt, we may be
1047 ! interrupting a low level interrupt thread. If so, compute its interval
1048 ! and update its cumulative counter.
1049 lduh [THREAD_REG + T_FLAGS], %o4
1050 andcc %o4, T_INTR_THREAD, %g0
1051 bz,pt %xcc, 4f
1052 nop
1053
1054 ! We have interrupted an interrupt thread. Take timestamp, compute
1055 ! interval, update cumulative counter.
1056
1057 ! Check t_intr_start. If it is zero, either intr_thread() or
1058 ! current_thread() (at a lower PIL, of course) already did
1059 ! the accounting for the underlying interrupt thread.
1060 ldx [THREAD_REG + T_INTR_START], %o5
1061 brz,pn %o5, 4f
1062 nop
1063
1064 stx %g0, [THREAD_REG + T_INTR_START]
1065 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
1066 sub %o4, %o5, %o5 ! o5 has the interval
1067
1068 ! Check for Energy Star mode
1069 lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor
1070 cmp %o4, 1
1071 bg,a,pn %xcc, 2f
1072 mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1
1073 2:
1074 ldub [THREAD_REG + T_PIL], %o4
1075 sllx %o4, 4, %o4 ! PIL index to byte offset
1076 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large
1077 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately
1078 ldx [%o3 + %o4], %l2 ! old counter in l2
1079 add %l2, %o5, %l2 ! new counter in l2
1080 stx %l2, [%o3 + %o4] ! store new counter
1081
1082 ! Also update intracct[]
1083 lduh [%o3 + CPU_MSTATE], %o4
1084 sllx %o4, 3, %o4
1085 add %o4, CPU_INTRACCT, %o4
1086 ldx [%o3 + %o4], %l2
1087 add %l2, %o5, %l2
1088 stx %l2, [%o3 + %o4]
1089 4:
1090 !
1091 ! Handle high-level interrupts on separate interrupt stack.
1092 ! No other high-level interrupts are active, so switch to int stack.
1093 !
1094 mov %sp, %l1
1095 ldn [%o3 + CPU_INTR_STACK], %l3
1096 sub %l3, STACK_BIAS, %sp
1097
1098 5:
1099 #ifdef DEBUG
1100 !
1101 ! ASSERT(%o2 > LOCK_LEVEL)
1102 !
1103 cmp %o2, LOCK_LEVEL
1104 bg,pt %xcc, 3f
1105 nop
1106 mov CE_PANIC, %o0
1107 sethi %hi(current_thread_wrong_pil), %o1
1108 call cmn_err ! %o2 has the %pil already
1109 or %o1, %lo(current_thread_wrong_pil), %o1
1110 #endif
1111 3:
1112 ! Store starting timestamp for this PIL in CPU structure at
1113 ! cpu.cpu_m.pil_high_start[PIL - (LOCK_LEVEL + 1)]
1114 sub %o2, LOCK_LEVEL + 1, %o4 ! convert PIL to array index
1115 sllx %o4, 3, %o4 ! index to byte offset
1116 add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large
1117 add %o4, MCPU_PIL_HIGH_START, %o4
1118 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %l2)
1119 stx %o5, [%o3 + %o4]
1120
1121 wrpr %g0, %o2, %pil ! enable interrupts
1122
1123 !
1124 ! call the handler
1125 !
1126 SERVE_INTR_PRE(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
1127 1:
1128 SERVE_INTR(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
1129
1130 brz,a,pt %o2, 0f ! if %o2, more intrs await
1131 rdpr %pil, %o2 ! delay annulled
1132 SERVE_INTR_NEXT(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
1133 ba 1b
1134 nop
1135 0:
1136 wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15)
1137
1138 cmp %o2, PIL_15
1139 bne,pt %xcc, 3f
1140 nop
1141
1142 sethi %hi(cpc_level15_inum), %o1
1143 ldx [%o1 + %lo(cpc_level15_inum)], %o1 ! arg for intr_enqueue_req
1144 brz %o1, 3f
1145 nop
1146
1147 rdpr %pstate, %g5
1148 andn %g5, PSTATE_IE, %g1
1149 wrpr %g0, %g1, %pstate ! Disable vec interrupts
1150
1151 call intr_enqueue_req ! preserves %g5
1152 mov PIL_15, %o0
1153
1154 ! clear perfcntr overflow
1155 mov 1, %o0
1156 sllx %o0, PIL_15, %o0
1157 wr %o0, CLEAR_SOFTINT
1158
1159 wrpr %g0, %g5, %pstate ! Enable vec interrupts
1160
1161 3:
1162 cmp %o2, PIL_14
1163 be tick_rtt ! cpu-specific tick processing
1164 nop
1165 .global current_thread_complete
1166 current_thread_complete:
1167 !
1168 ! Register usage:
1169 !
1170 ! %l1 = stack pointer
1171 ! %l2 = CPU_INTR_ACTV >> (LOCK_LEVEL + 1)
1172 ! %o2 = PIL
1173 ! %o3 = CPU pointer
1174 ! %o4, %o5, %l3, %l4, %l5 = scratch
1175 !
1176 ldn [THREAD_REG + T_CPU], %o3
1177 !
1178 ! Clear bit for this level in CPU's interrupt active bitmask.
1179 !
1180 ld [%o3 + CPU_INTR_ACTV], %l2
1181 mov 1, %o5
1182 sll %o5, %o2, %o5
1183 #ifdef DEBUG
1184 !
1185 ! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
1186 !
1187 andcc %l2, %o5, %g0
1188 bnz,pt %xcc, 0f
1189 nop
1190 ! Do not call panic if a panic is already in progress.
1191 sethi %hi(panic_quiesce), %l2
1192 ld [%l2 + %lo(panic_quiesce)], %l2
1193 brnz,pn %l2, 0f
1194 nop
1195 sethi %hi(current_thread_actv_bit_not_set), %o0
1196 call panic
1197 or %o0, %lo(current_thread_actv_bit_not_set), %o0
1198 0:
1199 #endif /* DEBUG */
1200 andn %l2, %o5, %l2
1201 st %l2, [%o3 + CPU_INTR_ACTV]
1202
1203 ! Take timestamp, compute interval, update cumulative counter.
1204 sub %o2, LOCK_LEVEL + 1, %o4 ! PIL to array index
1205 sllx %o4, 3, %o4 ! index to byte offset
1206 add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large
1207 add %o4, MCPU_PIL_HIGH_START, %o4
1208 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %o0)
1209 ldx [%o3 + %o4], %o0
1210 #ifdef DEBUG
1211 ! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0)
1212 brnz,pt %o0, 9f
1213 nop
1214 ! Don't panic if a panic is already in progress.
1215 sethi %hi(panic_quiesce), %l2
1216 ld [%l2 + %lo(panic_quiesce)], %l2
1217 brnz,pn %l2, 9f
1218 nop
1219 sethi %hi(current_thread_timestamp_zero), %o0
1220 call panic
1221 or %o0, %lo(current_thread_timestamp_zero), %o0
1222 9:
1223 #endif /* DEBUG */
1224 stx %g0, [%o3 + %o4]
1225 sub %o5, %o0, %o5 ! interval in o5
1226
1227 ! Check for Energy Star mode
1228 lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor
1229 cmp %o4, 1
1230 bg,a,pn %xcc, 2f
1231 mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1
1232 2:
1233 sllx %o2, 4, %o4 ! PIL index to byte offset
1234 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT too large
1235 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately
1236 ldx [%o3 + %o4], %o0 ! old counter in o0
1237 add %o0, %o5, %o0 ! new counter in o0
1238 stx %o0, [%o3 + %o4] ! store new counter
1239
1240 ! Also update intracct[]
1241 lduh [%o3 + CPU_MSTATE], %o4
1242 sllx %o4, 3, %o4
1243 add %o4, CPU_INTRACCT, %o4
1244 ldx [%o3 + %o4], %o0
1245 add %o0, %o5, %o0
1246 stx %o0, [%o3 + %o4]
1247
1248 !
1249 ! get back on current thread's stack
1250 !
1251 srl %l2, LOCK_LEVEL + 1, %l2
1252 tst %l2 ! any more high-level ints?
1253 movz %xcc, %l1, %sp
1254 !
1255 ! Current register usage:
1256 ! o2 = PIL
1257 ! o3 = CPU pointer
1258 ! l0 = return address
1259 ! l2 = intr_actv shifted right
1260 !
1261 bz,pt %xcc, 3f ! if l2 was zero, no more ints
1262 nop
1263 !
1264 ! We found another high-level interrupt active below the one that just
1265 ! returned. Store a starting timestamp for it in the CPU structure.
1266 !
1267 ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
1268 ! interrupted high-level interrupt.
1269 ! Create mask for cpu_intr_actv. Begin by looking for bits set
1270 ! at one level below the current PIL. Since %l2 contains the active
1271 ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
1272 ! at bit (current_pil - (LOCK_LEVEL + 2)).
1273 ! %l1 = mask, %o5 = index of bit set in mask
1274 !
1275 mov 1, %l1
1276 sub %o2, LOCK_LEVEL + 2, %o5
1277 sll %l1, %o5, %l1 ! l1 = mask for level
1278 1:
1279 #ifdef DEBUG
1280 ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
1281 brnz,pt %l1, 9f
1282 nop
1283 sethi %hi(current_thread_nested_PIL_not_found), %o0
1284 call panic
1285 or %o0, %lo(current_thread_nested_PIL_not_found), %o0
1286 9:
1287 #endif /* DEBUG */
1288 andcc %l1, %l2, %g0 ! test mask against high-level bits of
1289 bnz %xcc, 2f ! cpu_intr_actv
1290 nop
1291 srl %l1, 1, %l1 ! No match. Try next lower PIL.
1292 ba,pt %xcc, 1b
1293 sub %o5, 1, %o5 ! delay - decrement PIL
1294 2:
1295 sll %o5, 3, %o5 ! convert array index to byte offset
1296 add %o5, CPU_MCPU, %o5 ! CPU_PIL_HIGH_START is too large
1297 add %o5, MCPU_PIL_HIGH_START, %o5
1298 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
1299 ! Another high-level interrupt is active below this one, so
1300 ! there is no need to check for an interrupt thread. That will be
1301 ! done by the lowest priority high-level interrupt active.
1302 ba,pt %xcc, 7f
1303 stx %o4, [%o3 + %o5] ! delay - store timestamp
1304 3:
1305 ! If we haven't interrupted another high-level interrupt, we may have
1306 ! interrupted a low level interrupt thread. If so, store a starting
1307 ! timestamp in its thread structure.
1308 lduh [THREAD_REG + T_FLAGS], %o4
1309 andcc %o4, T_INTR_THREAD, %g0
1310 bz,pt %xcc, 7f
1311 nop
1312
1313 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
1314 stx %o4, [THREAD_REG + T_INTR_START]
1315
1316 7:
1317 sub %o2, LOCK_LEVEL + 1, %o4
1318 sll %o4, CPTRSHIFT, %o5
1319
1320 ! Check on_trap saved area and restore as needed
1321 add %o5, CPU_OTD, %o4
1322 ldn [%o3 + %o4], %l2
1323 brz,pt %l2, no_ontrp_restore
1324 nop
1325 stn %l2, [THREAD_REG + T_ONTRAP] ! restore
1326 stn %g0, [%o3 + %o4] ! clear
1327
1328 no_ontrp_restore:
1329 ! Check on_fault saved area and restore as needed
1330 add %o5, CPU_OFD, %o4
1331 ldn [%o3 + %o4], %l2
1332 brz,pt %l2, 8f
1333 nop
1334 stn %l2, [THREAD_REG + T_ONFAULT] ! restore
1335 stn %g0, [%o3 + %o4] ! clear
1336 add %o5, CPU_LFD, %o4
1337 ldn [%o3 + %o4], %l2
1338 stn %l2, [THREAD_REG + T_LOFAULT] ! restore
1339 stn %g0, [%o3 + %o4] ! clear
1340
1341
1342 8:
1343 ! Enable interrupts and return
1344 jmp %l0 + 8
1345 wrpr %g0, %o2, %pil ! enable interrupts
1346 SET_SIZE(current_thread)
1347
1348
1349 #ifdef DEBUG
1350 current_thread_wrong_pil:
1351 .asciz "current_thread: unexpected pil level: %d"
1352 current_thread_actv_bit_set:
1353 .asciz "current_thread(): cpu_intr_actv bit already set for PIL"
1354 current_thread_actv_bit_not_set:
1355 .asciz "current_thread(): cpu_intr_actv bit not set for PIL"
1356 current_thread_nested_pil_zero:
1357 .asciz "current_thread(): timestamp zero for nested PIL %d"
1358 current_thread_timestamp_zero:
1359 .asciz "current_thread(): timestamp zero upon handler return"
1360 current_thread_nested_PIL_not_found:
1361 .asciz "current_thread: couldn't find nested high-level PIL"
1362 #endif /* DEBUG */
1363
1364 /*
1365 * Return a thread's interrupt level.
1366 * Since this isn't saved anywhere but in %l4 on interrupt entry, we
1367 * must dig it out of the save area.
1368 *
1369 * Caller 'swears' that this really is an interrupt thread.
1370 *
1371 * int
1372 * intr_level(t)
1373 * kthread_id_t t;
1374 */
1375
1376 ENTRY_NP(intr_level)
1377 retl
1378 ldub [%o0 + T_PIL], %o0 ! return saved pil
1379 SET_SIZE(intr_level)
1380
1381 ENTRY_NP(disable_pil_intr)
1382 rdpr %pil, %o0
1383 retl
1384 wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15)
1385 SET_SIZE(disable_pil_intr)
1386
1387 ENTRY_NP(enable_pil_intr)
1388 retl
1389 wrpr %o0, %pil
1390 SET_SIZE(enable_pil_intr)
1391
1392 ENTRY_NP(disable_vec_intr)
1393 rdpr %pstate, %o0
1394 andn %o0, PSTATE_IE, %g1
1395 retl
1396 wrpr %g0, %g1, %pstate ! disable interrupt
1397 SET_SIZE(disable_vec_intr)
1398
1399 ENTRY_NP(enable_vec_intr)
1400 retl
1401 wrpr %g0, %o0, %pstate
1402 SET_SIZE(enable_vec_intr)
1403
1404 ENTRY_NP(cbe_level14)
1405 save %sp, -SA(MINFRAME), %sp ! get a new window
1406 !
1407 ! Make sure that this is from TICK_COMPARE; if not just return
1408 !
1409 rd SOFTINT, %l1
1410 set (TICK_INT_MASK | STICK_INT_MASK), %o2
1411 andcc %l1, %o2, %g0
1412 bz,pn %icc, 2f
1413 nop
1414
1415 CPU_ADDR(%o1, %o2)
1416 call cyclic_fire
1417 mov %o1, %o0
1418 2:
1419 ret
1420 restore %g0, 1, %o0
1421 SET_SIZE(cbe_level14)
1422
1423
1424 ENTRY_NP(kdi_setsoftint)
1425 save %sp, -SA(MINFRAME), %sp ! get a new window
1426 rdpr %pstate, %l5
1427 andn %l5, PSTATE_IE, %l1
1428 wrpr %l1, %pstate ! disable interrupt
1429 !
1430 ! We have a pointer to an interrupt vector data structure.
1431 ! Put the request on the cpu's softint priority list and
1432 ! set %set_softint.
1433 !
1434 ! Register usage
1435 ! %i0 - pointer to intr_vec_t (iv)
1436 ! %l2 - requested pil
1437 ! %l4 - cpu
1438 ! %l5 - pstate
1439 ! %l1, %l3, %l6 - temps
1440 !
1441 ! check if a softint is pending for this softint,
1442 ! if one is pending, don't bother queuing another.
1443 !
1444 lduh [%i0 + IV_FLAGS], %l1 ! %l1 = iv->iv_flags
1445 and %l1, IV_SOFTINT_PEND, %l6 ! %l6 = iv->iv_flags & IV_SOFTINT_PEND
1446 brnz,pn %l6, 4f ! branch if softint is already pending
1447 or %l1, IV_SOFTINT_PEND, %l2
1448 sth %l2, [%i0 + IV_FLAGS] ! Set IV_SOFTINT_PEND flag
1449
1450 CPU_ADDR(%l4, %l2) ! %l4 = cpu
1451 lduh [%i0 + IV_PIL], %l2 ! %l2 = iv->iv_pil
1452
1453 !
1454 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
1455 !
1456 sll %l2, CPTRSHIFT, %l0 ! %l0 = offset to pil entry
1457 add %l4, INTR_TAIL, %l6 ! %l6 = &cpu->m_cpu.intr_tail
1458 ldn [%l6 + %l0], %l1 ! %l1 = cpu->m_cpu.intr_tail[pil]
1459 ! current tail (ct)
1460 brz,pt %l1, 2f ! branch if current tail is NULL
1461 stn %i0, [%l6 + %l0] ! make intr_vec_t (iv) as new tail
1462 !
1463 ! there's pending intr_vec_t already
1464 !
1465 lduh [%l1 + IV_FLAGS], %l6 ! %l6 = ct->iv_flags
1466 and %l6, IV_SOFTINT_MT, %l6 ! %l6 = ct->iv_flags & IV_SOFTINT_MT
1467 brz,pt %l6, 1f ! check for Multi target softint flag
1468 add %l1, IV_PIL_NEXT, %l3 ! %l3 = &ct->iv_pil_next
1469 ld [%l4 + CPU_ID], %l6 ! for multi target softint, use cpuid
1470 sll %l6, CPTRSHIFT, %l6 ! calculate offset address from cpuid
1471 add %l3, %l6, %l3 ! %l3 = &ct->iv_xpil_next[cpuid]
1472 1:
1473 !
1474 ! update old tail
1475 !
1476 ba,pt %xcc, 3f
1477 stn %i0, [%l3] ! [%l3] = iv, set pil_next field
1478 2:
1479 !
1480 ! no pending intr_vec_t; make intr_vec_t as new head
1481 !
1482 add %l4, INTR_HEAD, %l6 ! %l6 = &cpu->m_cpu.intr_head[pil]
1483 stn %i0, [%l6 + %l0] ! cpu->m_cpu.intr_head[pil] = iv
1484 3:
1485 !
1486 ! Write %set_softint with (1<<pil) to cause a "pil" level trap
1487 !
1488 mov 1, %l1 ! %l1 = 1
1489 sll %l1, %l2, %l1 ! %l1 = 1 << pil
1490 wr %l1, SET_SOFTINT ! trigger required pil softint
1491 4:
1492 wrpr %g0, %l5, %pstate ! %pstate = saved %pstate (in %l5)
1493 ret
1494 restore
1495 SET_SIZE(kdi_setsoftint)
1496
1497 !
1498 ! Register usage
1499 ! Arguments:
1500 ! %g1 - Pointer to intr_vec_t (iv)
1501 !
1502 ! Internal:
1503 ! %g2 - pil
1504 ! %g4 - cpu
1505 ! %g3,%g5-g7 - temps
1506 !
1507 ENTRY_NP(setsoftint_tl1)
1508 !
1509 ! We have a pointer to an interrupt vector data structure.
1510 ! Put the request on the cpu's softint priority list and
1511 ! set %set_softint.
1512 !
1513 CPU_ADDR(%g4, %g2) ! %g4 = cpu
1514 lduh [%g1 + IV_PIL], %g2 ! %g2 = iv->iv_pil
1515
1516 !
1517 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
1518 !
1519 sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry
1520 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail
1521 ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil]
1522 ! current tail (ct)
1523 brz,pt %g5, 1f ! branch if current tail is NULL
1524 stn %g1, [%g6 + %g7] ! make intr_rec_t (iv) as new tail
1525 !
1526 ! there's pending intr_vec_t already
1527 !
1528 lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags
1529 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT
1530 brz,pt %g6, 0f ! check for Multi target softint flag
1531 add %g5, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next
1532 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid
1533 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid
1534 add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid]
1535 0:
1536 !
1537 ! update old tail
1538 !
1539 ba,pt %xcc, 2f
1540 stn %g1, [%g3] ! [%g3] = iv, set pil_next field
1541 1:
1542 !
1543 ! no pending intr_vec_t; make intr_vec_t as new head
1544 !
1545 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil]
1546 stn %g1, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv
1547 2:
1548 #ifdef TRAPTRACE
1549 TRACE_PTR(%g5, %g6)
1550 GET_TRACE_TICK(%g6, %g3)
1551 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick
1552 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
1553 rdpr %tt, %g6
1554 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt
1555 rdpr %tpc, %g6
1556 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc
1557 rdpr %tstate, %g6
1558 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1559 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp
1560 stna %g1, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv
1561 ldn [%g1 + IV_PIL_NEXT], %g6 !
1562 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = iv->iv_pil_next
1563 add %g4, INTR_HEAD, %g6
1564 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil]
1565 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil]
1566 add %g4, INTR_TAIL, %g6
1567 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil]
1568 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil]
1569 stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil
1570 TRACE_NEXT(%g5, %g6, %g3)
1571 #endif /* TRAPTRACE */
1572 !
1573 ! Write %set_softint with (1<<pil) to cause a "pil" level trap
1574 !
1575 mov 1, %g5 ! %g5 = 1
1576 sll %g5, %g2, %g5 ! %g5 = 1 << pil
1577 wr %g5, SET_SOFTINT ! trigger required pil softint
1578 retry
1579 SET_SIZE(setsoftint_tl1)
1580
1581 !
1582 ! Register usage
1583 ! Arguments:
1584 ! %g1 - inumber
1585 !
1586 ! Internal:
1587 ! %g1 - softint pil mask
1588 ! %g2 - pil of intr_vec_t
1589 ! %g3 - pointer to current intr_vec_t (iv)
1590 ! %g4 - cpu
1591 ! %g5, %g6,%g7 - temps
1592 !
1593 ENTRY_NP(setvecint_tl1)
1594 !
1595 ! Verify the inumber received (should be inum < MAXIVNUM).
1596 !
1597 set MAXIVNUM, %g2
1598 cmp %g1, %g2
1599 bgeu,pn %xcc, .no_ivintr
1600 clr %g2 ! expected in .no_ivintr
1601
1602 !
1603 ! Fetch data from intr_vec_table according to the inum.
1604 !
1605 ! We have an interrupt number. Fetch the interrupt vector requests
1606 ! from the interrupt vector table for a given interrupt number and
1607 ! insert them into cpu's softint priority lists and set %set_softint.
1608 !
1609 set intr_vec_table, %g5 ! %g5 = intr_vec_table
1610 sll %g1, CPTRSHIFT, %g6 ! %g6 = offset to inum entry in table
1611 add %g5, %g6, %g5 ! %g5 = &intr_vec_table[inum]
1612 ldn [%g5], %g3 ! %g3 = pointer to first entry of
1613 ! intr_vec_t list
1614
1615 ! Verify the first intr_vec_t pointer for a given inum and it should
1616 ! not be NULL. This used to be guarded by DEBUG but broken drivers can
1617 ! cause spurious tick interrupts when the softint register is programmed
1618 ! with 1 << 0 at the end of this routine. Now we always check for a
1619 ! valid intr_vec_t pointer.
1620 brz,pn %g3, .no_ivintr
1621 nop
1622
1623 !
1624 ! Traverse the intr_vec_t link list, put each item on to corresponding
1625 ! CPU softint priority queue, and compose the final softint pil mask.
1626 !
1627 ! At this point:
1628 ! %g3 = intr_vec_table[inum]
1629 !
1630 CPU_ADDR(%g4, %g2) ! %g4 = cpu
1631 mov %g0, %g1 ! %g1 = 0, initialize pil mask to 0
1632 0:
1633 !
1634 ! Insert next intr_vec_t (iv) to appropriate cpu's softint priority list
1635 !
1636 ! At this point:
1637 ! %g1 = softint pil mask
1638 ! %g3 = pointer to next intr_vec_t (iv)
1639 ! %g4 = cpu
1640 !
1641 lduh [%g3 + IV_PIL], %g2 ! %g2 = iv->iv_pil
1642 sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry
1643 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail
1644 ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil]
1645 ! current tail (ct)
1646 brz,pt %g5, 2f ! branch if current tail is NULL
1647 stn %g3, [%g6 + %g7] ! make intr_vec_t (iv) as new tail
1648 ! cpu->m_cpu.intr_tail[pil] = iv
1649 !
1650 ! there's pending intr_vec_t already
1651 !
1652 lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags
1653 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT
1654 brz,pt %g6, 1f ! check for Multi target softint flag
1655 add %g5, IV_PIL_NEXT, %g5 ! %g5 = &ct->iv_pil_next
1656 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid
1657 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid
1658 add %g5, %g6, %g5 ! %g5 = &ct->iv_xpil_next[cpuid]
1659 1:
1660 !
1661 ! update old tail
1662 !
1663 ba,pt %xcc, 3f
1664 stn %g3, [%g5] ! [%g5] = iv, set pil_next field
1665 2:
1666 !
1667 ! no pending intr_vec_t; make intr_vec_t as new head
1668 !
1669 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil]
1670 stn %g3, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv
1671 3:
1672 #ifdef TRAPTRACE
1673 TRACE_PTR(%g5, %g6)
1674 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
1675 rdpr %tt, %g6
1676 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt`
1677 rdpr %tpc, %g6
1678 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc
1679 rdpr %tstate, %g6
1680 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1681 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp
1682 stna %g3, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv
1683 stna %g1, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = pil mask
1684 add %g4, INTR_HEAD, %g6
1685 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil]
1686 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil]
1687 add %g4, INTR_TAIL, %g6
1688 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil]
1689 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil]
1690 stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil
1691 GET_TRACE_TICK(%g6, %g7)
1692 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick
1693 TRACE_NEXT(%g5, %g6, %g7)
1694 #endif /* TRAPTRACE */
1695 mov 1, %g6 ! %g6 = 1
1696 sll %g6, %g2, %g6 ! %g6 = 1 << pil
1697 or %g1, %g6, %g1 ! %g1 |= (1 << pil), pil mask
1698 ldn [%g3 + IV_VEC_NEXT], %g3 ! %g3 = pointer to next intr_vec_t (iv)
1699 brnz,pn %g3, 0b ! iv->iv_vec_next is non NULL, goto 0b
1700 nop
1701 wr %g1, SET_SOFTINT ! triggered one or more pil softints
1702 retry
1703
1704 .no_ivintr:
1705 ! no_ivintr: arguments: rp, inum (%g1), pil (%g2 == 0)
1706 mov %g2, %g3
1707 mov %g1, %g2
1708 set no_ivintr, %g1
1709 ba,pt %xcc, sys_trap
1710 mov PIL_15, %g4
1711 SET_SIZE(setvecint_tl1)
1712
1713 ENTRY_NP(wr_clr_softint)
1714 retl
1715 wr %o0, CLEAR_SOFTINT
1716 SET_SIZE(wr_clr_softint)
1717
1718 /*
1719 * intr_enqueue_req
1720 *
1721 * %o0 - pil
1722 * %o1 - pointer to intr_vec_t (iv)
1723 * %o5 - preserved
1724 * %g5 - preserved
1725 */
1726 ENTRY_NP(intr_enqueue_req)
1727 !
1728 CPU_ADDR(%g4, %g1) ! %g4 = cpu
1729
1730 !
1731 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
1732 !
1733 sll %o0, CPTRSHIFT, %o0 ! %o0 = offset to pil entry
1734 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail
1735 ldn [%o0 + %g6], %g1 ! %g1 = cpu->m_cpu.intr_tail[pil]
1736 ! current tail (ct)
1737 brz,pt %g1, 2f ! branch if current tail is NULL
1738 stn %o1, [%g6 + %o0] ! make intr_vec_t (iv) as new tail
1739
1740 !
1741 ! there's pending intr_vec_t already
1742 !
1743 lduh [%g1 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags
1744 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT
1745 brz,pt %g6, 1f ! check for Multi target softint flag
1746 add %g1, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next
1747 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid
1748 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid
1749 add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid]
1750 1:
1751 !
1752 ! update old tail
1753 !
1754 ba,pt %xcc, 3f
1755 stn %o1, [%g3] ! {%g5] = iv, set pil_next field
1756 2:
1757 !
1758 ! no intr_vec_t's queued so make intr_vec_t as new head
1759 !
1760 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil]
1761 stn %o1, [%g6 + %o0] ! cpu->m_cpu.intr_head[pil] = iv
1762 3:
1763 retl
1764 nop
1765 SET_SIZE(intr_enqueue_req)
1766
1767 /*
1768 * Set CPU's base SPL level, based on which interrupt levels are active.
1769 * Called at spl7 or above.
1770 */
1771
1772 ENTRY_NP(set_base_spl)
1773 ldn [THREAD_REG + T_CPU], %o2 ! load CPU pointer
1774 ld [%o2 + CPU_INTR_ACTV], %o5 ! load active interrupts mask
1775
1776 /*
1777 * WARNING: non-standard callinq sequence; do not call from C
1778 * %o2 = pointer to CPU
1779 * %o5 = updated CPU_INTR_ACTV
1780 */
1781 _intr_set_spl: ! intr_thread_exit enters here
1782 !
1783 ! Determine highest interrupt level active. Several could be blocked
1784 ! at higher levels than this one, so must convert flags to a PIL
1785 ! Normally nothing will be blocked, so test this first.
1786 !
1787 brz,pt %o5, 1f ! nothing active
1788 sra %o5, 11, %o3 ! delay - set %o3 to bits 15-11
1789 set _intr_flag_table, %o1
1790 tst %o3 ! see if any of the bits set
1791 ldub [%o1 + %o3], %o3 ! load bit number
1792 bnz,a,pn %xcc, 1f ! yes, add 10 and we're done
1793 add %o3, 11-1, %o3 ! delay - add bit number - 1
1794
1795 sra %o5, 6, %o3 ! test bits 10-6
1796 tst %o3
1797 ldub [%o1 + %o3], %o3
1798 bnz,a,pn %xcc, 1f
1799 add %o3, 6-1, %o3
1800
1801 sra %o5, 1, %o3 ! test bits 5-1
1802 ldub [%o1 + %o3], %o3
1803
1804 !
1805 ! highest interrupt level number active is in %l6
1806 !
1807 1:
1808 retl
1809 st %o3, [%o2 + CPU_BASE_SPL] ! delay - store base priority
1810 SET_SIZE(set_base_spl)
1811
1812 /*
1813 * Table that finds the most significant bit set in a five bit field.
1814 * Each entry is the high-order bit number + 1 of it's index in the table.
1815 * This read-only data is in the text segment.
1816 */
1817 _intr_flag_table:
1818 .byte 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4
1819 .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
1820 .align 4
1821
1822 /*
1823 * int
1824 * intr_passivate(from, to)
1825 * kthread_id_t from; interrupt thread
1826 * kthread_id_t to; interrupted thread
1827 */
1828
1829 ENTRY_NP(intr_passivate)
1830 save %sp, -SA(MINFRAME), %sp ! get a new window
1831
1832 flushw ! force register windows to stack
1833 !
1834 ! restore registers from the base of the stack of the interrupt thread.
1835 !
1836 ldn [%i0 + T_STACK], %i2 ! get stack save area pointer
1837 ldn [%i2 + (0*GREGSIZE)], %l0 ! load locals
1838 ldn [%i2 + (1*GREGSIZE)], %l1
1839 ldn [%i2 + (2*GREGSIZE)], %l2
1840 ldn [%i2 + (3*GREGSIZE)], %l3
1841 ldn [%i2 + (4*GREGSIZE)], %l4
1842 ldn [%i2 + (5*GREGSIZE)], %l5
1843 ldn [%i2 + (6*GREGSIZE)], %l6
1844 ldn [%i2 + (7*GREGSIZE)], %l7
1845 ldn [%i2 + (8*GREGSIZE)], %o0 ! put ins from stack in outs
1846 ldn [%i2 + (9*GREGSIZE)], %o1
1847 ldn [%i2 + (10*GREGSIZE)], %o2
1848 ldn [%i2 + (11*GREGSIZE)], %o3
1849 ldn [%i2 + (12*GREGSIZE)], %o4
1850 ldn [%i2 + (13*GREGSIZE)], %o5
1851 ldn [%i2 + (14*GREGSIZE)], %i4
1852 ! copy stack/pointer without using %sp
1853 ldn [%i2 + (15*GREGSIZE)], %i5
1854 !
1855 ! put registers into the save area at the top of the interrupted
1856 ! thread's stack, pointed to by %l7 in the save area just loaded.
1857 !
1858 ldn [%i1 + T_SP], %i3 ! get stack save area pointer
1859 stn %l0, [%i3 + STACK_BIAS + (0*GREGSIZE)] ! save locals
1860 stn %l1, [%i3 + STACK_BIAS + (1*GREGSIZE)]
1861 stn %l2, [%i3 + STACK_BIAS + (2*GREGSIZE)]
1862 stn %l3, [%i3 + STACK_BIAS + (3*GREGSIZE)]
1863 stn %l4, [%i3 + STACK_BIAS + (4*GREGSIZE)]
1864 stn %l5, [%i3 + STACK_BIAS + (5*GREGSIZE)]
1865 stn %l6, [%i3 + STACK_BIAS + (6*GREGSIZE)]
1866 stn %l7, [%i3 + STACK_BIAS + (7*GREGSIZE)]
1867 stn %o0, [%i3 + STACK_BIAS + (8*GREGSIZE)] ! save ins using outs
1868 stn %o1, [%i3 + STACK_BIAS + (9*GREGSIZE)]
1869 stn %o2, [%i3 + STACK_BIAS + (10*GREGSIZE)]
1870 stn %o3, [%i3 + STACK_BIAS + (11*GREGSIZE)]
1871 stn %o4, [%i3 + STACK_BIAS + (12*GREGSIZE)]
1872 stn %o5, [%i3 + STACK_BIAS + (13*GREGSIZE)]
1873 stn %i4, [%i3 + STACK_BIAS + (14*GREGSIZE)]
1874 ! fp, %i7 copied using %i4
1875 stn %i5, [%i3 + STACK_BIAS + (15*GREGSIZE)]
1876 stn %g0, [%i2 + ((8+6)*GREGSIZE)]
1877 ! clear fp in save area
1878
1879 ! load saved pil for return
1880 ldub [%i0 + T_PIL], %i0
1881 ret
1882 restore
1883 SET_SIZE(intr_passivate)
1884
1885 /*
1886 * intr_get_time() is a resource for interrupt handlers to determine how
1887 * much time has been spent handling the current interrupt. Such a function
1888 * is needed because higher level interrupts can arrive during the
1889 * processing of an interrupt, thus making direct comparisons of %tick by
1890 * the handler inaccurate. intr_get_time() only returns time spent in the
1891 * current interrupt handler.
1892 *
1893 * The caller must be calling from an interrupt handler running at a pil
1894 * below or at lock level. Timings are not provided for high-level
1895 * interrupts.
1896 *
1897 * The first time intr_get_time() is called while handling an interrupt,
1898 * it returns the time since the interrupt handler was invoked. Subsequent
1899 * calls will return the time since the prior call to intr_get_time(). Time
1900 * is returned as ticks, adjusted for any clock divisor due to power
1901 * management. Use tick2ns() to convert ticks to nsec. Warning: ticks may
1902 * not be the same across CPUs.
1903 *
1904 * Theory Of Intrstat[][]:
1905 *
1906 * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
1907 * uint64_ts per pil.
1908 *
1909 * intrstat[pil][0] is a cumulative count of the number of ticks spent
1910 * handling all interrupts at the specified pil on this CPU. It is
1911 * exported via kstats to the user.
1912 *
1913 * intrstat[pil][1] is always a count of ticks less than or equal to the
1914 * value in [0]. The difference between [1] and [0] is the value returned
1915 * by a call to intr_get_time(). At the start of interrupt processing,
1916 * [0] and [1] will be equal (or nearly so). As the interrupt consumes
1917 * time, [0] will increase, but [1] will remain the same. A call to
1918 * intr_get_time() will return the difference, then update [1] to be the
1919 * same as [0]. Future calls will return the time since the last call.
1920 * Finally, when the interrupt completes, [1] is updated to the same as [0].
1921 *
1922 * Implementation:
1923 *
1924 * intr_get_time() works much like a higher level interrupt arriving. It
1925 * "checkpoints" the timing information by incrementing intrstat[pil][0]
1926 * to include elapsed running time, and by setting t_intr_start to %tick.
1927 * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
1928 * and updates intrstat[pil][1] to be the same as the new value of
1929 * intrstat[pil][0].
1930 *
1931 * In the normal handling of interrupts, after an interrupt handler returns
1932 * and the code in intr_thread() updates intrstat[pil][0], it then sets
1933 * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
1934 * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
1935 * is 0.
1936 *
1937 * Whenever interrupts arrive on a CPU which is handling a lower pil
1938 * interrupt, they update the lower pil's [0] to show time spent in the
1939 * handler that they've interrupted. This results in a growing discrepancy
1940 * between [0] and [1], which is returned the next time intr_get_time() is
1941 * called. Time spent in the higher-pil interrupt will not be returned in
1942 * the next intr_get_time() call from the original interrupt, because
1943 * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
1944 */
1945 ENTRY_NP(intr_get_time)
1946 #ifdef DEBUG
1947 !
1948 ! Lots of asserts, but just check panic_quiesce first.
1949 ! Don't bother with lots of tests if we're just ignoring them.
1950 !
1951 sethi %hi(panic_quiesce), %o0
1952 ld [%o0 + %lo(panic_quiesce)], %o0
1953 brnz,pn %o0, 2f
1954 nop
1955 !
1956 ! ASSERT(%pil <= LOCK_LEVEL)
1957 !
1958 rdpr %pil, %o1
1959 cmp %o1, LOCK_LEVEL
1960 ble,pt %xcc, 0f
1961 sethi %hi(intr_get_time_high_pil), %o0 ! delay
1962 call panic
1963 or %o0, %lo(intr_get_time_high_pil), %o0
1964 0:
1965 !
1966 ! ASSERT((t_flags & T_INTR_THREAD) != 0 && t_pil > 0)
1967 !
1968 lduh [THREAD_REG + T_FLAGS], %o2
1969 andcc %o2, T_INTR_THREAD, %g0
1970 bz,pn %xcc, 1f
1971 ldub [THREAD_REG + T_PIL], %o1 ! delay
1972 brnz,pt %o1, 0f
1973 1:
1974 sethi %hi(intr_get_time_not_intr), %o0
1975 call panic
1976 or %o0, %lo(intr_get_time_not_intr), %o0
1977 0:
1978 !
1979 ! ASSERT(t_intr_start != 0)
1980 !
1981 ldx [THREAD_REG + T_INTR_START], %o1
1982 brnz,pt %o1, 2f
1983 sethi %hi(intr_get_time_no_start_time), %o0 ! delay
1984 call panic
1985 or %o0, %lo(intr_get_time_no_start_time), %o0
1986 2:
1987 #endif /* DEBUG */
1988 !
1989 ! %o0 = elapsed time and return value
1990 ! %o1 = pil
1991 ! %o2 = scratch
1992 ! %o3 = scratch
1993 ! %o4 = scratch
1994 ! %o5 = cpu
1995 !
1996 wrpr %g0, PIL_MAX, %pil ! make this easy -- block normal intrs
1997 ldn [THREAD_REG + T_CPU], %o5
1998 ldub [THREAD_REG + T_PIL], %o1
1999 ldx [THREAD_REG + T_INTR_START], %o3 ! %o3 = t_intr_start
2000 !
2001 ! Calculate elapsed time since t_intr_start. Update t_intr_start,
2002 ! get delta, and multiply by cpu_divisor if necessary.
2003 !
2004 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o2, %o0)
2005 stx %o2, [THREAD_REG + T_INTR_START]
2006 sub %o2, %o3, %o0
2007
2008 lduh [%o5 + CPU_DIVISOR], %o4
2009 cmp %o4, 1
2010 bg,a,pn %xcc, 1f
2011 mulx %o0, %o4, %o0 ! multiply interval by clock divisor iff > 1
2012 1:
2013 ! Update intracct[]
2014 lduh [%o5 + CPU_MSTATE], %o4
2015 sllx %o4, 3, %o4
2016 add %o4, CPU_INTRACCT, %o4
2017 ldx [%o5 + %o4], %o2
2018 add %o2, %o0, %o2
2019 stx %o2, [%o5 + %o4]
2020
2021 !
2022 ! Increment cpu_m.intrstat[pil][0]. Calculate elapsed time since
2023 ! cpu_m.intrstat[pil][1], which is either when the interrupt was
2024 ! first entered, or the last time intr_get_time() was invoked. Then
2025 ! update cpu_m.intrstat[pil][1] to match [0].
2026 !
2027 sllx %o1, 4, %o3
2028 add %o3, CPU_MCPU, %o3
2029 add %o3, MCPU_INTRSTAT, %o3
2030 add %o3, %o5, %o3 ! %o3 = cpu_m.intrstat[pil][0]
2031 ldx [%o3], %o2
2032 add %o2, %o0, %o2 ! %o2 = new value for intrstat
2033 stx %o2, [%o3]
2034 ldx [%o3 + 8], %o4 ! %o4 = cpu_m.intrstat[pil][1]
2035 sub %o2, %o4, %o0 ! %o0 is elapsed time since %o4
2036 stx %o2, [%o3 + 8] ! make [1] match [0], resetting time
2037
2038 ld [%o5 + CPU_BASE_SPL], %o2 ! restore %pil to the greater
2039 cmp %o2, %o1 ! of either our pil %o1 or
2040 movl %xcc, %o1, %o2 ! cpu_base_spl.
2041 retl
2042 wrpr %g0, %o2, %pil
2043 SET_SIZE(intr_get_time)
2044
2045 #ifdef DEBUG
2046 intr_get_time_high_pil:
2047 .asciz "intr_get_time(): %pil > LOCK_LEVEL"
2048 intr_get_time_not_intr:
2049 .asciz "intr_get_time(): not called from an interrupt thread"
2050 intr_get_time_no_start_time:
2051 .asciz "intr_get_time(): t_intr_start == 0"
2052 #endif /* DEBUG */