1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #if !defined(lint)
28 #include "assym.h"
29 #endif /* !lint */
30 #include <sys/asm_linkage.h>
31 #include <sys/privregs.h>
32 #include <sys/sun4asi.h>
33 #include <sys/machasi.h>
34 #include <sys/hypervisor_api.h>
35 #include <sys/machtrap.h>
36 #include <sys/machthread.h>
37 #include <sys/machbrand.h>
38 #include <sys/pcb.h>
39 #include <sys/pte.h>
40 #include <sys/mmu.h>
41 #include <sys/machpcb.h>
42 #include <sys/async.h>
43 #include <sys/intreg.h>
44 #include <sys/scb.h>
45 #include <sys/psr_compat.h>
46 #include <sys/syscall.h>
47 #include <sys/machparam.h>
48 #include <sys/traptrace.h>
49 #include <vm/hat_sfmmu.h>
50 #include <sys/archsystm.h>
51 #include <sys/utrap.h>
52 #include <sys/clock.h>
53 #include <sys/intr.h>
54 #include <sys/fpu/fpu_simulator.h>
55 #include <vm/seg_spt.h>
56
57 /*
58 * WARNING: If you add a fast trap handler which can be invoked by a
59 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
60 * instead of "done" instruction to return back to the user mode. See
61 * comments for the "fast_trap_done" entry point for more information.
62 *
63 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
64 * cases where you always want to process any pending interrupts before
65 * returning back to the user mode.
66 */
67 #define FAST_TRAP_DONE \
68 ba,a fast_trap_done
69
70 #define FAST_TRAP_DONE_CHK_INTR \
71 ba,a fast_trap_done_chk_intr
72
73 /*
74 * SPARC V9 Trap Table
75 *
76 * Most of the trap handlers are made from common building
77 * blocks, and some are instantiated multiple times within
78 * the trap table. So, I build a bunch of macros, then
79 * populate the table using only the macros.
80 *
81 * Many macros branch to sys_trap. Its calling convention is:
82 * %g1 kernel trap handler
83 * %g2, %g3 args for above
84 * %g4 desire %pil
85 */
86
87 #ifdef TRAPTRACE
88
89 /*
90 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
91 */
92 #define TT_TRACE(label) \
93 ba label ;\
94 rd %pc, %g7
95 #define TT_TRACE_INS 2
96
97 #define TT_TRACE_L(label) \
98 ba label ;\
99 rd %pc, %l4 ;\
100 clr %l4
101 #define TT_TRACE_L_INS 3
102
103 #else
104
105 #define TT_TRACE(label)
106 #define TT_TRACE_INS 0
107
108 #define TT_TRACE_L(label)
109 #define TT_TRACE_L_INS 0
110
111 #endif
112
113 /*
114 * This first set are funneled to trap() with %tt as the type.
115 * Trap will then either panic or send the user a signal.
116 */
117 /*
118 * NOT is used for traps that just shouldn't happen.
119 * It comes in both single and quadruple flavors.
120 */
121 #if !defined(lint)
122 .global trap
123 #endif /* !lint */
124 #define NOT \
125 TT_TRACE(trace_gen) ;\
126 set trap, %g1 ;\
127 rdpr %tt, %g3 ;\
128 ba,pt %xcc, sys_trap ;\
129 sub %g0, 1, %g4 ;\
130 .align 32
131 #define NOT4 NOT; NOT; NOT; NOT
132
133 #define NOTP \
134 TT_TRACE(trace_gen) ;\
135 ba,pt %xcc, ptl1_panic ;\
136 mov PTL1_BAD_TRAP, %g1 ;\
137 .align 32
138 #define NOTP4 NOTP; NOTP; NOTP; NOTP
139
140
141 /*
142 * BAD is used for trap vectors we don't have a kernel
143 * handler for.
144 * It also comes in single and quadruple versions.
145 */
146 #define BAD NOT
147 #define BAD4 NOT4
148
149 #define DONE \
150 done; \
151 .align 32
152
153 /*
154 * TRAP vectors to the trap() function.
155 * It's main use is for user errors.
156 */
157 #if !defined(lint)
158 .global trap
159 #endif /* !lint */
160 #define TRAP(arg) \
161 TT_TRACE(trace_gen) ;\
162 set trap, %g1 ;\
163 mov arg, %g3 ;\
164 ba,pt %xcc, sys_trap ;\
165 sub %g0, 1, %g4 ;\
166 .align 32
167
168 /*
169 * SYSCALL is used for unsupported syscall interfaces (with 'which'
170 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
171 * 'which' set to 'syscall_trap32').
172 *
173 * The SYSCALL_TRAP* macros are used for syscall entry points.
174 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
175 * is used to support ILP32. Each macro can only be used once
176 * since they each define a symbol. The symbols are used as hot patch
177 * points by the brand infrastructure to dynamically enable and disable
178 * brand syscall interposition. See the comments around BRAND_CALLBACK
179 * and brand_plat_interposition_enable() for more information.
180 */
181 #define SYSCALL_NOTT(which) \
182 set (which), %g1 ;\
183 ba,pt %xcc, user_trap ;\
184 sub %g0, 1, %g4 ;\
185 .align 32
186
187 #define SYSCALL(which) \
188 TT_TRACE(trace_gen) ;\
189 SYSCALL_NOTT(which)
190
191 #define SYSCALL_TRAP32 \
192 TT_TRACE(trace_gen) ;\
193 ALTENTRY(syscall_trap32_patch_point) \
194 SYSCALL_NOTT(syscall_trap32)
195
196 #define SYSCALL_TRAP \
197 TT_TRACE(trace_gen) ;\
198 ALTENTRY(syscall_trap_patch_point) \
199 SYSCALL_NOTT(syscall_trap)
200
201 /*
202 * GOTO just jumps to a label.
203 * It's used for things that can be fixed without going thru sys_trap.
204 */
205 #define GOTO(label) \
206 .global label ;\
207 ba,a label ;\
208 .empty ;\
209 .align 32
210
211 /*
212 * GOTO_TT just jumps to a label.
213 * correctable ECC error traps at level 0 and 1 will use this macro.
214 * It's used for things that can be fixed without going thru sys_trap.
215 */
216 #define GOTO_TT(label, ttlabel) \
217 .global label ;\
218 TT_TRACE(ttlabel) ;\
219 ba,a label ;\
220 .empty ;\
221 .align 32
222
223 /*
224 * Privileged traps
225 * Takes breakpoint if privileged, calls trap() if not.
226 */
227 #define PRIV(label) \
228 rdpr %tstate, %g1 ;\
229 btst TSTATE_PRIV, %g1 ;\
230 bnz label ;\
231 rdpr %tt, %g3 ;\
232 set trap, %g1 ;\
233 ba,pt %xcc, sys_trap ;\
234 sub %g0, 1, %g4 ;\
235 .align 32
236
237
238 /*
239 * DTrace traps.
240 */
241 #define DTRACE_PID \
242 .global dtrace_pid_probe ;\
243 set dtrace_pid_probe, %g1 ;\
244 ba,pt %xcc, user_trap ;\
245 sub %g0, 1, %g4 ;\
246 .align 32
247
248 #define DTRACE_RETURN \
249 .global dtrace_return_probe ;\
250 set dtrace_return_probe, %g1 ;\
251 ba,pt %xcc, user_trap ;\
252 sub %g0, 1, %g4 ;\
253 .align 32
254
255 /*
256 * REGISTER WINDOW MANAGEMENT MACROS
257 */
258
259 /*
260 * various convenient units of padding
261 */
262 #define SKIP(n) .skip 4*(n)
263
264 /*
265 * CLEAN_WINDOW is the simple handler for cleaning a register window.
266 */
267 #define CLEAN_WINDOW \
268 TT_TRACE_L(trace_win) ;\
269 rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\
270 clr %l0; clr %l1; clr %l2; clr %l3 ;\
271 clr %l4; clr %l5; clr %l6; clr %l7 ;\
272 clr %o0; clr %o1; clr %o2; clr %o3 ;\
273 clr %o4; clr %o5; clr %o6; clr %o7 ;\
274 retry; .align 128
275
276 #if !defined(lint)
277
278 /*
279 * If we get an unresolved tlb miss while in a window handler, the fault
280 * handler will resume execution at the last instruction of the window
281 * hander, instead of delivering the fault to the kernel. Spill handlers
282 * use this to spill windows into the wbuf.
283 *
284 * The mixed handler works by checking %sp, and branching to the correct
285 * handler. This is done by branching back to label 1: for 32b frames,
286 * or label 2: for 64b frames; which implies the handler order is: 32b,
287 * 64b, mixed. The 1: and 2: labels are offset into the routines to
288 * allow the branchs' delay slots to contain useful instructions.
289 */
290
291 /*
292 * SPILL_32bit spills a 32-bit-wide kernel register window. It
293 * assumes that the kernel context and the nucleus context are the
294 * same. The stack pointer is required to be eight-byte aligned even
295 * though this code only needs it to be four-byte aligned.
296 */
297 #define SPILL_32bit(tail) \
298 srl %sp, 0, %sp ;\
299 1: st %l0, [%sp + 0] ;\
300 st %l1, [%sp + 4] ;\
301 st %l2, [%sp + 8] ;\
302 st %l3, [%sp + 12] ;\
303 st %l4, [%sp + 16] ;\
304 st %l5, [%sp + 20] ;\
305 st %l6, [%sp + 24] ;\
306 st %l7, [%sp + 28] ;\
307 st %i0, [%sp + 32] ;\
308 st %i1, [%sp + 36] ;\
309 st %i2, [%sp + 40] ;\
310 st %i3, [%sp + 44] ;\
311 st %i4, [%sp + 48] ;\
312 st %i5, [%sp + 52] ;\
313 st %i6, [%sp + 56] ;\
314 st %i7, [%sp + 60] ;\
315 TT_TRACE_L(trace_win) ;\
316 saved ;\
317 retry ;\
318 SKIP(31-19-TT_TRACE_L_INS) ;\
319 ba,a,pt %xcc, fault_32bit_/**/tail ;\
320 .empty
321
322 /*
323 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
324 * wide address space via the designated asi. It is used to spill
325 * non-kernel windows. The stack pointer is required to be eight-byte
326 * aligned even though this code only needs it to be four-byte
327 * aligned.
328 */
329 #define SPILL_32bit_asi(asi_num, tail) \
330 srl %sp, 0, %sp ;\
331 1: sta %l0, [%sp + %g0]asi_num ;\
332 mov 4, %g1 ;\
333 sta %l1, [%sp + %g1]asi_num ;\
334 mov 8, %g2 ;\
335 sta %l2, [%sp + %g2]asi_num ;\
336 mov 12, %g3 ;\
337 sta %l3, [%sp + %g3]asi_num ;\
338 add %sp, 16, %g4 ;\
339 sta %l4, [%g4 + %g0]asi_num ;\
340 sta %l5, [%g4 + %g1]asi_num ;\
341 sta %l6, [%g4 + %g2]asi_num ;\
342 sta %l7, [%g4 + %g3]asi_num ;\
343 add %g4, 16, %g4 ;\
344 sta %i0, [%g4 + %g0]asi_num ;\
345 sta %i1, [%g4 + %g1]asi_num ;\
346 sta %i2, [%g4 + %g2]asi_num ;\
347 sta %i3, [%g4 + %g3]asi_num ;\
348 add %g4, 16, %g4 ;\
349 sta %i4, [%g4 + %g0]asi_num ;\
350 sta %i5, [%g4 + %g1]asi_num ;\
351 sta %i6, [%g4 + %g2]asi_num ;\
352 sta %i7, [%g4 + %g3]asi_num ;\
353 TT_TRACE_L(trace_win) ;\
354 saved ;\
355 retry ;\
356 SKIP(31-25-TT_TRACE_L_INS) ;\
357 ba,a,pt %xcc, fault_32bit_/**/tail ;\
358 .empty
359
360 #define SPILL_32bit_tt1(asi_num, tail) \
361 ba,a,pt %xcc, fault_32bit_/**/tail ;\
362 .empty ;\
363 .align 128
364
365
366 /*
367 * FILL_32bit fills a 32-bit-wide kernel register window. It assumes
368 * that the kernel context and the nucleus context are the same. The
369 * stack pointer is required to be eight-byte aligned even though this
370 * code only needs it to be four-byte aligned.
371 */
372 #define FILL_32bit(tail) \
373 srl %sp, 0, %sp ;\
374 1: TT_TRACE_L(trace_win) ;\
375 ld [%sp + 0], %l0 ;\
376 ld [%sp + 4], %l1 ;\
377 ld [%sp + 8], %l2 ;\
378 ld [%sp + 12], %l3 ;\
379 ld [%sp + 16], %l4 ;\
380 ld [%sp + 20], %l5 ;\
381 ld [%sp + 24], %l6 ;\
382 ld [%sp + 28], %l7 ;\
383 ld [%sp + 32], %i0 ;\
384 ld [%sp + 36], %i1 ;\
385 ld [%sp + 40], %i2 ;\
386 ld [%sp + 44], %i3 ;\
387 ld [%sp + 48], %i4 ;\
388 ld [%sp + 52], %i5 ;\
389 ld [%sp + 56], %i6 ;\
390 ld [%sp + 60], %i7 ;\
391 restored ;\
392 retry ;\
393 SKIP(31-19-TT_TRACE_L_INS) ;\
394 ba,a,pt %xcc, fault_32bit_/**/tail ;\
395 .empty
396
397 /*
398 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
399 * wide address space via the designated asi. It is used to fill
400 * non-kernel windows. The stack pointer is required to be eight-byte
401 * aligned even though this code only needs it to be four-byte
402 * aligned.
403 */
404 #define FILL_32bit_asi(asi_num, tail) \
405 srl %sp, 0, %sp ;\
406 1: TT_TRACE_L(trace_win) ;\
407 mov 4, %g1 ;\
408 lda [%sp + %g0]asi_num, %l0 ;\
409 mov 8, %g2 ;\
410 lda [%sp + %g1]asi_num, %l1 ;\
411 mov 12, %g3 ;\
412 lda [%sp + %g2]asi_num, %l2 ;\
413 lda [%sp + %g3]asi_num, %l3 ;\
414 add %sp, 16, %g4 ;\
415 lda [%g4 + %g0]asi_num, %l4 ;\
416 lda [%g4 + %g1]asi_num, %l5 ;\
417 lda [%g4 + %g2]asi_num, %l6 ;\
418 lda [%g4 + %g3]asi_num, %l7 ;\
419 add %g4, 16, %g4 ;\
420 lda [%g4 + %g0]asi_num, %i0 ;\
421 lda [%g4 + %g1]asi_num, %i1 ;\
422 lda [%g4 + %g2]asi_num, %i2 ;\
423 lda [%g4 + %g3]asi_num, %i3 ;\
424 add %g4, 16, %g4 ;\
425 lda [%g4 + %g0]asi_num, %i4 ;\
426 lda [%g4 + %g1]asi_num, %i5 ;\
427 lda [%g4 + %g2]asi_num, %i6 ;\
428 lda [%g4 + %g3]asi_num, %i7 ;\
429 restored ;\
430 retry ;\
431 SKIP(31-25-TT_TRACE_L_INS) ;\
432 ba,a,pt %xcc, fault_32bit_/**/tail ;\
433 .empty
434
435
436 /*
437 * SPILL_64bit spills a 64-bit-wide kernel register window. It
438 * assumes that the kernel context and the nucleus context are the
439 * same. The stack pointer is required to be eight-byte aligned.
440 */
441 #define SPILL_64bit(tail) \
442 2: stx %l0, [%sp + V9BIAS64 + 0] ;\
443 stx %l1, [%sp + V9BIAS64 + 8] ;\
444 stx %l2, [%sp + V9BIAS64 + 16] ;\
445 stx %l3, [%sp + V9BIAS64 + 24] ;\
446 stx %l4, [%sp + V9BIAS64 + 32] ;\
447 stx %l5, [%sp + V9BIAS64 + 40] ;\
448 stx %l6, [%sp + V9BIAS64 + 48] ;\
449 stx %l7, [%sp + V9BIAS64 + 56] ;\
450 stx %i0, [%sp + V9BIAS64 + 64] ;\
451 stx %i1, [%sp + V9BIAS64 + 72] ;\
452 stx %i2, [%sp + V9BIAS64 + 80] ;\
453 stx %i3, [%sp + V9BIAS64 + 88] ;\
454 stx %i4, [%sp + V9BIAS64 + 96] ;\
455 stx %i5, [%sp + V9BIAS64 + 104] ;\
456 stx %i6, [%sp + V9BIAS64 + 112] ;\
457 stx %i7, [%sp + V9BIAS64 + 120] ;\
458 TT_TRACE_L(trace_win) ;\
459 saved ;\
460 retry ;\
461 SKIP(31-18-TT_TRACE_L_INS) ;\
462 ba,a,pt %xcc, fault_64bit_/**/tail ;\
463 .empty
464
465 #define SPILL_64bit_ktt1(tail) \
466 ba,a,pt %xcc, fault_64bit_/**/tail ;\
467 .empty ;\
468 .align 128
469
470 #define SPILL_mixed_ktt1(tail) \
471 btst 1, %sp ;\
472 bz,a,pt %xcc, fault_32bit_/**/tail ;\
473 srl %sp, 0, %sp ;\
474 ba,a,pt %xcc, fault_64bit_/**/tail ;\
475 .empty ;\
476 .align 128
477
478 /*
479 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
480 * wide address space via the designated asi. It is used to spill
481 * non-kernel windows. The stack pointer is required to be eight-byte
482 * aligned.
483 */
484 #define SPILL_64bit_asi(asi_num, tail) \
485 mov 0 + V9BIAS64, %g1 ;\
486 2: stxa %l0, [%sp + %g1]asi_num ;\
487 mov 8 + V9BIAS64, %g2 ;\
488 stxa %l1, [%sp + %g2]asi_num ;\
489 mov 16 + V9BIAS64, %g3 ;\
490 stxa %l2, [%sp + %g3]asi_num ;\
491 mov 24 + V9BIAS64, %g4 ;\
492 stxa %l3, [%sp + %g4]asi_num ;\
493 add %sp, 32, %g5 ;\
494 stxa %l4, [%g5 + %g1]asi_num ;\
495 stxa %l5, [%g5 + %g2]asi_num ;\
496 stxa %l6, [%g5 + %g3]asi_num ;\
497 stxa %l7, [%g5 + %g4]asi_num ;\
498 add %g5, 32, %g5 ;\
499 stxa %i0, [%g5 + %g1]asi_num ;\
500 stxa %i1, [%g5 + %g2]asi_num ;\
501 stxa %i2, [%g5 + %g3]asi_num ;\
502 stxa %i3, [%g5 + %g4]asi_num ;\
503 add %g5, 32, %g5 ;\
504 stxa %i4, [%g5 + %g1]asi_num ;\
505 stxa %i5, [%g5 + %g2]asi_num ;\
506 stxa %i6, [%g5 + %g3]asi_num ;\
507 stxa %i7, [%g5 + %g4]asi_num ;\
508 TT_TRACE_L(trace_win) ;\
509 saved ;\
510 retry ;\
511 SKIP(31-25-TT_TRACE_L_INS) ;\
512 ba,a,pt %xcc, fault_64bit_/**/tail ;\
513 .empty
514
515 #define SPILL_64bit_tt1(asi_num, tail) \
516 ba,a,pt %xcc, fault_64bit_/**/tail ;\
517 .empty ;\
518 .align 128
519
520 /*
521 * FILL_64bit fills a 64-bit-wide kernel register window. It assumes
522 * that the kernel context and the nucleus context are the same. The
523 * stack pointer is required to be eight-byte aligned.
524 */
525 #define FILL_64bit(tail) \
526 2: TT_TRACE_L(trace_win) ;\
527 ldx [%sp + V9BIAS64 + 0], %l0 ;\
528 ldx [%sp + V9BIAS64 + 8], %l1 ;\
529 ldx [%sp + V9BIAS64 + 16], %l2 ;\
530 ldx [%sp + V9BIAS64 + 24], %l3 ;\
531 ldx [%sp + V9BIAS64 + 32], %l4 ;\
532 ldx [%sp + V9BIAS64 + 40], %l5 ;\
533 ldx [%sp + V9BIAS64 + 48], %l6 ;\
534 ldx [%sp + V9BIAS64 + 56], %l7 ;\
535 ldx [%sp + V9BIAS64 + 64], %i0 ;\
536 ldx [%sp + V9BIAS64 + 72], %i1 ;\
537 ldx [%sp + V9BIAS64 + 80], %i2 ;\
538 ldx [%sp + V9BIAS64 + 88], %i3 ;\
539 ldx [%sp + V9BIAS64 + 96], %i4 ;\
540 ldx [%sp + V9BIAS64 + 104], %i5 ;\
541 ldx [%sp + V9BIAS64 + 112], %i6 ;\
542 ldx [%sp + V9BIAS64 + 120], %i7 ;\
543 restored ;\
544 retry ;\
545 SKIP(31-18-TT_TRACE_L_INS) ;\
546 ba,a,pt %xcc, fault_64bit_/**/tail ;\
547 .empty
548
549 /*
550 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
551 * wide address space via the designated asi. It is used to fill
552 * non-kernel windows. The stack pointer is required to be eight-byte
553 * aligned.
554 */
555 #define FILL_64bit_asi(asi_num, tail) \
556 mov V9BIAS64 + 0, %g1 ;\
557 2: TT_TRACE_L(trace_win) ;\
558 ldxa [%sp + %g1]asi_num, %l0 ;\
559 mov V9BIAS64 + 8, %g2 ;\
560 ldxa [%sp + %g2]asi_num, %l1 ;\
561 mov V9BIAS64 + 16, %g3 ;\
562 ldxa [%sp + %g3]asi_num, %l2 ;\
563 mov V9BIAS64 + 24, %g4 ;\
564 ldxa [%sp + %g4]asi_num, %l3 ;\
565 add %sp, 32, %g5 ;\
566 ldxa [%g5 + %g1]asi_num, %l4 ;\
567 ldxa [%g5 + %g2]asi_num, %l5 ;\
568 ldxa [%g5 + %g3]asi_num, %l6 ;\
569 ldxa [%g5 + %g4]asi_num, %l7 ;\
570 add %g5, 32, %g5 ;\
571 ldxa [%g5 + %g1]asi_num, %i0 ;\
572 ldxa [%g5 + %g2]asi_num, %i1 ;\
573 ldxa [%g5 + %g3]asi_num, %i2 ;\
574 ldxa [%g5 + %g4]asi_num, %i3 ;\
575 add %g5, 32, %g5 ;\
576 ldxa [%g5 + %g1]asi_num, %i4 ;\
577 ldxa [%g5 + %g2]asi_num, %i5 ;\
578 ldxa [%g5 + %g3]asi_num, %i6 ;\
579 ldxa [%g5 + %g4]asi_num, %i7 ;\
580 restored ;\
581 retry ;\
582 SKIP(31-25-TT_TRACE_L_INS) ;\
583 ba,a,pt %xcc, fault_64bit_/**/tail ;\
584 .empty
585
586
587 #endif /* !lint */
588
589 /*
590 * SPILL_mixed spills either size window, depending on
591 * whether %sp is even or odd, to a 32-bit address space.
592 * This may only be used in conjunction with SPILL_32bit/
593 * FILL_64bit.
594 * Clear upper 32 bits of %sp if it is odd.
595 * We won't need to clear them in 64 bit kernel.
596 */
597 #define SPILL_mixed \
598 btst 1, %sp ;\
599 bz,a,pt %xcc, 1b ;\
600 srl %sp, 0, %sp ;\
601 ba,pt %xcc, 2b ;\
602 nop ;\
603 .align 128
604
605 /*
606 * FILL_mixed(ASI) fills either size window, depending on
607 * whether %sp is even or odd, from a 32-bit address space.
608 * This may only be used in conjunction with FILL_32bit/
609 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
610 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
611 * attention should be paid to the instructions that belong
612 * in the delay slots of the branches depending on the type
613 * of fill handler being branched to.
614 * Clear upper 32 bits of %sp if it is odd.
615 * We won't need to clear them in 64 bit kernel.
616 */
617 #define FILL_mixed \
618 btst 1, %sp ;\
619 bz,a,pt %xcc, 1b ;\
620 srl %sp, 0, %sp ;\
621 ba,pt %xcc, 2b ;\
622 nop ;\
623 .align 128
624
625
626 /*
627 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
628 * respectively, into the address space via the designated asi. The
629 * unbiased stack pointer is required to be eight-byte aligned (even for
630 * the 32-bit case even though this code does not require such strict
631 * alignment).
632 *
633 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
634 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
635 * will cause cwp + 2 to be spilled but will not clean cwp + 1. That
636 * window may contain kernel data so in user_rtt we set wstate to call
637 * these spill handlers on the first user spill trap. These handler then
638 * spill the appropriate window but also back up a window and clean the
639 * window that didn't get a cleanwin trap.
640 */
641 #define SPILL_32clean(asi_num, tail) \
642 srl %sp, 0, %sp ;\
643 sta %l0, [%sp + %g0]asi_num ;\
644 mov 4, %g1 ;\
645 sta %l1, [%sp + %g1]asi_num ;\
646 mov 8, %g2 ;\
647 sta %l2, [%sp + %g2]asi_num ;\
648 mov 12, %g3 ;\
649 sta %l3, [%sp + %g3]asi_num ;\
650 add %sp, 16, %g4 ;\
651 sta %l4, [%g4 + %g0]asi_num ;\
652 sta %l5, [%g4 + %g1]asi_num ;\
653 sta %l6, [%g4 + %g2]asi_num ;\
654 sta %l7, [%g4 + %g3]asi_num ;\
655 add %g4, 16, %g4 ;\
656 sta %i0, [%g4 + %g0]asi_num ;\
657 sta %i1, [%g4 + %g1]asi_num ;\
658 sta %i2, [%g4 + %g2]asi_num ;\
659 sta %i3, [%g4 + %g3]asi_num ;\
660 add %g4, 16, %g4 ;\
661 sta %i4, [%g4 + %g0]asi_num ;\
662 sta %i5, [%g4 + %g1]asi_num ;\
663 sta %i6, [%g4 + %g2]asi_num ;\
664 sta %i7, [%g4 + %g3]asi_num ;\
665 TT_TRACE_L(trace_win) ;\
666 b .spill_clean ;\
667 mov WSTATE_USER32, %g7 ;\
668 SKIP(31-25-TT_TRACE_L_INS) ;\
669 ba,a,pt %xcc, fault_32bit_/**/tail ;\
670 .empty
671
672 #define SPILL_64clean(asi_num, tail) \
673 mov 0 + V9BIAS64, %g1 ;\
674 stxa %l0, [%sp + %g1]asi_num ;\
675 mov 8 + V9BIAS64, %g2 ;\
676 stxa %l1, [%sp + %g2]asi_num ;\
677 mov 16 + V9BIAS64, %g3 ;\
678 stxa %l2, [%sp + %g3]asi_num ;\
679 mov 24 + V9BIAS64, %g4 ;\
680 stxa %l3, [%sp + %g4]asi_num ;\
681 add %sp, 32, %g5 ;\
682 stxa %l4, [%g5 + %g1]asi_num ;\
683 stxa %l5, [%g5 + %g2]asi_num ;\
684 stxa %l6, [%g5 + %g3]asi_num ;\
685 stxa %l7, [%g5 + %g4]asi_num ;\
686 add %g5, 32, %g5 ;\
687 stxa %i0, [%g5 + %g1]asi_num ;\
688 stxa %i1, [%g5 + %g2]asi_num ;\
689 stxa %i2, [%g5 + %g3]asi_num ;\
690 stxa %i3, [%g5 + %g4]asi_num ;\
691 add %g5, 32, %g5 ;\
692 stxa %i4, [%g5 + %g1]asi_num ;\
693 stxa %i5, [%g5 + %g2]asi_num ;\
694 stxa %i6, [%g5 + %g3]asi_num ;\
695 stxa %i7, [%g5 + %g4]asi_num ;\
696 TT_TRACE_L(trace_win) ;\
697 b .spill_clean ;\
698 mov WSTATE_USER64, %g7 ;\
699 SKIP(31-25-TT_TRACE_L_INS) ;\
700 ba,a,pt %xcc, fault_64bit_/**/tail ;\
701 .empty
702
703
704 /*
705 * Floating point disabled.
706 */
707 #define FP_DISABLED_TRAP \
708 TT_TRACE(trace_gen) ;\
709 ba,pt %xcc,.fp_disabled ;\
710 nop ;\
711 .align 32
712
713 /*
714 * Floating point exceptions.
715 */
716 #define FP_IEEE_TRAP \
717 TT_TRACE(trace_gen) ;\
718 ba,pt %xcc,.fp_ieee_exception ;\
719 nop ;\
720 .align 32
721
722 #define FP_TRAP \
723 TT_TRACE(trace_gen) ;\
724 ba,pt %xcc,.fp_exception ;\
725 nop ;\
726 .align 32
727
728 #if !defined(lint)
729
730 /*
731 * ECACHE_ECC error traps at level 0 and level 1
732 */
733 #define ECACHE_ECC(table_name) \
734 .global table_name ;\
735 table_name: ;\
736 membar #Sync ;\
737 set trap, %g1 ;\
738 rdpr %tt, %g3 ;\
739 ba,pt %xcc, sys_trap ;\
740 sub %g0, 1, %g4 ;\
741 .align 32
742
743 #endif /* !lint */
744
745 /*
746 * illegal instruction trap
747 */
748 #define ILLTRAP_INSTR \
749 membar #Sync ;\
750 TT_TRACE(trace_gen) ;\
751 or %g0, P_UTRAP4, %g2 ;\
752 or %g0, T_UNIMP_INSTR, %g3 ;\
753 sethi %hi(.check_v9utrap), %g4 ;\
754 jmp %g4 + %lo(.check_v9utrap) ;\
755 nop ;\
756 .align 32
757
758 /*
759 * tag overflow trap
760 */
761 #define TAG_OVERFLOW \
762 TT_TRACE(trace_gen) ;\
763 or %g0, P_UTRAP10, %g2 ;\
764 or %g0, T_TAG_OVERFLOW, %g3 ;\
765 sethi %hi(.check_v9utrap), %g4 ;\
766 jmp %g4 + %lo(.check_v9utrap) ;\
767 nop ;\
768 .align 32
769
770 /*
771 * divide by zero trap
772 */
773 #define DIV_BY_ZERO \
774 TT_TRACE(trace_gen) ;\
775 or %g0, P_UTRAP11, %g2 ;\
776 or %g0, T_IDIV0, %g3 ;\
777 sethi %hi(.check_v9utrap), %g4 ;\
778 jmp %g4 + %lo(.check_v9utrap) ;\
779 nop ;\
780 .align 32
781
782 /*
783 * trap instruction for V9 user trap handlers
784 */
785 #define TRAP_INSTR \
786 TT_TRACE(trace_gen) ;\
787 or %g0, T_SOFTWARE_TRAP, %g3 ;\
788 sethi %hi(.check_v9utrap), %g4 ;\
789 jmp %g4 + %lo(.check_v9utrap) ;\
790 nop ;\
791 .align 32
792 #define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
793
794 /*
795 * LEVEL_INTERRUPT is for level N interrupts.
796 * VECTOR_INTERRUPT is for the vector trap.
797 */
798 #define LEVEL_INTERRUPT(level) \
799 .global tt_pil/**/level ;\
800 tt_pil/**/level: ;\
801 ba,pt %xcc, pil_interrupt ;\
802 mov level, %g4 ;\
803 .align 32
804
805 #define LEVEL14_INTERRUPT \
806 ba pil14_interrupt ;\
807 mov PIL_14, %g4 ;\
808 .align 32
809
810 #define LEVEL15_INTERRUPT \
811 ba pil15_interrupt ;\
812 mov PIL_15, %g4 ;\
813 .align 32
814
815 #define CPU_MONDO \
816 ba,a,pt %xcc, cpu_mondo ;\
817 .align 32
818
819 #define DEV_MONDO \
820 ba,a,pt %xcc, dev_mondo ;\
821 .align 32
822
823 /*
824 * We take over the rtba after we set our trap table and
825 * fault status area. The watchdog reset trap is now handled by the OS.
826 */
827 #define WATCHDOG_RESET \
828 mov PTL1_BAD_WATCHDOG, %g1 ;\
829 ba,a,pt %xcc, .watchdog_trap ;\
830 .align 32
831
832 /*
833 * RED is for traps that use the red mode handler.
834 * We should never see these either.
835 */
836 #define RED \
837 mov PTL1_BAD_RED, %g1 ;\
838 ba,a,pt %xcc, .watchdog_trap ;\
839 .align 32
840
841
842 /*
843 * MMU Trap Handlers.
844 */
845
846 /*
847 * synthesize for trap(): SFSR in %g3
848 */
849 #define IMMU_EXCEPTION \
850 MMU_FAULT_STATUS_AREA(%g3) ;\
851 rdpr %tpc, %g2 ;\
852 ldx [%g3 + MMFSA_I_TYPE], %g1 ;\
853 ldx [%g3 + MMFSA_I_CTX], %g3 ;\
854 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
855 or %g3, %g1, %g3 ;\
856 ba,pt %xcc, .mmu_exception_end ;\
857 mov T_INSTR_EXCEPTION, %g1 ;\
858 .align 32
859
860 /*
861 * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3
862 */
863 #define DMMU_EXCEPTION \
864 ba,a,pt %xcc, .dmmu_exception ;\
865 .align 32
866
867 /*
868 * synthesize for trap(): SFAR in %g2, SFSR in %g3
869 */
870 #define DMMU_EXC_AG_PRIV \
871 MMU_FAULT_STATUS_AREA(%g3) ;\
872 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\
873 /* Fault type not available in MMU fault status area */ ;\
874 mov MMFSA_F_PRVACT, %g1 ;\
875 ldx [%g3 + MMFSA_D_CTX], %g3 ;\
876 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
877 ba,pt %xcc, .mmu_priv_exception ;\
878 or %g3, %g1, %g3 ;\
879 .align 32
880
881 /*
882 * synthesize for trap(): SFAR in %g2, SFSR in %g3
883 */
884 #define DMMU_EXC_AG_NOT_ALIGNED \
885 MMU_FAULT_STATUS_AREA(%g3) ;\
886 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\
887 /* Fault type not available in MMU fault status area */ ;\
888 mov MMFSA_F_UNALIGN, %g1 ;\
889 ldx [%g3 + MMFSA_D_CTX], %g3 ;\
890 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
891 ba,pt %xcc, .mmu_exception_not_aligned ;\
892 or %g3, %g1, %g3 /* SFSR */ ;\
893 .align 32
894 /*
895 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
896 */
897
898 /*
899 * synthesize for trap(): SFAR in %g2, SFSR in %g3
900 */
901 #define DMMU_EXC_LDDF_NOT_ALIGNED \
902 ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned ;\
903 .align 32
904 /*
905 * synthesize for trap(): SFAR in %g2, SFSR in %g3
906 */
907 #define DMMU_EXC_STDF_NOT_ALIGNED \
908 ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned ;\
909 .align 32
910
911 #if defined(cscope)
912 /*
913 * Define labels to direct cscope quickly to labels that
914 * are generated by macro expansion of DTLB_MISS().
915 */
916 .global tt0_dtlbmiss
917 tt0_dtlbmiss:
918 .global tt1_dtlbmiss
919 tt1_dtlbmiss:
920 nop
921 #endif
922
923 /*
924 * Data miss handler (must be exactly 32 instructions)
925 *
926 * This handler is invoked only if the hypervisor has been instructed
927 * not to do any TSB walk.
928 *
929 * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss
930 * handler.
931 *
932 * User TLB miss handling depends upon whether a user process has one or
933 * two TSBs. User TSB information (physical base and size code) is kept
934 * in two dedicated scratchpad registers. Absence of a user TSB (primarily
935 * second TSB) is indicated by a negative value (-1) in that register.
936 */
937
938 /*
939 * synthesize for miss handler: pseudo-tag access in %g2 (with context "type"
940 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
941 */
942 #define DTLB_MISS(table_name) ;\
943 .global table_name/**/_dtlbmiss ;\
944 table_name/**/_dtlbmiss: ;\
945 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
946 cmp %g3, INVALID_CONTEXT ;\
947 ble,pn %xcc, sfmmu_kdtlb_miss ;\
948 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
949 mov SCRATCHPAD_UTSBREG2, %g1 ;\
950 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\
951 brgez,pn %g1, sfmmu_udtlb_slowpath /* branch if 2 TSBs */ ;\
952 nop ;\
953 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\
954 ba,pt %xcc, sfmmu_udtlb_fastpath /* no 4M TSB, miss */ ;\
955 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
956 .align 128
957
958
959 #if defined(cscope)
960 /*
961 * Define labels to direct cscope quickly to labels that
962 * are generated by macro expansion of ITLB_MISS().
963 */
964 .global tt0_itlbmiss
965 tt0_itlbmiss:
966 .global tt1_itlbmiss
967 tt1_itlbmiss:
968 nop
969 #endif
970
971 /*
972 * Instruction miss handler.
973 *
974 * This handler is invoked only if the hypervisor has been instructed
975 * not to do any TSB walk.
976 *
977 * ldda instructions will have their ASI patched
978 * by sfmmu_patch_ktsb at runtime.
979 * MUST be EXACTLY 32 instructions or we'll break.
980 */
981
982 /*
983 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
984 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
985 */
986 #define ITLB_MISS(table_name) \
987 .global table_name/**/_itlbmiss ;\
988 table_name/**/_itlbmiss: ;\
989 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
990 cmp %g3, INVALID_CONTEXT ;\
991 ble,pn %xcc, sfmmu_kitlb_miss ;\
992 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
993 mov SCRATCHPAD_UTSBREG2, %g1 ;\
994 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\
995 brgez,pn %g1, sfmmu_uitlb_slowpath /* branch if 2 TSBs */ ;\
996 nop ;\
997 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\
998 ba,pt %xcc, sfmmu_uitlb_fastpath /* no 4M TSB, miss */ ;\
999 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
1000 .align 128
1001
1002 #define DTSB_MISS \
1003 GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu)
1004
1005 #define ITSB_MISS \
1006 GOTO_TT(sfmmu_slow_immu_miss,trace_immu)
1007
1008 /*
1009 * This macro is the first level handler for fast protection faults.
1010 * It first demaps the tlb entry which generated the fault and then
1011 * attempts to set the modify bit on the hash. It needs to be
1012 * exactly 32 instructions.
1013 */
1014 /*
1015 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1016 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1017 */
1018 #define DTLB_PROT \
1019 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
1020 /* ;\
1021 * g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\
1022 * g3 = ctx type (0, 1, or 2) ;\
1023 */ ;\
1024 TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\
1025 /* clobbers g1 and g6 XXXQ? */ ;\
1026 brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\
1027 nop ;\
1028 ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\
1029 .align 128
1030
1031 #define DMMU_EXCEPTION_TL1 ;\
1032 ba,a,pt %xcc, mmu_trap_tl1 ;\
1033 .align 32
1034
1035 #define MISALIGN_ADDR_TL1 ;\
1036 ba,a,pt %xcc, mmu_trap_tl1 ;\
1037 .align 32
1038
1039 /*
1040 * Trace a tsb hit
1041 * g1 = tsbe pointer (in/clobbered)
1042 * g2 = tag access register (in)
1043 * g3 - g4 = scratch (clobbered)
1044 * g5 = tsbe data (in)
1045 * g6 = scratch (clobbered)
1046 * g7 = pc we jumped here from (in)
1047 * ttextra = value to OR in to trap type (%tt) (in)
1048 */
1049 #ifdef TRAPTRACE
1050 #define TRACE_TSBHIT(ttextra) \
1051 membar #Sync ;\
1052 sethi %hi(FLUSH_ADDR), %g6 ;\
1053 flush %g6 ;\
1054 TRACE_PTR(%g3, %g6) ;\
1055 GET_TRACE_TICK(%g6, %g4) ;\
1056 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\
1057 stna %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\
1058 stna %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\
1059 rdpr %tnpc, %g6 ;\
1060 stna %g6, [%g3 + TRAP_ENT_F2]%asi ;\
1061 stna %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\
1062 stna %g0, [%g3 + TRAP_ENT_F4]%asi ;\
1063 rdpr %tpc, %g6 ;\
1064 stna %g6, [%g3 + TRAP_ENT_TPC]%asi ;\
1065 TRACE_SAVE_TL_GL_REGS(%g3, %g6) ;\
1066 rdpr %tt, %g6 ;\
1067 or %g6, (ttextra), %g1 ;\
1068 stha %g1, [%g3 + TRAP_ENT_TT]%asi ;\
1069 MMU_FAULT_STATUS_AREA(%g4) ;\
1070 mov MMFSA_D_ADDR, %g1 ;\
1071 cmp %g6, FAST_IMMU_MISS_TT ;\
1072 move %xcc, MMFSA_I_ADDR, %g1 ;\
1073 cmp %g6, T_INSTR_MMU_MISS ;\
1074 move %xcc, MMFSA_I_ADDR, %g1 ;\
1075 ldx [%g4 + %g1], %g1 ;\
1076 stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */ ;\
1077 mov MMFSA_D_CTX, %g1 ;\
1078 cmp %g6, FAST_IMMU_MISS_TT ;\
1079 move %xcc, MMFSA_I_CTX, %g1 ;\
1080 cmp %g6, T_INSTR_MMU_MISS ;\
1081 move %xcc, MMFSA_I_CTX, %g1 ;\
1082 ldx [%g4 + %g1], %g1 ;\
1083 stna %g1, [%g3 + TRAP_ENT_TR]%asi ;\
1084 TRACE_NEXT(%g3, %g4, %g6)
1085 #else
1086 #define TRACE_TSBHIT(ttextra)
1087 #endif
1088
1089
1090 #if defined(lint)
1091
1092 struct scb trap_table;
1093 struct scb scb; /* trap_table/scb are the same object */
1094
1095 #else /* lint */
1096
1097 /*
1098 * =======================================================================
1099 * SPARC V9 TRAP TABLE
1100 *
1101 * The trap table is divided into two halves: the first half is used when
1102 * taking traps when TL=0; the second half is used when taking traps from
1103 * TL>0. Note that handlers in the second half of the table might not be able
1104 * to make the same assumptions as handlers in the first half of the table.
1105 *
1106 * Worst case trap nesting so far:
1107 *
1108 * at TL=0 client issues software trap requesting service
1109 * at TL=1 nucleus wants a register window
1110 * at TL=2 register window clean/spill/fill takes a TLB miss
1111 * at TL=3 processing TLB miss
1112 * at TL=4 handle asynchronous error
1113 *
1114 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1115 *
1116 * =======================================================================
1117 */
1118 .section ".text"
1119 .align 4
1120 .global trap_table, scb, trap_table0, trap_table1, etrap_table
1121 .type trap_table, #object
1122 .type trap_table0, #object
1123 .type trap_table1, #object
1124 .type scb, #object
1125 trap_table:
1126 scb:
1127 trap_table0:
1128 /* hardware traps */
1129 NOT; /* 000 reserved */
1130 RED; /* 001 power on reset */
1131 WATCHDOG_RESET; /* 002 watchdog reset */
1132 RED; /* 003 externally initiated reset */
1133 RED; /* 004 software initiated reset */
1134 RED; /* 005 red mode exception */
1135 NOT; NOT; /* 006 - 007 reserved */
1136 IMMU_EXCEPTION; /* 008 instruction access exception */
1137 ITSB_MISS; /* 009 instruction access MMU miss */
1138 NOT; /* 00A reserved */
1139 NOT; NOT4; /* 00B - 00F reserved */
1140 ILLTRAP_INSTR; /* 010 illegal instruction */
1141 TRAP(T_PRIV_INSTR); /* 011 privileged opcode */
1142 TRAP(T_UNIMP_LDD); /* 012 unimplemented LDD */
1143 TRAP(T_UNIMP_STD); /* 013 unimplemented STD */
1144 NOT4; NOT4; NOT4; /* 014 - 01F reserved */
1145 FP_DISABLED_TRAP; /* 020 fp disabled */
1146 FP_IEEE_TRAP; /* 021 fp exception ieee 754 */
1147 FP_TRAP; /* 022 fp exception other */
1148 TAG_OVERFLOW; /* 023 tag overflow */
1149 CLEAN_WINDOW; /* 024 - 027 clean window */
1150 DIV_BY_ZERO; /* 028 division by zero */
1151 NOT; /* 029 internal processor error */
1152 NOT; NOT; NOT4; /* 02A - 02F reserved */
1153 DMMU_EXCEPTION; /* 030 data access exception */
1154 DTSB_MISS; /* 031 data access MMU miss */
1155 NOT; /* 032 reserved */
1156 NOT; /* 033 data access protection */
1157 DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */
1158 DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */
1159 DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */
1160 DMMU_EXC_AG_PRIV; /* 037 privileged action */
1161 NOT; /* 038 LDQF mem address not aligned */
1162 NOT; /* 039 STQF mem address not aligned */
1163 NOT; NOT; NOT4; /* 03A - 03F reserved */
1164 NOT; /* 040 async data error */
1165 LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */
1166 LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */
1167 LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */
1168 LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */
1169 LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */
1170 LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */
1171 LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */
1172 LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */
1173 LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */
1174 LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */
1175 LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */
1176 LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */
1177 LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */
1178 LEVEL14_INTERRUPT; /* 04E interrupt level 14 */
1179 LEVEL15_INTERRUPT; /* 04F interrupt level 15 */
1180 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */
1181 NOT; /* 060 interrupt vector */
1182 GOTO(kmdb_trap); /* 061 PA watchpoint */
1183 GOTO(kmdb_trap); /* 062 VA watchpoint */
1184 NOT; /* 063 reserved */
1185 ITLB_MISS(tt0); /* 064 instruction access MMU miss */
1186 DTLB_MISS(tt0); /* 068 data access MMU miss */
1187 DTLB_PROT; /* 06C data access protection */
1188 NOT; /* 070 reserved */
1189 NOT; /* 071 reserved */
1190 NOT; /* 072 reserved */
1191 NOT; /* 073 reserved */
1192 NOT4; NOT4 /* 074 - 07B reserved */
1193 CPU_MONDO; /* 07C cpu_mondo */
1194 DEV_MONDO; /* 07D dev_mondo */
1195 GOTO_TT(resumable_error, trace_gen); /* 07E resumable error */
1196 GOTO_TT(nonresumable_error, trace_gen); /* 07F non-reasumable error */
1197 NOT4; /* 080 spill 0 normal */
1198 SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */
1199 SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */
1200 SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */
1201 SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */
1202 SPILL_32bit(not); /* 094 spill 5 normal */
1203 SPILL_64bit(not); /* 098 spill 6 normal */
1204 SPILL_mixed; /* 09C spill 7 normal */
1205 NOT4; /* 0A0 spill 0 other */
1206 SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */
1207 SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */
1208 SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */
1209 SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */
1210 NOT4; /* 0B4 spill 5 other */
1211 NOT4; /* 0B8 spill 6 other */
1212 NOT4; /* 0BC spill 7 other */
1213 NOT4; /* 0C0 fill 0 normal */
1214 FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */
1215 FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */
1216 FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */
1217 FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */
1218 FILL_32bit(not); /* 0D4 fill 5 normal */
1219 FILL_64bit(not); /* 0D8 fill 6 normal */
1220 FILL_mixed; /* 0DC fill 7 normal */
1221 NOT4; /* 0E0 fill 0 other */
1222 NOT4; /* 0E4 fill 1 other */
1223 NOT4; /* 0E8 fill 2 other */
1224 NOT4; /* 0EC fill 3 other */
1225 NOT4; /* 0F0 fill 4 other */
1226 NOT4; /* 0F4 fill 5 other */
1227 NOT4; /* 0F8 fill 6 other */
1228 NOT4; /* 0FC fill 7 other */
1229 /* user traps */
1230 GOTO(syscall_trap_4x); /* 100 old system call */
1231 TRAP(T_BREAKPOINT); /* 101 user breakpoint */
1232 TRAP(T_DIV0); /* 102 user divide by zero */
1233 GOTO(.flushw); /* 103 flush windows */
1234 GOTO(.clean_windows); /* 104 clean windows */
1235 BAD; /* 105 range check ?? */
1236 GOTO(.fix_alignment); /* 106 do unaligned references */
1237 BAD; /* 107 unused */
1238 SYSCALL_TRAP32; /* 108 ILP32 system call on LP64 */
1239 GOTO(set_trap0_addr); /* 109 set trap0 address */
1240 BAD; BAD; BAD4; /* 10A - 10F unused */
1241 TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */
1242 GOTO(.getcc); /* 120 get condition codes */
1243 GOTO(.setcc); /* 121 set condition codes */
1244 GOTO(.getpsr); /* 122 get psr */
1245 GOTO(.setpsr); /* 123 set psr (some fields) */
1246 GOTO(get_timestamp); /* 124 get timestamp */
1247 GOTO(get_virtime); /* 125 get lwp virtual time */
1248 PRIV(self_xcall); /* 126 self xcall */
1249 GOTO(get_hrestime); /* 127 get hrestime */
1250 BAD; /* 128 ST_SETV9STACK */
1251 GOTO(.getlgrp); /* 129 get lgrpid */
1252 BAD; BAD; BAD4; /* 12A - 12F unused */
1253 BAD4; BAD4; /* 130 - 137 unused */
1254 DTRACE_PID; /* 138 dtrace pid tracing provider */
1255 BAD; /* 139 unused */
1256 DTRACE_RETURN; /* 13A dtrace pid return probe */
1257 BAD; BAD4; /* 13B - 13F unused */
1258 SYSCALL_TRAP; /* 140 LP64 system call */
1259 SYSCALL(nosys); /* 141 unused system call trap */
1260 #ifdef DEBUG_USER_TRAPTRACECTL
1261 GOTO(.traptrace_freeze); /* 142 freeze traptrace */
1262 GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */
1263 #else
1264 SYSCALL(nosys); /* 142 unused system call trap */
1265 SYSCALL(nosys); /* 143 unused system call trap */
1266 #endif
1267 BAD4; BAD4; BAD4; /* 144 - 14F unused */
1268 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
1269 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
1270 BAD; /* 170 - unused */
1271 BAD; /* 171 - unused */
1272 BAD; BAD; /* 172 - 173 unused */
1273 BAD4; BAD4; /* 174 - 17B unused */
1274 #ifdef PTL1_PANIC_DEBUG
1275 mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1276 /* 17C test ptl1_panic */
1277 #else
1278 BAD; /* 17C unused */
1279 #endif /* PTL1_PANIC_DEBUG */
1280 PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */
1281 PRIV(kmdb_trap); /* 17E kmdb breakpoint */
1282 PRIV(obp_bpt); /* 17F obp breakpoint */
1283 /* reserved */
1284 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
1285 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
1286 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
1287 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
1288 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
1289 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
1290 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
1291 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
1292 .size trap_table0, (.-trap_table0)
1293 trap_table1:
1294 NOT4; NOT4; /* 000 - 007 unused */
1295 NOT; /* 008 instruction access exception */
1296 ITSB_MISS; /* 009 instruction access MMU miss */
1297 NOT; /* 00A reserved */
1298 NOT; NOT4; /* 00B - 00F unused */
1299 NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */
1300 NOT4; /* 020 - 023 unused */
1301 CLEAN_WINDOW; /* 024 - 027 clean window */
1302 NOT4; NOT4; /* 028 - 02F unused */
1303 DMMU_EXCEPTION_TL1; /* 030 data access exception */
1304 DTSB_MISS; /* 031 data access MMU miss */
1305 NOT; /* 032 reserved */
1306 NOT; /* 033 unused */
1307 MISALIGN_ADDR_TL1; /* 034 mem address not aligned */
1308 NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */
1309 NOT4; NOT4; NOT4; NOT4; /* 040 - 04F unused */
1310 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */
1311 NOT; /* 060 unused */
1312 GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */
1313 GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */
1314 NOT; /* 063 reserved */
1315 ITLB_MISS(tt1); /* 064 instruction access MMU miss */
1316 DTLB_MISS(tt1); /* 068 data access MMU miss */
1317 DTLB_PROT; /* 06C data access protection */
1318 NOT; /* 070 reserved */
1319 NOT; /* 071 reserved */
1320 NOT; /* 072 reserved */
1321 NOT; /* 073 reserved */
1322 NOT4; NOT4; /* 074 - 07B reserved */
1323 NOT; /* 07C reserved */
1324 NOT; /* 07D reserved */
1325 NOT; /* 07E resumable error */
1326 GOTO_TT(nonresumable_error, trace_gen); /* 07F nonresumable error */
1327 NOTP4; /* 080 spill 0 normal */
1328 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */
1329 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */
1330 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */
1331 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */
1332 NOTP4; /* 094 spill 5 normal */
1333 SPILL_64bit_ktt1(sk); /* 098 spill 6 normal */
1334 SPILL_mixed_ktt1(sk); /* 09C spill 7 normal */
1335 NOTP4; /* 0A0 spill 0 other */
1336 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */
1337 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */
1338 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */
1339 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */
1340 NOTP4; /* 0B4 spill 5 other */
1341 NOTP4; /* 0B8 spill 6 other */
1342 NOTP4; /* 0BC spill 7 other */
1343 NOT4; /* 0C0 fill 0 normal */
1344 NOT4; /* 0C4 fill 1 normal */
1345 NOT4; /* 0C8 fill 2 normal */
1346 NOT4; /* 0CC fill 3 normal */
1347 NOT4; /* 0D0 fill 4 normal */
1348 NOT4; /* 0D4 fill 5 normal */
1349 NOT4; /* 0D8 fill 6 normal */
1350 NOT4; /* 0DC fill 7 normal */
1351 NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */
1352 NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */
1353 /*
1354 * Code running at TL>0 does not use soft traps, so
1355 * we can truncate the table here.
1356 * However:
1357 * sun4v uses (hypervisor) ta instructions at TL > 0, so
1358 * provide a safety net for now.
1359 */
1360 /* soft traps */
1361 BAD4; BAD4; BAD4; BAD4; /* 100 - 10F unused */
1362 BAD4; BAD4; BAD4; BAD4; /* 110 - 11F unused */
1363 BAD4; BAD4; BAD4; BAD4; /* 120 - 12F unused */
1364 BAD4; BAD4; BAD4; BAD4; /* 130 - 13F unused */
1365 BAD4; BAD4; BAD4; BAD4; /* 140 - 14F unused */
1366 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
1367 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
1368 BAD4; BAD4; BAD4; BAD4; /* 170 - 17F unused */
1369 /* reserved */
1370 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
1371 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
1372 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
1373 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
1374 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
1375 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
1376 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
1377 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
1378 etrap_table:
1379 .size trap_table1, (.-trap_table1)
1380 .size trap_table, (.-trap_table)
1381 .size scb, (.-scb)
1382
1383 /*
1384 * We get to exec_fault in the case of an instruction miss and tte
1385 * has no execute bit set. We go to tl0 to handle it.
1386 *
1387 * g1 = tsbe pointer (in/clobbered)
1388 * g2 = tag access register (in)
1389 * g3 - g4 = scratch (clobbered)
1390 * g5 = tsbe data (in)
1391 * g6 = scratch (clobbered)
1392 * g7 = pc we jumped here from (in)
1393 */
1394 /*
1395 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1396 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1397 */
1398 ALTENTRY(exec_fault)
1399 TRACE_TSBHIT(TT_MMU_EXEC)
1400 MMU_FAULT_STATUS_AREA(%g4)
1401 ldx [%g4 + MMFSA_I_ADDR], %g2 /* g2 = address */
1402 ldx [%g4 + MMFSA_I_CTX], %g3 /* g3 = ctx */
1403 srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry
1404 cmp %g3, USER_CONTEXT_TYPE
1405 sllx %g2, MMU_PAGESHIFT, %g2
1406 movgu %icc, USER_CONTEXT_TYPE, %g3
1407 or %g2, %g3, %g2 /* TAG_ACCESS */
1408 mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype
1409 set trap, %g1
1410 ba,pt %xcc, sys_trap
1411 mov -1, %g4
1412
1413 .mmu_exception_not_aligned:
1414 /* %g2 = sfar, %g3 = sfsr */
1415 rdpr %tstate, %g1
1416 btst TSTATE_PRIV, %g1
1417 bnz,pn %icc, 2f
1418 nop
1419 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1420 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1421 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1422 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1423 brz,pt %g5, 2f
1424 nop
1425 ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap?
1426 brz,pn %g5, 2f
1427 nop
1428 btst 1, %sp
1429 bz,pt %xcc, 1f ! 32 bit user program
1430 nop
1431 ba,pt %xcc, .setup_v9utrap ! 64 bit user program
1432 nop
1433 1:
1434 ba,pt %xcc, .setup_utrap
1435 or %g2, %g0, %g7
1436 2:
1437 ba,pt %xcc, .mmu_exception_end
1438 mov T_ALIGNMENT, %g1
1439
1440 .mmu_priv_exception:
1441 rdpr %tstate, %g1
1442 btst TSTATE_PRIV, %g1
1443 bnz,pn %icc, 1f
1444 nop
1445 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1446 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1447 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1448 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1449 brz,pt %g5, 1f
1450 nop
1451 ldn [%g5 + P_UTRAP16], %g5
1452 brnz,pt %g5, .setup_v9utrap
1453 nop
1454 1:
1455 mov T_PRIV_INSTR, %g1
1456
1457 .mmu_exception_end:
1458 CPU_INDEX(%g4, %g5)
1459 set cpu_core, %g5
1460 sllx %g4, CPU_CORE_SHIFT, %g4
1461 add %g4, %g5, %g4
1462 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5
1463 andcc %g5, CPU_DTRACE_NOFAULT, %g0
1464 bz 1f
1465 or %g5, CPU_DTRACE_BADADDR, %g5
1466 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS]
1467 done
1468
1469 1:
1470 sllx %g3, 32, %g3
1471 or %g3, %g1, %g3
1472 set trap, %g1
1473 ba,pt %xcc, sys_trap
1474 sub %g0, 1, %g4
1475
1476 .fp_disabled:
1477 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1478 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1479 rdpr %tstate, %g4
1480 btst TSTATE_PRIV, %g4
1481 bnz,a,pn %icc, ptl1_panic
1482 mov PTL1_BAD_FPTRAP, %g1
1483
1484 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1485 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1486 brz,a,pt %g5, 2f
1487 nop
1488 ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap?
1489 brz,a,pn %g5, 2f
1490 nop
1491 btst 1, %sp
1492 bz,a,pt %xcc, 1f ! 32 bit user program
1493 nop
1494 ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program
1495 nop
1496 1:
1497 ba,pt %xcc, .setup_utrap
1498 or %g0, %g0, %g7
1499 2:
1500 set fp_disabled, %g1
1501 ba,pt %xcc, sys_trap
1502 sub %g0, 1, %g4
1503
1504 .fp_ieee_exception:
1505 rdpr %tstate, %g1
1506 btst TSTATE_PRIV, %g1
1507 bnz,a,pn %icc, ptl1_panic
1508 mov PTL1_BAD_FPTRAP, %g1
1509 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1510 stx %fsr, [%g1 + CPU_TMP1]
1511 ldx [%g1 + CPU_TMP1], %g2
1512 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1513 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1514 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1515 brz,a,pt %g5, 1f
1516 nop
1517 ldn [%g5 + P_UTRAP8], %g5
1518 brnz,a,pt %g5, .setup_v9utrap
1519 nop
1520 1:
1521 set _fp_ieee_exception, %g1
1522 ba,pt %xcc, sys_trap
1523 sub %g0, 1, %g4
1524
1525 /*
1526 * Register Inputs:
1527 * %g5 user trap handler
1528 * %g7 misaligned addr - for alignment traps only
1529 */
1530 .setup_utrap:
1531 set trap, %g1 ! setup in case we go
1532 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1533 sub %g0, 1, %g4 ! the save instruction below
1534
1535 /*
1536 * If the DTrace pid provider is single stepping a copied-out
1537 * instruction, t->t_dtrace_step will be set. In that case we need
1538 * to abort the single-stepping (since execution of the instruction
1539 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1540 */
1541 save %sp, -SA(MINFRAME32), %sp ! window for trap handler
1542 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1543 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1544 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1545 rdpr %tnpc, %l2 ! arg1 == tnpc
1546 brz,pt %g2, 1f
1547 rdpr %tpc, %l1 ! arg0 == tpc
1548
1549 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1550 ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step)
1551 brz,pt %g2, 1f
1552 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1553 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1554 1:
1555 mov %g7, %l3 ! arg2 == misaligned address
1556
1557 rdpr %tstate, %g1 ! cwp for trap handler
1558 rdpr %cwp, %g4
1559 bclr TSTATE_CWP_MASK, %g1
1560 wrpr %g1, %g4, %tstate
1561 wrpr %g0, %g5, %tnpc ! trap handler address
1562 FAST_TRAP_DONE
1563 /* NOTREACHED */
1564
1565 .check_v9utrap:
1566 rdpr %tstate, %g1
1567 btst TSTATE_PRIV, %g1
1568 bnz,a,pn %icc, 3f
1569 nop
1570 CPU_ADDR(%g4, %g1) ! load CPU struct addr
1571 ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer
1572 ldn [%g5 + T_PROCP], %g5 ! load proc pointer
1573 ldn [%g5 + P_UTRAPS], %g5 ! are there utraps?
1574
1575 cmp %g3, T_SOFTWARE_TRAP
1576 bne,a,pt %icc, 1f
1577 nop
1578
1579 brz,pt %g5, 3f ! if p_utraps == NULL goto trap()
1580 rdpr %tt, %g3 ! delay - get actual hw trap type
1581
1582 sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1583 ba,pt %icc, 2f
1584 smul %g1, CPTRSIZE, %g2
1585 1:
1586 brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap()
1587 nop
1588
1589 cmp %g3, T_UNIMP_INSTR
1590 bne,a,pt %icc, 2f
1591 nop
1592
1593 mov 1, %g1
1594 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1595 rdpr %tpc, %g1 ! ld trapping instruction using
1596 lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault
1597 st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1598
1599 sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction
1600 andcc %g1, %g4, %g4 ! and instruction with mask
1601 bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP
1602 nop ! fall thru to setup
1603 2:
1604 ldn [%g5 + %g2], %g5
1605 brnz,a,pt %g5, .setup_v9utrap
1606 nop
1607 3:
1608 set trap, %g1
1609 ba,pt %xcc, sys_trap
1610 sub %g0, 1, %g4
1611 /* NOTREACHED */
1612
1613 /*
1614 * Register Inputs:
1615 * %g5 user trap handler
1616 */
1617 .setup_v9utrap:
1618 set trap, %g1 ! setup in case we go
1619 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1620 sub %g0, 1, %g4 ! the save instruction below
1621
1622 /*
1623 * If the DTrace pid provider is single stepping a copied-out
1624 * instruction, t->t_dtrace_step will be set. In that case we need
1625 * to abort the single-stepping (since execution of the instruction
1626 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1627 */
1628 save %sp, -SA(MINFRAME64), %sp ! window for trap handler
1629 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1630 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1631 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1632 rdpr %tnpc, %l7 ! arg1 == tnpc
1633 brz,pt %g2, 1f
1634 rdpr %tpc, %l6 ! arg0 == tpc
1635
1636 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1637 ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step)
1638 brz,pt %g2, 1f
1639 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1640 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1641 1:
1642 rdpr %tstate, %g2 ! cwp for trap handler
1643 rdpr %cwp, %g4
1644 bclr TSTATE_CWP_MASK, %g2
1645 wrpr %g2, %g4, %tstate
1646
1647 ldn [%g1 + T_PROCP], %g4 ! load proc pointer
1648 ldn [%g4 + P_AS], %g4 ! load as pointer
1649 ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit
1650 cmp %l7, %g4 ! check for single-step set
1651 bne,pt %xcc, 4f
1652 nop
1653 ldn [%g1 + T_LWP], %g1 ! load klwp pointer
1654 ld [%g1 + PCB_STEP], %g4 ! load single-step flag
1655 cmp %g4, STEP_ACTIVE ! step flags set in pcb?
1656 bne,pt %icc, 4f
1657 nop
1658 stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb
1659 mov %l7, %g4 ! on entry to precise user trap
1660 add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc
1661 ! at time of trap
1662 wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS,
1663 ! %g4 == userlimit
1664 FAST_TRAP_DONE
1665 /* NOTREACHED */
1666 4:
1667 wrpr %g0, %g5, %tnpc ! trap handler address
1668 FAST_TRAP_DONE_CHK_INTR
1669 /* NOTREACHED */
1670
1671 .fp_exception:
1672 CPU_ADDR(%g1, %g4)
1673 stx %fsr, [%g1 + CPU_TMP1]
1674 ldx [%g1 + CPU_TMP1], %g2
1675
1676 /*
1677 * Cheetah takes unfinished_FPop trap for certain range of operands
1678 * to the "fitos" instruction. Instead of going through the slow
1679 * software emulation path, we try to simulate the "fitos" instruction
1680 * via "fitod" and "fdtos" provided the following conditions are met:
1681 *
1682 * fpu_exists is set (if DEBUG)
1683 * not in privileged mode
1684 * ftt is unfinished_FPop
1685 * NXM IEEE trap is not enabled
1686 * instruction at %tpc is "fitos"
1687 *
1688 * Usage:
1689 * %g1 per cpu address
1690 * %g2 %fsr
1691 * %g6 user instruction
1692 *
1693 * Note that we can take a memory access related trap while trying
1694 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1695 * flag to catch those traps and let the SFMMU code deal with page
1696 * fault and data access exception.
1697 */
1698 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1699 sethi %hi(fpu_exists), %g7
1700 ld [%g7 + %lo(fpu_exists)], %g7
1701 brz,pn %g7, .fp_exception_cont
1702 nop
1703 #endif
1704 rdpr %tstate, %g7 ! branch if in privileged mode
1705 btst TSTATE_PRIV, %g7
1706 bnz,pn %xcc, .fp_exception_cont
1707 srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr
1708 and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1709 cmp %g7, FTT_UNFIN
1710 set FSR_TEM_NX, %g5
1711 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop
1712 andcc %g2, %g5, %g0
1713 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled
1714 rdpr %tpc, %g5 ! get faulting PC
1715
1716 or %g0, 1, %g7
1717 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
1718 lda [%g5]ASI_USER, %g6 ! get user's instruction
1719 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
1720
1721 set FITOS_INSTR_MASK, %g7
1722 and %g6, %g7, %g7
1723 set FITOS_INSTR, %g5
1724 cmp %g7, %g5
1725 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR
1726 nop
1727
1728 /*
1729 * This is unfinished FPops trap for "fitos" instruction. We
1730 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1731 * sequence.
1732 *
1733 * We need a temporary FP register to do the conversion. Since
1734 * both source and destination operands for the "fitos" instruction
1735 * have to be within %f0-%f31, we use an FP register from the upper
1736 * half to guarantee that it won't collide with the source or the
1737 * dest operand. However, we do have to save and restore its value.
1738 *
1739 * We use %d62 as a temporary FP register for the conversion and
1740 * branch to appropriate instruction within the conversion tables
1741 * based upon the rs2 and rd values.
1742 */
1743
1744 std %d62, [%g1 + CPU_TMP1] ! save original value
1745
1746 srl %g6, FITOS_RS2_SHIFT, %g7
1747 and %g7, FITOS_REG_MASK, %g7
1748 set _fitos_fitod_table, %g4
1749 sllx %g7, 2, %g7
1750 jmp %g4 + %g7
1751 ba,pt %xcc, _fitos_fitod_done
1752 .empty
1753
1754 _fitos_fitod_table:
1755 fitod %f0, %d62
1756 fitod %f1, %d62
1757 fitod %f2, %d62
1758 fitod %f3, %d62
1759 fitod %f4, %d62
1760 fitod %f5, %d62
1761 fitod %f6, %d62
1762 fitod %f7, %d62
1763 fitod %f8, %d62
1764 fitod %f9, %d62
1765 fitod %f10, %d62
1766 fitod %f11, %d62
1767 fitod %f12, %d62
1768 fitod %f13, %d62
1769 fitod %f14, %d62
1770 fitod %f15, %d62
1771 fitod %f16, %d62
1772 fitod %f17, %d62
1773 fitod %f18, %d62
1774 fitod %f19, %d62
1775 fitod %f20, %d62
1776 fitod %f21, %d62
1777 fitod %f22, %d62
1778 fitod %f23, %d62
1779 fitod %f24, %d62
1780 fitod %f25, %d62
1781 fitod %f26, %d62
1782 fitod %f27, %d62
1783 fitod %f28, %d62
1784 fitod %f29, %d62
1785 fitod %f30, %d62
1786 fitod %f31, %d62
1787 _fitos_fitod_done:
1788
1789 /*
1790 * Now convert data back into single precision
1791 */
1792 srl %g6, FITOS_RD_SHIFT, %g7
1793 and %g7, FITOS_REG_MASK, %g7
1794 set _fitos_fdtos_table, %g4
1795 sllx %g7, 2, %g7
1796 jmp %g4 + %g7
1797 ba,pt %xcc, _fitos_fdtos_done
1798 .empty
1799
1800 _fitos_fdtos_table:
1801 fdtos %d62, %f0
1802 fdtos %d62, %f1
1803 fdtos %d62, %f2
1804 fdtos %d62, %f3
1805 fdtos %d62, %f4
1806 fdtos %d62, %f5
1807 fdtos %d62, %f6
1808 fdtos %d62, %f7
1809 fdtos %d62, %f8
1810 fdtos %d62, %f9
1811 fdtos %d62, %f10
1812 fdtos %d62, %f11
1813 fdtos %d62, %f12
1814 fdtos %d62, %f13
1815 fdtos %d62, %f14
1816 fdtos %d62, %f15
1817 fdtos %d62, %f16
1818 fdtos %d62, %f17
1819 fdtos %d62, %f18
1820 fdtos %d62, %f19
1821 fdtos %d62, %f20
1822 fdtos %d62, %f21
1823 fdtos %d62, %f22
1824 fdtos %d62, %f23
1825 fdtos %d62, %f24
1826 fdtos %d62, %f25
1827 fdtos %d62, %f26
1828 fdtos %d62, %f27
1829 fdtos %d62, %f28
1830 fdtos %d62, %f29
1831 fdtos %d62, %f30
1832 fdtos %d62, %f31
1833 _fitos_fdtos_done:
1834
1835 ldd [%g1 + CPU_TMP1], %d62 ! restore %d62
1836
1837 #if DEBUG
1838 /*
1839 * Update FPop_unfinished trap kstat
1840 */
1841 set fpustat+FPUSTAT_UNFIN_KSTAT, %g7
1842 ldx [%g7], %g5
1843 1:
1844 add %g5, 1, %g6
1845
1846 casxa [%g7] ASI_N, %g5, %g6
1847 cmp %g5, %g6
1848 bne,a,pn %xcc, 1b
1849 or %g0, %g6, %g5
1850
1851 /*
1852 * Update fpu_sim_fitos kstat
1853 */
1854 set fpuinfo+FPUINFO_FITOS_KSTAT, %g7
1855 ldx [%g7], %g5
1856 1:
1857 add %g5, 1, %g6
1858
1859 casxa [%g7] ASI_N, %g5, %g6
1860 cmp %g5, %g6
1861 bne,a,pn %xcc, 1b
1862 or %g0, %g6, %g5
1863 #endif /* DEBUG */
1864
1865 FAST_TRAP_DONE
1866
1867 .fp_exception_cont:
1868 /*
1869 * Let _fp_exception deal with simulating FPop instruction.
1870 * Note that we need to pass %fsr in %g2 (already read above).
1871 */
1872
1873 set _fp_exception, %g1
1874 ba,pt %xcc, sys_trap
1875 sub %g0, 1, %g4
1876
1877
1878 /*
1879 * Register windows
1880 */
1881 .flushw:
1882 .clean_windows:
1883 rdpr %tnpc, %g1
1884 wrpr %g1, %tpc
1885 add %g1, 4, %g1
1886 wrpr %g1, %tnpc
1887 set trap, %g1
1888 mov T_FLUSH_PCB, %g3
1889 ba,pt %xcc, sys_trap
1890 sub %g0, 1, %g4
1891
1892 /*
1893 * .spill_clean: clean the previous window, restore the wstate, and
1894 * "done".
1895 *
1896 * Entry: %g7 contains new wstate
1897 */
1898 .spill_clean:
1899 sethi %hi(nwin_minus_one), %g5
1900 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
1901 rdpr %cwp, %g6 ! %g6 = %cwp
1902 deccc %g6 ! %g6--
1903 movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1
1904 wrpr %g6, %cwp
1905 TT_TRACE_L(trace_win)
1906 clr %l0
1907 clr %l1
1908 clr %l2
1909 clr %l3
1910 clr %l4
1911 clr %l5
1912 clr %l6
1913 clr %l7
1914 wrpr %g0, %g7, %wstate
1915 saved
1916 retry ! restores correct %cwp
1917
1918 .fix_alignment:
1919 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
1920 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1921 ldn [%g1 + T_PROCP], %g1
1922 mov 1, %g2
1923 stb %g2, [%g1 + P_FIXALIGNMENT]
1924 FAST_TRAP_DONE
1925
1926 #define STDF_REG(REG, ADDR, TMP) \
1927 sll REG, 3, REG ;\
1928 mark1: set start1, TMP ;\
1929 jmp REG + TMP ;\
1930 nop ;\
1931 start1: ba,pt %xcc, done1 ;\
1932 std %f0, [ADDR + CPU_TMP1] ;\
1933 ba,pt %xcc, done1 ;\
1934 std %f32, [ADDR + CPU_TMP1] ;\
1935 ba,pt %xcc, done1 ;\
1936 std %f2, [ADDR + CPU_TMP1] ;\
1937 ba,pt %xcc, done1 ;\
1938 std %f34, [ADDR + CPU_TMP1] ;\
1939 ba,pt %xcc, done1 ;\
1940 std %f4, [ADDR + CPU_TMP1] ;\
1941 ba,pt %xcc, done1 ;\
1942 std %f36, [ADDR + CPU_TMP1] ;\
1943 ba,pt %xcc, done1 ;\
1944 std %f6, [ADDR + CPU_TMP1] ;\
1945 ba,pt %xcc, done1 ;\
1946 std %f38, [ADDR + CPU_TMP1] ;\
1947 ba,pt %xcc, done1 ;\
1948 std %f8, [ADDR + CPU_TMP1] ;\
1949 ba,pt %xcc, done1 ;\
1950 std %f40, [ADDR + CPU_TMP1] ;\
1951 ba,pt %xcc, done1 ;\
1952 std %f10, [ADDR + CPU_TMP1] ;\
1953 ba,pt %xcc, done1 ;\
1954 std %f42, [ADDR + CPU_TMP1] ;\
1955 ba,pt %xcc, done1 ;\
1956 std %f12, [ADDR + CPU_TMP1] ;\
1957 ba,pt %xcc, done1 ;\
1958 std %f44, [ADDR + CPU_TMP1] ;\
1959 ba,pt %xcc, done1 ;\
1960 std %f14, [ADDR + CPU_TMP1] ;\
1961 ba,pt %xcc, done1 ;\
1962 std %f46, [ADDR + CPU_TMP1] ;\
1963 ba,pt %xcc, done1 ;\
1964 std %f16, [ADDR + CPU_TMP1] ;\
1965 ba,pt %xcc, done1 ;\
1966 std %f48, [ADDR + CPU_TMP1] ;\
1967 ba,pt %xcc, done1 ;\
1968 std %f18, [ADDR + CPU_TMP1] ;\
1969 ba,pt %xcc, done1 ;\
1970 std %f50, [ADDR + CPU_TMP1] ;\
1971 ba,pt %xcc, done1 ;\
1972 std %f20, [ADDR + CPU_TMP1] ;\
1973 ba,pt %xcc, done1 ;\
1974 std %f52, [ADDR + CPU_TMP1] ;\
1975 ba,pt %xcc, done1 ;\
1976 std %f22, [ADDR + CPU_TMP1] ;\
1977 ba,pt %xcc, done1 ;\
1978 std %f54, [ADDR + CPU_TMP1] ;\
1979 ba,pt %xcc, done1 ;\
1980 std %f24, [ADDR + CPU_TMP1] ;\
1981 ba,pt %xcc, done1 ;\
1982 std %f56, [ADDR + CPU_TMP1] ;\
1983 ba,pt %xcc, done1 ;\
1984 std %f26, [ADDR + CPU_TMP1] ;\
1985 ba,pt %xcc, done1 ;\
1986 std %f58, [ADDR + CPU_TMP1] ;\
1987 ba,pt %xcc, done1 ;\
1988 std %f28, [ADDR + CPU_TMP1] ;\
1989 ba,pt %xcc, done1 ;\
1990 std %f60, [ADDR + CPU_TMP1] ;\
1991 ba,pt %xcc, done1 ;\
1992 std %f30, [ADDR + CPU_TMP1] ;\
1993 ba,pt %xcc, done1 ;\
1994 std %f62, [ADDR + CPU_TMP1] ;\
1995 done1:
1996
1997 #define LDDF_REG(REG, ADDR, TMP) \
1998 sll REG, 3, REG ;\
1999 mark2: set start2, TMP ;\
2000 jmp REG + TMP ;\
2001 nop ;\
2002 start2: ba,pt %xcc, done2 ;\
2003 ldd [ADDR + CPU_TMP1], %f0 ;\
2004 ba,pt %xcc, done2 ;\
2005 ldd [ADDR + CPU_TMP1], %f32 ;\
2006 ba,pt %xcc, done2 ;\
2007 ldd [ADDR + CPU_TMP1], %f2 ;\
2008 ba,pt %xcc, done2 ;\
2009 ldd [ADDR + CPU_TMP1], %f34 ;\
2010 ba,pt %xcc, done2 ;\
2011 ldd [ADDR + CPU_TMP1], %f4 ;\
2012 ba,pt %xcc, done2 ;\
2013 ldd [ADDR + CPU_TMP1], %f36 ;\
2014 ba,pt %xcc, done2 ;\
2015 ldd [ADDR + CPU_TMP1], %f6 ;\
2016 ba,pt %xcc, done2 ;\
2017 ldd [ADDR + CPU_TMP1], %f38 ;\
2018 ba,pt %xcc, done2 ;\
2019 ldd [ADDR + CPU_TMP1], %f8 ;\
2020 ba,pt %xcc, done2 ;\
2021 ldd [ADDR + CPU_TMP1], %f40 ;\
2022 ba,pt %xcc, done2 ;\
2023 ldd [ADDR + CPU_TMP1], %f10 ;\
2024 ba,pt %xcc, done2 ;\
2025 ldd [ADDR + CPU_TMP1], %f42 ;\
2026 ba,pt %xcc, done2 ;\
2027 ldd [ADDR + CPU_TMP1], %f12 ;\
2028 ba,pt %xcc, done2 ;\
2029 ldd [ADDR + CPU_TMP1], %f44 ;\
2030 ba,pt %xcc, done2 ;\
2031 ldd [ADDR + CPU_TMP1], %f14 ;\
2032 ba,pt %xcc, done2 ;\
2033 ldd [ADDR + CPU_TMP1], %f46 ;\
2034 ba,pt %xcc, done2 ;\
2035 ldd [ADDR + CPU_TMP1], %f16 ;\
2036 ba,pt %xcc, done2 ;\
2037 ldd [ADDR + CPU_TMP1], %f48 ;\
2038 ba,pt %xcc, done2 ;\
2039 ldd [ADDR + CPU_TMP1], %f18 ;\
2040 ba,pt %xcc, done2 ;\
2041 ldd [ADDR + CPU_TMP1], %f50 ;\
2042 ba,pt %xcc, done2 ;\
2043 ldd [ADDR + CPU_TMP1], %f20 ;\
2044 ba,pt %xcc, done2 ;\
2045 ldd [ADDR + CPU_TMP1], %f52 ;\
2046 ba,pt %xcc, done2 ;\
2047 ldd [ADDR + CPU_TMP1], %f22 ;\
2048 ba,pt %xcc, done2 ;\
2049 ldd [ADDR + CPU_TMP1], %f54 ;\
2050 ba,pt %xcc, done2 ;\
2051 ldd [ADDR + CPU_TMP1], %f24 ;\
2052 ba,pt %xcc, done2 ;\
2053 ldd [ADDR + CPU_TMP1], %f56 ;\
2054 ba,pt %xcc, done2 ;\
2055 ldd [ADDR + CPU_TMP1], %f26 ;\
2056 ba,pt %xcc, done2 ;\
2057 ldd [ADDR + CPU_TMP1], %f58 ;\
2058 ba,pt %xcc, done2 ;\
2059 ldd [ADDR + CPU_TMP1], %f28 ;\
2060 ba,pt %xcc, done2 ;\
2061 ldd [ADDR + CPU_TMP1], %f60 ;\
2062 ba,pt %xcc, done2 ;\
2063 ldd [ADDR + CPU_TMP1], %f30 ;\
2064 ba,pt %xcc, done2 ;\
2065 ldd [ADDR + CPU_TMP1], %f62 ;\
2066 done2:
2067
2068 .lddf_exception_not_aligned:
2069 /* %g2 = sfar, %g3 = sfsr */
2070 mov %g2, %g5 ! stash sfar
2071 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2072 sethi %hi(fpu_exists), %g2 ! check fpu_exists
2073 ld [%g2 + %lo(fpu_exists)], %g2
2074 brz,a,pn %g2, 4f
2075 nop
2076 #endif
2077 CPU_ADDR(%g1, %g4)
2078 or %g0, 1, %g4
2079 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2080
2081 rdpr %tpc, %g2
2082 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction
2083 srl %g6, 23, %g1 ! using ldda or not?
2084 and %g1, 1, %g1
2085 brz,a,pt %g1, 2f ! check for ldda instruction
2086 nop
2087 srl %g6, 13, %g1 ! check immflag
2088 and %g1, 1, %g1
2089 rdpr %tstate, %g2 ! %tstate in %g2
2090 brnz,a,pn %g1, 1f
2091 srl %g2, 31, %g1 ! get asi from %tstate
2092 srl %g6, 5, %g1 ! get asi from instruction
2093 and %g1, 0xFF, %g1 ! imm_asi field
2094 1:
2095 cmp %g1, ASI_P ! primary address space
2096 be,a,pt %icc, 2f
2097 nop
2098 cmp %g1, ASI_PNF ! primary no fault address space
2099 be,a,pt %icc, 2f
2100 nop
2101 cmp %g1, ASI_S ! secondary address space
2102 be,a,pt %icc, 2f
2103 nop
2104 cmp %g1, ASI_SNF ! secondary no fault address space
2105 bne,a,pn %icc, 3f
2106 nop
2107 2:
2108 lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data
2109 add %g5, 4, %g5 ! increment misaligned data address
2110 lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data
2111
2112 sllx %g7, 32, %g7
2113 or %g5, %g7, %g5 ! combine data
2114 CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis
2115 stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1
2116
2117 srl %g6, 25, %g3 ! %g6 has the instruction
2118 and %g3, 0x1F, %g3 ! %g3 has rd
2119 LDDF_REG(%g3, %g7, %g4)
2120
2121 CPU_ADDR(%g1, %g4)
2122 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2123 FAST_TRAP_DONE
2124 3:
2125 CPU_ADDR(%g1, %g4)
2126 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2127 4:
2128 set T_USER, %g3 ! trap type in %g3
2129 or %g3, T_LDDF_ALIGN, %g3
2130 mov %g5, %g2 ! misaligned vaddr in %g2
2131 set fpu_trap, %g1 ! goto C for the little and
2132 ba,pt %xcc, sys_trap ! no fault little asi's
2133 sub %g0, 1, %g4
2134
2135 .stdf_exception_not_aligned:
2136 /* %g2 = sfar, %g3 = sfsr */
2137 mov %g2, %g5
2138
2139 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2140 sethi %hi(fpu_exists), %g7 ! check fpu_exists
2141 ld [%g7 + %lo(fpu_exists)], %g3
2142 brz,a,pn %g3, 4f
2143 nop
2144 #endif
2145 CPU_ADDR(%g1, %g4)
2146 or %g0, 1, %g4
2147 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2148
2149 rdpr %tpc, %g2
2150 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction
2151
2152 srl %g6, 23, %g1 ! using stda or not?
2153 and %g1, 1, %g1
2154 brz,a,pt %g1, 2f ! check for stda instruction
2155 nop
2156 srl %g6, 13, %g1 ! check immflag
2157 and %g1, 1, %g1
2158 rdpr %tstate, %g2 ! %tstate in %g2
2159 brnz,a,pn %g1, 1f
2160 srl %g2, 31, %g1 ! get asi from %tstate
2161 srl %g6, 5, %g1 ! get asi from instruction
2162 and %g1, 0xff, %g1 ! imm_asi field
2163 1:
2164 cmp %g1, ASI_P ! primary address space
2165 be,a,pt %icc, 2f
2166 nop
2167 cmp %g1, ASI_S ! secondary address space
2168 bne,a,pn %icc, 3f
2169 nop
2170 2:
2171 srl %g6, 25, %g6
2172 and %g6, 0x1F, %g6 ! %g6 has rd
2173 CPU_ADDR(%g7, %g1)
2174 STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP)
2175
2176 ldx [%g7 + CPU_TMP1], %g6
2177 srlx %g6, 32, %g7
2178 stuwa %g7, [%g5]ASI_USER ! first half
2179 add %g5, 4, %g5 ! increment misaligned data address
2180 stuwa %g6, [%g5]ASI_USER ! second half
2181
2182 CPU_ADDR(%g1, %g4)
2183 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2184 FAST_TRAP_DONE
2185 3:
2186 CPU_ADDR(%g1, %g4)
2187 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2188 4:
2189 set T_USER, %g3 ! trap type in %g3
2190 or %g3, T_STDF_ALIGN, %g3
2191 mov %g5, %g2 ! misaligned vaddr in %g2
2192 set fpu_trap, %g1 ! goto C for the little and
2193 ba,pt %xcc, sys_trap ! nofault little asi's
2194 sub %g0, 1, %g4
2195
2196 #ifdef DEBUG_USER_TRAPTRACECTL
2197
2198 .traptrace_freeze:
2199 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2200 TT_TRACE_L(trace_win)
2201 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2202 set trap_freeze, %g1
2203 mov 1, %g2
2204 st %g2, [%g1]
2205 FAST_TRAP_DONE
2206
2207 .traptrace_unfreeze:
2208 set trap_freeze, %g1
2209 st %g0, [%g1]
2210 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2211 TT_TRACE_L(trace_win)
2212 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2213 FAST_TRAP_DONE
2214
2215 #endif /* DEBUG_USER_TRAPTRACECTL */
2216
2217 .getcc:
2218 CPU_ADDR(%g1, %g2)
2219 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2220 rdpr %tstate, %g3 ! get tstate
2221 srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2222 set PSR_ICC, %g2
2223 and %o0, %g2, %o0 ! mask out the rest
2224 srl %o0, PSR_ICC_SHIFT, %o0 ! right justify
2225 wrpr %g0, 0, %gl
2226 mov %o0, %g1 ! move ccr to normal %g1
2227 wrpr %g0, 1, %gl
2228 ! cannot assume globals retained their values after increasing %gl
2229 CPU_ADDR(%g1, %g2)
2230 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2231 FAST_TRAP_DONE
2232
2233 .setcc:
2234 CPU_ADDR(%g1, %g2)
2235 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2236 wrpr %g0, 0, %gl
2237 mov %g1, %o0
2238 wrpr %g0, 1, %gl
2239 ! cannot assume globals retained their values after increasing %gl
2240 CPU_ADDR(%g1, %g2)
2241 sll %o0, PSR_ICC_SHIFT, %g2
2242 set PSR_ICC, %g3
2243 and %g2, %g3, %g2 ! mask out rest
2244 sllx %g2, PSR_TSTATE_CC_SHIFT, %g2
2245 rdpr %tstate, %g3 ! get tstate
2246 srl %g3, 0, %g3 ! clear upper word
2247 or %g3, %g2, %g3 ! or in new bits
2248 wrpr %g3, %tstate
2249 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2250 FAST_TRAP_DONE
2251
2252 /*
2253 * getpsr(void)
2254 * Note that the xcc part of the ccr is not provided.
2255 * The V8 code shows why the V9 trap is not faster:
2256 * #define GETPSR_TRAP() \
2257 * mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2258 */
2259
2260 .type .getpsr, #function
2261 .getpsr:
2262 rdpr %tstate, %g1 ! get tstate
2263 srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2264 set PSR_ICC, %g2
2265 and %o0, %g2, %o0 ! mask out the rest
2266
2267 rd %fprs, %g1 ! get fprs
2268 and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower
2269 sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef
2270 or %o0, %g2, %o0 ! or result into psr.ef
2271
2272 set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef
2273 or %o0, %g2, %o0 ! or psr.impl/ver
2274 FAST_TRAP_DONE
2275 SET_SIZE(.getpsr)
2276
2277 /*
2278 * setpsr(newpsr)
2279 * Note that there is no support for ccr.xcc in the V9 code.
2280 */
2281
2282 .type .setpsr, #function
2283 .setpsr:
2284 rdpr %tstate, %g1 ! get tstate
2285 ! setx TSTATE_V8_UBITS, %g2
2286 or %g0, CCR_ICC, %g3
2287 sllx %g3, TSTATE_CCR_SHIFT, %g2
2288
2289 andn %g1, %g2, %g1 ! zero current user bits
2290 set PSR_ICC, %g2
2291 and %g2, %o0, %g2 ! clear all but psr.icc bits
2292 sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc
2293 wrpr %g1, %g3, %tstate ! write tstate
2294
2295 set PSR_EF, %g2
2296 and %g2, %o0, %g2 ! clear all but fp enable bit
2297 srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef
2298 wr %g0, %g4, %fprs ! write fprs
2299
2300 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1
2301 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2302 ldn [%g2 + T_LWP], %g3 ! load klwp pointer
2303 ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer
2304 stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs
2305 srlx %g4, 2, %g4 ! shift fef value to bit 0
2306 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en
2307 FAST_TRAP_DONE
2308 SET_SIZE(.setpsr)
2309
2310 /*
2311 * getlgrp
2312 * get home lgrpid on which the calling thread is currently executing.
2313 */
2314 .type .getlgrp, #function
2315 .getlgrp:
2316 ! Thanks for the incredibly helpful comments
2317 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2318 ld [%g1 + CPU_ID], %o0 ! load cpu_id
2319 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2320 ldn [%g2 + T_LPL], %g2 ! load lpl pointer
2321 ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
2322 sra %g1, 0, %o1
2323 FAST_TRAP_DONE
2324 SET_SIZE(.getlgrp)
2325
2326 /*
2327 * Entry for old 4.x trap (trap 0).
2328 */
2329 ENTRY_NP(syscall_trap_4x)
2330 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2331 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2332 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2333 ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr
2334 brz,pn %g2, 1f ! has it been set?
2335 st %l0, [%g1 + CPU_TMP1] ! delay - save some locals
2336 st %l1, [%g1 + CPU_TMP2]
2337 rdpr %tnpc, %l1 ! save old tnpc
2338 wrpr %g0, %g2, %tnpc ! setup tnpc
2339
2340 mov %g1, %l0 ! save CPU struct addr
2341 wrpr %g0, 0, %gl
2342 mov %l1, %g6 ! pass tnpc to user code in %g6
2343 wrpr %g0, 1, %gl
2344 ld [%l0 + CPU_TMP2], %l1 ! restore locals
2345 ld [%l0 + CPU_TMP1], %l0
2346 FAST_TRAP_DONE_CHK_INTR
2347 1:
2348 !
2349 ! check for old syscall mmap which is the only different one which
2350 ! must be the same. Others are handled in the compatibility library.
2351 !
2352 mov %g1, %l0 ! save CPU struct addr
2353 wrpr %g0, 0, %gl
2354 cmp %g1, OSYS_mmap ! compare to old 4.x mmap
2355 movz %icc, SYS_mmap, %g1
2356 wrpr %g0, 1, %gl
2357 ld [%l0 + CPU_TMP1], %l0
2358 SYSCALL(syscall_trap32)
2359 SET_SIZE(syscall_trap_4x)
2360
2361 /*
2362 * Handler for software trap 9.
2363 * Set trap0 emulation address for old 4.x system call trap.
2364 * XXX - this should be a system call.
2365 */
2366 ENTRY_NP(set_trap0_addr)
2367 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2368 st %l0, [%g1 + CPU_TMP1] ! save some locals
2369 st %l1, [%g1 + CPU_TMP2]
2370 mov %g1, %l0 ! preserve CPU addr
2371 wrpr %g0, 0, %gl
2372 mov %g1, %l1
2373 wrpr %g0, 1, %gl
2374 ! cannot assume globals retained their values after increasing %gl
2375 ldn [%l0 + CPU_THREAD], %g2 ! load thread pointer
2376 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2377 andn %l1, 3, %l1 ! force alignment
2378 st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr
2379 ld [%l0 + CPU_TMP2], %l1 ! restore locals
2380 ld [%l0 + CPU_TMP1], %l0
2381 FAST_TRAP_DONE
2382 SET_SIZE(set_trap0_addr)
2383
2384 /*
2385 * mmu_trap_tl1
2386 * trap handler for unexpected mmu traps.
2387 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2388 * case we go to fpu_trap or a user trap from the window handler, in which
2389 * case we go save the state on the pcb. Otherwise, we go to ptl1_panic.
2390 */
2391 .type mmu_trap_tl1, #function
2392 mmu_trap_tl1:
2393 #ifdef TRAPTRACE
2394 TRACE_PTR(%g5, %g6)
2395 GET_TRACE_TICK(%g6, %g7)
2396 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2397 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2398 rdpr %tt, %g6
2399 stha %g6, [%g5 + TRAP_ENT_TT]%asi
2400 rdpr %tstate, %g6
2401 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi
2402 stna %sp, [%g5 + TRAP_ENT_SP]%asi
2403 stna %g0, [%g5 + TRAP_ENT_TR]%asi
2404 rdpr %tpc, %g6
2405 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2406 MMU_FAULT_STATUS_AREA(%g6)
2407 ldx [%g6 + MMFSA_D_ADDR], %g6
2408 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! MMU fault address
2409 CPU_PADDR(%g7, %g6);
2410 add %g7, CPU_TL1_HDLR, %g7
2411 lda [%g7]ASI_MEM, %g6
2412 stna %g6, [%g5 + TRAP_ENT_F2]%asi
2413 MMU_FAULT_STATUS_AREA(%g6)
2414 ldx [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant?
2415 ldx [%g6 + MMFSA_D_CTX], %g6
2416 sllx %g6, SFSR_CTX_SHIFT, %g6
2417 or %g6, %g7, %g6
2418 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type
2419 set 0xdeadbeef, %g6
2420 stna %g6, [%g5 + TRAP_ENT_F4]%asi
2421 TRACE_NEXT(%g5, %g6, %g7)
2422 #endif /* TRAPTRACE */
2423 CPU_PADDR(%g7, %g6);
2424 add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA)
2425 lda [%g7]ASI_MEM, %g6
2426 brz,a,pt %g6, 1f
2427 nop
2428 sta %g0, [%g7]ASI_MEM
2429 ! XXXQ need to setup registers for sfmmu_mmu_trap?
2430 ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults
2431 1:
2432 rdpr %tpc, %g7
2433 /* in user_rtt? */
2434 set rtt_fill_start, %g6
2435 cmp %g7, %g6
2436 blu,pn %xcc, 6f
2437 .empty
2438 set rtt_fill_end, %g6
2439 cmp %g7, %g6
2440 bgeu,pn %xcc, 6f
2441 nop
2442 set fault_rtt_fn1, %g7
2443 ba,a 7f
2444 6:
2445 ! check to see if the trap pc is in a window spill/fill handling
2446 rdpr %tpc, %g7
2447 /* tpc should be in the trap table */
2448 set trap_table, %g6
2449 cmp %g7, %g6
2450 blu,a,pn %xcc, ptl1_panic
2451 mov PTL1_BAD_MMUTRAP, %g1
2452 set etrap_table, %g6
2453 cmp %g7, %g6
2454 bgeu,a,pn %xcc, ptl1_panic
2455 mov PTL1_BAD_MMUTRAP, %g1
2456 ! pc is inside the trap table, convert to trap type
2457 srl %g7, 5, %g6 ! XXXQ need #define
2458 and %g6, 0x1ff, %g6 ! XXXQ need #define
2459 ! and check for a window trap type
2460 and %g6, WTRAP_TTMASK, %g6
2461 cmp %g6, WTRAP_TYPE
2462 bne,a,pn %xcc, ptl1_panic
2463 mov PTL1_BAD_MMUTRAP, %g1
2464 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */
2465 add %g7, WTRAP_FAULTOFF, %g7
2466
2467 7:
2468 ! Arguments are passed in the global set active after the
2469 ! 'done' instruction. Before switching sets, must save
2470 ! the calculated next pc
2471 wrpr %g0, %g7, %tnpc
2472 wrpr %g0, 1, %gl
2473 rdpr %tt, %g5
2474 MMU_FAULT_STATUS_AREA(%g7)
2475 cmp %g5, T_ALIGNMENT
2476 be,pn %xcc, 1f
2477 ldx [%g7 + MMFSA_D_ADDR], %g6
2478 ldx [%g7 + MMFSA_D_CTX], %g7
2479 srlx %g6, MMU_PAGESHIFT, %g6 /* align address */
2480 cmp %g7, USER_CONTEXT_TYPE
2481 sllx %g6, MMU_PAGESHIFT, %g6
2482 movgu %icc, USER_CONTEXT_TYPE, %g7
2483 or %g6, %g7, %g6 /* TAG_ACCESS */
2484 1:
2485 done
2486 SET_SIZE(mmu_trap_tl1)
2487
2488 /*
2489 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These
2490 * traps are valid only when kmdb is loaded. When the debugger is active,
2491 * the code below is rewritten to transfer control to the appropriate
2492 * debugger entry points.
2493 */
2494 .global kmdb_trap
2495 .align 8
2496 kmdb_trap:
2497 ba,a trap_table0
2498 jmp %g1 + 0
2499 nop
2500
2501 .global kmdb_trap_tl1
2502 .align 8
2503 kmdb_trap_tl1:
2504 ba,a trap_table0
2505 jmp %g1 + 0
2506 nop
2507
2508 /*
2509 * This entry is copied from OBP's trap table during boot.
2510 */
2511 .global obp_bpt
2512 .align 8
2513 obp_bpt:
2514 NOT
2515
2516
2517
2518 #ifdef TRAPTRACE
2519 /*
2520 * TRAPTRACE support.
2521 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2522 * Return is done by "jmp %g7 + 4".
2523 */
2524
2525 trace_dmmu:
2526 TRACE_PTR(%g3, %g6)
2527 GET_TRACE_TICK(%g6, %g5)
2528 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2529 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2530 rdpr %tt, %g6
2531 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2532 rdpr %tstate, %g6
2533 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2534 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2535 rdpr %tpc, %g6
2536 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2537 MMU_FAULT_STATUS_AREA(%g6)
2538 ldx [%g6 + MMFSA_D_ADDR], %g4
2539 stxa %g4, [%g3 + TRAP_ENT_TR]%asi
2540 ldx [%g6 + MMFSA_D_CTX], %g4
2541 stxa %g4, [%g3 + TRAP_ENT_F1]%asi
2542 ldx [%g6 + MMFSA_D_TYPE], %g4
2543 stxa %g4, [%g3 + TRAP_ENT_F2]%asi
2544 stxa %g6, [%g3 + TRAP_ENT_F3]%asi
2545 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2546 TRACE_NEXT(%g3, %g4, %g5)
2547 jmp %g7 + 4
2548 nop
2549
2550 trace_immu:
2551 TRACE_PTR(%g3, %g6)
2552 GET_TRACE_TICK(%g6, %g5)
2553 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2554 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2555 rdpr %tt, %g6
2556 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2557 rdpr %tstate, %g6
2558 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2559 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2560 rdpr %tpc, %g6
2561 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2562 MMU_FAULT_STATUS_AREA(%g6)
2563 ldx [%g6 + MMFSA_I_ADDR], %g4
2564 stxa %g4, [%g3 + TRAP_ENT_TR]%asi
2565 ldx [%g6 + MMFSA_I_CTX], %g4
2566 stxa %g4, [%g3 + TRAP_ENT_F1]%asi
2567 ldx [%g6 + MMFSA_I_TYPE], %g4
2568 stxa %g4, [%g3 + TRAP_ENT_F2]%asi
2569 stxa %g6, [%g3 + TRAP_ENT_F3]%asi
2570 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2571 TRACE_NEXT(%g3, %g4, %g5)
2572 jmp %g7 + 4
2573 nop
2574
2575 trace_gen:
2576 TRACE_PTR(%g3, %g6)
2577 GET_TRACE_TICK(%g6, %g5)
2578 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2579 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2580 rdpr %tt, %g6
2581 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2582 rdpr %tstate, %g6
2583 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2584 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2585 rdpr %tpc, %g6
2586 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2587 stna %g0, [%g3 + TRAP_ENT_TR]%asi
2588 stna %g0, [%g3 + TRAP_ENT_F1]%asi
2589 stna %g0, [%g3 + TRAP_ENT_F2]%asi
2590 stna %g0, [%g3 + TRAP_ENT_F3]%asi
2591 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2592 TRACE_NEXT(%g3, %g4, %g5)
2593 jmp %g7 + 4
2594 nop
2595
2596 trace_win:
2597 TRACE_WIN_INFO(0, %l0, %l1, %l2)
2598 ! Keep the locals as clean as possible, caller cleans %l4
2599 clr %l2
2600 clr %l1
2601 jmp %l4 + 4
2602 clr %l0
2603
2604 /*
2605 * Trace a tsb hit
2606 * g1 = tsbe pointer (in/clobbered)
2607 * g2 = tag access register (in)
2608 * g3 - g4 = scratch (clobbered)
2609 * g5 = tsbe data (in)
2610 * g6 = scratch (clobbered)
2611 * g7 = pc we jumped here from (in)
2612 */
2613
2614 ! Do not disturb %g5, it will be used after the trace
2615 ALTENTRY(trace_tsbhit)
2616 TRACE_TSBHIT(0)
2617 jmp %g7 + 4
2618 nop
2619
2620 /*
2621 * Trace a TSB miss
2622 *
2623 * g1 = tsb8k pointer (in)
2624 * g2 = tag access register (in)
2625 * g3 = tsb4m pointer (in)
2626 * g4 = tsbe tag (in/clobbered)
2627 * g5 - g6 = scratch (clobbered)
2628 * g7 = pc we jumped here from (in)
2629 */
2630 .global trace_tsbmiss
2631 trace_tsbmiss:
2632 membar #Sync
2633 sethi %hi(FLUSH_ADDR), %g6
2634 flush %g6
2635 TRACE_PTR(%g5, %g6)
2636 stna %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access
2637 stna %g4, [%g5 + TRAP_ENT_F1]%asi ! XXX? tsb tag
2638 GET_TRACE_TICK(%g6, %g4)
2639 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2640 rdpr %tnpc, %g6
2641 stna %g6, [%g5 + TRAP_ENT_F2]%asi
2642 stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer
2643 rdpr %tpc, %g6
2644 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2645 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2646 rdpr %tt, %g6
2647 or %g6, TT_MMU_MISS, %g4
2648 stha %g4, [%g5 + TRAP_ENT_TT]%asi
2649 mov MMFSA_D_ADDR, %g4
2650 cmp %g6, FAST_IMMU_MISS_TT
2651 move %xcc, MMFSA_I_ADDR, %g4
2652 cmp %g6, T_INSTR_MMU_MISS
2653 move %xcc, MMFSA_I_ADDR, %g4
2654 MMU_FAULT_STATUS_AREA(%g6)
2655 ldx [%g6 + %g4], %g6
2656 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target
2657 cmp %g4, MMFSA_D_ADDR
2658 move %xcc, MMFSA_D_CTX, %g4
2659 movne %xcc, MMFSA_I_CTX, %g4
2660 MMU_FAULT_STATUS_AREA(%g6)
2661 ldx [%g6 + %g4], %g6
2662 stxa %g6, [%g5 + TRAP_ENT_F4]%asi ! context ID
2663 stna %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer
2664 TRACE_NEXT(%g5, %g4, %g6)
2665 jmp %g7 + 4
2666 nop
2667
2668 /*
2669 * g2 = tag access register (in)
2670 * g3 = ctx type (0, 1 or 2) (in) (not used)
2671 */
2672 trace_dataprot:
2673 membar #Sync
2674 sethi %hi(FLUSH_ADDR), %g6
2675 flush %g6
2676 TRACE_PTR(%g1, %g6)
2677 GET_TRACE_TICK(%g6, %g4)
2678 stxa %g6, [%g1 + TRAP_ENT_TICK]%asi
2679 rdpr %tpc, %g6
2680 stna %g6, [%g1 + TRAP_ENT_TPC]%asi
2681 rdpr %tstate, %g6
2682 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi
2683 stna %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg
2684 stna %g0, [%g1 + TRAP_ENT_F1]%asi
2685 stna %g0, [%g1 + TRAP_ENT_F2]%asi
2686 stna %g0, [%g1 + TRAP_ENT_F3]%asi
2687 stna %g0, [%g1 + TRAP_ENT_F4]%asi
2688 TRACE_SAVE_TL_GL_REGS(%g1, %g6)
2689 rdpr %tt, %g6
2690 stha %g6, [%g1 + TRAP_ENT_TT]%asi
2691 mov MMFSA_D_CTX, %g4
2692 cmp %g6, FAST_IMMU_MISS_TT
2693 move %xcc, MMFSA_I_CTX, %g4
2694 cmp %g6, T_INSTR_MMU_MISS
2695 move %xcc, MMFSA_I_CTX, %g4
2696 MMU_FAULT_STATUS_AREA(%g6)
2697 ldx [%g6 + %g4], %g6
2698 stxa %g6, [%g1 + TRAP_ENT_TR]%asi ! context ID
2699 TRACE_NEXT(%g1, %g4, %g5)
2700 jmp %g7 + 4
2701 nop
2702
2703 #endif /* TRAPTRACE */
2704
2705 /*
2706 * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE
2707 * HV service, which requires the return target to be specified as a VA
2708 * since we are enabling the MMU. We set the target to ptl1_panic.
2709 */
2710
2711 .type .watchdog_trap, #function
2712 .watchdog_trap:
2713 mov 1, %o0
2714 setx ptl1_panic, %g2, %o1
2715 mov MMU_ENABLE, %o5
2716 ta FAST_TRAP
2717 done
2718 SET_SIZE(.watchdog_trap)
2719 /*
2720 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2721 */
2722 .type .dmmu_exc_lddf_not_aligned, #function
2723 .dmmu_exc_lddf_not_aligned:
2724 MMU_FAULT_STATUS_AREA(%g3)
2725 ldx [%g3 + MMFSA_D_ADDR], %g2
2726 /* Fault type not available in MMU fault status area */
2727 mov MMFSA_F_UNALIGN, %g1
2728 ldx [%g3 + MMFSA_D_CTX], %g3
2729 sllx %g3, SFSR_CTX_SHIFT, %g3
2730 btst 1, %sp
2731 bnz,pt %xcc, .lddf_exception_not_aligned
2732 or %g3, %g1, %g3 /* SFSR */
2733 ba,a,pt %xcc, .mmu_exception_not_aligned
2734 SET_SIZE(.dmmu_exc_lddf_not_aligned)
2735
2736 /*
2737 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2738 */
2739 .type .dmmu_exc_stdf_not_aligned, #function
2740 .dmmu_exc_stdf_not_aligned:
2741 MMU_FAULT_STATUS_AREA(%g3)
2742 ldx [%g3 + MMFSA_D_ADDR], %g2
2743 /* Fault type not available in MMU fault status area */
2744 mov MMFSA_F_UNALIGN, %g1
2745 ldx [%g3 + MMFSA_D_CTX], %g3
2746 sllx %g3, SFSR_CTX_SHIFT, %g3
2747 btst 1, %sp
2748 bnz,pt %xcc, .stdf_exception_not_aligned
2749 or %g3, %g1, %g3 /* SFSR */
2750 ba,a,pt %xcc, .mmu_exception_not_aligned
2751 SET_SIZE(.dmmu_exc_stdf_not_aligned)
2752
2753 .type .dmmu_exception, #function
2754 .dmmu_exception:
2755 MMU_FAULT_STATUS_AREA(%g3)
2756 ldx [%g3 + MMFSA_D_ADDR], %g2
2757 ldx [%g3 + MMFSA_D_TYPE], %g1
2758 ldx [%g3 + MMFSA_D_CTX], %g4
2759 srlx %g2, MMU_PAGESHIFT, %g2 /* align address */
2760 sllx %g2, MMU_PAGESHIFT, %g2
2761 sllx %g4, SFSR_CTX_SHIFT, %g3
2762 or %g3, %g1, %g3 /* SFSR */
2763 cmp %g4, USER_CONTEXT_TYPE
2764 movgeu %icc, USER_CONTEXT_TYPE, %g4
2765 or %g2, %g4, %g2 /* TAG_ACCESS */
2766 ba,pt %xcc, .mmu_exception_end
2767 mov T_DATA_EXCEPTION, %g1
2768 SET_SIZE(.dmmu_exception)
2769
2770 .align 32
2771 .global pil15_epilogue
2772 pil15_epilogue:
2773 ba pil_interrupt_common
2774 nop
2775 .align 32
2776
2777 /*
2778 * fast_trap_done, fast_trap_done_chk_intr:
2779 *
2780 * Due to the design of UltraSPARC pipeline, pending interrupts are not
2781 * taken immediately after a RETRY or DONE instruction which causes IE to
2782 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2783 * to execute first before taking any interrupts. If that instruction
2784 * results in other traps, and if the corresponding trap handler runs
2785 * entirely at TL=1 with interrupts disabled, then pending interrupts
2786 * won't be taken until after yet another instruction following the %tpc
2787 * or %tnpc.
2788 *
2789 * A malicious user program can use this feature to block out interrupts
2790 * for extended durations, which can result in send_mondo_timeout kernel
2791 * panic.
2792 *
2793 * This problem is addressed by servicing any pending interrupts via
2794 * sys_trap before returning back to the user mode from a fast trap
2795 * handler. The "done" instruction within a fast trap handler, which
2796 * runs entirely at TL=1 with interrupts disabled, is replaced with the
2797 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2798 * entry point.
2799 *
2800 * We check for any pending interrupts here and force a sys_trap to
2801 * service those interrupts, if any. To minimize overhead, pending
2802 * interrupts are checked if the %tpc happens to be at 16K boundary,
2803 * which allows a malicious program to execute at most 4K consecutive
2804 * instructions before we service any pending interrupts. If a worst
2805 * case fast trap handler takes about 2 usec, then interrupts will be
2806 * blocked for at most 8 msec, less than a clock tick.
2807 *
2808 * For the cases where we don't know if the %tpc will cross a 16K
2809 * boundary, we can't use the above optimization and always process
2810 * any pending interrupts via fast_frap_done_chk_intr entry point.
2811 *
2812 * Entry Conditions:
2813 * %pstate am:0 priv:1 ie:0
2814 * globals are AG (not normal globals)
2815 */
2816
2817 .global fast_trap_done, fast_trap_done_chk_intr
2818 fast_trap_done:
2819 rdpr %tpc, %g5
2820 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff
2821 andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc
2822 bz,pn %icc, 1f ! branch if zero (lower 32 bits only)
2823 nop
2824 done
2825
2826 fast_trap_done_chk_intr:
2827 1: rd SOFTINT, %g6
2828 brnz,pn %g6, 2f ! branch if any pending intr
2829 nop
2830 done
2831
2832 2:
2833 /*
2834 * We get here if there are any pending interrupts.
2835 * Adjust %tpc/%tnpc as we'll be resuming via "retry"
2836 * instruction.
2837 */
2838 rdpr %tnpc, %g5
2839 wrpr %g0, %g5, %tpc
2840 add %g5, 4, %g5
2841 wrpr %g0, %g5, %tnpc
2842
2843 /*
2844 * Force a dummy sys_trap call so that interrupts can be serviced.
2845 */
2846 set fast_trap_dummy_call, %g1
2847 ba,pt %xcc, sys_trap
2848 mov -1, %g4
2849
2850 fast_trap_dummy_call:
2851 retl
2852 nop
2853
2854 /*
2855 * Currently the brand syscall interposition code is not enabled by
2856 * default. Instead, when a branded zone is first booted the brand
2857 * infrastructure will patch the trap table so that the syscall
2858 * entry points are redirected to syscall_wrapper32 and syscall_wrapper
2859 * for ILP32 and LP64 syscalls respectively. this is done in
2860 * brand_plat_interposition_enable(). Note that the syscall wrappers
2861 * below do not collect any trap trace data since the syscall hot patch
2862 * points are reached after trap trace data has already been collected.
2863 */
2864 #define BRAND_CALLBACK(callback_id) \
2865 CPU_ADDR(%g2, %g1) /* load CPU struct addr to %g2 */ ;\
2866 ldn [%g2 + CPU_THREAD], %g3 /* load thread pointer */ ;\
2867 ldn [%g3 + T_PROCP], %g3 /* get proc pointer */ ;\
2868 ldn [%g3 + P_BRAND], %g3 /* get brand pointer */ ;\
2869 brz %g3, 1f /* No brand? No callback. */ ;\
2870 nop ;\
2871 ldn [%g3 + B_MACHOPS], %g3 /* get machops list */ ;\
2872 ldn [%g3 + (callback_id << 3)], %g3 ;\
2873 brz %g3, 1f ;\
2874 /* \
2875 * This isn't pretty. We want a low-latency way for the callback \
2876 * routine to decline to do anything. We just pass in an address \
2877 * the routine can directly jmp back to, pretending that nothing \
2878 * has happened. \
2879 * \
2880 * %g1: return address (where the brand handler jumps back to) \
2881 * %g2: address of CPU structure \
2882 * %g3: address of brand handler (where we will jump to) \
2883 */ \
2884 mov %pc, %g1 ;\
2885 add %g1, 16, %g1 ;\
2886 jmp %g3 ;\
2887 nop ;\
2888 1:
2889
2890 ENTRY_NP(syscall_wrapper32)
2891 BRAND_CALLBACK(BRAND_CB_SYSCALL32)
2892 SYSCALL_NOTT(syscall_trap32)
2893 SET_SIZE(syscall_wrapper32)
2894
2895 ENTRY_NP(syscall_wrapper)
2896 BRAND_CALLBACK(BRAND_CB_SYSCALL)
2897 SYSCALL_NOTT(syscall_trap)
2898 SET_SIZE(syscall_wrapper)
2899
2900 #endif /* lint */