1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include "assym.h"
28 #include <sys/asm_linkage.h>
29 #include <sys/privregs.h>
30 #include <sys/sun4asi.h>
31 #include <sys/machasi.h>
32 #include <sys/hypervisor_api.h>
33 #include <sys/machtrap.h>
34 #include <sys/machthread.h>
35 #include <sys/machbrand.h>
36 #include <sys/pcb.h>
37 #include <sys/pte.h>
38 #include <sys/mmu.h>
39 #include <sys/machpcb.h>
40 #include <sys/async.h>
41 #include <sys/intreg.h>
42 #include <sys/scb.h>
43 #include <sys/psr_compat.h>
44 #include <sys/syscall.h>
45 #include <sys/machparam.h>
46 #include <sys/traptrace.h>
47 #include <vm/hat_sfmmu.h>
48 #include <sys/archsystm.h>
49 #include <sys/utrap.h>
50 #include <sys/clock.h>
51 #include <sys/intr.h>
52 #include <sys/fpu/fpu_simulator.h>
53 #include <vm/seg_spt.h>
54
55 /*
56 * WARNING: If you add a fast trap handler which can be invoked by a
57 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
58 * instead of "done" instruction to return back to the user mode. See
59 * comments for the "fast_trap_done" entry point for more information.
60 *
61 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
62 * cases where you always want to process any pending interrupts before
63 * returning back to the user mode.
64 */
65 #define FAST_TRAP_DONE \
66 ba,a fast_trap_done
67
68 #define FAST_TRAP_DONE_CHK_INTR \
69 ba,a fast_trap_done_chk_intr
70
71 /*
72 * SPARC V9 Trap Table
73 *
74 * Most of the trap handlers are made from common building
75 * blocks, and some are instantiated multiple times within
76 * the trap table. So, I build a bunch of macros, then
77 * populate the table using only the macros.
78 *
79 * Many macros branch to sys_trap. Its calling convention is:
80 * %g1 kernel trap handler
81 * %g2, %g3 args for above
82 * %g4 desire %pil
83 */
84
85 #ifdef TRAPTRACE
86
87 /*
88 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
89 */
90 #define TT_TRACE(label) \
91 ba label ;\
92 rd %pc, %g7
93 #define TT_TRACE_INS 2
94
95 #define TT_TRACE_L(label) \
96 ba label ;\
97 rd %pc, %l4 ;\
98 clr %l4
99 #define TT_TRACE_L_INS 3
100
101 #else
102
103 #define TT_TRACE(label)
104 #define TT_TRACE_INS 0
105
106 #define TT_TRACE_L(label)
107 #define TT_TRACE_L_INS 0
108
109 #endif
110
111 /*
112 * This first set are funneled to trap() with %tt as the type.
113 * Trap will then either panic or send the user a signal.
114 */
115 /*
116 * NOT is used for traps that just shouldn't happen.
117 * It comes in both single and quadruple flavors.
118 */
119 .global trap
120 #define NOT \
121 TT_TRACE(trace_gen) ;\
122 set trap, %g1 ;\
123 rdpr %tt, %g3 ;\
124 ba,pt %xcc, sys_trap ;\
125 sub %g0, 1, %g4 ;\
126 .align 32
127 #define NOT4 NOT; NOT; NOT; NOT
128
129 #define NOTP \
130 TT_TRACE(trace_gen) ;\
131 ba,pt %xcc, ptl1_panic ;\
132 mov PTL1_BAD_TRAP, %g1 ;\
133 .align 32
134 #define NOTP4 NOTP; NOTP; NOTP; NOTP
135
136
137 /*
138 * BAD is used for trap vectors we don't have a kernel
139 * handler for.
140 * It also comes in single and quadruple versions.
141 */
142 #define BAD NOT
143 #define BAD4 NOT4
144
145 #define DONE \
146 done; \
147 .align 32
148
149 /*
150 * TRAP vectors to the trap() function.
151 * It's main use is for user errors.
152 */
153 .global trap
154 #define TRAP(arg) \
155 TT_TRACE(trace_gen) ;\
156 set trap, %g1 ;\
157 mov arg, %g3 ;\
158 ba,pt %xcc, sys_trap ;\
159 sub %g0, 1, %g4 ;\
160 .align 32
161
162 /*
163 * SYSCALL is used for unsupported syscall interfaces (with 'which'
164 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
165 * 'which' set to 'syscall_trap32').
166 *
167 * The SYSCALL_TRAP* macros are used for syscall entry points.
168 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
169 * is used to support ILP32. Each macro can only be used once
170 * since they each define a symbol. The symbols are used as hot patch
171 * points by the brand infrastructure to dynamically enable and disable
172 * brand syscall interposition. See the comments around BRAND_CALLBACK
173 * and brand_plat_interposition_enable() for more information.
174 */
175 #define SYSCALL_NOTT(which) \
176 set (which), %g1 ;\
177 ba,pt %xcc, user_trap ;\
178 sub %g0, 1, %g4 ;\
179 .align 32
180
181 #define SYSCALL(which) \
182 TT_TRACE(trace_gen) ;\
183 SYSCALL_NOTT(which)
184
185 #define SYSCALL_TRAP32 \
186 TT_TRACE(trace_gen) ;\
187 ALTENTRY(syscall_trap32_patch_point) \
188 SYSCALL_NOTT(syscall_trap32)
189
190 #define SYSCALL_TRAP \
191 TT_TRACE(trace_gen) ;\
192 ALTENTRY(syscall_trap_patch_point) \
193 SYSCALL_NOTT(syscall_trap)
194
195 /*
196 * GOTO just jumps to a label.
197 * It's used for things that can be fixed without going thru sys_trap.
198 */
199 #define GOTO(label) \
200 .global label ;\
201 ba,a label ;\
202 .empty ;\
203 .align 32
204
205 /*
206 * GOTO_TT just jumps to a label.
207 * correctable ECC error traps at level 0 and 1 will use this macro.
208 * It's used for things that can be fixed without going thru sys_trap.
209 */
210 #define GOTO_TT(label, ttlabel) \
211 .global label ;\
212 TT_TRACE(ttlabel) ;\
213 ba,a label ;\
214 .empty ;\
215 .align 32
216
217 /*
218 * Privileged traps
219 * Takes breakpoint if privileged, calls trap() if not.
220 */
221 #define PRIV(label) \
222 rdpr %tstate, %g1 ;\
223 btst TSTATE_PRIV, %g1 ;\
224 bnz label ;\
225 rdpr %tt, %g3 ;\
226 set trap, %g1 ;\
227 ba,pt %xcc, sys_trap ;\
228 sub %g0, 1, %g4 ;\
229 .align 32
230
231
232 /*
233 * DTrace traps.
234 */
235 #define DTRACE_PID \
236 .global dtrace_pid_probe ;\
237 set dtrace_pid_probe, %g1 ;\
238 ba,pt %xcc, user_trap ;\
239 sub %g0, 1, %g4 ;\
240 .align 32
241
242 #define DTRACE_RETURN \
243 .global dtrace_return_probe ;\
244 set dtrace_return_probe, %g1 ;\
245 ba,pt %xcc, user_trap ;\
246 sub %g0, 1, %g4 ;\
247 .align 32
248
249 /*
250 * REGISTER WINDOW MANAGEMENT MACROS
251 */
252
253 /*
254 * various convenient units of padding
255 */
256 #define SKIP(n) .skip 4*(n)
257
258 /*
259 * CLEAN_WINDOW is the simple handler for cleaning a register window.
260 */
261 #define CLEAN_WINDOW \
262 TT_TRACE_L(trace_win) ;\
263 rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\
264 clr %l0; clr %l1; clr %l2; clr %l3 ;\
265 clr %l4; clr %l5; clr %l6; clr %l7 ;\
266 clr %o0; clr %o1; clr %o2; clr %o3 ;\
267 clr %o4; clr %o5; clr %o6; clr %o7 ;\
268 retry; .align 128
269
270 /*
271 * If we get an unresolved tlb miss while in a window handler, the fault
272 * handler will resume execution at the last instruction of the window
273 * hander, instead of delivering the fault to the kernel. Spill handlers
274 * use this to spill windows into the wbuf.
275 *
276 * The mixed handler works by checking %sp, and branching to the correct
277 * handler. This is done by branching back to label 1: for 32b frames,
278 * or label 2: for 64b frames; which implies the handler order is: 32b,
279 * 64b, mixed. The 1: and 2: labels are offset into the routines to
280 * allow the branchs' delay slots to contain useful instructions.
281 */
282
283 /*
284 * SPILL_32bit spills a 32-bit-wide kernel register window. It
285 * assumes that the kernel context and the nucleus context are the
286 * same. The stack pointer is required to be eight-byte aligned even
287 * though this code only needs it to be four-byte aligned.
288 */
289 #define SPILL_32bit(tail) \
290 srl %sp, 0, %sp ;\
291 1: st %l0, [%sp + 0] ;\
292 st %l1, [%sp + 4] ;\
293 st %l2, [%sp + 8] ;\
294 st %l3, [%sp + 12] ;\
295 st %l4, [%sp + 16] ;\
296 st %l5, [%sp + 20] ;\
297 st %l6, [%sp + 24] ;\
298 st %l7, [%sp + 28] ;\
299 st %i0, [%sp + 32] ;\
300 st %i1, [%sp + 36] ;\
301 st %i2, [%sp + 40] ;\
302 st %i3, [%sp + 44] ;\
303 st %i4, [%sp + 48] ;\
304 st %i5, [%sp + 52] ;\
305 st %i6, [%sp + 56] ;\
306 st %i7, [%sp + 60] ;\
307 TT_TRACE_L(trace_win) ;\
308 saved ;\
309 retry ;\
310 SKIP(31-19-TT_TRACE_L_INS) ;\
311 ba,a,pt %xcc, fault_32bit_/**/tail ;\
312 .empty
313
314 /*
315 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
316 * wide address space via the designated asi. It is used to spill
317 * non-kernel windows. The stack pointer is required to be eight-byte
318 * aligned even though this code only needs it to be four-byte
319 * aligned.
320 */
321 #define SPILL_32bit_asi(asi_num, tail) \
322 srl %sp, 0, %sp ;\
323 1: sta %l0, [%sp + %g0]asi_num ;\
324 mov 4, %g1 ;\
325 sta %l1, [%sp + %g1]asi_num ;\
326 mov 8, %g2 ;\
327 sta %l2, [%sp + %g2]asi_num ;\
328 mov 12, %g3 ;\
329 sta %l3, [%sp + %g3]asi_num ;\
330 add %sp, 16, %g4 ;\
331 sta %l4, [%g4 + %g0]asi_num ;\
332 sta %l5, [%g4 + %g1]asi_num ;\
333 sta %l6, [%g4 + %g2]asi_num ;\
334 sta %l7, [%g4 + %g3]asi_num ;\
335 add %g4, 16, %g4 ;\
336 sta %i0, [%g4 + %g0]asi_num ;\
337 sta %i1, [%g4 + %g1]asi_num ;\
338 sta %i2, [%g4 + %g2]asi_num ;\
339 sta %i3, [%g4 + %g3]asi_num ;\
340 add %g4, 16, %g4 ;\
341 sta %i4, [%g4 + %g0]asi_num ;\
342 sta %i5, [%g4 + %g1]asi_num ;\
343 sta %i6, [%g4 + %g2]asi_num ;\
344 sta %i7, [%g4 + %g3]asi_num ;\
345 TT_TRACE_L(trace_win) ;\
346 saved ;\
347 retry ;\
348 SKIP(31-25-TT_TRACE_L_INS) ;\
349 ba,a,pt %xcc, fault_32bit_/**/tail ;\
350 .empty
351
352 #define SPILL_32bit_tt1(asi_num, tail) \
353 ba,a,pt %xcc, fault_32bit_/**/tail ;\
354 .empty ;\
355 .align 128
356
357
358 /*
359 * FILL_32bit fills a 32-bit-wide kernel register window. It assumes
360 * that the kernel context and the nucleus context are the same. The
361 * stack pointer is required to be eight-byte aligned even though this
362 * code only needs it to be four-byte aligned.
363 */
364 #define FILL_32bit(tail) \
365 srl %sp, 0, %sp ;\
366 1: TT_TRACE_L(trace_win) ;\
367 ld [%sp + 0], %l0 ;\
368 ld [%sp + 4], %l1 ;\
369 ld [%sp + 8], %l2 ;\
370 ld [%sp + 12], %l3 ;\
371 ld [%sp + 16], %l4 ;\
372 ld [%sp + 20], %l5 ;\
373 ld [%sp + 24], %l6 ;\
374 ld [%sp + 28], %l7 ;\
375 ld [%sp + 32], %i0 ;\
376 ld [%sp + 36], %i1 ;\
377 ld [%sp + 40], %i2 ;\
378 ld [%sp + 44], %i3 ;\
379 ld [%sp + 48], %i4 ;\
380 ld [%sp + 52], %i5 ;\
381 ld [%sp + 56], %i6 ;\
382 ld [%sp + 60], %i7 ;\
383 restored ;\
384 retry ;\
385 SKIP(31-19-TT_TRACE_L_INS) ;\
386 ba,a,pt %xcc, fault_32bit_/**/tail ;\
387 .empty
388
389 /*
390 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
391 * wide address space via the designated asi. It is used to fill
392 * non-kernel windows. The stack pointer is required to be eight-byte
393 * aligned even though this code only needs it to be four-byte
394 * aligned.
395 */
396 #define FILL_32bit_asi(asi_num, tail) \
397 srl %sp, 0, %sp ;\
398 1: TT_TRACE_L(trace_win) ;\
399 mov 4, %g1 ;\
400 lda [%sp + %g0]asi_num, %l0 ;\
401 mov 8, %g2 ;\
402 lda [%sp + %g1]asi_num, %l1 ;\
403 mov 12, %g3 ;\
404 lda [%sp + %g2]asi_num, %l2 ;\
405 lda [%sp + %g3]asi_num, %l3 ;\
406 add %sp, 16, %g4 ;\
407 lda [%g4 + %g0]asi_num, %l4 ;\
408 lda [%g4 + %g1]asi_num, %l5 ;\
409 lda [%g4 + %g2]asi_num, %l6 ;\
410 lda [%g4 + %g3]asi_num, %l7 ;\
411 add %g4, 16, %g4 ;\
412 lda [%g4 + %g0]asi_num, %i0 ;\
413 lda [%g4 + %g1]asi_num, %i1 ;\
414 lda [%g4 + %g2]asi_num, %i2 ;\
415 lda [%g4 + %g3]asi_num, %i3 ;\
416 add %g4, 16, %g4 ;\
417 lda [%g4 + %g0]asi_num, %i4 ;\
418 lda [%g4 + %g1]asi_num, %i5 ;\
419 lda [%g4 + %g2]asi_num, %i6 ;\
420 lda [%g4 + %g3]asi_num, %i7 ;\
421 restored ;\
422 retry ;\
423 SKIP(31-25-TT_TRACE_L_INS) ;\
424 ba,a,pt %xcc, fault_32bit_/**/tail ;\
425 .empty
426
427
428 /*
429 * SPILL_64bit spills a 64-bit-wide kernel register window. It
430 * assumes that the kernel context and the nucleus context are the
431 * same. The stack pointer is required to be eight-byte aligned.
432 */
433 #define SPILL_64bit(tail) \
434 2: stx %l0, [%sp + V9BIAS64 + 0] ;\
435 stx %l1, [%sp + V9BIAS64 + 8] ;\
436 stx %l2, [%sp + V9BIAS64 + 16] ;\
437 stx %l3, [%sp + V9BIAS64 + 24] ;\
438 stx %l4, [%sp + V9BIAS64 + 32] ;\
439 stx %l5, [%sp + V9BIAS64 + 40] ;\
440 stx %l6, [%sp + V9BIAS64 + 48] ;\
441 stx %l7, [%sp + V9BIAS64 + 56] ;\
442 stx %i0, [%sp + V9BIAS64 + 64] ;\
443 stx %i1, [%sp + V9BIAS64 + 72] ;\
444 stx %i2, [%sp + V9BIAS64 + 80] ;\
445 stx %i3, [%sp + V9BIAS64 + 88] ;\
446 stx %i4, [%sp + V9BIAS64 + 96] ;\
447 stx %i5, [%sp + V9BIAS64 + 104] ;\
448 stx %i6, [%sp + V9BIAS64 + 112] ;\
449 stx %i7, [%sp + V9BIAS64 + 120] ;\
450 TT_TRACE_L(trace_win) ;\
451 saved ;\
452 retry ;\
453 SKIP(31-18-TT_TRACE_L_INS) ;\
454 ba,a,pt %xcc, fault_64bit_/**/tail ;\
455 .empty
456
457 #define SPILL_64bit_ktt1(tail) \
458 ba,a,pt %xcc, fault_64bit_/**/tail ;\
459 .empty ;\
460 .align 128
461
462 #define SPILL_mixed_ktt1(tail) \
463 btst 1, %sp ;\
464 bz,a,pt %xcc, fault_32bit_/**/tail ;\
465 srl %sp, 0, %sp ;\
466 ba,a,pt %xcc, fault_64bit_/**/tail ;\
467 .empty ;\
468 .align 128
469
470 /*
471 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
472 * wide address space via the designated asi. It is used to spill
473 * non-kernel windows. The stack pointer is required to be eight-byte
474 * aligned.
475 */
476 #define SPILL_64bit_asi(asi_num, tail) \
477 mov 0 + V9BIAS64, %g1 ;\
478 2: stxa %l0, [%sp + %g1]asi_num ;\
479 mov 8 + V9BIAS64, %g2 ;\
480 stxa %l1, [%sp + %g2]asi_num ;\
481 mov 16 + V9BIAS64, %g3 ;\
482 stxa %l2, [%sp + %g3]asi_num ;\
483 mov 24 + V9BIAS64, %g4 ;\
484 stxa %l3, [%sp + %g4]asi_num ;\
485 add %sp, 32, %g5 ;\
486 stxa %l4, [%g5 + %g1]asi_num ;\
487 stxa %l5, [%g5 + %g2]asi_num ;\
488 stxa %l6, [%g5 + %g3]asi_num ;\
489 stxa %l7, [%g5 + %g4]asi_num ;\
490 add %g5, 32, %g5 ;\
491 stxa %i0, [%g5 + %g1]asi_num ;\
492 stxa %i1, [%g5 + %g2]asi_num ;\
493 stxa %i2, [%g5 + %g3]asi_num ;\
494 stxa %i3, [%g5 + %g4]asi_num ;\
495 add %g5, 32, %g5 ;\
496 stxa %i4, [%g5 + %g1]asi_num ;\
497 stxa %i5, [%g5 + %g2]asi_num ;\
498 stxa %i6, [%g5 + %g3]asi_num ;\
499 stxa %i7, [%g5 + %g4]asi_num ;\
500 TT_TRACE_L(trace_win) ;\
501 saved ;\
502 retry ;\
503 SKIP(31-25-TT_TRACE_L_INS) ;\
504 ba,a,pt %xcc, fault_64bit_/**/tail ;\
505 .empty
506
507 #define SPILL_64bit_tt1(asi_num, tail) \
508 ba,a,pt %xcc, fault_64bit_/**/tail ;\
509 .empty ;\
510 .align 128
511
512 /*
513 * FILL_64bit fills a 64-bit-wide kernel register window. It assumes
514 * that the kernel context and the nucleus context are the same. The
515 * stack pointer is required to be eight-byte aligned.
516 */
517 #define FILL_64bit(tail) \
518 2: TT_TRACE_L(trace_win) ;\
519 ldx [%sp + V9BIAS64 + 0], %l0 ;\
520 ldx [%sp + V9BIAS64 + 8], %l1 ;\
521 ldx [%sp + V9BIAS64 + 16], %l2 ;\
522 ldx [%sp + V9BIAS64 + 24], %l3 ;\
523 ldx [%sp + V9BIAS64 + 32], %l4 ;\
524 ldx [%sp + V9BIAS64 + 40], %l5 ;\
525 ldx [%sp + V9BIAS64 + 48], %l6 ;\
526 ldx [%sp + V9BIAS64 + 56], %l7 ;\
527 ldx [%sp + V9BIAS64 + 64], %i0 ;\
528 ldx [%sp + V9BIAS64 + 72], %i1 ;\
529 ldx [%sp + V9BIAS64 + 80], %i2 ;\
530 ldx [%sp + V9BIAS64 + 88], %i3 ;\
531 ldx [%sp + V9BIAS64 + 96], %i4 ;\
532 ldx [%sp + V9BIAS64 + 104], %i5 ;\
533 ldx [%sp + V9BIAS64 + 112], %i6 ;\
534 ldx [%sp + V9BIAS64 + 120], %i7 ;\
535 restored ;\
536 retry ;\
537 SKIP(31-18-TT_TRACE_L_INS) ;\
538 ba,a,pt %xcc, fault_64bit_/**/tail ;\
539 .empty
540
541 /*
542 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
543 * wide address space via the designated asi. It is used to fill
544 * non-kernel windows. The stack pointer is required to be eight-byte
545 * aligned.
546 */
547 #define FILL_64bit_asi(asi_num, tail) \
548 mov V9BIAS64 + 0, %g1 ;\
549 2: TT_TRACE_L(trace_win) ;\
550 ldxa [%sp + %g1]asi_num, %l0 ;\
551 mov V9BIAS64 + 8, %g2 ;\
552 ldxa [%sp + %g2]asi_num, %l1 ;\
553 mov V9BIAS64 + 16, %g3 ;\
554 ldxa [%sp + %g3]asi_num, %l2 ;\
555 mov V9BIAS64 + 24, %g4 ;\
556 ldxa [%sp + %g4]asi_num, %l3 ;\
557 add %sp, 32, %g5 ;\
558 ldxa [%g5 + %g1]asi_num, %l4 ;\
559 ldxa [%g5 + %g2]asi_num, %l5 ;\
560 ldxa [%g5 + %g3]asi_num, %l6 ;\
561 ldxa [%g5 + %g4]asi_num, %l7 ;\
562 add %g5, 32, %g5 ;\
563 ldxa [%g5 + %g1]asi_num, %i0 ;\
564 ldxa [%g5 + %g2]asi_num, %i1 ;\
565 ldxa [%g5 + %g3]asi_num, %i2 ;\
566 ldxa [%g5 + %g4]asi_num, %i3 ;\
567 add %g5, 32, %g5 ;\
568 ldxa [%g5 + %g1]asi_num, %i4 ;\
569 ldxa [%g5 + %g2]asi_num, %i5 ;\
570 ldxa [%g5 + %g3]asi_num, %i6 ;\
571 ldxa [%g5 + %g4]asi_num, %i7 ;\
572 restored ;\
573 retry ;\
574 SKIP(31-25-TT_TRACE_L_INS) ;\
575 ba,a,pt %xcc, fault_64bit_/**/tail ;\
576 .empty
577
578
579 /*
580 * SPILL_mixed spills either size window, depending on
581 * whether %sp is even or odd, to a 32-bit address space.
582 * This may only be used in conjunction with SPILL_32bit/
583 * FILL_64bit.
584 * Clear upper 32 bits of %sp if it is odd.
585 * We won't need to clear them in 64 bit kernel.
586 */
587 #define SPILL_mixed \
588 btst 1, %sp ;\
589 bz,a,pt %xcc, 1b ;\
590 srl %sp, 0, %sp ;\
591 ba,pt %xcc, 2b ;\
592 nop ;\
593 .align 128
594
595 /*
596 * FILL_mixed(ASI) fills either size window, depending on
597 * whether %sp is even or odd, from a 32-bit address space.
598 * This may only be used in conjunction with FILL_32bit/
599 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
600 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
601 * attention should be paid to the instructions that belong
602 * in the delay slots of the branches depending on the type
603 * of fill handler being branched to.
604 * Clear upper 32 bits of %sp if it is odd.
605 * We won't need to clear them in 64 bit kernel.
606 */
607 #define FILL_mixed \
608 btst 1, %sp ;\
609 bz,a,pt %xcc, 1b ;\
610 srl %sp, 0, %sp ;\
611 ba,pt %xcc, 2b ;\
612 nop ;\
613 .align 128
614
615
616 /*
617 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
618 * respectively, into the address space via the designated asi. The
619 * unbiased stack pointer is required to be eight-byte aligned (even for
620 * the 32-bit case even though this code does not require such strict
621 * alignment).
622 *
623 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
624 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
625 * will cause cwp + 2 to be spilled but will not clean cwp + 1. That
626 * window may contain kernel data so in user_rtt we set wstate to call
627 * these spill handlers on the first user spill trap. These handler then
628 * spill the appropriate window but also back up a window and clean the
629 * window that didn't get a cleanwin trap.
630 */
631 #define SPILL_32clean(asi_num, tail) \
632 srl %sp, 0, %sp ;\
633 sta %l0, [%sp + %g0]asi_num ;\
634 mov 4, %g1 ;\
635 sta %l1, [%sp + %g1]asi_num ;\
636 mov 8, %g2 ;\
637 sta %l2, [%sp + %g2]asi_num ;\
638 mov 12, %g3 ;\
639 sta %l3, [%sp + %g3]asi_num ;\
640 add %sp, 16, %g4 ;\
641 sta %l4, [%g4 + %g0]asi_num ;\
642 sta %l5, [%g4 + %g1]asi_num ;\
643 sta %l6, [%g4 + %g2]asi_num ;\
644 sta %l7, [%g4 + %g3]asi_num ;\
645 add %g4, 16, %g4 ;\
646 sta %i0, [%g4 + %g0]asi_num ;\
647 sta %i1, [%g4 + %g1]asi_num ;\
648 sta %i2, [%g4 + %g2]asi_num ;\
649 sta %i3, [%g4 + %g3]asi_num ;\
650 add %g4, 16, %g4 ;\
651 sta %i4, [%g4 + %g0]asi_num ;\
652 sta %i5, [%g4 + %g1]asi_num ;\
653 sta %i6, [%g4 + %g2]asi_num ;\
654 sta %i7, [%g4 + %g3]asi_num ;\
655 TT_TRACE_L(trace_win) ;\
656 b .spill_clean ;\
657 mov WSTATE_USER32, %g7 ;\
658 SKIP(31-25-TT_TRACE_L_INS) ;\
659 ba,a,pt %xcc, fault_32bit_/**/tail ;\
660 .empty
661
662 #define SPILL_64clean(asi_num, tail) \
663 mov 0 + V9BIAS64, %g1 ;\
664 stxa %l0, [%sp + %g1]asi_num ;\
665 mov 8 + V9BIAS64, %g2 ;\
666 stxa %l1, [%sp + %g2]asi_num ;\
667 mov 16 + V9BIAS64, %g3 ;\
668 stxa %l2, [%sp + %g3]asi_num ;\
669 mov 24 + V9BIAS64, %g4 ;\
670 stxa %l3, [%sp + %g4]asi_num ;\
671 add %sp, 32, %g5 ;\
672 stxa %l4, [%g5 + %g1]asi_num ;\
673 stxa %l5, [%g5 + %g2]asi_num ;\
674 stxa %l6, [%g5 + %g3]asi_num ;\
675 stxa %l7, [%g5 + %g4]asi_num ;\
676 add %g5, 32, %g5 ;\
677 stxa %i0, [%g5 + %g1]asi_num ;\
678 stxa %i1, [%g5 + %g2]asi_num ;\
679 stxa %i2, [%g5 + %g3]asi_num ;\
680 stxa %i3, [%g5 + %g4]asi_num ;\
681 add %g5, 32, %g5 ;\
682 stxa %i4, [%g5 + %g1]asi_num ;\
683 stxa %i5, [%g5 + %g2]asi_num ;\
684 stxa %i6, [%g5 + %g3]asi_num ;\
685 stxa %i7, [%g5 + %g4]asi_num ;\
686 TT_TRACE_L(trace_win) ;\
687 b .spill_clean ;\
688 mov WSTATE_USER64, %g7 ;\
689 SKIP(31-25-TT_TRACE_L_INS) ;\
690 ba,a,pt %xcc, fault_64bit_/**/tail ;\
691 .empty
692
693
694 /*
695 * Floating point disabled.
696 */
697 #define FP_DISABLED_TRAP \
698 TT_TRACE(trace_gen) ;\
699 ba,pt %xcc,.fp_disabled ;\
700 nop ;\
701 .align 32
702
703 /*
704 * Floating point exceptions.
705 */
706 #define FP_IEEE_TRAP \
707 TT_TRACE(trace_gen) ;\
708 ba,pt %xcc,.fp_ieee_exception ;\
709 nop ;\
710 .align 32
711
712 #define FP_TRAP \
713 TT_TRACE(trace_gen) ;\
714 ba,pt %xcc,.fp_exception ;\
715 nop ;\
716 .align 32
717
718 /*
719 * ECACHE_ECC error traps at level 0 and level 1
720 */
721 #define ECACHE_ECC(table_name) \
722 .global table_name ;\
723 table_name: ;\
724 membar #Sync ;\
725 set trap, %g1 ;\
726 rdpr %tt, %g3 ;\
727 ba,pt %xcc, sys_trap ;\
728 sub %g0, 1, %g4 ;\
729 .align 32
730
731 /*
732 * illegal instruction trap
733 */
734 #define ILLTRAP_INSTR \
735 membar #Sync ;\
736 TT_TRACE(trace_gen) ;\
737 or %g0, P_UTRAP4, %g2 ;\
738 or %g0, T_UNIMP_INSTR, %g3 ;\
739 sethi %hi(.check_v9utrap), %g4 ;\
740 jmp %g4 + %lo(.check_v9utrap) ;\
741 nop ;\
742 .align 32
743
744 /*
745 * tag overflow trap
746 */
747 #define TAG_OVERFLOW \
748 TT_TRACE(trace_gen) ;\
749 or %g0, P_UTRAP10, %g2 ;\
750 or %g0, T_TAG_OVERFLOW, %g3 ;\
751 sethi %hi(.check_v9utrap), %g4 ;\
752 jmp %g4 + %lo(.check_v9utrap) ;\
753 nop ;\
754 .align 32
755
756 /*
757 * divide by zero trap
758 */
759 #define DIV_BY_ZERO \
760 TT_TRACE(trace_gen) ;\
761 or %g0, P_UTRAP11, %g2 ;\
762 or %g0, T_IDIV0, %g3 ;\
763 sethi %hi(.check_v9utrap), %g4 ;\
764 jmp %g4 + %lo(.check_v9utrap) ;\
765 nop ;\
766 .align 32
767
768 /*
769 * trap instruction for V9 user trap handlers
770 */
771 #define TRAP_INSTR \
772 TT_TRACE(trace_gen) ;\
773 or %g0, T_SOFTWARE_TRAP, %g3 ;\
774 sethi %hi(.check_v9utrap), %g4 ;\
775 jmp %g4 + %lo(.check_v9utrap) ;\
776 nop ;\
777 .align 32
778 #define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
779
780 /*
781 * LEVEL_INTERRUPT is for level N interrupts.
782 * VECTOR_INTERRUPT is for the vector trap.
783 */
784 #define LEVEL_INTERRUPT(level) \
785 .global tt_pil/**/level ;\
786 tt_pil/**/level: ;\
787 ba,pt %xcc, pil_interrupt ;\
788 mov level, %g4 ;\
789 .align 32
790
791 #define LEVEL14_INTERRUPT \
792 ba pil14_interrupt ;\
793 mov PIL_14, %g4 ;\
794 .align 32
795
796 #define LEVEL15_INTERRUPT \
797 ba pil15_interrupt ;\
798 mov PIL_15, %g4 ;\
799 .align 32
800
801 #define CPU_MONDO \
802 ba,a,pt %xcc, cpu_mondo ;\
803 .align 32
804
805 #define DEV_MONDO \
806 ba,a,pt %xcc, dev_mondo ;\
807 .align 32
808
809 /*
810 * We take over the rtba after we set our trap table and
811 * fault status area. The watchdog reset trap is now handled by the OS.
812 */
813 #define WATCHDOG_RESET \
814 mov PTL1_BAD_WATCHDOG, %g1 ;\
815 ba,a,pt %xcc, .watchdog_trap ;\
816 .align 32
817
818 /*
819 * RED is for traps that use the red mode handler.
820 * We should never see these either.
821 */
822 #define RED \
823 mov PTL1_BAD_RED, %g1 ;\
824 ba,a,pt %xcc, .watchdog_trap ;\
825 .align 32
826
827
828 /*
829 * MMU Trap Handlers.
830 */
831
832 /*
833 * synthesize for trap(): SFSR in %g3
834 */
835 #define IMMU_EXCEPTION \
836 MMU_FAULT_STATUS_AREA(%g3) ;\
837 rdpr %tpc, %g2 ;\
838 ldx [%g3 + MMFSA_I_TYPE], %g1 ;\
839 ldx [%g3 + MMFSA_I_CTX], %g3 ;\
840 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
841 or %g3, %g1, %g3 ;\
842 ba,pt %xcc, .mmu_exception_end ;\
843 mov T_INSTR_EXCEPTION, %g1 ;\
844 .align 32
845
846 /*
847 * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3
848 */
849 #define DMMU_EXCEPTION \
850 ba,a,pt %xcc, .dmmu_exception ;\
851 .align 32
852
853 /*
854 * synthesize for trap(): SFAR in %g2, SFSR in %g3
855 */
856 #define DMMU_EXC_AG_PRIV \
857 MMU_FAULT_STATUS_AREA(%g3) ;\
858 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\
859 /* Fault type not available in MMU fault status area */ ;\
860 mov MMFSA_F_PRVACT, %g1 ;\
861 ldx [%g3 + MMFSA_D_CTX], %g3 ;\
862 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
863 ba,pt %xcc, .mmu_priv_exception ;\
864 or %g3, %g1, %g3 ;\
865 .align 32
866
867 /*
868 * synthesize for trap(): SFAR in %g2, SFSR in %g3
869 */
870 #define DMMU_EXC_AG_NOT_ALIGNED \
871 MMU_FAULT_STATUS_AREA(%g3) ;\
872 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\
873 /* Fault type not available in MMU fault status area */ ;\
874 mov MMFSA_F_UNALIGN, %g1 ;\
875 ldx [%g3 + MMFSA_D_CTX], %g3 ;\
876 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\
877 ba,pt %xcc, .mmu_exception_not_aligned ;\
878 or %g3, %g1, %g3 /* SFSR */ ;\
879 .align 32
880 /*
881 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
882 */
883
884 /*
885 * synthesize for trap(): SFAR in %g2, SFSR in %g3
886 */
887 #define DMMU_EXC_LDDF_NOT_ALIGNED \
888 ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned ;\
889 .align 32
890 /*
891 * synthesize for trap(): SFAR in %g2, SFSR in %g3
892 */
893 #define DMMU_EXC_STDF_NOT_ALIGNED \
894 ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned ;\
895 .align 32
896
897 #if defined(cscope)
898 /*
899 * Define labels to direct cscope quickly to labels that
900 * are generated by macro expansion of DTLB_MISS().
901 */
902 .global tt0_dtlbmiss
903 tt0_dtlbmiss:
904 .global tt1_dtlbmiss
905 tt1_dtlbmiss:
906 nop
907 #endif
908
909 /*
910 * Data miss handler (must be exactly 32 instructions)
911 *
912 * This handler is invoked only if the hypervisor has been instructed
913 * not to do any TSB walk.
914 *
915 * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss
916 * handler.
917 *
918 * User TLB miss handling depends upon whether a user process has one or
919 * two TSBs. User TSB information (physical base and size code) is kept
920 * in two dedicated scratchpad registers. Absence of a user TSB (primarily
921 * second TSB) is indicated by a negative value (-1) in that register.
922 */
923
924 /*
925 * synthesize for miss handler: pseudo-tag access in %g2 (with context "type"
926 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
927 */
928 #define DTLB_MISS(table_name) ;\
929 .global table_name/**/_dtlbmiss ;\
930 table_name/**/_dtlbmiss: ;\
931 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
932 cmp %g3, INVALID_CONTEXT ;\
933 ble,pn %xcc, sfmmu_kdtlb_miss ;\
934 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
935 mov SCRATCHPAD_UTSBREG2, %g1 ;\
936 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\
937 brgez,pn %g1, sfmmu_udtlb_slowpath /* branch if 2 TSBs */ ;\
938 nop ;\
939 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\
940 ba,pt %xcc, sfmmu_udtlb_fastpath /* no 4M TSB, miss */ ;\
941 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
942 .align 128
943
944
945 #if defined(cscope)
946 /*
947 * Define labels to direct cscope quickly to labels that
948 * are generated by macro expansion of ITLB_MISS().
949 */
950 .global tt0_itlbmiss
951 tt0_itlbmiss:
952 .global tt1_itlbmiss
953 tt1_itlbmiss:
954 nop
955 #endif
956
957 /*
958 * Instruction miss handler.
959 *
960 * This handler is invoked only if the hypervisor has been instructed
961 * not to do any TSB walk.
962 *
963 * ldda instructions will have their ASI patched
964 * by sfmmu_patch_ktsb at runtime.
965 * MUST be EXACTLY 32 instructions or we'll break.
966 */
967
968 /*
969 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
970 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
971 */
972 #define ITLB_MISS(table_name) \
973 .global table_name/**/_itlbmiss ;\
974 table_name/**/_itlbmiss: ;\
975 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
976 cmp %g3, INVALID_CONTEXT ;\
977 ble,pn %xcc, sfmmu_kitlb_miss ;\
978 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
979 mov SCRATCHPAD_UTSBREG2, %g1 ;\
980 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\
981 brgez,pn %g1, sfmmu_uitlb_slowpath /* branch if 2 TSBs */ ;\
982 nop ;\
983 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\
984 ba,pt %xcc, sfmmu_uitlb_fastpath /* no 4M TSB, miss */ ;\
985 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
986 .align 128
987
988 #define DTSB_MISS \
989 GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu)
990
991 #define ITSB_MISS \
992 GOTO_TT(sfmmu_slow_immu_miss,trace_immu)
993
994 /*
995 * This macro is the first level handler for fast protection faults.
996 * It first demaps the tlb entry which generated the fault and then
997 * attempts to set the modify bit on the hash. It needs to be
998 * exactly 32 instructions.
999 */
1000 /*
1001 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1002 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1003 */
1004 #define DTLB_PROT \
1005 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\
1006 /* ;\
1007 * g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\
1008 * g3 = ctx type (0, 1, or 2) ;\
1009 */ ;\
1010 TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\
1011 /* clobbers g1 and g6 XXXQ? */ ;\
1012 brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\
1013 nop ;\
1014 ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\
1015 .align 128
1016
1017 #define DMMU_EXCEPTION_TL1 ;\
1018 ba,a,pt %xcc, mmu_trap_tl1 ;\
1019 .align 32
1020
1021 #define MISALIGN_ADDR_TL1 ;\
1022 ba,a,pt %xcc, mmu_trap_tl1 ;\
1023 .align 32
1024
1025 /*
1026 * Trace a tsb hit
1027 * g1 = tsbe pointer (in/clobbered)
1028 * g2 = tag access register (in)
1029 * g3 - g4 = scratch (clobbered)
1030 * g5 = tsbe data (in)
1031 * g6 = scratch (clobbered)
1032 * g7 = pc we jumped here from (in)
1033 * ttextra = value to OR in to trap type (%tt) (in)
1034 */
1035 #ifdef TRAPTRACE
1036 #define TRACE_TSBHIT(ttextra) \
1037 membar #Sync ;\
1038 sethi %hi(FLUSH_ADDR), %g6 ;\
1039 flush %g6 ;\
1040 TRACE_PTR(%g3, %g6) ;\
1041 GET_TRACE_TICK(%g6, %g4) ;\
1042 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\
1043 stna %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\
1044 stna %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\
1045 rdpr %tnpc, %g6 ;\
1046 stna %g6, [%g3 + TRAP_ENT_F2]%asi ;\
1047 stna %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\
1048 stna %g0, [%g3 + TRAP_ENT_F4]%asi ;\
1049 rdpr %tpc, %g6 ;\
1050 stna %g6, [%g3 + TRAP_ENT_TPC]%asi ;\
1051 TRACE_SAVE_TL_GL_REGS(%g3, %g6) ;\
1052 rdpr %tt, %g6 ;\
1053 or %g6, (ttextra), %g1 ;\
1054 stha %g1, [%g3 + TRAP_ENT_TT]%asi ;\
1055 MMU_FAULT_STATUS_AREA(%g4) ;\
1056 mov MMFSA_D_ADDR, %g1 ;\
1057 cmp %g6, FAST_IMMU_MISS_TT ;\
1058 move %xcc, MMFSA_I_ADDR, %g1 ;\
1059 cmp %g6, T_INSTR_MMU_MISS ;\
1060 move %xcc, MMFSA_I_ADDR, %g1 ;\
1061 ldx [%g4 + %g1], %g1 ;\
1062 stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */ ;\
1063 mov MMFSA_D_CTX, %g1 ;\
1064 cmp %g6, FAST_IMMU_MISS_TT ;\
1065 move %xcc, MMFSA_I_CTX, %g1 ;\
1066 cmp %g6, T_INSTR_MMU_MISS ;\
1067 move %xcc, MMFSA_I_CTX, %g1 ;\
1068 ldx [%g4 + %g1], %g1 ;\
1069 stna %g1, [%g3 + TRAP_ENT_TR]%asi ;\
1070 TRACE_NEXT(%g3, %g4, %g6)
1071 #else
1072 #define TRACE_TSBHIT(ttextra)
1073 #endif
1074
1075
1076 /*
1077 * =======================================================================
1078 * SPARC V9 TRAP TABLE
1079 *
1080 * The trap table is divided into two halves: the first half is used when
1081 * taking traps when TL=0; the second half is used when taking traps from
1082 * TL>0. Note that handlers in the second half of the table might not be able
1083 * to make the same assumptions as handlers in the first half of the table.
1084 *
1085 * Worst case trap nesting so far:
1086 *
1087 * at TL=0 client issues software trap requesting service
1088 * at TL=1 nucleus wants a register window
1089 * at TL=2 register window clean/spill/fill takes a TLB miss
1090 * at TL=3 processing TLB miss
1091 * at TL=4 handle asynchronous error
1092 *
1093 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1094 *
1095 * =======================================================================
1096 */
1097 .section ".text"
1098 .align 4
1099 .global trap_table, scb, trap_table0, trap_table1, etrap_table
1100 .type trap_table, #object
1101 .type trap_table0, #object
1102 .type trap_table1, #object
1103 .type scb, #object
1104 trap_table:
1105 scb:
1106 trap_table0:
1107 /* hardware traps */
1108 NOT; /* 000 reserved */
1109 RED; /* 001 power on reset */
1110 WATCHDOG_RESET; /* 002 watchdog reset */
1111 RED; /* 003 externally initiated reset */
1112 RED; /* 004 software initiated reset */
1113 RED; /* 005 red mode exception */
1114 NOT; NOT; /* 006 - 007 reserved */
1115 IMMU_EXCEPTION; /* 008 instruction access exception */
1116 ITSB_MISS; /* 009 instruction access MMU miss */
1117 NOT; /* 00A reserved */
1118 NOT; NOT4; /* 00B - 00F reserved */
1119 ILLTRAP_INSTR; /* 010 illegal instruction */
1120 TRAP(T_PRIV_INSTR); /* 011 privileged opcode */
1121 TRAP(T_UNIMP_LDD); /* 012 unimplemented LDD */
1122 TRAP(T_UNIMP_STD); /* 013 unimplemented STD */
1123 NOT4; NOT4; NOT4; /* 014 - 01F reserved */
1124 FP_DISABLED_TRAP; /* 020 fp disabled */
1125 FP_IEEE_TRAP; /* 021 fp exception ieee 754 */
1126 FP_TRAP; /* 022 fp exception other */
1127 TAG_OVERFLOW; /* 023 tag overflow */
1128 CLEAN_WINDOW; /* 024 - 027 clean window */
1129 DIV_BY_ZERO; /* 028 division by zero */
1130 NOT; /* 029 internal processor error */
1131 NOT; NOT; NOT4; /* 02A - 02F reserved */
1132 DMMU_EXCEPTION; /* 030 data access exception */
1133 DTSB_MISS; /* 031 data access MMU miss */
1134 NOT; /* 032 reserved */
1135 NOT; /* 033 data access protection */
1136 DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */
1137 DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */
1138 DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */
1139 DMMU_EXC_AG_PRIV; /* 037 privileged action */
1140 NOT; /* 038 LDQF mem address not aligned */
1141 NOT; /* 039 STQF mem address not aligned */
1142 NOT; NOT; NOT4; /* 03A - 03F reserved */
1143 NOT; /* 040 async data error */
1144 LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */
1145 LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */
1146 LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */
1147 LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */
1148 LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */
1149 LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */
1150 LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */
1151 LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */
1152 LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */
1153 LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */
1154 LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */
1155 LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */
1156 LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */
1157 LEVEL14_INTERRUPT; /* 04E interrupt level 14 */
1158 LEVEL15_INTERRUPT; /* 04F interrupt level 15 */
1159 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */
1160 NOT; /* 060 interrupt vector */
1161 GOTO(kmdb_trap); /* 061 PA watchpoint */
1162 GOTO(kmdb_trap); /* 062 VA watchpoint */
1163 NOT; /* 063 reserved */
1164 ITLB_MISS(tt0); /* 064 instruction access MMU miss */
1165 DTLB_MISS(tt0); /* 068 data access MMU miss */
1166 DTLB_PROT; /* 06C data access protection */
1167 NOT; /* 070 reserved */
1168 NOT; /* 071 reserved */
1169 NOT; /* 072 reserved */
1170 NOT; /* 073 reserved */
1171 NOT4; NOT4 /* 074 - 07B reserved */
1172 CPU_MONDO; /* 07C cpu_mondo */
1173 DEV_MONDO; /* 07D dev_mondo */
1174 GOTO_TT(resumable_error, trace_gen); /* 07E resumable error */
1175 GOTO_TT(nonresumable_error, trace_gen); /* 07F non-reasumable error */
1176 NOT4; /* 080 spill 0 normal */
1177 SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */
1178 SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */
1179 SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */
1180 SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */
1181 SPILL_32bit(not); /* 094 spill 5 normal */
1182 SPILL_64bit(not); /* 098 spill 6 normal */
1183 SPILL_mixed; /* 09C spill 7 normal */
1184 NOT4; /* 0A0 spill 0 other */
1185 SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */
1186 SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */
1187 SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */
1188 SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */
1189 NOT4; /* 0B4 spill 5 other */
1190 NOT4; /* 0B8 spill 6 other */
1191 NOT4; /* 0BC spill 7 other */
1192 NOT4; /* 0C0 fill 0 normal */
1193 FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */
1194 FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */
1195 FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */
1196 FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */
1197 FILL_32bit(not); /* 0D4 fill 5 normal */
1198 FILL_64bit(not); /* 0D8 fill 6 normal */
1199 FILL_mixed; /* 0DC fill 7 normal */
1200 NOT4; /* 0E0 fill 0 other */
1201 NOT4; /* 0E4 fill 1 other */
1202 NOT4; /* 0E8 fill 2 other */
1203 NOT4; /* 0EC fill 3 other */
1204 NOT4; /* 0F0 fill 4 other */
1205 NOT4; /* 0F4 fill 5 other */
1206 NOT4; /* 0F8 fill 6 other */
1207 NOT4; /* 0FC fill 7 other */
1208 /* user traps */
1209 GOTO(syscall_trap_4x); /* 100 old system call */
1210 TRAP(T_BREAKPOINT); /* 101 user breakpoint */
1211 TRAP(T_DIV0); /* 102 user divide by zero */
1212 GOTO(.flushw); /* 103 flush windows */
1213 GOTO(.clean_windows); /* 104 clean windows */
1214 BAD; /* 105 range check ?? */
1215 GOTO(.fix_alignment); /* 106 do unaligned references */
1216 BAD; /* 107 unused */
1217 SYSCALL_TRAP32; /* 108 ILP32 system call on LP64 */
1218 GOTO(set_trap0_addr); /* 109 set trap0 address */
1219 BAD; BAD; BAD4; /* 10A - 10F unused */
1220 TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */
1221 GOTO(.getcc); /* 120 get condition codes */
1222 GOTO(.setcc); /* 121 set condition codes */
1223 GOTO(.getpsr); /* 122 get psr */
1224 GOTO(.setpsr); /* 123 set psr (some fields) */
1225 GOTO(get_timestamp); /* 124 get timestamp */
1226 GOTO(get_virtime); /* 125 get lwp virtual time */
1227 PRIV(self_xcall); /* 126 self xcall */
1228 GOTO(get_hrestime); /* 127 get hrestime */
1229 BAD; /* 128 ST_SETV9STACK */
1230 GOTO(.getlgrp); /* 129 get lgrpid */
1231 BAD; BAD; BAD4; /* 12A - 12F unused */
1232 BAD4; BAD4; /* 130 - 137 unused */
1233 DTRACE_PID; /* 138 dtrace pid tracing provider */
1234 BAD; /* 139 unused */
1235 DTRACE_RETURN; /* 13A dtrace pid return probe */
1236 BAD; BAD4; /* 13B - 13F unused */
1237 SYSCALL_TRAP; /* 140 LP64 system call */
1238 SYSCALL(nosys); /* 141 unused system call trap */
1239 #ifdef DEBUG_USER_TRAPTRACECTL
1240 GOTO(.traptrace_freeze); /* 142 freeze traptrace */
1241 GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */
1242 #else
1243 SYSCALL(nosys); /* 142 unused system call trap */
1244 SYSCALL(nosys); /* 143 unused system call trap */
1245 #endif
1246 BAD4; BAD4; BAD4; /* 144 - 14F unused */
1247 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
1248 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
1249 BAD; /* 170 - unused */
1250 BAD; /* 171 - unused */
1251 BAD; BAD; /* 172 - 173 unused */
1252 BAD4; BAD4; /* 174 - 17B unused */
1253 #ifdef PTL1_PANIC_DEBUG
1254 mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1255 /* 17C test ptl1_panic */
1256 #else
1257 BAD; /* 17C unused */
1258 #endif /* PTL1_PANIC_DEBUG */
1259 PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */
1260 PRIV(kmdb_trap); /* 17E kmdb breakpoint */
1261 PRIV(obp_bpt); /* 17F obp breakpoint */
1262 /* reserved */
1263 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
1264 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
1265 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
1266 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
1267 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
1268 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
1269 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
1270 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
1271 .size trap_table0, (.-trap_table0)
1272 trap_table1:
1273 NOT4; NOT4; /* 000 - 007 unused */
1274 NOT; /* 008 instruction access exception */
1275 ITSB_MISS; /* 009 instruction access MMU miss */
1276 NOT; /* 00A reserved */
1277 NOT; NOT4; /* 00B - 00F unused */
1278 NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */
1279 NOT4; /* 020 - 023 unused */
1280 CLEAN_WINDOW; /* 024 - 027 clean window */
1281 NOT4; NOT4; /* 028 - 02F unused */
1282 DMMU_EXCEPTION_TL1; /* 030 data access exception */
1283 DTSB_MISS; /* 031 data access MMU miss */
1284 NOT; /* 032 reserved */
1285 NOT; /* 033 unused */
1286 MISALIGN_ADDR_TL1; /* 034 mem address not aligned */
1287 NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */
1288 NOT4; NOT4; NOT4; NOT4; /* 040 - 04F unused */
1289 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */
1290 NOT; /* 060 unused */
1291 GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */
1292 GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */
1293 NOT; /* 063 reserved */
1294 ITLB_MISS(tt1); /* 064 instruction access MMU miss */
1295 DTLB_MISS(tt1); /* 068 data access MMU miss */
1296 DTLB_PROT; /* 06C data access protection */
1297 NOT; /* 070 reserved */
1298 NOT; /* 071 reserved */
1299 NOT; /* 072 reserved */
1300 NOT; /* 073 reserved */
1301 NOT4; NOT4; /* 074 - 07B reserved */
1302 NOT; /* 07C reserved */
1303 NOT; /* 07D reserved */
1304 NOT; /* 07E resumable error */
1305 GOTO_TT(nonresumable_error, trace_gen); /* 07F nonresumable error */
1306 NOTP4; /* 080 spill 0 normal */
1307 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */
1308 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */
1309 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */
1310 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */
1311 NOTP4; /* 094 spill 5 normal */
1312 SPILL_64bit_ktt1(sk); /* 098 spill 6 normal */
1313 SPILL_mixed_ktt1(sk); /* 09C spill 7 normal */
1314 NOTP4; /* 0A0 spill 0 other */
1315 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */
1316 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */
1317 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */
1318 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */
1319 NOTP4; /* 0B4 spill 5 other */
1320 NOTP4; /* 0B8 spill 6 other */
1321 NOTP4; /* 0BC spill 7 other */
1322 NOT4; /* 0C0 fill 0 normal */
1323 NOT4; /* 0C4 fill 1 normal */
1324 NOT4; /* 0C8 fill 2 normal */
1325 NOT4; /* 0CC fill 3 normal */
1326 NOT4; /* 0D0 fill 4 normal */
1327 NOT4; /* 0D4 fill 5 normal */
1328 NOT4; /* 0D8 fill 6 normal */
1329 NOT4; /* 0DC fill 7 normal */
1330 NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */
1331 NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */
1332 /*
1333 * Code running at TL>0 does not use soft traps, so
1334 * we can truncate the table here.
1335 * However:
1336 * sun4v uses (hypervisor) ta instructions at TL > 0, so
1337 * provide a safety net for now.
1338 */
1339 /* soft traps */
1340 BAD4; BAD4; BAD4; BAD4; /* 100 - 10F unused */
1341 BAD4; BAD4; BAD4; BAD4; /* 110 - 11F unused */
1342 BAD4; BAD4; BAD4; BAD4; /* 120 - 12F unused */
1343 BAD4; BAD4; BAD4; BAD4; /* 130 - 13F unused */
1344 BAD4; BAD4; BAD4; BAD4; /* 140 - 14F unused */
1345 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
1346 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
1347 BAD4; BAD4; BAD4; BAD4; /* 170 - 17F unused */
1348 /* reserved */
1349 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
1350 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
1351 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
1352 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
1353 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
1354 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
1355 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
1356 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
1357 etrap_table:
1358 .size trap_table1, (.-trap_table1)
1359 .size trap_table, (.-trap_table)
1360 .size scb, (.-scb)
1361
1362 /*
1363 * We get to exec_fault in the case of an instruction miss and tte
1364 * has no execute bit set. We go to tl0 to handle it.
1365 *
1366 * g1 = tsbe pointer (in/clobbered)
1367 * g2 = tag access register (in)
1368 * g3 - g4 = scratch (clobbered)
1369 * g5 = tsbe data (in)
1370 * g6 = scratch (clobbered)
1371 * g7 = pc we jumped here from (in)
1372 */
1373 /*
1374 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1375 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1376 */
1377 ALTENTRY(exec_fault)
1378 TRACE_TSBHIT(TT_MMU_EXEC)
1379 MMU_FAULT_STATUS_AREA(%g4)
1380 ldx [%g4 + MMFSA_I_ADDR], %g2 /* g2 = address */
1381 ldx [%g4 + MMFSA_I_CTX], %g3 /* g3 = ctx */
1382 srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry
1383 cmp %g3, USER_CONTEXT_TYPE
1384 sllx %g2, MMU_PAGESHIFT, %g2
1385 movgu %icc, USER_CONTEXT_TYPE, %g3
1386 or %g2, %g3, %g2 /* TAG_ACCESS */
1387 mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype
1388 set trap, %g1
1389 ba,pt %xcc, sys_trap
1390 mov -1, %g4
1391
1392 .mmu_exception_not_aligned:
1393 /* %g2 = sfar, %g3 = sfsr */
1394 rdpr %tstate, %g1
1395 btst TSTATE_PRIV, %g1
1396 bnz,pn %icc, 2f
1397 nop
1398 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1399 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1400 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1401 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1402 brz,pt %g5, 2f
1403 nop
1404 ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap?
1405 brz,pn %g5, 2f
1406 nop
1407 btst 1, %sp
1408 bz,pt %xcc, 1f ! 32 bit user program
1409 nop
1410 ba,pt %xcc, .setup_v9utrap ! 64 bit user program
1411 nop
1412 1:
1413 ba,pt %xcc, .setup_utrap
1414 or %g2, %g0, %g7
1415 2:
1416 ba,pt %xcc, .mmu_exception_end
1417 mov T_ALIGNMENT, %g1
1418
1419 .mmu_priv_exception:
1420 rdpr %tstate, %g1
1421 btst TSTATE_PRIV, %g1
1422 bnz,pn %icc, 1f
1423 nop
1424 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1425 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1426 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1427 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1428 brz,pt %g5, 1f
1429 nop
1430 ldn [%g5 + P_UTRAP16], %g5
1431 brnz,pt %g5, .setup_v9utrap
1432 nop
1433 1:
1434 mov T_PRIV_INSTR, %g1
1435
1436 .mmu_exception_end:
1437 CPU_INDEX(%g4, %g5)
1438 set cpu_core, %g5
1439 sllx %g4, CPU_CORE_SHIFT, %g4
1440 add %g4, %g5, %g4
1441 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5
1442 andcc %g5, CPU_DTRACE_NOFAULT, %g0
1443 bz 1f
1444 or %g5, CPU_DTRACE_BADADDR, %g5
1445 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS]
1446 done
1447
1448 1:
1449 sllx %g3, 32, %g3
1450 or %g3, %g1, %g3
1451 set trap, %g1
1452 ba,pt %xcc, sys_trap
1453 sub %g0, 1, %g4
1454
1455 .fp_disabled:
1456 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1457 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1458 rdpr %tstate, %g4
1459 btst TSTATE_PRIV, %g4
1460 bnz,a,pn %icc, ptl1_panic
1461 mov PTL1_BAD_FPTRAP, %g1
1462
1463 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1464 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1465 brz,a,pt %g5, 2f
1466 nop
1467 ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap?
1468 brz,a,pn %g5, 2f
1469 nop
1470 btst 1, %sp
1471 bz,a,pt %xcc, 1f ! 32 bit user program
1472 nop
1473 ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program
1474 nop
1475 1:
1476 ba,pt %xcc, .setup_utrap
1477 or %g0, %g0, %g7
1478 2:
1479 set fp_disabled, %g1
1480 ba,pt %xcc, sys_trap
1481 sub %g0, 1, %g4
1482
1483 .fp_ieee_exception:
1484 rdpr %tstate, %g1
1485 btst TSTATE_PRIV, %g1
1486 bnz,a,pn %icc, ptl1_panic
1487 mov PTL1_BAD_FPTRAP, %g1
1488 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1489 stx %fsr, [%g1 + CPU_TMP1]
1490 ldx [%g1 + CPU_TMP1], %g2
1491 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1492 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1493 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1494 brz,a,pt %g5, 1f
1495 nop
1496 ldn [%g5 + P_UTRAP8], %g5
1497 brnz,a,pt %g5, .setup_v9utrap
1498 nop
1499 1:
1500 set _fp_ieee_exception, %g1
1501 ba,pt %xcc, sys_trap
1502 sub %g0, 1, %g4
1503
1504 /*
1505 * Register Inputs:
1506 * %g5 user trap handler
1507 * %g7 misaligned addr - for alignment traps only
1508 */
1509 .setup_utrap:
1510 set trap, %g1 ! setup in case we go
1511 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1512 sub %g0, 1, %g4 ! the save instruction below
1513
1514 /*
1515 * If the DTrace pid provider is single stepping a copied-out
1516 * instruction, t->t_dtrace_step will be set. In that case we need
1517 * to abort the single-stepping (since execution of the instruction
1518 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1519 */
1520 save %sp, -SA(MINFRAME32), %sp ! window for trap handler
1521 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1522 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1523 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1524 rdpr %tnpc, %l2 ! arg1 == tnpc
1525 brz,pt %g2, 1f
1526 rdpr %tpc, %l1 ! arg0 == tpc
1527
1528 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1529 ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step)
1530 brz,pt %g2, 1f
1531 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1532 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1533 1:
1534 mov %g7, %l3 ! arg2 == misaligned address
1535
1536 rdpr %tstate, %g1 ! cwp for trap handler
1537 rdpr %cwp, %g4
1538 bclr TSTATE_CWP_MASK, %g1
1539 wrpr %g1, %g4, %tstate
1540 wrpr %g0, %g5, %tnpc ! trap handler address
1541 FAST_TRAP_DONE
1542 /* NOTREACHED */
1543
1544 .check_v9utrap:
1545 rdpr %tstate, %g1
1546 btst TSTATE_PRIV, %g1
1547 bnz,a,pn %icc, 3f
1548 nop
1549 CPU_ADDR(%g4, %g1) ! load CPU struct addr
1550 ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer
1551 ldn [%g5 + T_PROCP], %g5 ! load proc pointer
1552 ldn [%g5 + P_UTRAPS], %g5 ! are there utraps?
1553
1554 cmp %g3, T_SOFTWARE_TRAP
1555 bne,a,pt %icc, 1f
1556 nop
1557
1558 brz,pt %g5, 3f ! if p_utraps == NULL goto trap()
1559 rdpr %tt, %g3 ! delay - get actual hw trap type
1560
1561 sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1562 ba,pt %icc, 2f
1563 smul %g1, CPTRSIZE, %g2
1564 1:
1565 brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap()
1566 nop
1567
1568 cmp %g3, T_UNIMP_INSTR
1569 bne,a,pt %icc, 2f
1570 nop
1571
1572 mov 1, %g1
1573 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1574 rdpr %tpc, %g1 ! ld trapping instruction using
1575 lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault
1576 st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1577
1578 sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction
1579 andcc %g1, %g4, %g4 ! and instruction with mask
1580 bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP
1581 nop ! fall thru to setup
1582 2:
1583 ldn [%g5 + %g2], %g5
1584 brnz,a,pt %g5, .setup_v9utrap
1585 nop
1586 3:
1587 set trap, %g1
1588 ba,pt %xcc, sys_trap
1589 sub %g0, 1, %g4
1590 /* NOTREACHED */
1591
1592 /*
1593 * Register Inputs:
1594 * %g5 user trap handler
1595 */
1596 .setup_v9utrap:
1597 set trap, %g1 ! setup in case we go
1598 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1599 sub %g0, 1, %g4 ! the save instruction below
1600
1601 /*
1602 * If the DTrace pid provider is single stepping a copied-out
1603 * instruction, t->t_dtrace_step will be set. In that case we need
1604 * to abort the single-stepping (since execution of the instruction
1605 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1606 */
1607 save %sp, -SA(MINFRAME64), %sp ! window for trap handler
1608 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1609 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1610 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1611 rdpr %tnpc, %l7 ! arg1 == tnpc
1612 brz,pt %g2, 1f
1613 rdpr %tpc, %l6 ! arg0 == tpc
1614
1615 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1616 ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step)
1617 brz,pt %g2, 1f
1618 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1619 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1620 1:
1621 rdpr %tstate, %g2 ! cwp for trap handler
1622 rdpr %cwp, %g4
1623 bclr TSTATE_CWP_MASK, %g2
1624 wrpr %g2, %g4, %tstate
1625
1626 ldn [%g1 + T_PROCP], %g4 ! load proc pointer
1627 ldn [%g4 + P_AS], %g4 ! load as pointer
1628 ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit
1629 cmp %l7, %g4 ! check for single-step set
1630 bne,pt %xcc, 4f
1631 nop
1632 ldn [%g1 + T_LWP], %g1 ! load klwp pointer
1633 ld [%g1 + PCB_STEP], %g4 ! load single-step flag
1634 cmp %g4, STEP_ACTIVE ! step flags set in pcb?
1635 bne,pt %icc, 4f
1636 nop
1637 stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb
1638 mov %l7, %g4 ! on entry to precise user trap
1639 add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc
1640 ! at time of trap
1641 wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS,
1642 ! %g4 == userlimit
1643 FAST_TRAP_DONE
1644 /* NOTREACHED */
1645 4:
1646 wrpr %g0, %g5, %tnpc ! trap handler address
1647 FAST_TRAP_DONE_CHK_INTR
1648 /* NOTREACHED */
1649
1650 .fp_exception:
1651 CPU_ADDR(%g1, %g4)
1652 stx %fsr, [%g1 + CPU_TMP1]
1653 ldx [%g1 + CPU_TMP1], %g2
1654
1655 /*
1656 * Cheetah takes unfinished_FPop trap for certain range of operands
1657 * to the "fitos" instruction. Instead of going through the slow
1658 * software emulation path, we try to simulate the "fitos" instruction
1659 * via "fitod" and "fdtos" provided the following conditions are met:
1660 *
1661 * fpu_exists is set (if DEBUG)
1662 * not in privileged mode
1663 * ftt is unfinished_FPop
1664 * NXM IEEE trap is not enabled
1665 * instruction at %tpc is "fitos"
1666 *
1667 * Usage:
1668 * %g1 per cpu address
1669 * %g2 %fsr
1670 * %g6 user instruction
1671 *
1672 * Note that we can take a memory access related trap while trying
1673 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1674 * flag to catch those traps and let the SFMMU code deal with page
1675 * fault and data access exception.
1676 */
1677 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1678 sethi %hi(fpu_exists), %g7
1679 ld [%g7 + %lo(fpu_exists)], %g7
1680 brz,pn %g7, .fp_exception_cont
1681 nop
1682 #endif
1683 rdpr %tstate, %g7 ! branch if in privileged mode
1684 btst TSTATE_PRIV, %g7
1685 bnz,pn %xcc, .fp_exception_cont
1686 srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr
1687 and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1688 cmp %g7, FTT_UNFIN
1689 set FSR_TEM_NX, %g5
1690 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop
1691 andcc %g2, %g5, %g0
1692 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled
1693 rdpr %tpc, %g5 ! get faulting PC
1694
1695 or %g0, 1, %g7
1696 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
1697 lda [%g5]ASI_USER, %g6 ! get user's instruction
1698 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
1699
1700 set FITOS_INSTR_MASK, %g7
1701 and %g6, %g7, %g7
1702 set FITOS_INSTR, %g5
1703 cmp %g7, %g5
1704 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR
1705 nop
1706
1707 /*
1708 * This is unfinished FPops trap for "fitos" instruction. We
1709 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1710 * sequence.
1711 *
1712 * We need a temporary FP register to do the conversion. Since
1713 * both source and destination operands for the "fitos" instruction
1714 * have to be within %f0-%f31, we use an FP register from the upper
1715 * half to guarantee that it won't collide with the source or the
1716 * dest operand. However, we do have to save and restore its value.
1717 *
1718 * We use %d62 as a temporary FP register for the conversion and
1719 * branch to appropriate instruction within the conversion tables
1720 * based upon the rs2 and rd values.
1721 */
1722
1723 std %d62, [%g1 + CPU_TMP1] ! save original value
1724
1725 srl %g6, FITOS_RS2_SHIFT, %g7
1726 and %g7, FITOS_REG_MASK, %g7
1727 set _fitos_fitod_table, %g4
1728 sllx %g7, 2, %g7
1729 jmp %g4 + %g7
1730 ba,pt %xcc, _fitos_fitod_done
1731 .empty
1732
1733 _fitos_fitod_table:
1734 fitod %f0, %d62
1735 fitod %f1, %d62
1736 fitod %f2, %d62
1737 fitod %f3, %d62
1738 fitod %f4, %d62
1739 fitod %f5, %d62
1740 fitod %f6, %d62
1741 fitod %f7, %d62
1742 fitod %f8, %d62
1743 fitod %f9, %d62
1744 fitod %f10, %d62
1745 fitod %f11, %d62
1746 fitod %f12, %d62
1747 fitod %f13, %d62
1748 fitod %f14, %d62
1749 fitod %f15, %d62
1750 fitod %f16, %d62
1751 fitod %f17, %d62
1752 fitod %f18, %d62
1753 fitod %f19, %d62
1754 fitod %f20, %d62
1755 fitod %f21, %d62
1756 fitod %f22, %d62
1757 fitod %f23, %d62
1758 fitod %f24, %d62
1759 fitod %f25, %d62
1760 fitod %f26, %d62
1761 fitod %f27, %d62
1762 fitod %f28, %d62
1763 fitod %f29, %d62
1764 fitod %f30, %d62
1765 fitod %f31, %d62
1766 _fitos_fitod_done:
1767
1768 /*
1769 * Now convert data back into single precision
1770 */
1771 srl %g6, FITOS_RD_SHIFT, %g7
1772 and %g7, FITOS_REG_MASK, %g7
1773 set _fitos_fdtos_table, %g4
1774 sllx %g7, 2, %g7
1775 jmp %g4 + %g7
1776 ba,pt %xcc, _fitos_fdtos_done
1777 .empty
1778
1779 _fitos_fdtos_table:
1780 fdtos %d62, %f0
1781 fdtos %d62, %f1
1782 fdtos %d62, %f2
1783 fdtos %d62, %f3
1784 fdtos %d62, %f4
1785 fdtos %d62, %f5
1786 fdtos %d62, %f6
1787 fdtos %d62, %f7
1788 fdtos %d62, %f8
1789 fdtos %d62, %f9
1790 fdtos %d62, %f10
1791 fdtos %d62, %f11
1792 fdtos %d62, %f12
1793 fdtos %d62, %f13
1794 fdtos %d62, %f14
1795 fdtos %d62, %f15
1796 fdtos %d62, %f16
1797 fdtos %d62, %f17
1798 fdtos %d62, %f18
1799 fdtos %d62, %f19
1800 fdtos %d62, %f20
1801 fdtos %d62, %f21
1802 fdtos %d62, %f22
1803 fdtos %d62, %f23
1804 fdtos %d62, %f24
1805 fdtos %d62, %f25
1806 fdtos %d62, %f26
1807 fdtos %d62, %f27
1808 fdtos %d62, %f28
1809 fdtos %d62, %f29
1810 fdtos %d62, %f30
1811 fdtos %d62, %f31
1812 _fitos_fdtos_done:
1813
1814 ldd [%g1 + CPU_TMP1], %d62 ! restore %d62
1815
1816 #if DEBUG
1817 /*
1818 * Update FPop_unfinished trap kstat
1819 */
1820 set fpustat+FPUSTAT_UNFIN_KSTAT, %g7
1821 ldx [%g7], %g5
1822 1:
1823 add %g5, 1, %g6
1824
1825 casxa [%g7] ASI_N, %g5, %g6
1826 cmp %g5, %g6
1827 bne,a,pn %xcc, 1b
1828 or %g0, %g6, %g5
1829
1830 /*
1831 * Update fpu_sim_fitos kstat
1832 */
1833 set fpuinfo+FPUINFO_FITOS_KSTAT, %g7
1834 ldx [%g7], %g5
1835 1:
1836 add %g5, 1, %g6
1837
1838 casxa [%g7] ASI_N, %g5, %g6
1839 cmp %g5, %g6
1840 bne,a,pn %xcc, 1b
1841 or %g0, %g6, %g5
1842 #endif /* DEBUG */
1843
1844 FAST_TRAP_DONE
1845
1846 .fp_exception_cont:
1847 /*
1848 * Let _fp_exception deal with simulating FPop instruction.
1849 * Note that we need to pass %fsr in %g2 (already read above).
1850 */
1851
1852 set _fp_exception, %g1
1853 ba,pt %xcc, sys_trap
1854 sub %g0, 1, %g4
1855
1856
1857 /*
1858 * Register windows
1859 */
1860 .flushw:
1861 .clean_windows:
1862 rdpr %tnpc, %g1
1863 wrpr %g1, %tpc
1864 add %g1, 4, %g1
1865 wrpr %g1, %tnpc
1866 set trap, %g1
1867 mov T_FLUSH_PCB, %g3
1868 ba,pt %xcc, sys_trap
1869 sub %g0, 1, %g4
1870
1871 /*
1872 * .spill_clean: clean the previous window, restore the wstate, and
1873 * "done".
1874 *
1875 * Entry: %g7 contains new wstate
1876 */
1877 .spill_clean:
1878 sethi %hi(nwin_minus_one), %g5
1879 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
1880 rdpr %cwp, %g6 ! %g6 = %cwp
1881 deccc %g6 ! %g6--
1882 movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1
1883 wrpr %g6, %cwp
1884 TT_TRACE_L(trace_win)
1885 clr %l0
1886 clr %l1
1887 clr %l2
1888 clr %l3
1889 clr %l4
1890 clr %l5
1891 clr %l6
1892 clr %l7
1893 wrpr %g0, %g7, %wstate
1894 saved
1895 retry ! restores correct %cwp
1896
1897 .fix_alignment:
1898 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
1899 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1900 ldn [%g1 + T_PROCP], %g1
1901 mov 1, %g2
1902 stb %g2, [%g1 + P_FIXALIGNMENT]
1903 FAST_TRAP_DONE
1904
1905 #define STDF_REG(REG, ADDR, TMP) \
1906 sll REG, 3, REG ;\
1907 mark1: set start1, TMP ;\
1908 jmp REG + TMP ;\
1909 nop ;\
1910 start1: ba,pt %xcc, done1 ;\
1911 std %f0, [ADDR + CPU_TMP1] ;\
1912 ba,pt %xcc, done1 ;\
1913 std %f32, [ADDR + CPU_TMP1] ;\
1914 ba,pt %xcc, done1 ;\
1915 std %f2, [ADDR + CPU_TMP1] ;\
1916 ba,pt %xcc, done1 ;\
1917 std %f34, [ADDR + CPU_TMP1] ;\
1918 ba,pt %xcc, done1 ;\
1919 std %f4, [ADDR + CPU_TMP1] ;\
1920 ba,pt %xcc, done1 ;\
1921 std %f36, [ADDR + CPU_TMP1] ;\
1922 ba,pt %xcc, done1 ;\
1923 std %f6, [ADDR + CPU_TMP1] ;\
1924 ba,pt %xcc, done1 ;\
1925 std %f38, [ADDR + CPU_TMP1] ;\
1926 ba,pt %xcc, done1 ;\
1927 std %f8, [ADDR + CPU_TMP1] ;\
1928 ba,pt %xcc, done1 ;\
1929 std %f40, [ADDR + CPU_TMP1] ;\
1930 ba,pt %xcc, done1 ;\
1931 std %f10, [ADDR + CPU_TMP1] ;\
1932 ba,pt %xcc, done1 ;\
1933 std %f42, [ADDR + CPU_TMP1] ;\
1934 ba,pt %xcc, done1 ;\
1935 std %f12, [ADDR + CPU_TMP1] ;\
1936 ba,pt %xcc, done1 ;\
1937 std %f44, [ADDR + CPU_TMP1] ;\
1938 ba,pt %xcc, done1 ;\
1939 std %f14, [ADDR + CPU_TMP1] ;\
1940 ba,pt %xcc, done1 ;\
1941 std %f46, [ADDR + CPU_TMP1] ;\
1942 ba,pt %xcc, done1 ;\
1943 std %f16, [ADDR + CPU_TMP1] ;\
1944 ba,pt %xcc, done1 ;\
1945 std %f48, [ADDR + CPU_TMP1] ;\
1946 ba,pt %xcc, done1 ;\
1947 std %f18, [ADDR + CPU_TMP1] ;\
1948 ba,pt %xcc, done1 ;\
1949 std %f50, [ADDR + CPU_TMP1] ;\
1950 ba,pt %xcc, done1 ;\
1951 std %f20, [ADDR + CPU_TMP1] ;\
1952 ba,pt %xcc, done1 ;\
1953 std %f52, [ADDR + CPU_TMP1] ;\
1954 ba,pt %xcc, done1 ;\
1955 std %f22, [ADDR + CPU_TMP1] ;\
1956 ba,pt %xcc, done1 ;\
1957 std %f54, [ADDR + CPU_TMP1] ;\
1958 ba,pt %xcc, done1 ;\
1959 std %f24, [ADDR + CPU_TMP1] ;\
1960 ba,pt %xcc, done1 ;\
1961 std %f56, [ADDR + CPU_TMP1] ;\
1962 ba,pt %xcc, done1 ;\
1963 std %f26, [ADDR + CPU_TMP1] ;\
1964 ba,pt %xcc, done1 ;\
1965 std %f58, [ADDR + CPU_TMP1] ;\
1966 ba,pt %xcc, done1 ;\
1967 std %f28, [ADDR + CPU_TMP1] ;\
1968 ba,pt %xcc, done1 ;\
1969 std %f60, [ADDR + CPU_TMP1] ;\
1970 ba,pt %xcc, done1 ;\
1971 std %f30, [ADDR + CPU_TMP1] ;\
1972 ba,pt %xcc, done1 ;\
1973 std %f62, [ADDR + CPU_TMP1] ;\
1974 done1:
1975
1976 #define LDDF_REG(REG, ADDR, TMP) \
1977 sll REG, 3, REG ;\
1978 mark2: set start2, TMP ;\
1979 jmp REG + TMP ;\
1980 nop ;\
1981 start2: ba,pt %xcc, done2 ;\
1982 ldd [ADDR + CPU_TMP1], %f0 ;\
1983 ba,pt %xcc, done2 ;\
1984 ldd [ADDR + CPU_TMP1], %f32 ;\
1985 ba,pt %xcc, done2 ;\
1986 ldd [ADDR + CPU_TMP1], %f2 ;\
1987 ba,pt %xcc, done2 ;\
1988 ldd [ADDR + CPU_TMP1], %f34 ;\
1989 ba,pt %xcc, done2 ;\
1990 ldd [ADDR + CPU_TMP1], %f4 ;\
1991 ba,pt %xcc, done2 ;\
1992 ldd [ADDR + CPU_TMP1], %f36 ;\
1993 ba,pt %xcc, done2 ;\
1994 ldd [ADDR + CPU_TMP1], %f6 ;\
1995 ba,pt %xcc, done2 ;\
1996 ldd [ADDR + CPU_TMP1], %f38 ;\
1997 ba,pt %xcc, done2 ;\
1998 ldd [ADDR + CPU_TMP1], %f8 ;\
1999 ba,pt %xcc, done2 ;\
2000 ldd [ADDR + CPU_TMP1], %f40 ;\
2001 ba,pt %xcc, done2 ;\
2002 ldd [ADDR + CPU_TMP1], %f10 ;\
2003 ba,pt %xcc, done2 ;\
2004 ldd [ADDR + CPU_TMP1], %f42 ;\
2005 ba,pt %xcc, done2 ;\
2006 ldd [ADDR + CPU_TMP1], %f12 ;\
2007 ba,pt %xcc, done2 ;\
2008 ldd [ADDR + CPU_TMP1], %f44 ;\
2009 ba,pt %xcc, done2 ;\
2010 ldd [ADDR + CPU_TMP1], %f14 ;\
2011 ba,pt %xcc, done2 ;\
2012 ldd [ADDR + CPU_TMP1], %f46 ;\
2013 ba,pt %xcc, done2 ;\
2014 ldd [ADDR + CPU_TMP1], %f16 ;\
2015 ba,pt %xcc, done2 ;\
2016 ldd [ADDR + CPU_TMP1], %f48 ;\
2017 ba,pt %xcc, done2 ;\
2018 ldd [ADDR + CPU_TMP1], %f18 ;\
2019 ba,pt %xcc, done2 ;\
2020 ldd [ADDR + CPU_TMP1], %f50 ;\
2021 ba,pt %xcc, done2 ;\
2022 ldd [ADDR + CPU_TMP1], %f20 ;\
2023 ba,pt %xcc, done2 ;\
2024 ldd [ADDR + CPU_TMP1], %f52 ;\
2025 ba,pt %xcc, done2 ;\
2026 ldd [ADDR + CPU_TMP1], %f22 ;\
2027 ba,pt %xcc, done2 ;\
2028 ldd [ADDR + CPU_TMP1], %f54 ;\
2029 ba,pt %xcc, done2 ;\
2030 ldd [ADDR + CPU_TMP1], %f24 ;\
2031 ba,pt %xcc, done2 ;\
2032 ldd [ADDR + CPU_TMP1], %f56 ;\
2033 ba,pt %xcc, done2 ;\
2034 ldd [ADDR + CPU_TMP1], %f26 ;\
2035 ba,pt %xcc, done2 ;\
2036 ldd [ADDR + CPU_TMP1], %f58 ;\
2037 ba,pt %xcc, done2 ;\
2038 ldd [ADDR + CPU_TMP1], %f28 ;\
2039 ba,pt %xcc, done2 ;\
2040 ldd [ADDR + CPU_TMP1], %f60 ;\
2041 ba,pt %xcc, done2 ;\
2042 ldd [ADDR + CPU_TMP1], %f30 ;\
2043 ba,pt %xcc, done2 ;\
2044 ldd [ADDR + CPU_TMP1], %f62 ;\
2045 done2:
2046
2047 .lddf_exception_not_aligned:
2048 /* %g2 = sfar, %g3 = sfsr */
2049 mov %g2, %g5 ! stash sfar
2050 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2051 sethi %hi(fpu_exists), %g2 ! check fpu_exists
2052 ld [%g2 + %lo(fpu_exists)], %g2
2053 brz,a,pn %g2, 4f
2054 nop
2055 #endif
2056 CPU_ADDR(%g1, %g4)
2057 or %g0, 1, %g4
2058 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2059
2060 rdpr %tpc, %g2
2061 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction
2062 srl %g6, 23, %g1 ! using ldda or not?
2063 and %g1, 1, %g1
2064 brz,a,pt %g1, 2f ! check for ldda instruction
2065 nop
2066 srl %g6, 13, %g1 ! check immflag
2067 and %g1, 1, %g1
2068 rdpr %tstate, %g2 ! %tstate in %g2
2069 brnz,a,pn %g1, 1f
2070 srl %g2, 31, %g1 ! get asi from %tstate
2071 srl %g6, 5, %g1 ! get asi from instruction
2072 and %g1, 0xFF, %g1 ! imm_asi field
2073 1:
2074 cmp %g1, ASI_P ! primary address space
2075 be,a,pt %icc, 2f
2076 nop
2077 cmp %g1, ASI_PNF ! primary no fault address space
2078 be,a,pt %icc, 2f
2079 nop
2080 cmp %g1, ASI_S ! secondary address space
2081 be,a,pt %icc, 2f
2082 nop
2083 cmp %g1, ASI_SNF ! secondary no fault address space
2084 bne,a,pn %icc, 3f
2085 nop
2086 2:
2087 lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data
2088 add %g5, 4, %g5 ! increment misaligned data address
2089 lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data
2090
2091 sllx %g7, 32, %g7
2092 or %g5, %g7, %g5 ! combine data
2093 CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis
2094 stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1
2095
2096 srl %g6, 25, %g3 ! %g6 has the instruction
2097 and %g3, 0x1F, %g3 ! %g3 has rd
2098 LDDF_REG(%g3, %g7, %g4)
2099
2100 CPU_ADDR(%g1, %g4)
2101 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2102 FAST_TRAP_DONE
2103 3:
2104 CPU_ADDR(%g1, %g4)
2105 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2106 4:
2107 set T_USER, %g3 ! trap type in %g3
2108 or %g3, T_LDDF_ALIGN, %g3
2109 mov %g5, %g2 ! misaligned vaddr in %g2
2110 set fpu_trap, %g1 ! goto C for the little and
2111 ba,pt %xcc, sys_trap ! no fault little asi's
2112 sub %g0, 1, %g4
2113
2114 .stdf_exception_not_aligned:
2115 /* %g2 = sfar, %g3 = sfsr */
2116 mov %g2, %g5
2117
2118 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2119 sethi %hi(fpu_exists), %g7 ! check fpu_exists
2120 ld [%g7 + %lo(fpu_exists)], %g3
2121 brz,a,pn %g3, 4f
2122 nop
2123 #endif
2124 CPU_ADDR(%g1, %g4)
2125 or %g0, 1, %g4
2126 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2127
2128 rdpr %tpc, %g2
2129 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction
2130
2131 srl %g6, 23, %g1 ! using stda or not?
2132 and %g1, 1, %g1
2133 brz,a,pt %g1, 2f ! check for stda instruction
2134 nop
2135 srl %g6, 13, %g1 ! check immflag
2136 and %g1, 1, %g1
2137 rdpr %tstate, %g2 ! %tstate in %g2
2138 brnz,a,pn %g1, 1f
2139 srl %g2, 31, %g1 ! get asi from %tstate
2140 srl %g6, 5, %g1 ! get asi from instruction
2141 and %g1, 0xff, %g1 ! imm_asi field
2142 1:
2143 cmp %g1, ASI_P ! primary address space
2144 be,a,pt %icc, 2f
2145 nop
2146 cmp %g1, ASI_S ! secondary address space
2147 bne,a,pn %icc, 3f
2148 nop
2149 2:
2150 srl %g6, 25, %g6
2151 and %g6, 0x1F, %g6 ! %g6 has rd
2152 CPU_ADDR(%g7, %g1)
2153 STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP)
2154
2155 ldx [%g7 + CPU_TMP1], %g6
2156 srlx %g6, 32, %g7
2157 stuwa %g7, [%g5]ASI_USER ! first half
2158 add %g5, 4, %g5 ! increment misaligned data address
2159 stuwa %g6, [%g5]ASI_USER ! second half
2160
2161 CPU_ADDR(%g1, %g4)
2162 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2163 FAST_TRAP_DONE
2164 3:
2165 CPU_ADDR(%g1, %g4)
2166 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2167 4:
2168 set T_USER, %g3 ! trap type in %g3
2169 or %g3, T_STDF_ALIGN, %g3
2170 mov %g5, %g2 ! misaligned vaddr in %g2
2171 set fpu_trap, %g1 ! goto C for the little and
2172 ba,pt %xcc, sys_trap ! nofault little asi's
2173 sub %g0, 1, %g4
2174
2175 #ifdef DEBUG_USER_TRAPTRACECTL
2176
2177 .traptrace_freeze:
2178 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2179 TT_TRACE_L(trace_win)
2180 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2181 set trap_freeze, %g1
2182 mov 1, %g2
2183 st %g2, [%g1]
2184 FAST_TRAP_DONE
2185
2186 .traptrace_unfreeze:
2187 set trap_freeze, %g1
2188 st %g0, [%g1]
2189 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2190 TT_TRACE_L(trace_win)
2191 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2192 FAST_TRAP_DONE
2193
2194 #endif /* DEBUG_USER_TRAPTRACECTL */
2195
2196 .getcc:
2197 CPU_ADDR(%g1, %g2)
2198 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2199 rdpr %tstate, %g3 ! get tstate
2200 srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2201 set PSR_ICC, %g2
2202 and %o0, %g2, %o0 ! mask out the rest
2203 srl %o0, PSR_ICC_SHIFT, %o0 ! right justify
2204 wrpr %g0, 0, %gl
2205 mov %o0, %g1 ! move ccr to normal %g1
2206 wrpr %g0, 1, %gl
2207 ! cannot assume globals retained their values after increasing %gl
2208 CPU_ADDR(%g1, %g2)
2209 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2210 FAST_TRAP_DONE
2211
2212 .setcc:
2213 CPU_ADDR(%g1, %g2)
2214 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2215 wrpr %g0, 0, %gl
2216 mov %g1, %o0
2217 wrpr %g0, 1, %gl
2218 ! cannot assume globals retained their values after increasing %gl
2219 CPU_ADDR(%g1, %g2)
2220 sll %o0, PSR_ICC_SHIFT, %g2
2221 set PSR_ICC, %g3
2222 and %g2, %g3, %g2 ! mask out rest
2223 sllx %g2, PSR_TSTATE_CC_SHIFT, %g2
2224 rdpr %tstate, %g3 ! get tstate
2225 srl %g3, 0, %g3 ! clear upper word
2226 or %g3, %g2, %g3 ! or in new bits
2227 wrpr %g3, %tstate
2228 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2229 FAST_TRAP_DONE
2230
2231 /*
2232 * getpsr(void)
2233 * Note that the xcc part of the ccr is not provided.
2234 * The V8 code shows why the V9 trap is not faster:
2235 * #define GETPSR_TRAP() \
2236 * mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2237 */
2238
2239 .type .getpsr, #function
2240 .getpsr:
2241 rdpr %tstate, %g1 ! get tstate
2242 srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2243 set PSR_ICC, %g2
2244 and %o0, %g2, %o0 ! mask out the rest
2245
2246 rd %fprs, %g1 ! get fprs
2247 and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower
2248 sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef
2249 or %o0, %g2, %o0 ! or result into psr.ef
2250
2251 set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef
2252 or %o0, %g2, %o0 ! or psr.impl/ver
2253 FAST_TRAP_DONE
2254 SET_SIZE(.getpsr)
2255
2256 /*
2257 * setpsr(newpsr)
2258 * Note that there is no support for ccr.xcc in the V9 code.
2259 */
2260
2261 .type .setpsr, #function
2262 .setpsr:
2263 rdpr %tstate, %g1 ! get tstate
2264 ! setx TSTATE_V8_UBITS, %g2
2265 or %g0, CCR_ICC, %g3
2266 sllx %g3, TSTATE_CCR_SHIFT, %g2
2267
2268 andn %g1, %g2, %g1 ! zero current user bits
2269 set PSR_ICC, %g2
2270 and %g2, %o0, %g2 ! clear all but psr.icc bits
2271 sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc
2272 wrpr %g1, %g3, %tstate ! write tstate
2273
2274 set PSR_EF, %g2
2275 and %g2, %o0, %g2 ! clear all but fp enable bit
2276 srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef
2277 wr %g0, %g4, %fprs ! write fprs
2278
2279 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1
2280 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2281 ldn [%g2 + T_LWP], %g3 ! load klwp pointer
2282 ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer
2283 stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs
2284 srlx %g4, 2, %g4 ! shift fef value to bit 0
2285 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en
2286 FAST_TRAP_DONE
2287 SET_SIZE(.setpsr)
2288
2289 /*
2290 * getlgrp
2291 * get home lgrpid on which the calling thread is currently executing.
2292 */
2293 .type .getlgrp, #function
2294 .getlgrp:
2295 ! Thanks for the incredibly helpful comments
2296 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2297 ld [%g1 + CPU_ID], %o0 ! load cpu_id
2298 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2299 ldn [%g2 + T_LPL], %g2 ! load lpl pointer
2300 ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
2301 sra %g1, 0, %o1
2302 FAST_TRAP_DONE
2303 SET_SIZE(.getlgrp)
2304
2305 /*
2306 * Entry for old 4.x trap (trap 0).
2307 */
2308 ENTRY_NP(syscall_trap_4x)
2309 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2310 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2311 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2312 ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr
2313 brz,pn %g2, 1f ! has it been set?
2314 st %l0, [%g1 + CPU_TMP1] ! delay - save some locals
2315 st %l1, [%g1 + CPU_TMP2]
2316 rdpr %tnpc, %l1 ! save old tnpc
2317 wrpr %g0, %g2, %tnpc ! setup tnpc
2318
2319 mov %g1, %l0 ! save CPU struct addr
2320 wrpr %g0, 0, %gl
2321 mov %l1, %g6 ! pass tnpc to user code in %g6
2322 wrpr %g0, 1, %gl
2323 ld [%l0 + CPU_TMP2], %l1 ! restore locals
2324 ld [%l0 + CPU_TMP1], %l0
2325 FAST_TRAP_DONE_CHK_INTR
2326 1:
2327 !
2328 ! check for old syscall mmap which is the only different one which
2329 ! must be the same. Others are handled in the compatibility library.
2330 !
2331 mov %g1, %l0 ! save CPU struct addr
2332 wrpr %g0, 0, %gl
2333 cmp %g1, OSYS_mmap ! compare to old 4.x mmap
2334 movz %icc, SYS_mmap, %g1
2335 wrpr %g0, 1, %gl
2336 ld [%l0 + CPU_TMP1], %l0
2337 SYSCALL(syscall_trap32)
2338 SET_SIZE(syscall_trap_4x)
2339
2340 /*
2341 * Handler for software trap 9.
2342 * Set trap0 emulation address for old 4.x system call trap.
2343 * XXX - this should be a system call.
2344 */
2345 ENTRY_NP(set_trap0_addr)
2346 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2347 st %l0, [%g1 + CPU_TMP1] ! save some locals
2348 st %l1, [%g1 + CPU_TMP2]
2349 mov %g1, %l0 ! preserve CPU addr
2350 wrpr %g0, 0, %gl
2351 mov %g1, %l1
2352 wrpr %g0, 1, %gl
2353 ! cannot assume globals retained their values after increasing %gl
2354 ldn [%l0 + CPU_THREAD], %g2 ! load thread pointer
2355 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2356 andn %l1, 3, %l1 ! force alignment
2357 st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr
2358 ld [%l0 + CPU_TMP2], %l1 ! restore locals
2359 ld [%l0 + CPU_TMP1], %l0
2360 FAST_TRAP_DONE
2361 SET_SIZE(set_trap0_addr)
2362
2363 /*
2364 * mmu_trap_tl1
2365 * trap handler for unexpected mmu traps.
2366 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2367 * case we go to fpu_trap or a user trap from the window handler, in which
2368 * case we go save the state on the pcb. Otherwise, we go to ptl1_panic.
2369 */
2370 .type mmu_trap_tl1, #function
2371 mmu_trap_tl1:
2372 #ifdef TRAPTRACE
2373 TRACE_PTR(%g5, %g6)
2374 GET_TRACE_TICK(%g6, %g7)
2375 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2376 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2377 rdpr %tt, %g6
2378 stha %g6, [%g5 + TRAP_ENT_TT]%asi
2379 rdpr %tstate, %g6
2380 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi
2381 stna %sp, [%g5 + TRAP_ENT_SP]%asi
2382 stna %g0, [%g5 + TRAP_ENT_TR]%asi
2383 rdpr %tpc, %g6
2384 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2385 MMU_FAULT_STATUS_AREA(%g6)
2386 ldx [%g6 + MMFSA_D_ADDR], %g6
2387 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! MMU fault address
2388 CPU_PADDR(%g7, %g6);
2389 add %g7, CPU_TL1_HDLR, %g7
2390 lda [%g7]ASI_MEM, %g6
2391 stna %g6, [%g5 + TRAP_ENT_F2]%asi
2392 MMU_FAULT_STATUS_AREA(%g6)
2393 ldx [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant?
2394 ldx [%g6 + MMFSA_D_CTX], %g6
2395 sllx %g6, SFSR_CTX_SHIFT, %g6
2396 or %g6, %g7, %g6
2397 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type
2398 set 0xdeadbeef, %g6
2399 stna %g6, [%g5 + TRAP_ENT_F4]%asi
2400 TRACE_NEXT(%g5, %g6, %g7)
2401 #endif /* TRAPTRACE */
2402 CPU_PADDR(%g7, %g6);
2403 add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA)
2404 lda [%g7]ASI_MEM, %g6
2405 brz,a,pt %g6, 1f
2406 nop
2407 sta %g0, [%g7]ASI_MEM
2408 ! XXXQ need to setup registers for sfmmu_mmu_trap?
2409 ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults
2410 1:
2411 rdpr %tpc, %g7
2412 /* in user_rtt? */
2413 set rtt_fill_start, %g6
2414 cmp %g7, %g6
2415 blu,pn %xcc, 6f
2416 .empty
2417 set rtt_fill_end, %g6
2418 cmp %g7, %g6
2419 bgeu,pn %xcc, 6f
2420 nop
2421 set fault_rtt_fn1, %g7
2422 ba,a 7f
2423 6:
2424 ! check to see if the trap pc is in a window spill/fill handling
2425 rdpr %tpc, %g7
2426 /* tpc should be in the trap table */
2427 set trap_table, %g6
2428 cmp %g7, %g6
2429 blu,a,pn %xcc, ptl1_panic
2430 mov PTL1_BAD_MMUTRAP, %g1
2431 set etrap_table, %g6
2432 cmp %g7, %g6
2433 bgeu,a,pn %xcc, ptl1_panic
2434 mov PTL1_BAD_MMUTRAP, %g1
2435 ! pc is inside the trap table, convert to trap type
2436 srl %g7, 5, %g6 ! XXXQ need #define
2437 and %g6, 0x1ff, %g6 ! XXXQ need #define
2438 ! and check for a window trap type
2439 and %g6, WTRAP_TTMASK, %g6
2440 cmp %g6, WTRAP_TYPE
2441 bne,a,pn %xcc, ptl1_panic
2442 mov PTL1_BAD_MMUTRAP, %g1
2443 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */
2444 add %g7, WTRAP_FAULTOFF, %g7
2445
2446 7:
2447 ! Arguments are passed in the global set active after the
2448 ! 'done' instruction. Before switching sets, must save
2449 ! the calculated next pc
2450 wrpr %g0, %g7, %tnpc
2451 wrpr %g0, 1, %gl
2452 rdpr %tt, %g5
2453 MMU_FAULT_STATUS_AREA(%g7)
2454 cmp %g5, T_ALIGNMENT
2455 be,pn %xcc, 1f
2456 ldx [%g7 + MMFSA_D_ADDR], %g6
2457 ldx [%g7 + MMFSA_D_CTX], %g7
2458 srlx %g6, MMU_PAGESHIFT, %g6 /* align address */
2459 cmp %g7, USER_CONTEXT_TYPE
2460 sllx %g6, MMU_PAGESHIFT, %g6
2461 movgu %icc, USER_CONTEXT_TYPE, %g7
2462 or %g6, %g7, %g6 /* TAG_ACCESS */
2463 1:
2464 done
2465 SET_SIZE(mmu_trap_tl1)
2466
2467 /*
2468 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These
2469 * traps are valid only when kmdb is loaded. When the debugger is active,
2470 * the code below is rewritten to transfer control to the appropriate
2471 * debugger entry points.
2472 */
2473 .global kmdb_trap
2474 .align 8
2475 kmdb_trap:
2476 ba,a trap_table0
2477 jmp %g1 + 0
2478 nop
2479
2480 .global kmdb_trap_tl1
2481 .align 8
2482 kmdb_trap_tl1:
2483 ba,a trap_table0
2484 jmp %g1 + 0
2485 nop
2486
2487 /*
2488 * This entry is copied from OBP's trap table during boot.
2489 */
2490 .global obp_bpt
2491 .align 8
2492 obp_bpt:
2493 NOT
2494
2495
2496
2497 #ifdef TRAPTRACE
2498 /*
2499 * TRAPTRACE support.
2500 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2501 * Return is done by "jmp %g7 + 4".
2502 */
2503
2504 trace_dmmu:
2505 TRACE_PTR(%g3, %g6)
2506 GET_TRACE_TICK(%g6, %g5)
2507 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2508 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2509 rdpr %tt, %g6
2510 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2511 rdpr %tstate, %g6
2512 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2513 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2514 rdpr %tpc, %g6
2515 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2516 MMU_FAULT_STATUS_AREA(%g6)
2517 ldx [%g6 + MMFSA_D_ADDR], %g4
2518 stxa %g4, [%g3 + TRAP_ENT_TR]%asi
2519 ldx [%g6 + MMFSA_D_CTX], %g4
2520 stxa %g4, [%g3 + TRAP_ENT_F1]%asi
2521 ldx [%g6 + MMFSA_D_TYPE], %g4
2522 stxa %g4, [%g3 + TRAP_ENT_F2]%asi
2523 stxa %g6, [%g3 + TRAP_ENT_F3]%asi
2524 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2525 TRACE_NEXT(%g3, %g4, %g5)
2526 jmp %g7 + 4
2527 nop
2528
2529 trace_immu:
2530 TRACE_PTR(%g3, %g6)
2531 GET_TRACE_TICK(%g6, %g5)
2532 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2533 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2534 rdpr %tt, %g6
2535 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2536 rdpr %tstate, %g6
2537 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2538 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2539 rdpr %tpc, %g6
2540 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2541 MMU_FAULT_STATUS_AREA(%g6)
2542 ldx [%g6 + MMFSA_I_ADDR], %g4
2543 stxa %g4, [%g3 + TRAP_ENT_TR]%asi
2544 ldx [%g6 + MMFSA_I_CTX], %g4
2545 stxa %g4, [%g3 + TRAP_ENT_F1]%asi
2546 ldx [%g6 + MMFSA_I_TYPE], %g4
2547 stxa %g4, [%g3 + TRAP_ENT_F2]%asi
2548 stxa %g6, [%g3 + TRAP_ENT_F3]%asi
2549 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2550 TRACE_NEXT(%g3, %g4, %g5)
2551 jmp %g7 + 4
2552 nop
2553
2554 trace_gen:
2555 TRACE_PTR(%g3, %g6)
2556 GET_TRACE_TICK(%g6, %g5)
2557 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2558 TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2559 rdpr %tt, %g6
2560 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2561 rdpr %tstate, %g6
2562 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2563 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2564 rdpr %tpc, %g6
2565 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2566 stna %g0, [%g3 + TRAP_ENT_TR]%asi
2567 stna %g0, [%g3 + TRAP_ENT_F1]%asi
2568 stna %g0, [%g3 + TRAP_ENT_F2]%asi
2569 stna %g0, [%g3 + TRAP_ENT_F3]%asi
2570 stna %g0, [%g3 + TRAP_ENT_F4]%asi
2571 TRACE_NEXT(%g3, %g4, %g5)
2572 jmp %g7 + 4
2573 nop
2574
2575 trace_win:
2576 TRACE_WIN_INFO(0, %l0, %l1, %l2)
2577 ! Keep the locals as clean as possible, caller cleans %l4
2578 clr %l2
2579 clr %l1
2580 jmp %l4 + 4
2581 clr %l0
2582
2583 /*
2584 * Trace a tsb hit
2585 * g1 = tsbe pointer (in/clobbered)
2586 * g2 = tag access register (in)
2587 * g3 - g4 = scratch (clobbered)
2588 * g5 = tsbe data (in)
2589 * g6 = scratch (clobbered)
2590 * g7 = pc we jumped here from (in)
2591 */
2592
2593 ! Do not disturb %g5, it will be used after the trace
2594 ALTENTRY(trace_tsbhit)
2595 TRACE_TSBHIT(0)
2596 jmp %g7 + 4
2597 nop
2598
2599 /*
2600 * Trace a TSB miss
2601 *
2602 * g1 = tsb8k pointer (in)
2603 * g2 = tag access register (in)
2604 * g3 = tsb4m pointer (in)
2605 * g4 = tsbe tag (in/clobbered)
2606 * g5 - g6 = scratch (clobbered)
2607 * g7 = pc we jumped here from (in)
2608 */
2609 .global trace_tsbmiss
2610 trace_tsbmiss:
2611 membar #Sync
2612 sethi %hi(FLUSH_ADDR), %g6
2613 flush %g6
2614 TRACE_PTR(%g5, %g6)
2615 stna %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access
2616 stna %g4, [%g5 + TRAP_ENT_F1]%asi ! XXX? tsb tag
2617 GET_TRACE_TICK(%g6, %g4)
2618 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2619 rdpr %tnpc, %g6
2620 stna %g6, [%g5 + TRAP_ENT_F2]%asi
2621 stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer
2622 rdpr %tpc, %g6
2623 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2624 TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2625 rdpr %tt, %g6
2626 or %g6, TT_MMU_MISS, %g4
2627 stha %g4, [%g5 + TRAP_ENT_TT]%asi
2628 mov MMFSA_D_ADDR, %g4
2629 cmp %g6, FAST_IMMU_MISS_TT
2630 move %xcc, MMFSA_I_ADDR, %g4
2631 cmp %g6, T_INSTR_MMU_MISS
2632 move %xcc, MMFSA_I_ADDR, %g4
2633 MMU_FAULT_STATUS_AREA(%g6)
2634 ldx [%g6 + %g4], %g6
2635 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target
2636 cmp %g4, MMFSA_D_ADDR
2637 move %xcc, MMFSA_D_CTX, %g4
2638 movne %xcc, MMFSA_I_CTX, %g4
2639 MMU_FAULT_STATUS_AREA(%g6)
2640 ldx [%g6 + %g4], %g6
2641 stxa %g6, [%g5 + TRAP_ENT_F4]%asi ! context ID
2642 stna %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer
2643 TRACE_NEXT(%g5, %g4, %g6)
2644 jmp %g7 + 4
2645 nop
2646
2647 /*
2648 * g2 = tag access register (in)
2649 * g3 = ctx type (0, 1 or 2) (in) (not used)
2650 */
2651 trace_dataprot:
2652 membar #Sync
2653 sethi %hi(FLUSH_ADDR), %g6
2654 flush %g6
2655 TRACE_PTR(%g1, %g6)
2656 GET_TRACE_TICK(%g6, %g4)
2657 stxa %g6, [%g1 + TRAP_ENT_TICK]%asi
2658 rdpr %tpc, %g6
2659 stna %g6, [%g1 + TRAP_ENT_TPC]%asi
2660 rdpr %tstate, %g6
2661 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi
2662 stna %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg
2663 stna %g0, [%g1 + TRAP_ENT_F1]%asi
2664 stna %g0, [%g1 + TRAP_ENT_F2]%asi
2665 stna %g0, [%g1 + TRAP_ENT_F3]%asi
2666 stna %g0, [%g1 + TRAP_ENT_F4]%asi
2667 TRACE_SAVE_TL_GL_REGS(%g1, %g6)
2668 rdpr %tt, %g6
2669 stha %g6, [%g1 + TRAP_ENT_TT]%asi
2670 mov MMFSA_D_CTX, %g4
2671 cmp %g6, FAST_IMMU_MISS_TT
2672 move %xcc, MMFSA_I_CTX, %g4
2673 cmp %g6, T_INSTR_MMU_MISS
2674 move %xcc, MMFSA_I_CTX, %g4
2675 MMU_FAULT_STATUS_AREA(%g6)
2676 ldx [%g6 + %g4], %g6
2677 stxa %g6, [%g1 + TRAP_ENT_TR]%asi ! context ID
2678 TRACE_NEXT(%g1, %g4, %g5)
2679 jmp %g7 + 4
2680 nop
2681
2682 #endif /* TRAPTRACE */
2683
2684 /*
2685 * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE
2686 * HV service, which requires the return target to be specified as a VA
2687 * since we are enabling the MMU. We set the target to ptl1_panic.
2688 */
2689
2690 .type .watchdog_trap, #function
2691 .watchdog_trap:
2692 mov 1, %o0
2693 setx ptl1_panic, %g2, %o1
2694 mov MMU_ENABLE, %o5
2695 ta FAST_TRAP
2696 done
2697 SET_SIZE(.watchdog_trap)
2698 /*
2699 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2700 */
2701 .type .dmmu_exc_lddf_not_aligned, #function
2702 .dmmu_exc_lddf_not_aligned:
2703 MMU_FAULT_STATUS_AREA(%g3)
2704 ldx [%g3 + MMFSA_D_ADDR], %g2
2705 /* Fault type not available in MMU fault status area */
2706 mov MMFSA_F_UNALIGN, %g1
2707 ldx [%g3 + MMFSA_D_CTX], %g3
2708 sllx %g3, SFSR_CTX_SHIFT, %g3
2709 btst 1, %sp
2710 bnz,pt %xcc, .lddf_exception_not_aligned
2711 or %g3, %g1, %g3 /* SFSR */
2712 ba,a,pt %xcc, .mmu_exception_not_aligned
2713 SET_SIZE(.dmmu_exc_lddf_not_aligned)
2714
2715 /*
2716 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2717 */
2718 .type .dmmu_exc_stdf_not_aligned, #function
2719 .dmmu_exc_stdf_not_aligned:
2720 MMU_FAULT_STATUS_AREA(%g3)
2721 ldx [%g3 + MMFSA_D_ADDR], %g2
2722 /* Fault type not available in MMU fault status area */
2723 mov MMFSA_F_UNALIGN, %g1
2724 ldx [%g3 + MMFSA_D_CTX], %g3
2725 sllx %g3, SFSR_CTX_SHIFT, %g3
2726 btst 1, %sp
2727 bnz,pt %xcc, .stdf_exception_not_aligned
2728 or %g3, %g1, %g3 /* SFSR */
2729 ba,a,pt %xcc, .mmu_exception_not_aligned
2730 SET_SIZE(.dmmu_exc_stdf_not_aligned)
2731
2732 .type .dmmu_exception, #function
2733 .dmmu_exception:
2734 MMU_FAULT_STATUS_AREA(%g3)
2735 ldx [%g3 + MMFSA_D_ADDR], %g2
2736 ldx [%g3 + MMFSA_D_TYPE], %g1
2737 ldx [%g3 + MMFSA_D_CTX], %g4
2738 srlx %g2, MMU_PAGESHIFT, %g2 /* align address */
2739 sllx %g2, MMU_PAGESHIFT, %g2
2740 sllx %g4, SFSR_CTX_SHIFT, %g3
2741 or %g3, %g1, %g3 /* SFSR */
2742 cmp %g4, USER_CONTEXT_TYPE
2743 movgeu %icc, USER_CONTEXT_TYPE, %g4
2744 or %g2, %g4, %g2 /* TAG_ACCESS */
2745 ba,pt %xcc, .mmu_exception_end
2746 mov T_DATA_EXCEPTION, %g1
2747 SET_SIZE(.dmmu_exception)
2748
2749 .align 32
2750 .global pil15_epilogue
2751 pil15_epilogue:
2752 ba pil_interrupt_common
2753 nop
2754 .align 32
2755
2756 /*
2757 * fast_trap_done, fast_trap_done_chk_intr:
2758 *
2759 * Due to the design of UltraSPARC pipeline, pending interrupts are not
2760 * taken immediately after a RETRY or DONE instruction which causes IE to
2761 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2762 * to execute first before taking any interrupts. If that instruction
2763 * results in other traps, and if the corresponding trap handler runs
2764 * entirely at TL=1 with interrupts disabled, then pending interrupts
2765 * won't be taken until after yet another instruction following the %tpc
2766 * or %tnpc.
2767 *
2768 * A malicious user program can use this feature to block out interrupts
2769 * for extended durations, which can result in send_mondo_timeout kernel
2770 * panic.
2771 *
2772 * This problem is addressed by servicing any pending interrupts via
2773 * sys_trap before returning back to the user mode from a fast trap
2774 * handler. The "done" instruction within a fast trap handler, which
2775 * runs entirely at TL=1 with interrupts disabled, is replaced with the
2776 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2777 * entry point.
2778 *
2779 * We check for any pending interrupts here and force a sys_trap to
2780 * service those interrupts, if any. To minimize overhead, pending
2781 * interrupts are checked if the %tpc happens to be at 16K boundary,
2782 * which allows a malicious program to execute at most 4K consecutive
2783 * instructions before we service any pending interrupts. If a worst
2784 * case fast trap handler takes about 2 usec, then interrupts will be
2785 * blocked for at most 8 msec, less than a clock tick.
2786 *
2787 * For the cases where we don't know if the %tpc will cross a 16K
2788 * boundary, we can't use the above optimization and always process
2789 * any pending interrupts via fast_frap_done_chk_intr entry point.
2790 *
2791 * Entry Conditions:
2792 * %pstate am:0 priv:1 ie:0
2793 * globals are AG (not normal globals)
2794 */
2795
2796 .global fast_trap_done, fast_trap_done_chk_intr
2797 fast_trap_done:
2798 rdpr %tpc, %g5
2799 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff
2800 andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc
2801 bz,pn %icc, 1f ! branch if zero (lower 32 bits only)
2802 nop
2803 done
2804
2805 fast_trap_done_chk_intr:
2806 1: rd SOFTINT, %g6
2807 brnz,pn %g6, 2f ! branch if any pending intr
2808 nop
2809 done
2810
2811 2:
2812 /*
2813 * We get here if there are any pending interrupts.
2814 * Adjust %tpc/%tnpc as we'll be resuming via "retry"
2815 * instruction.
2816 */
2817 rdpr %tnpc, %g5
2818 wrpr %g0, %g5, %tpc
2819 add %g5, 4, %g5
2820 wrpr %g0, %g5, %tnpc
2821
2822 /*
2823 * Force a dummy sys_trap call so that interrupts can be serviced.
2824 */
2825 set fast_trap_dummy_call, %g1
2826 ba,pt %xcc, sys_trap
2827 mov -1, %g4
2828
2829 fast_trap_dummy_call:
2830 retl
2831 nop
2832
2833 /*
2834 * Currently the brand syscall interposition code is not enabled by
2835 * default. Instead, when a branded zone is first booted the brand
2836 * infrastructure will patch the trap table so that the syscall
2837 * entry points are redirected to syscall_wrapper32 and syscall_wrapper
2838 * for ILP32 and LP64 syscalls respectively. this is done in
2839 * brand_plat_interposition_enable(). Note that the syscall wrappers
2840 * below do not collect any trap trace data since the syscall hot patch
2841 * points are reached after trap trace data has already been collected.
2842 */
2843 #define BRAND_CALLBACK(callback_id) \
2844 CPU_ADDR(%g2, %g1) /* load CPU struct addr to %g2 */ ;\
2845 ldn [%g2 + CPU_THREAD], %g3 /* load thread pointer */ ;\
2846 ldn [%g3 + T_PROCP], %g3 /* get proc pointer */ ;\
2847 ldn [%g3 + P_BRAND], %g3 /* get brand pointer */ ;\
2848 brz %g3, 1f /* No brand? No callback. */ ;\
2849 nop ;\
2850 ldn [%g3 + B_MACHOPS], %g3 /* get machops list */ ;\
2851 ldn [%g3 + (callback_id << 3)], %g3 ;\
2852 brz %g3, 1f ;\
2853 /* \
2854 * This isn't pretty. We want a low-latency way for the callback \
2855 * routine to decline to do anything. We just pass in an address \
2856 * the routine can directly jmp back to, pretending that nothing \
2857 * has happened. \
2858 * \
2859 * %g1: return address (where the brand handler jumps back to) \
2860 * %g2: address of CPU structure \
2861 * %g3: address of brand handler (where we will jump to) \
2862 */ \
2863 mov %pc, %g1 ;\
2864 add %g1, 16, %g1 ;\
2865 jmp %g3 ;\
2866 nop ;\
2867 1:
2868
2869 ENTRY_NP(syscall_wrapper32)
2870 BRAND_CALLBACK(BRAND_CB_SYSCALL32)
2871 SYSCALL_NOTT(syscall_trap32)
2872 SET_SIZE(syscall_wrapper32)
2873
2874 ENTRY_NP(syscall_wrapper)
2875 BRAND_CALLBACK(BRAND_CB_SYSCALL)
2876 SYSCALL_NOTT(syscall_trap)
2877 SET_SIZE(syscall_wrapper)
2878