1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #if !defined(lint)
27 #include "assym.h"
28 #endif /* !lint */
29 #include <sys/asm_linkage.h>
30 #include <sys/privregs.h>
31 #include <sys/sun4asi.h>
32 #include <sys/spitregs.h>
33 #include <sys/cheetahregs.h>
34 #include <sys/machtrap.h>
35 #include <sys/machthread.h>
36 #include <sys/machbrand.h>
37 #include <sys/pcb.h>
38 #include <sys/pte.h>
39 #include <sys/mmu.h>
40 #include <sys/machpcb.h>
41 #include <sys/async.h>
42 #include <sys/intreg.h>
43 #include <sys/scb.h>
44 #include <sys/psr_compat.h>
45 #include <sys/syscall.h>
46 #include <sys/machparam.h>
47 #include <sys/traptrace.h>
48 #include <vm/hat_sfmmu.h>
49 #include <sys/archsystm.h>
50 #include <sys/utrap.h>
51 #include <sys/clock.h>
52 #include <sys/intr.h>
53 #include <sys/fpu/fpu_simulator.h>
54 #include <vm/seg_spt.h>
55
56 /*
57 * WARNING: If you add a fast trap handler which can be invoked by a
58 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
59 * instead of "done" instruction to return back to the user mode. See
60 * comments for the "fast_trap_done" entry point for more information.
61 *
62 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
63 * cases where you always want to process any pending interrupts before
64 * returning back to the user mode.
65 */
66 #define FAST_TRAP_DONE \
67 ba,a fast_trap_done
68
69 #define FAST_TRAP_DONE_CHK_INTR \
70 ba,a fast_trap_done_chk_intr
71
72 /*
73 * SPARC V9 Trap Table
74 *
75 * Most of the trap handlers are made from common building
76 * blocks, and some are instantiated multiple times within
77 * the trap table. So, I build a bunch of macros, then
78 * populate the table using only the macros.
79 *
80 * Many macros branch to sys_trap. Its calling convention is:
81 * %g1 kernel trap handler
82 * %g2, %g3 args for above
83 * %g4 desire %pil
84 */
85
86 #ifdef TRAPTRACE
87
88 /*
89 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
90 */
91 #define TT_TRACE(label) \
92 ba label ;\
93 rd %pc, %g7
94 #define TT_TRACE_INS 2
95
96 #define TT_TRACE_L(label) \
97 ba label ;\
98 rd %pc, %l4 ;\
99 clr %l4
100 #define TT_TRACE_L_INS 3
101
102 #else
103
104 #define TT_TRACE(label)
105 #define TT_TRACE_INS 0
106
107 #define TT_TRACE_L(label)
108 #define TT_TRACE_L_INS 0
109
110 #endif
111
112 /*
113 * This first set are funneled to trap() with %tt as the type.
114 * Trap will then either panic or send the user a signal.
115 */
116 /*
117 * NOT is used for traps that just shouldn't happen.
118 * It comes in both single and quadruple flavors.
119 */
120 #if !defined(lint)
121 .global trap
122 #endif /* !lint */
123 #define NOT \
124 TT_TRACE(trace_gen) ;\
125 set trap, %g1 ;\
126 rdpr %tt, %g3 ;\
127 ba,pt %xcc, sys_trap ;\
128 sub %g0, 1, %g4 ;\
129 .align 32
130 #define NOT4 NOT; NOT; NOT; NOT
131 /*
132 * RED is for traps that use the red mode handler.
133 * We should never see these either.
134 */
135 #define RED NOT
136 /*
137 * BAD is used for trap vectors we don't have a kernel
138 * handler for.
139 * It also comes in single and quadruple versions.
140 */
141 #define BAD NOT
142 #define BAD4 NOT4
143
144 #define DONE \
145 done; \
146 .align 32
147
148 /*
149 * TRAP vectors to the trap() function.
150 * It's main use is for user errors.
151 */
152 #if !defined(lint)
153 .global trap
154 #endif /* !lint */
155 #define TRAP(arg) \
156 TT_TRACE(trace_gen) ;\
157 set trap, %g1 ;\
158 mov arg, %g3 ;\
159 ba,pt %xcc, sys_trap ;\
160 sub %g0, 1, %g4 ;\
161 .align 32
162
163 /*
164 * SYSCALL is used for unsupported syscall interfaces (with 'which'
165 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
166 * 'which' set to 'syscall_trap32').
167 *
168 * The SYSCALL_TRAP* macros are used for syscall entry points.
169 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
170 * is used to support ILP32. Each macro can only be used once
171 * since they each define a symbol. The symbols are used as hot patch
172 * points by the brand infrastructure to dynamically enable and disable
173 * brand syscall interposition. See the comments around BRAND_CALLBACK
174 * and brand_plat_interposition_enable() for more information.
175 */
176 #define SYSCALL_NOTT(which) \
177 set (which), %g1 ;\
178 ba,pt %xcc, sys_trap ;\
179 sub %g0, 1, %g4 ;\
180 .align 32
181
182 #define SYSCALL(which) \
183 TT_TRACE(trace_gen) ;\
184 SYSCALL_NOTT(which)
185
186 #define SYSCALL_TRAP32 \
187 TT_TRACE(trace_gen) ;\
188 ALTENTRY(syscall_trap32_patch_point) \
189 SYSCALL_NOTT(syscall_trap32)
190
191 #define SYSCALL_TRAP \
192 TT_TRACE(trace_gen) ;\
193 ALTENTRY(syscall_trap_patch_point) \
194 SYSCALL_NOTT(syscall_trap)
195
196 #define FLUSHW(h_name) \
197 .global h_name ;\
198 h_name: ;\
199 set trap, %g1 ;\
200 mov T_FLUSHW, %g3 ;\
201 sub %g0, 1, %g4 ;\
202 save ;\
203 flushw ;\
204 restore ;\
205 FAST_TRAP_DONE ;\
206 .align 32
207
208 /*
209 * GOTO just jumps to a label.
210 * It's used for things that can be fixed without going thru sys_trap.
211 */
212 #define GOTO(label) \
213 .global label ;\
214 ba,a label ;\
215 .empty ;\
216 .align 32
217
218 /*
219 * GOTO_TT just jumps to a label.
220 * correctable ECC error traps at level 0 and 1 will use this macro.
221 * It's used for things that can be fixed without going thru sys_trap.
222 */
223 #define GOTO_TT(label, ttlabel) \
224 .global label ;\
225 TT_TRACE(ttlabel) ;\
226 ba,a label ;\
227 .empty ;\
228 .align 32
229
230 /*
231 * Privileged traps
232 * Takes breakpoint if privileged, calls trap() if not.
233 */
234 #define PRIV(label) \
235 rdpr %tstate, %g1 ;\
236 btst TSTATE_PRIV, %g1 ;\
237 bnz label ;\
238 rdpr %tt, %g3 ;\
239 set trap, %g1 ;\
240 ba,pt %xcc, sys_trap ;\
241 sub %g0, 1, %g4 ;\
242 .align 32
243
244
245 /*
246 * DTrace traps.
247 */
248 #define DTRACE_PID \
249 .global dtrace_pid_probe ;\
250 set dtrace_pid_probe, %g1 ;\
251 ba,pt %xcc, user_trap ;\
252 sub %g0, 1, %g4 ;\
253 .align 32
254
255 #define DTRACE_RETURN \
256 .global dtrace_return_probe ;\
257 set dtrace_return_probe, %g1 ;\
258 ba,pt %xcc, user_trap ;\
259 sub %g0, 1, %g4 ;\
260 .align 32
261
262 /*
263 * REGISTER WINDOW MANAGEMENT MACROS
264 */
265
266 /*
267 * various convenient units of padding
268 */
269 #define SKIP(n) .skip 4*(n)
270
271 /*
272 * CLEAN_WINDOW is the simple handler for cleaning a register window.
273 */
274 #define CLEAN_WINDOW \
275 TT_TRACE_L(trace_win) ;\
276 rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\
277 clr %l0; clr %l1; clr %l2; clr %l3 ;\
278 clr %l4; clr %l5; clr %l6; clr %l7 ;\
279 clr %o0; clr %o1; clr %o2; clr %o3 ;\
280 clr %o4; clr %o5; clr %o6; clr %o7 ;\
281 retry; .align 128
282
283 #if !defined(lint)
284
285 /*
286 * If we get an unresolved tlb miss while in a window handler, the fault
287 * handler will resume execution at the last instruction of the window
288 * hander, instead of delivering the fault to the kernel. Spill handlers
289 * use this to spill windows into the wbuf.
290 *
291 * The mixed handler works by checking %sp, and branching to the correct
292 * handler. This is done by branching back to label 1: for 32b frames,
293 * or label 2: for 64b frames; which implies the handler order is: 32b,
294 * 64b, mixed. The 1: and 2: labels are offset into the routines to
295 * allow the branchs' delay slots to contain useful instructions.
296 */
297
298 /*
299 * SPILL_32bit spills a 32-bit-wide kernel register window. It
300 * assumes that the kernel context and the nucleus context are the
301 * same. The stack pointer is required to be eight-byte aligned even
302 * though this code only needs it to be four-byte aligned.
303 */
304 #define SPILL_32bit(tail) \
305 srl %sp, 0, %sp ;\
306 1: st %l0, [%sp + 0] ;\
307 st %l1, [%sp + 4] ;\
308 st %l2, [%sp + 8] ;\
309 st %l3, [%sp + 12] ;\
310 st %l4, [%sp + 16] ;\
311 st %l5, [%sp + 20] ;\
312 st %l6, [%sp + 24] ;\
313 st %l7, [%sp + 28] ;\
314 st %i0, [%sp + 32] ;\
315 st %i1, [%sp + 36] ;\
316 st %i2, [%sp + 40] ;\
317 st %i3, [%sp + 44] ;\
318 st %i4, [%sp + 48] ;\
319 st %i5, [%sp + 52] ;\
320 st %i6, [%sp + 56] ;\
321 st %i7, [%sp + 60] ;\
322 TT_TRACE_L(trace_win) ;\
323 saved ;\
324 retry ;\
325 SKIP(31-19-TT_TRACE_L_INS) ;\
326 ba,a,pt %xcc, fault_32bit_/**/tail ;\
327 .empty
328
329 /*
330 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
331 * wide address space via the designated asi. It is used to spill
332 * non-kernel windows. The stack pointer is required to be eight-byte
333 * aligned even though this code only needs it to be four-byte
334 * aligned.
335 */
336 #define SPILL_32bit_asi(asi_num, tail) \
337 srl %sp, 0, %sp ;\
338 1: sta %l0, [%sp + %g0]asi_num ;\
339 mov 4, %g1 ;\
340 sta %l1, [%sp + %g1]asi_num ;\
341 mov 8, %g2 ;\
342 sta %l2, [%sp + %g2]asi_num ;\
343 mov 12, %g3 ;\
344 sta %l3, [%sp + %g3]asi_num ;\
345 add %sp, 16, %g4 ;\
346 sta %l4, [%g4 + %g0]asi_num ;\
347 sta %l5, [%g4 + %g1]asi_num ;\
348 sta %l6, [%g4 + %g2]asi_num ;\
349 sta %l7, [%g4 + %g3]asi_num ;\
350 add %g4, 16, %g4 ;\
351 sta %i0, [%g4 + %g0]asi_num ;\
352 sta %i1, [%g4 + %g1]asi_num ;\
353 sta %i2, [%g4 + %g2]asi_num ;\
354 sta %i3, [%g4 + %g3]asi_num ;\
355 add %g4, 16, %g4 ;\
356 sta %i4, [%g4 + %g0]asi_num ;\
357 sta %i5, [%g4 + %g1]asi_num ;\
358 sta %i6, [%g4 + %g2]asi_num ;\
359 sta %i7, [%g4 + %g3]asi_num ;\
360 TT_TRACE_L(trace_win) ;\
361 saved ;\
362 retry ;\
363 SKIP(31-25-TT_TRACE_L_INS) ;\
364 ba,a,pt %xcc, fault_32bit_/**/tail ;\
365 .empty
366
367 /*
368 * SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit
369 * wide address space via the designated asi. It is used to spill
370 * windows at tl>1 where performance isn't the primary concern and
371 * where we don't want to use unnecessary registers. The stack
372 * pointer is required to be eight-byte aligned even though this code
373 * only needs it to be four-byte aligned.
374 */
375 #define SPILL_32bit_tt1(asi_num, tail) \
376 mov asi_num, %asi ;\
377 1: srl %sp, 0, %sp ;\
378 sta %l0, [%sp + 0]%asi ;\
379 sta %l1, [%sp + 4]%asi ;\
380 sta %l2, [%sp + 8]%asi ;\
381 sta %l3, [%sp + 12]%asi ;\
382 sta %l4, [%sp + 16]%asi ;\
383 sta %l5, [%sp + 20]%asi ;\
384 sta %l6, [%sp + 24]%asi ;\
385 sta %l7, [%sp + 28]%asi ;\
386 sta %i0, [%sp + 32]%asi ;\
387 sta %i1, [%sp + 36]%asi ;\
388 sta %i2, [%sp + 40]%asi ;\
389 sta %i3, [%sp + 44]%asi ;\
390 sta %i4, [%sp + 48]%asi ;\
391 sta %i5, [%sp + 52]%asi ;\
392 sta %i6, [%sp + 56]%asi ;\
393 sta %i7, [%sp + 60]%asi ;\
394 TT_TRACE_L(trace_win) ;\
395 saved ;\
396 retry ;\
397 SKIP(31-20-TT_TRACE_L_INS) ;\
398 ba,a,pt %xcc, fault_32bit_/**/tail ;\
399 .empty
400
401
402 /*
403 * FILL_32bit fills a 32-bit-wide kernel register window. It assumes
404 * that the kernel context and the nucleus context are the same. The
405 * stack pointer is required to be eight-byte aligned even though this
406 * code only needs it to be four-byte aligned.
407 */
408 #define FILL_32bit(tail) \
409 srl %sp, 0, %sp ;\
410 1: TT_TRACE_L(trace_win) ;\
411 ld [%sp + 0], %l0 ;\
412 ld [%sp + 4], %l1 ;\
413 ld [%sp + 8], %l2 ;\
414 ld [%sp + 12], %l3 ;\
415 ld [%sp + 16], %l4 ;\
416 ld [%sp + 20], %l5 ;\
417 ld [%sp + 24], %l6 ;\
418 ld [%sp + 28], %l7 ;\
419 ld [%sp + 32], %i0 ;\
420 ld [%sp + 36], %i1 ;\
421 ld [%sp + 40], %i2 ;\
422 ld [%sp + 44], %i3 ;\
423 ld [%sp + 48], %i4 ;\
424 ld [%sp + 52], %i5 ;\
425 ld [%sp + 56], %i6 ;\
426 ld [%sp + 60], %i7 ;\
427 restored ;\
428 retry ;\
429 SKIP(31-19-TT_TRACE_L_INS) ;\
430 ba,a,pt %xcc, fault_32bit_/**/tail ;\
431 .empty
432
433 /*
434 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
435 * wide address space via the designated asi. It is used to fill
436 * non-kernel windows. The stack pointer is required to be eight-byte
437 * aligned even though this code only needs it to be four-byte
438 * aligned.
439 */
440 #define FILL_32bit_asi(asi_num, tail) \
441 srl %sp, 0, %sp ;\
442 1: TT_TRACE_L(trace_win) ;\
443 mov 4, %g1 ;\
444 lda [%sp + %g0]asi_num, %l0 ;\
445 mov 8, %g2 ;\
446 lda [%sp + %g1]asi_num, %l1 ;\
447 mov 12, %g3 ;\
448 lda [%sp + %g2]asi_num, %l2 ;\
449 lda [%sp + %g3]asi_num, %l3 ;\
450 add %sp, 16, %g4 ;\
451 lda [%g4 + %g0]asi_num, %l4 ;\
452 lda [%g4 + %g1]asi_num, %l5 ;\
453 lda [%g4 + %g2]asi_num, %l6 ;\
454 lda [%g4 + %g3]asi_num, %l7 ;\
455 add %g4, 16, %g4 ;\
456 lda [%g4 + %g0]asi_num, %i0 ;\
457 lda [%g4 + %g1]asi_num, %i1 ;\
458 lda [%g4 + %g2]asi_num, %i2 ;\
459 lda [%g4 + %g3]asi_num, %i3 ;\
460 add %g4, 16, %g4 ;\
461 lda [%g4 + %g0]asi_num, %i4 ;\
462 lda [%g4 + %g1]asi_num, %i5 ;\
463 lda [%g4 + %g2]asi_num, %i6 ;\
464 lda [%g4 + %g3]asi_num, %i7 ;\
465 restored ;\
466 retry ;\
467 SKIP(31-25-TT_TRACE_L_INS) ;\
468 ba,a,pt %xcc, fault_32bit_/**/tail ;\
469 .empty
470
471 /*
472 * FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit
473 * wide address space via the designated asi. It is used to fill
474 * windows at tl>1 where performance isn't the primary concern and
475 * where we don't want to use unnecessary registers. The stack
476 * pointer is required to be eight-byte aligned even though this code
477 * only needs it to be four-byte aligned.
478 */
479 #define FILL_32bit_tt1(asi_num, tail) \
480 mov asi_num, %asi ;\
481 1: srl %sp, 0, %sp ;\
482 TT_TRACE_L(trace_win) ;\
483 lda [%sp + 0]%asi, %l0 ;\
484 lda [%sp + 4]%asi, %l1 ;\
485 lda [%sp + 8]%asi, %l2 ;\
486 lda [%sp + 12]%asi, %l3 ;\
487 lda [%sp + 16]%asi, %l4 ;\
488 lda [%sp + 20]%asi, %l5 ;\
489 lda [%sp + 24]%asi, %l6 ;\
490 lda [%sp + 28]%asi, %l7 ;\
491 lda [%sp + 32]%asi, %i0 ;\
492 lda [%sp + 36]%asi, %i1 ;\
493 lda [%sp + 40]%asi, %i2 ;\
494 lda [%sp + 44]%asi, %i3 ;\
495 lda [%sp + 48]%asi, %i4 ;\
496 lda [%sp + 52]%asi, %i5 ;\
497 lda [%sp + 56]%asi, %i6 ;\
498 lda [%sp + 60]%asi, %i7 ;\
499 restored ;\
500 retry ;\
501 SKIP(31-20-TT_TRACE_L_INS) ;\
502 ba,a,pt %xcc, fault_32bit_/**/tail ;\
503 .empty
504
505
506 /*
507 * SPILL_64bit spills a 64-bit-wide kernel register window. It
508 * assumes that the kernel context and the nucleus context are the
509 * same. The stack pointer is required to be eight-byte aligned.
510 */
511 #define SPILL_64bit(tail) \
512 2: stx %l0, [%sp + V9BIAS64 + 0] ;\
513 stx %l1, [%sp + V9BIAS64 + 8] ;\
514 stx %l2, [%sp + V9BIAS64 + 16] ;\
515 stx %l3, [%sp + V9BIAS64 + 24] ;\
516 stx %l4, [%sp + V9BIAS64 + 32] ;\
517 stx %l5, [%sp + V9BIAS64 + 40] ;\
518 stx %l6, [%sp + V9BIAS64 + 48] ;\
519 stx %l7, [%sp + V9BIAS64 + 56] ;\
520 stx %i0, [%sp + V9BIAS64 + 64] ;\
521 stx %i1, [%sp + V9BIAS64 + 72] ;\
522 stx %i2, [%sp + V9BIAS64 + 80] ;\
523 stx %i3, [%sp + V9BIAS64 + 88] ;\
524 stx %i4, [%sp + V9BIAS64 + 96] ;\
525 stx %i5, [%sp + V9BIAS64 + 104] ;\
526 stx %i6, [%sp + V9BIAS64 + 112] ;\
527 stx %i7, [%sp + V9BIAS64 + 120] ;\
528 TT_TRACE_L(trace_win) ;\
529 saved ;\
530 retry ;\
531 SKIP(31-18-TT_TRACE_L_INS) ;\
532 ba,a,pt %xcc, fault_64bit_/**/tail ;\
533 .empty
534
535 /*
536 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
537 * wide address space via the designated asi. It is used to spill
538 * non-kernel windows. The stack pointer is required to be eight-byte
539 * aligned.
540 */
541 #define SPILL_64bit_asi(asi_num, tail) \
542 mov 0 + V9BIAS64, %g1 ;\
543 2: stxa %l0, [%sp + %g1]asi_num ;\
544 mov 8 + V9BIAS64, %g2 ;\
545 stxa %l1, [%sp + %g2]asi_num ;\
546 mov 16 + V9BIAS64, %g3 ;\
547 stxa %l2, [%sp + %g3]asi_num ;\
548 mov 24 + V9BIAS64, %g4 ;\
549 stxa %l3, [%sp + %g4]asi_num ;\
550 add %sp, 32, %g5 ;\
551 stxa %l4, [%g5 + %g1]asi_num ;\
552 stxa %l5, [%g5 + %g2]asi_num ;\
553 stxa %l6, [%g5 + %g3]asi_num ;\
554 stxa %l7, [%g5 + %g4]asi_num ;\
555 add %g5, 32, %g5 ;\
556 stxa %i0, [%g5 + %g1]asi_num ;\
557 stxa %i1, [%g5 + %g2]asi_num ;\
558 stxa %i2, [%g5 + %g3]asi_num ;\
559 stxa %i3, [%g5 + %g4]asi_num ;\
560 add %g5, 32, %g5 ;\
561 stxa %i4, [%g5 + %g1]asi_num ;\
562 stxa %i5, [%g5 + %g2]asi_num ;\
563 stxa %i6, [%g5 + %g3]asi_num ;\
564 stxa %i7, [%g5 + %g4]asi_num ;\
565 TT_TRACE_L(trace_win) ;\
566 saved ;\
567 retry ;\
568 SKIP(31-25-TT_TRACE_L_INS) ;\
569 ba,a,pt %xcc, fault_64bit_/**/tail ;\
570 .empty
571
572 /*
573 * SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit
574 * wide address space via the designated asi. It is used to spill
575 * windows at tl>1 where performance isn't the primary concern and
576 * where we don't want to use unnecessary registers. The stack
577 * pointer is required to be eight-byte aligned.
578 */
579 #define SPILL_64bit_tt1(asi_num, tail) \
580 mov asi_num, %asi ;\
581 2: stxa %l0, [%sp + V9BIAS64 + 0]%asi ;\
582 stxa %l1, [%sp + V9BIAS64 + 8]%asi ;\
583 stxa %l2, [%sp + V9BIAS64 + 16]%asi ;\
584 stxa %l3, [%sp + V9BIAS64 + 24]%asi ;\
585 stxa %l4, [%sp + V9BIAS64 + 32]%asi ;\
586 stxa %l5, [%sp + V9BIAS64 + 40]%asi ;\
587 stxa %l6, [%sp + V9BIAS64 + 48]%asi ;\
588 stxa %l7, [%sp + V9BIAS64 + 56]%asi ;\
589 stxa %i0, [%sp + V9BIAS64 + 64]%asi ;\
590 stxa %i1, [%sp + V9BIAS64 + 72]%asi ;\
591 stxa %i2, [%sp + V9BIAS64 + 80]%asi ;\
592 stxa %i3, [%sp + V9BIAS64 + 88]%asi ;\
593 stxa %i4, [%sp + V9BIAS64 + 96]%asi ;\
594 stxa %i5, [%sp + V9BIAS64 + 104]%asi ;\
595 stxa %i6, [%sp + V9BIAS64 + 112]%asi ;\
596 stxa %i7, [%sp + V9BIAS64 + 120]%asi ;\
597 TT_TRACE_L(trace_win) ;\
598 saved ;\
599 retry ;\
600 SKIP(31-19-TT_TRACE_L_INS) ;\
601 ba,a,pt %xcc, fault_64bit_/**/tail ;\
602 .empty
603
604
605 /*
606 * FILL_64bit fills a 64-bit-wide kernel register window. It assumes
607 * that the kernel context and the nucleus context are the same. The
608 * stack pointer is required to be eight-byte aligned.
609 */
610 #define FILL_64bit(tail) \
611 2: TT_TRACE_L(trace_win) ;\
612 ldx [%sp + V9BIAS64 + 0], %l0 ;\
613 ldx [%sp + V9BIAS64 + 8], %l1 ;\
614 ldx [%sp + V9BIAS64 + 16], %l2 ;\
615 ldx [%sp + V9BIAS64 + 24], %l3 ;\
616 ldx [%sp + V9BIAS64 + 32], %l4 ;\
617 ldx [%sp + V9BIAS64 + 40], %l5 ;\
618 ldx [%sp + V9BIAS64 + 48], %l6 ;\
619 ldx [%sp + V9BIAS64 + 56], %l7 ;\
620 ldx [%sp + V9BIAS64 + 64], %i0 ;\
621 ldx [%sp + V9BIAS64 + 72], %i1 ;\
622 ldx [%sp + V9BIAS64 + 80], %i2 ;\
623 ldx [%sp + V9BIAS64 + 88], %i3 ;\
624 ldx [%sp + V9BIAS64 + 96], %i4 ;\
625 ldx [%sp + V9BIAS64 + 104], %i5 ;\
626 ldx [%sp + V9BIAS64 + 112], %i6 ;\
627 ldx [%sp + V9BIAS64 + 120], %i7 ;\
628 restored ;\
629 retry ;\
630 SKIP(31-18-TT_TRACE_L_INS) ;\
631 ba,a,pt %xcc, fault_64bit_/**/tail ;\
632 .empty
633
634 /*
635 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
636 * wide address space via the designated asi. It is used to fill
637 * non-kernel windows. The stack pointer is required to be eight-byte
638 * aligned.
639 */
640 #define FILL_64bit_asi(asi_num, tail) \
641 mov V9BIAS64 + 0, %g1 ;\
642 2: TT_TRACE_L(trace_win) ;\
643 ldxa [%sp + %g1]asi_num, %l0 ;\
644 mov V9BIAS64 + 8, %g2 ;\
645 ldxa [%sp + %g2]asi_num, %l1 ;\
646 mov V9BIAS64 + 16, %g3 ;\
647 ldxa [%sp + %g3]asi_num, %l2 ;\
648 mov V9BIAS64 + 24, %g4 ;\
649 ldxa [%sp + %g4]asi_num, %l3 ;\
650 add %sp, 32, %g5 ;\
651 ldxa [%g5 + %g1]asi_num, %l4 ;\
652 ldxa [%g5 + %g2]asi_num, %l5 ;\
653 ldxa [%g5 + %g3]asi_num, %l6 ;\
654 ldxa [%g5 + %g4]asi_num, %l7 ;\
655 add %g5, 32, %g5 ;\
656 ldxa [%g5 + %g1]asi_num, %i0 ;\
657 ldxa [%g5 + %g2]asi_num, %i1 ;\
658 ldxa [%g5 + %g3]asi_num, %i2 ;\
659 ldxa [%g5 + %g4]asi_num, %i3 ;\
660 add %g5, 32, %g5 ;\
661 ldxa [%g5 + %g1]asi_num, %i4 ;\
662 ldxa [%g5 + %g2]asi_num, %i5 ;\
663 ldxa [%g5 + %g3]asi_num, %i6 ;\
664 ldxa [%g5 + %g4]asi_num, %i7 ;\
665 restored ;\
666 retry ;\
667 SKIP(31-25-TT_TRACE_L_INS) ;\
668 ba,a,pt %xcc, fault_64bit_/**/tail ;\
669 .empty
670
671 /*
672 * FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit
673 * wide address space via the designated asi. It is used to fill
674 * windows at tl>1 where performance isn't the primary concern and
675 * where we don't want to use unnecessary registers. The stack
676 * pointer is required to be eight-byte aligned.
677 */
678 #define FILL_64bit_tt1(asi_num, tail) \
679 mov asi_num, %asi ;\
680 TT_TRACE_L(trace_win) ;\
681 ldxa [%sp + V9BIAS64 + 0]%asi, %l0 ;\
682 ldxa [%sp + V9BIAS64 + 8]%asi, %l1 ;\
683 ldxa [%sp + V9BIAS64 + 16]%asi, %l2 ;\
684 ldxa [%sp + V9BIAS64 + 24]%asi, %l3 ;\
685 ldxa [%sp + V9BIAS64 + 32]%asi, %l4 ;\
686 ldxa [%sp + V9BIAS64 + 40]%asi, %l5 ;\
687 ldxa [%sp + V9BIAS64 + 48]%asi, %l6 ;\
688 ldxa [%sp + V9BIAS64 + 56]%asi, %l7 ;\
689 ldxa [%sp + V9BIAS64 + 64]%asi, %i0 ;\
690 ldxa [%sp + V9BIAS64 + 72]%asi, %i1 ;\
691 ldxa [%sp + V9BIAS64 + 80]%asi, %i2 ;\
692 ldxa [%sp + V9BIAS64 + 88]%asi, %i3 ;\
693 ldxa [%sp + V9BIAS64 + 96]%asi, %i4 ;\
694 ldxa [%sp + V9BIAS64 + 104]%asi, %i5 ;\
695 ldxa [%sp + V9BIAS64 + 112]%asi, %i6 ;\
696 ldxa [%sp + V9BIAS64 + 120]%asi, %i7 ;\
697 restored ;\
698 retry ;\
699 SKIP(31-19-TT_TRACE_L_INS) ;\
700 ba,a,pt %xcc, fault_64bit_/**/tail ;\
701 .empty
702
703 #endif /* !lint */
704
705 /*
706 * SPILL_mixed spills either size window, depending on
707 * whether %sp is even or odd, to a 32-bit address space.
708 * This may only be used in conjunction with SPILL_32bit/
709 * SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be
710 * needed for use with SPILL_{32,64}bit_{tt1,asi}. Particular
711 * attention should be paid to the instructions that belong
712 * in the delay slots of the branches depending on the type
713 * of spill handler being branched to.
714 * Clear upper 32 bits of %sp if it is odd.
715 * We won't need to clear them in 64 bit kernel.
716 */
717 #define SPILL_mixed \
718 btst 1, %sp ;\
719 bz,a,pt %xcc, 1b ;\
720 srl %sp, 0, %sp ;\
721 ba,pt %xcc, 2b ;\
722 nop ;\
723 .align 128
724
725 /*
726 * FILL_mixed(ASI) fills either size window, depending on
727 * whether %sp is even or odd, from a 32-bit address space.
728 * This may only be used in conjunction with FILL_32bit/
729 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
730 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
731 * attention should be paid to the instructions that belong
732 * in the delay slots of the branches depending on the type
733 * of fill handler being branched to.
734 * Clear upper 32 bits of %sp if it is odd.
735 * We won't need to clear them in 64 bit kernel.
736 */
737 #define FILL_mixed \
738 btst 1, %sp ;\
739 bz,a,pt %xcc, 1b ;\
740 srl %sp, 0, %sp ;\
741 ba,pt %xcc, 2b ;\
742 nop ;\
743 .align 128
744
745
746 /*
747 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
748 * respectively, into the address space via the designated asi. The
749 * unbiased stack pointer is required to be eight-byte aligned (even for
750 * the 32-bit case even though this code does not require such strict
751 * alignment).
752 *
753 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
754 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
755 * will cause cwp + 2 to be spilled but will not clean cwp + 1. That
756 * window may contain kernel data so in user_rtt we set wstate to call
757 * these spill handlers on the first user spill trap. These handler then
758 * spill the appropriate window but also back up a window and clean the
759 * window that didn't get a cleanwin trap.
760 */
761 #define SPILL_32clean(asi_num, tail) \
762 srl %sp, 0, %sp ;\
763 sta %l0, [%sp + %g0]asi_num ;\
764 mov 4, %g1 ;\
765 sta %l1, [%sp + %g1]asi_num ;\
766 mov 8, %g2 ;\
767 sta %l2, [%sp + %g2]asi_num ;\
768 mov 12, %g3 ;\
769 sta %l3, [%sp + %g3]asi_num ;\
770 add %sp, 16, %g4 ;\
771 sta %l4, [%g4 + %g0]asi_num ;\
772 sta %l5, [%g4 + %g1]asi_num ;\
773 sta %l6, [%g4 + %g2]asi_num ;\
774 sta %l7, [%g4 + %g3]asi_num ;\
775 add %g4, 16, %g4 ;\
776 sta %i0, [%g4 + %g0]asi_num ;\
777 sta %i1, [%g4 + %g1]asi_num ;\
778 sta %i2, [%g4 + %g2]asi_num ;\
779 sta %i3, [%g4 + %g3]asi_num ;\
780 add %g4, 16, %g4 ;\
781 sta %i4, [%g4 + %g0]asi_num ;\
782 sta %i5, [%g4 + %g1]asi_num ;\
783 sta %i6, [%g4 + %g2]asi_num ;\
784 sta %i7, [%g4 + %g3]asi_num ;\
785 TT_TRACE_L(trace_win) ;\
786 b .spill_clean ;\
787 mov WSTATE_USER32, %g7 ;\
788 SKIP(31-25-TT_TRACE_L_INS) ;\
789 ba,a,pt %xcc, fault_32bit_/**/tail ;\
790 .empty
791
792 #define SPILL_64clean(asi_num, tail) \
793 mov 0 + V9BIAS64, %g1 ;\
794 stxa %l0, [%sp + %g1]asi_num ;\
795 mov 8 + V9BIAS64, %g2 ;\
796 stxa %l1, [%sp + %g2]asi_num ;\
797 mov 16 + V9BIAS64, %g3 ;\
798 stxa %l2, [%sp + %g3]asi_num ;\
799 mov 24 + V9BIAS64, %g4 ;\
800 stxa %l3, [%sp + %g4]asi_num ;\
801 add %sp, 32, %g5 ;\
802 stxa %l4, [%g5 + %g1]asi_num ;\
803 stxa %l5, [%g5 + %g2]asi_num ;\
804 stxa %l6, [%g5 + %g3]asi_num ;\
805 stxa %l7, [%g5 + %g4]asi_num ;\
806 add %g5, 32, %g5 ;\
807 stxa %i0, [%g5 + %g1]asi_num ;\
808 stxa %i1, [%g5 + %g2]asi_num ;\
809 stxa %i2, [%g5 + %g3]asi_num ;\
810 stxa %i3, [%g5 + %g4]asi_num ;\
811 add %g5, 32, %g5 ;\
812 stxa %i4, [%g5 + %g1]asi_num ;\
813 stxa %i5, [%g5 + %g2]asi_num ;\
814 stxa %i6, [%g5 + %g3]asi_num ;\
815 stxa %i7, [%g5 + %g4]asi_num ;\
816 TT_TRACE_L(trace_win) ;\
817 b .spill_clean ;\
818 mov WSTATE_USER64, %g7 ;\
819 SKIP(31-25-TT_TRACE_L_INS) ;\
820 ba,a,pt %xcc, fault_64bit_/**/tail ;\
821 .empty
822
823
824 /*
825 * Floating point disabled.
826 */
827 #define FP_DISABLED_TRAP \
828 TT_TRACE(trace_gen) ;\
829 ba,pt %xcc,.fp_disabled ;\
830 nop ;\
831 .align 32
832
833 /*
834 * Floating point exceptions.
835 */
836 #define FP_IEEE_TRAP \
837 TT_TRACE(trace_gen) ;\
838 ba,pt %xcc,.fp_ieee_exception ;\
839 nop ;\
840 .align 32
841
842 #define FP_TRAP \
843 TT_TRACE(trace_gen) ;\
844 ba,pt %xcc,.fp_exception ;\
845 nop ;\
846 .align 32
847
848 #if !defined(lint)
849 /*
850 * asynchronous traps at level 0 and level 1
851 *
852 * The first instruction must be a membar for UltraSPARC-III
853 * to stop RED state entry if the store queue has many
854 * pending bad stores (PRM, Chapter 11).
855 */
856 #define ASYNC_TRAP(ttype, ttlabel, table_name)\
857 .global table_name ;\
858 table_name: ;\
859 membar #Sync ;\
860 TT_TRACE(ttlabel) ;\
861 ba async_err ;\
862 mov ttype, %g5 ;\
863 .align 32
864
865 /*
866 * Defaults to BAD entry, but establishes label to be used for
867 * architecture-specific overwrite of trap table entry.
868 */
869 #define LABELED_BAD(table_name) \
870 .global table_name ;\
871 table_name: ;\
872 BAD
873
874 #endif /* !lint */
875
876 /*
877 * illegal instruction trap
878 */
879 #define ILLTRAP_INSTR \
880 membar #Sync ;\
881 TT_TRACE(trace_gen) ;\
882 or %g0, P_UTRAP4, %g2 ;\
883 or %g0, T_UNIMP_INSTR, %g3 ;\
884 sethi %hi(.check_v9utrap), %g4 ;\
885 jmp %g4 + %lo(.check_v9utrap) ;\
886 nop ;\
887 .align 32
888
889 /*
890 * tag overflow trap
891 */
892 #define TAG_OVERFLOW \
893 TT_TRACE(trace_gen) ;\
894 or %g0, P_UTRAP10, %g2 ;\
895 or %g0, T_TAG_OVERFLOW, %g3 ;\
896 sethi %hi(.check_v9utrap), %g4 ;\
897 jmp %g4 + %lo(.check_v9utrap) ;\
898 nop ;\
899 .align 32
900
901 /*
902 * divide by zero trap
903 */
904 #define DIV_BY_ZERO \
905 TT_TRACE(trace_gen) ;\
906 or %g0, P_UTRAP11, %g2 ;\
907 or %g0, T_IDIV0, %g3 ;\
908 sethi %hi(.check_v9utrap), %g4 ;\
909 jmp %g4 + %lo(.check_v9utrap) ;\
910 nop ;\
911 .align 32
912
913 /*
914 * trap instruction for V9 user trap handlers
915 */
916 #define TRAP_INSTR \
917 TT_TRACE(trace_gen) ;\
918 or %g0, T_SOFTWARE_TRAP, %g3 ;\
919 sethi %hi(.check_v9utrap), %g4 ;\
920 jmp %g4 + %lo(.check_v9utrap) ;\
921 nop ;\
922 .align 32
923 #define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
924
925 /*
926 * LEVEL_INTERRUPT is for level N interrupts.
927 * VECTOR_INTERRUPT is for the vector trap.
928 */
929 #define LEVEL_INTERRUPT(level) \
930 .global tt_pil/**/level ;\
931 tt_pil/**/level: ;\
932 ba,pt %xcc, pil_interrupt ;\
933 mov level, %g4 ;\
934 .align 32
935
936 #define LEVEL14_INTERRUPT \
937 ba pil14_interrupt ;\
938 mov PIL_14, %g4 ;\
939 .align 32
940
941 #define LEVEL15_INTERRUPT \
942 ba pil15_interrupt ;\
943 mov PIL_15, %g4 ;\
944 .align 32
945
946 #define VECTOR_INTERRUPT \
947 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g1 ;\
948 btst IRSR_BUSY, %g1 ;\
949 bnz,pt %xcc, vec_interrupt ;\
950 nop ;\
951 ba,a,pt %xcc, vec_intr_spurious ;\
952 .empty ;\
953 .align 32
954
955 /*
956 * MMU Trap Handlers.
957 */
958 #define SWITCH_GLOBALS /* mmu->alt, alt->mmu */ \
959 rdpr %pstate, %g5 ;\
960 wrpr %g5, PSTATE_MG | PSTATE_AG, %pstate
961
962 #define IMMU_EXCEPTION \
963 membar #Sync ;\
964 SWITCH_GLOBALS ;\
965 wr %g0, ASI_IMMU, %asi ;\
966 rdpr %tpc, %g2 ;\
967 ldxa [MMU_SFSR]%asi, %g3 ;\
968 ba,pt %xcc, .mmu_exception_end ;\
969 mov T_INSTR_EXCEPTION, %g1 ;\
970 .align 32
971
972 #define DMMU_EXCEPTION \
973 SWITCH_GLOBALS ;\
974 wr %g0, ASI_DMMU, %asi ;\
975 ldxa [MMU_TAG_ACCESS]%asi, %g2 ;\
976 ldxa [MMU_SFSR]%asi, %g3 ;\
977 ba,pt %xcc, .mmu_exception_end ;\
978 mov T_DATA_EXCEPTION, %g1 ;\
979 .align 32
980
981 #define DMMU_EXC_AG_PRIV \
982 wr %g0, ASI_DMMU, %asi ;\
983 ldxa [MMU_SFAR]%asi, %g2 ;\
984 ba,pt %xcc, .mmu_priv_exception ;\
985 ldxa [MMU_SFSR]%asi, %g3 ;\
986 .align 32
987
988 #define DMMU_EXC_AG_NOT_ALIGNED \
989 wr %g0, ASI_DMMU, %asi ;\
990 ldxa [MMU_SFAR]%asi, %g2 ;\
991 ba,pt %xcc, .mmu_exception_not_aligned ;\
992 ldxa [MMU_SFSR]%asi, %g3 ;\
993 .align 32
994
995 /*
996 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
997 */
998 #define DMMU_EXC_LDDF_NOT_ALIGNED \
999 btst 1, %sp ;\
1000 bnz,pt %xcc, .lddf_exception_not_aligned ;\
1001 wr %g0, ASI_DMMU, %asi ;\
1002 ldxa [MMU_SFAR]%asi, %g2 ;\
1003 ba,pt %xcc, .mmu_exception_not_aligned ;\
1004 ldxa [MMU_SFSR]%asi, %g3 ;\
1005 .align 32
1006
1007 #define DMMU_EXC_STDF_NOT_ALIGNED \
1008 btst 1, %sp ;\
1009 bnz,pt %xcc, .stdf_exception_not_aligned ;\
1010 wr %g0, ASI_DMMU, %asi ;\
1011 ldxa [MMU_SFAR]%asi, %g2 ;\
1012 ba,pt %xcc, .mmu_exception_not_aligned ;\
1013 ldxa [MMU_SFSR]%asi, %g3 ;\
1014 .align 32
1015
1016 /*
1017 * Flush the TLB using either the primary, secondary, or nucleus flush
1018 * operation based on whether the ctx from the tag access register matches
1019 * the primary or secondary context (flush the nucleus if neither matches).
1020 *
1021 * Requires a membar #Sync before next ld/st.
1022 * exits with:
1023 * g2 = tag access register
1024 * g3 = ctx number
1025 */
1026 #if TAGACC_CTX_MASK != CTXREG_CTX_MASK
1027 #error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
1028 #endif
1029 #define DTLB_DEMAP_ENTRY \
1030 mov MMU_TAG_ACCESS, %g1 ;\
1031 mov MMU_PCONTEXT, %g5 ;\
1032 ldxa [%g1]ASI_DMMU, %g2 ;\
1033 sethi %hi(TAGACC_CTX_MASK), %g4 ;\
1034 or %g4, %lo(TAGACC_CTX_MASK), %g4 ;\
1035 and %g2, %g4, %g3 /* g3 = ctx */ ;\
1036 ldxa [%g5]ASI_DMMU, %g6 /* g6 = primary ctx */ ;\
1037 and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\
1038 cmp %g3, %g6 ;\
1039 be,pt %xcc, 1f ;\
1040 andn %g2, %g4, %g1 /* ctx = primary */ ;\
1041 mov MMU_SCONTEXT, %g5 ;\
1042 ldxa [%g5]ASI_DMMU, %g6 /* g6 = secondary ctx */ ;\
1043 and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\
1044 cmp %g3, %g6 ;\
1045 be,a,pt %xcc, 1f ;\
1046 or %g1, DEMAP_SECOND, %g1 ;\
1047 or %g1, DEMAP_NUCLEUS, %g1 ;\
1048 1: stxa %g0, [%g1]ASI_DTLB_DEMAP /* MMU_DEMAP_PAGE */ ;\
1049 membar #Sync
1050
1051 #if defined(cscope)
1052 /*
1053 * Define labels to direct cscope quickly to labels that
1054 * are generated by macro expansion of DTLB_MISS().
1055 */
1056 .global tt0_dtlbmiss
1057 tt0_dtlbmiss:
1058 .global tt1_dtlbmiss
1059 tt1_dtlbmiss:
1060 nop
1061 #endif
1062
1063 /*
1064 * Needs to be exactly 32 instructions
1065 *
1066 * UTLB NOTE: If we don't hit on the 8k pointer then we branch
1067 * to a special 4M tsb handler. It would be nice if that handler
1068 * could live in this file but currently it seems better to allow
1069 * it to fall thru to sfmmu_tsb_miss.
1070 */
1071 #ifdef UTSB_PHYS
1072 #define DTLB_MISS(table_name) ;\
1073 .global table_name/**/_dtlbmiss ;\
1074 table_name/**/_dtlbmiss: ;\
1075 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
1076 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
1077 ldxa [%g6]ASI_DMMU, %g2 /* g2 = tag access */ ;\
1078 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
1079 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
1080 cmp %g3, INVALID_CONTEXT ;\
1081 ble,pn %xcc, sfmmu_kdtlb_miss ;\
1082 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 ISM predicted */ ;\
1083 brgz,pn %g6, sfmmu_udtlb_slowpath_ismpred ;\
1084 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
1085 ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, %g5 data */;\
1086 cmp %g4, %g7 ;\
1087 bne,pn %xcc, sfmmu_udtlb_slowpath_noismpred ;\
1088 nop ;\
1089 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
1090 stxa %g5, [%g0]ASI_DTLB_IN /* trapstat expects TTE */ ;\
1091 retry /* in %g5 */ ;\
1092 unimp 0 ;\
1093 unimp 0 ;\
1094 unimp 0 ;\
1095 unimp 0 ;\
1096 unimp 0 ;\
1097 unimp 0 ;\
1098 unimp 0 ;\
1099 unimp 0 ;\
1100 unimp 0 ;\
1101 unimp 0 ;\
1102 unimp 0 ;\
1103 unimp 0 ;\
1104 unimp 0 ;\
1105 unimp 0 ;\
1106 .align 128
1107
1108 #else /* UTSB_PHYS */
1109 #define DTLB_MISS(table_name) ;\
1110 .global table_name/**/_dtlbmiss ;\
1111 table_name/**/_dtlbmiss: ;\
1112 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
1113 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
1114 ldxa [%g6]ASI_DMMU, %g2 /* g2 = tag access */ ;\
1115 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
1116 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
1117 cmp %g3, INVALID_CONTEXT ;\
1118 ble,pn %xcc, sfmmu_kdtlb_miss ;\
1119 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
1120 brlz,pn %g1, sfmmu_udtlb_slowpath ;\
1121 nop ;\
1122 ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, %g5 data */ ;\
1123 cmp %g4, %g7 ;\
1124 bne,pn %xcc, sfmmu_tsb_miss_tt /* no 4M TSB, miss */ ;\
1125 mov -1, %g3 /* set 4M tsbe ptr to -1 */ ;\
1126 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
1127 stxa %g5, [%g0]ASI_DTLB_IN /* trapstat expects TTE */ ;\
1128 retry /* in %g5 */ ;\
1129 unimp 0 ;\
1130 unimp 0 ;\
1131 unimp 0 ;\
1132 unimp 0 ;\
1133 unimp 0 ;\
1134 unimp 0 ;\
1135 unimp 0 ;\
1136 unimp 0 ;\
1137 unimp 0 ;\
1138 unimp 0 ;\
1139 unimp 0 ;\
1140 unimp 0 ;\
1141 unimp 0 ;\
1142 unimp 0 ;\
1143 .align 128
1144 #endif /* UTSB_PHYS */
1145
1146 #if defined(cscope)
1147 /*
1148 * Define labels to direct cscope quickly to labels that
1149 * are generated by macro expansion of ITLB_MISS().
1150 */
1151 .global tt0_itlbmiss
1152 tt0_itlbmiss:
1153 .global tt1_itlbmiss
1154 tt1_itlbmiss:
1155 nop
1156 #endif
1157
1158 /*
1159 * Instruction miss handler.
1160 * ldda instructions will have their ASI patched
1161 * by sfmmu_patch_ktsb at runtime.
1162 * MUST be EXACTLY 32 instructions or we'll break.
1163 */
1164 #ifdef UTSB_PHYS
1165 #define ITLB_MISS(table_name) \
1166 .global table_name/**/_itlbmiss ;\
1167 table_name/**/_itlbmiss: ;\
1168 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
1169 ldxa [%g0]ASI_IMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
1170 ldxa [%g6]ASI_IMMU, %g2 /* g2 = tag access */ ;\
1171 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
1172 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
1173 cmp %g3, INVALID_CONTEXT ;\
1174 ble,pn %xcc, sfmmu_kitlb_miss ;\
1175 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
1176 ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
1177 cmp %g4, %g7 ;\
1178 bne,pn %xcc, sfmmu_uitlb_slowpath ;\
1179 andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */ ;\
1180 bz,pn %icc, exec_fault ;\
1181 nop ;\
1182 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
1183 stxa %g5, [%g0]ASI_ITLB_IN /* trapstat expects %g5 */ ;\
1184 retry ;\
1185 unimp 0 ;\
1186 unimp 0 ;\
1187 unimp 0 ;\
1188 unimp 0 ;\
1189 unimp 0 ;\
1190 unimp 0 ;\
1191 unimp 0 ;\
1192 unimp 0 ;\
1193 unimp 0 ;\
1194 unimp 0 ;\
1195 unimp 0 ;\
1196 unimp 0 ;\
1197 unimp 0 ;\
1198 unimp 0 ;\
1199 .align 128
1200
1201 #else /* UTSB_PHYS */
1202 #define ITLB_MISS(table_name) \
1203 .global table_name/**/_itlbmiss ;\
1204 table_name/**/_itlbmiss: ;\
1205 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\
1206 ldxa [%g0]ASI_IMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
1207 ldxa [%g6]ASI_IMMU, %g2 /* g2 = tag access */ ;\
1208 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
1209 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\
1210 cmp %g3, INVALID_CONTEXT ;\
1211 ble,pn %xcc, sfmmu_kitlb_miss ;\
1212 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\
1213 brlz,pn %g1, sfmmu_uitlb_slowpath /* if >1 TSB branch */ ;\
1214 nop ;\
1215 ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, g5 = data */ ;\
1216 cmp %g4, %g7 ;\
1217 bne,pn %xcc, sfmmu_tsb_miss_tt /* br if 8k ptr miss */ ;\
1218 mov -1, %g3 /* set 4M TSB ptr to -1 */ ;\
1219 andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */ ;\
1220 bz,pn %icc, exec_fault ;\
1221 nop ;\
1222 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\
1223 stxa %g5, [%g0]ASI_ITLB_IN /* trapstat expects %g5 */ ;\
1224 retry ;\
1225 unimp 0 ;\
1226 unimp 0 ;\
1227 unimp 0 ;\
1228 unimp 0 ;\
1229 unimp 0 ;\
1230 unimp 0 ;\
1231 unimp 0 ;\
1232 unimp 0 ;\
1233 unimp 0 ;\
1234 unimp 0 ;\
1235 unimp 0 ;\
1236 .align 128
1237 #endif /* UTSB_PHYS */
1238
1239
1240 /*
1241 * This macro is the first level handler for fast protection faults.
1242 * It first demaps the tlb entry which generated the fault and then
1243 * attempts to set the modify bit on the hash. It needs to be
1244 * exactly 32 instructions.
1245 */
1246 #define DTLB_PROT \
1247 DTLB_DEMAP_ENTRY /* 20 instructions */ ;\
1248 /* ;\
1249 * At this point: ;\
1250 * g1 = ???? ;\
1251 * g2 = tag access register ;\
1252 * g3 = ctx number ;\
1253 * g4 = ???? ;\
1254 */ ;\
1255 TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\
1256 /* clobbers g1 and g6 */ ;\
1257 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\
1258 brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\
1259 nop ;\
1260 ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\
1261 unimp 0 ;\
1262 unimp 0 ;\
1263 unimp 0 ;\
1264 unimp 0 ;\
1265 unimp 0 ;\
1266 unimp 0 ;\
1267 .align 128
1268
1269 #define DMMU_EXCEPTION_TL1 ;\
1270 SWITCH_GLOBALS ;\
1271 ba,a,pt %xcc, mmu_trap_tl1 ;\
1272 nop ;\
1273 .align 32
1274
1275 #define MISALIGN_ADDR_TL1 ;\
1276 ba,a,pt %xcc, mmu_trap_tl1 ;\
1277 nop ;\
1278 .align 32
1279
1280 /*
1281 * Trace a tsb hit
1282 * g1 = tsbe pointer (in/clobbered)
1283 * g2 = tag access register (in)
1284 * g3 - g4 = scratch (clobbered)
1285 * g5 = tsbe data (in)
1286 * g6 = scratch (clobbered)
1287 * g7 = pc we jumped here from (in)
1288 * ttextra = value to OR in to trap type (%tt) (in)
1289 */
1290 #ifdef TRAPTRACE
1291 #define TRACE_TSBHIT(ttextra) \
1292 membar #Sync ;\
1293 sethi %hi(FLUSH_ADDR), %g6 ;\
1294 flush %g6 ;\
1295 TRACE_PTR(%g3, %g6) ;\
1296 GET_TRACE_TICK(%g6, %g4) ;\
1297 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\
1298 stxa %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\
1299 stxa %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\
1300 rdpr %tnpc, %g6 ;\
1301 stxa %g6, [%g3 + TRAP_ENT_F2]%asi ;\
1302 stxa %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\
1303 stxa %g0, [%g3 + TRAP_ENT_F4]%asi ;\
1304 rdpr %tpc, %g6 ;\
1305 stxa %g6, [%g3 + TRAP_ENT_TPC]%asi ;\
1306 rdpr %tl, %g6 ;\
1307 stha %g6, [%g3 + TRAP_ENT_TL]%asi ;\
1308 rdpr %tt, %g6 ;\
1309 or %g6, (ttextra), %g6 ;\
1310 stha %g6, [%g3 + TRAP_ENT_TT]%asi ;\
1311 ldxa [%g0]ASI_IMMU, %g1 /* tag target */ ;\
1312 ldxa [%g0]ASI_DMMU, %g4 ;\
1313 cmp %g6, FAST_IMMU_MISS_TT ;\
1314 movne %icc, %g4, %g1 ;\
1315 stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */ ;\
1316 stxa %g0, [%g3 + TRAP_ENT_TR]%asi ;\
1317 TRACE_NEXT(%g3, %g4, %g6)
1318 #else
1319 #define TRACE_TSBHIT(ttextra)
1320 #endif
1321
1322 #if defined(lint)
1323
1324 struct scb trap_table;
1325 struct scb scb; /* trap_table/scb are the same object */
1326
1327 #else /* lint */
1328
1329 /*
1330 * =======================================================================
1331 * SPARC V9 TRAP TABLE
1332 *
1333 * The trap table is divided into two halves: the first half is used when
1334 * taking traps when TL=0; the second half is used when taking traps from
1335 * TL>0. Note that handlers in the second half of the table might not be able
1336 * to make the same assumptions as handlers in the first half of the table.
1337 *
1338 * Worst case trap nesting so far:
1339 *
1340 * at TL=0 client issues software trap requesting service
1341 * at TL=1 nucleus wants a register window
1342 * at TL=2 register window clean/spill/fill takes a TLB miss
1343 * at TL=3 processing TLB miss
1344 * at TL=4 handle asynchronous error
1345 *
1346 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1347 *
1348 * =======================================================================
1349 */
1350 .section ".text"
1351 .align 4
1352 .global trap_table, scb, trap_table0, trap_table1, etrap_table
1353 .type trap_table, #object
1354 .type scb, #object
1355 trap_table:
1356 scb:
1357 trap_table0:
1358 /* hardware traps */
1359 NOT; /* 000 reserved */
1360 RED; /* 001 power on reset */
1361 RED; /* 002 watchdog reset */
1362 RED; /* 003 externally initiated reset */
1363 RED; /* 004 software initiated reset */
1364 RED; /* 005 red mode exception */
1365 NOT; NOT; /* 006 - 007 reserved */
1366 IMMU_EXCEPTION; /* 008 instruction access exception */
1367 NOT; /* 009 instruction access MMU miss */
1368 ASYNC_TRAP(T_INSTR_ERROR, trace_gen, tt0_iae);
1369 /* 00A instruction access error */
1370 NOT; NOT4; /* 00B - 00F reserved */
1371 ILLTRAP_INSTR; /* 010 illegal instruction */
1372 TRAP(T_PRIV_INSTR); /* 011 privileged opcode */
1373 NOT; /* 012 unimplemented LDD */
1374 NOT; /* 013 unimplemented STD */
1375 NOT4; NOT4; NOT4; /* 014 - 01F reserved */
1376 FP_DISABLED_TRAP; /* 020 fp disabled */
1377 FP_IEEE_TRAP; /* 021 fp exception ieee 754 */
1378 FP_TRAP; /* 022 fp exception other */
1379 TAG_OVERFLOW; /* 023 tag overflow */
1380 CLEAN_WINDOW; /* 024 - 027 clean window */
1381 DIV_BY_ZERO; /* 028 division by zero */
1382 NOT; /* 029 internal processor error */
1383 NOT; NOT; NOT4; /* 02A - 02F reserved */
1384 DMMU_EXCEPTION; /* 030 data access exception */
1385 NOT; /* 031 data access MMU miss */
1386 ASYNC_TRAP(T_DATA_ERROR, trace_gen, tt0_dae);
1387 /* 032 data access error */
1388 NOT; /* 033 data access protection */
1389 DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */
1390 DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */
1391 DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */
1392 DMMU_EXC_AG_PRIV; /* 037 privileged action */
1393 NOT; /* 038 LDQF mem address not aligned */
1394 NOT; /* 039 STQF mem address not aligned */
1395 NOT; NOT; NOT4; /* 03A - 03F reserved */
1396 LABELED_BAD(tt0_asdat); /* 040 async data error */
1397 LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */
1398 LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */
1399 LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */
1400 LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */
1401 LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */
1402 LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */
1403 LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */
1404 LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */
1405 LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */
1406 LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */
1407 LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */
1408 LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */
1409 LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */
1410 LEVEL14_INTERRUPT; /* 04E interrupt level 14 */
1411 LEVEL15_INTERRUPT; /* 04F interrupt level 15 */
1412 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */
1413 VECTOR_INTERRUPT; /* 060 interrupt vector */
1414 GOTO(kmdb_trap); /* 061 PA watchpoint */
1415 GOTO(kmdb_trap); /* 062 VA watchpoint */
1416 GOTO_TT(ce_err, trace_gen); /* 063 corrected ECC error */
1417 ITLB_MISS(tt0); /* 064 instruction access MMU miss */
1418 DTLB_MISS(tt0); /* 068 data access MMU miss */
1419 DTLB_PROT; /* 06C data access protection */
1420 LABELED_BAD(tt0_fecc); /* 070 fast ecache ECC error */
1421 LABELED_BAD(tt0_dperr); /* 071 Cheetah+ dcache parity error */
1422 LABELED_BAD(tt0_iperr); /* 072 Cheetah+ icache parity error */
1423 NOT; /* 073 reserved */
1424 NOT4; NOT4; NOT4; /* 074 - 07F reserved */
1425 NOT4; /* 080 spill 0 normal */
1426 SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */
1427 SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */
1428 SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */
1429 SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */
1430 SPILL_32bit(not); /* 094 spill 5 normal */
1431 SPILL_64bit(not); /* 098 spill 6 normal */
1432 SPILL_mixed; /* 09C spill 7 normal */
1433 NOT4; /* 0A0 spill 0 other */
1434 SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */
1435 SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */
1436 SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */
1437 SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */
1438 NOT4; /* 0B4 spill 5 other */
1439 NOT4; /* 0B8 spill 6 other */
1440 NOT4; /* 0BC spill 7 other */
1441 NOT4; /* 0C0 fill 0 normal */
1442 FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */
1443 FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */
1444 FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */
1445 FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */
1446 FILL_32bit(not); /* 0D4 fill 5 normal */
1447 FILL_64bit(not); /* 0D8 fill 6 normal */
1448 FILL_mixed; /* 0DC fill 7 normal */
1449 NOT4; /* 0E0 fill 0 other */
1450 NOT4; /* 0E4 fill 1 other */
1451 NOT4; /* 0E8 fill 2 other */
1452 NOT4; /* 0EC fill 3 other */
1453 NOT4; /* 0F0 fill 4 other */
1454 NOT4; /* 0F4 fill 5 other */
1455 NOT4; /* 0F8 fill 6 other */
1456 NOT4; /* 0FC fill 7 other */
1457 /* user traps */
1458 GOTO(syscall_trap_4x); /* 100 old system call */
1459 TRAP(T_BREAKPOINT); /* 101 user breakpoint */
1460 TRAP(T_DIV0); /* 102 user divide by zero */
1461 FLUSHW(tt0_flushw); /* 103 flush windows */
1462 GOTO(.clean_windows); /* 104 clean windows */
1463 BAD; /* 105 range check ?? */
1464 GOTO(.fix_alignment); /* 106 do unaligned references */
1465 BAD; /* 107 unused */
1466 SYSCALL_TRAP32; /* 108 ILP32 system call on LP64 */
1467 GOTO(set_trap0_addr); /* 109 set trap0 address */
1468 BAD; BAD; BAD4; /* 10A - 10F unused */
1469 TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */
1470 GOTO(.getcc); /* 120 get condition codes */
1471 GOTO(.setcc); /* 121 set condition codes */
1472 GOTO(.getpsr); /* 122 get psr */
1473 GOTO(.setpsr); /* 123 set psr (some fields) */
1474 GOTO(get_timestamp); /* 124 get timestamp */
1475 GOTO(get_virtime); /* 125 get lwp virtual time */
1476 PRIV(self_xcall); /* 126 self xcall */
1477 GOTO(get_hrestime); /* 127 get hrestime */
1478 BAD; /* 128 ST_SETV9STACK */
1479 GOTO(.getlgrp); /* 129 get lgrpid */
1480 BAD; BAD; BAD4; /* 12A - 12F unused */
1481 BAD4; BAD4; /* 130 - 137 unused */
1482 DTRACE_PID; /* 138 dtrace pid tracing provider */
1483 BAD; /* 139 unused */
1484 DTRACE_RETURN; /* 13A dtrace pid return probe */
1485 BAD; BAD4; /* 13B - 13F unused */
1486 SYSCALL_TRAP; /* 140 LP64 system call */
1487 SYSCALL(nosys); /* 141 unused system call trap */
1488 #ifdef DEBUG_USER_TRAPTRACECTL
1489 GOTO(.traptrace_freeze); /* 142 freeze traptrace */
1490 GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */
1491 #else
1492 SYSCALL(nosys); /* 142 unused system call trap */
1493 SYSCALL(nosys); /* 143 unused system call trap */
1494 #endif
1495 BAD4; BAD4; BAD4; /* 144 - 14F unused */
1496 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */
1497 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */
1498 BAD; /* 170 - unused */
1499 BAD; /* 171 - unused */
1500 BAD; BAD; /* 172 - 173 unused */
1501 BAD4; BAD4; /* 174 - 17B unused */
1502 #ifdef PTL1_PANIC_DEBUG
1503 mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1504 /* 17C test ptl1_panic */
1505 #else
1506 BAD; /* 17C unused */
1507 #endif /* PTL1_PANIC_DEBUG */
1508 PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */
1509 PRIV(kmdb_trap); /* 17E kmdb breakpoint */
1510 PRIV(kctx_obp_bpt); /* 17F obp breakpoint */
1511 /* reserved */
1512 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */
1513 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */
1514 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */
1515 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */
1516 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */
1517 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */
1518 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */
1519 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */
1520 trap_table1:
1521 NOT4; NOT4; NOT; NOT; /* 000 - 009 unused */
1522 ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen, tt1_iae);
1523 /* 00A instruction access error */
1524 NOT; NOT4; /* 00B - 00F unused */
1525 NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */
1526 NOT4; /* 020 - 023 unused */
1527 CLEAN_WINDOW; /* 024 - 027 clean window */
1528 NOT4; NOT4; /* 028 - 02F unused */
1529 DMMU_EXCEPTION_TL1; /* 030 data access exception */
1530 NOT; /* 031 unused */
1531 ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen, tt1_dae);
1532 /* 032 data access error */
1533 NOT; /* 033 unused */
1534 MISALIGN_ADDR_TL1; /* 034 mem address not aligned */
1535 NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */
1536 LABELED_BAD(tt1_asdat); /* 040 async data error */
1537 NOT; NOT; NOT; /* 041 - 043 unused */
1538 NOT4; NOT4; NOT4; /* 044 - 04F unused */
1539 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */
1540 NOT; /* 060 unused */
1541 GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */
1542 GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */
1543 GOTO_TT(ce_err_tl1, trace_gen); /* 063 corrected ECC error */
1544 ITLB_MISS(tt1); /* 064 instruction access MMU miss */
1545 DTLB_MISS(tt1); /* 068 data access MMU miss */
1546 DTLB_PROT; /* 06C data access protection */
1547 LABELED_BAD(tt1_fecc); /* 070 fast ecache ECC error */
1548 LABELED_BAD(tt1_dperr); /* 071 Cheetah+ dcache parity error */
1549 LABELED_BAD(tt1_iperr); /* 072 Cheetah+ icache parity error */
1550 NOT; /* 073 reserved */
1551 NOT4; NOT4; NOT4; /* 074 - 07F reserved */
1552 NOT4; /* 080 spill 0 normal */
1553 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */
1554 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */
1555 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */
1556 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */
1557 SPILL_32bit(not); /* 094 spill 5 normal */
1558 SPILL_64bit(not); /* 098 spill 6 normal */
1559 SPILL_mixed; /* 09C spill 7 normal */
1560 NOT4; /* 0A0 spill 0 other */
1561 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */
1562 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */
1563 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */
1564 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */
1565 NOT4; /* 0B4 spill 5 other */
1566 NOT4; /* 0B8 spill 6 other */
1567 NOT4; /* 0BC spill 7 other */
1568 NOT4; /* 0C0 fill 0 normal */
1569 FILL_32bit_tt1(ASI_AIUP,fn1); /* 0C4 fill 1 normal */
1570 FILL_64bit_tt1(ASI_AIUP,fn1); /* 0C8 fill 2 normal */
1571 FILL_32bit_tt1(ASI_AIUP,fn1); /* 0CC fill 3 normal */
1572 FILL_64bit_tt1(ASI_AIUP,fn1); /* 0D0 fill 4 normal */
1573 FILL_32bit(not); /* 0D4 fill 5 normal */
1574 FILL_64bit(not); /* 0D8 fill 6 normal */
1575 FILL_mixed; /* 0DC fill 7 normal */
1576 NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */
1577 NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */
1578 LABELED_BAD(tt1_swtrap0); /* 100 fast ecache ECC error (cont) */
1579 LABELED_BAD(tt1_swtrap1); /* 101 Ch+ D$ parity error (cont) */
1580 LABELED_BAD(tt1_swtrap2); /* 102 Ch+ I$ parity error (cont) */
1581 NOT; /* 103 reserved */
1582 /*
1583 * We only reserve the above four special case soft traps for code running
1584 * at TL>0, so we can truncate the trap table here.
1585 */
1586 etrap_table:
1587 .size trap_table, (.-trap_table)
1588 .size scb, (.-scb)
1589
1590 /*
1591 * We get to exec_fault in the case of an instruction miss and tte
1592 * has no execute bit set. We go to tl0 to handle it.
1593 *
1594 * g1 = tsbe pointer (in/clobbered)
1595 * g2 = tag access register (in)
1596 * g3 - g4 = scratch (clobbered)
1597 * g5 = tsbe data (in)
1598 * g6 = scratch (clobbered)
1599 */
1600 ALTENTRY(exec_fault)
1601 TRACE_TSBHIT(0x200)
1602 SWITCH_GLOBALS
1603 mov MMU_TAG_ACCESS, %g4
1604 ldxa [%g4]ASI_IMMU, %g2 ! arg1 = addr
1605 mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype
1606 set trap, %g1
1607 ba,pt %xcc, sys_trap
1608 mov -1, %g4
1609
1610 .mmu_exception_not_aligned:
1611 rdpr %tstate, %g1
1612 btst TSTATE_PRIV, %g1
1613 bnz,pn %icc, 2f
1614 nop
1615 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1616 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1617 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1618 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1619 brz,pt %g5, 2f
1620 nop
1621 ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap?
1622 brz,pn %g5, 2f
1623 nop
1624 btst 1, %sp
1625 bz,pt %xcc, 1f ! 32 bit user program
1626 nop
1627 ba,pt %xcc, .setup_v9utrap ! 64 bit user program
1628 nop
1629 1:
1630 ba,pt %xcc, .setup_utrap
1631 or %g2, %g0, %g7
1632 2:
1633 ba,pt %xcc, .mmu_exception_end
1634 mov T_ALIGNMENT, %g1
1635
1636 .mmu_priv_exception:
1637 rdpr %tstate, %g1
1638 btst TSTATE_PRIV, %g1
1639 bnz,pn %icc, 1f
1640 nop
1641 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1642 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1643 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1644 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1645 brz,pt %g5, 1f
1646 nop
1647 ldn [%g5 + P_UTRAP16], %g5
1648 brnz,pt %g5, .setup_v9utrap
1649 nop
1650 1:
1651 mov T_PRIV_INSTR, %g1
1652
1653 .mmu_exception_end:
1654 CPU_INDEX(%g4, %g5)
1655 set cpu_core, %g5
1656 sllx %g4, CPU_CORE_SHIFT, %g4
1657 add %g4, %g5, %g4
1658 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5
1659 andcc %g5, CPU_DTRACE_NOFAULT, %g0
1660 bz %xcc, .mmu_exception_tlb_chk
1661 or %g5, CPU_DTRACE_BADADDR, %g5
1662 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS]
1663 done
1664
1665 .mmu_exception_tlb_chk:
1666 GET_CPU_IMPL(%g5) ! check SFSR.FT to see if this
1667 cmp %g5, PANTHER_IMPL ! is a TLB parity error. But
1668 bne 2f ! we only do this check while
1669 mov 1, %g4 ! running on Panther CPUs
1670 sllx %g4, PN_SFSR_PARITY_SHIFT, %g4 ! since US-I/II use the same
1671 andcc %g3, %g4, %g0 ! bit for something else which
1672 bz 2f ! will be handled later.
1673 nop
1674 .mmu_exception_is_tlb_parity:
1675 .weak itlb_parity_trap
1676 .weak dtlb_parity_trap
1677 set itlb_parity_trap, %g4
1678 cmp %g1, T_INSTR_EXCEPTION ! branch to the itlb or
1679 be 3f ! dtlb parity handler
1680 nop ! if this trap is due
1681 set dtlb_parity_trap, %g4
1682 cmp %g1, T_DATA_EXCEPTION ! to a IMMU exception
1683 be 3f ! or DMMU exception.
1684 nop
1685 2:
1686 sllx %g3, 32, %g3
1687 or %g3, %g1, %g3
1688 set trap, %g1
1689 ba,pt %xcc, sys_trap
1690 sub %g0, 1, %g4
1691 3:
1692 jmp %g4 ! off to the appropriate
1693 nop ! TLB parity handler
1694
1695 .fp_disabled:
1696 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1697 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1698 #ifdef SF_ERRATA_30 /* call causes fp-disabled */
1699 brz,a,pn %g1, 2f
1700 nop
1701 #endif
1702 rdpr %tstate, %g4
1703 btst TSTATE_PRIV, %g4
1704 #ifdef SF_ERRATA_30 /* call causes fp-disabled */
1705 bnz,pn %icc, 2f
1706 nop
1707 #else
1708 bnz,a,pn %icc, ptl1_panic
1709 mov PTL1_BAD_FPTRAP, %g1
1710 #endif
1711 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1712 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1713 brz,a,pt %g5, 2f
1714 nop
1715 ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap?
1716 brz,a,pn %g5, 2f
1717 nop
1718 btst 1, %sp
1719 bz,a,pt %xcc, 1f ! 32 bit user program
1720 nop
1721 ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program
1722 nop
1723 1:
1724 ba,pt %xcc, .setup_utrap
1725 or %g0, %g0, %g7
1726 2:
1727 set fp_disabled, %g1
1728 ba,pt %xcc, sys_trap
1729 sub %g0, 1, %g4
1730
1731 .fp_ieee_exception:
1732 rdpr %tstate, %g1
1733 btst TSTATE_PRIV, %g1
1734 bnz,a,pn %icc, ptl1_panic
1735 mov PTL1_BAD_FPTRAP, %g1
1736 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1737 stx %fsr, [%g1 + CPU_TMP1]
1738 ldx [%g1 + CPU_TMP1], %g2
1739 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1740 ldn [%g1 + T_PROCP], %g1 ! load proc pointer
1741 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
1742 brz,a,pt %g5, 1f
1743 nop
1744 ldn [%g5 + P_UTRAP8], %g5
1745 brnz,a,pt %g5, .setup_v9utrap
1746 nop
1747 1:
1748 set _fp_ieee_exception, %g1
1749 ba,pt %xcc, sys_trap
1750 sub %g0, 1, %g4
1751
1752 /*
1753 * Register Inputs:
1754 * %g5 user trap handler
1755 * %g7 misaligned addr - for alignment traps only
1756 */
1757 .setup_utrap:
1758 set trap, %g1 ! setup in case we go
1759 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1760 sub %g0, 1, %g4 ! the save instruction below
1761
1762 /*
1763 * If the DTrace pid provider is single stepping a copied-out
1764 * instruction, t->t_dtrace_step will be set. In that case we need
1765 * to abort the single-stepping (since execution of the instruction
1766 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1767 */
1768 save %sp, -SA(MINFRAME32), %sp ! window for trap handler
1769 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1770 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1771 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1772 rdpr %tnpc, %l2 ! arg1 == tnpc
1773 brz,pt %g2, 1f
1774 rdpr %tpc, %l1 ! arg0 == tpc
1775
1776 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1777 ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step)
1778 brz,pt %g2, 1f
1779 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1780 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1781 1:
1782 mov %g7, %l3 ! arg2 == misaligned address
1783
1784 rdpr %tstate, %g1 ! cwp for trap handler
1785 rdpr %cwp, %g4
1786 bclr TSTATE_CWP_MASK, %g1
1787 wrpr %g1, %g4, %tstate
1788 wrpr %g0, %g5, %tnpc ! trap handler address
1789 FAST_TRAP_DONE
1790 /* NOTREACHED */
1791
1792 .check_v9utrap:
1793 rdpr %tstate, %g1
1794 btst TSTATE_PRIV, %g1
1795 bnz,a,pn %icc, 3f
1796 nop
1797 CPU_ADDR(%g4, %g1) ! load CPU struct addr
1798 ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer
1799 ldn [%g5 + T_PROCP], %g5 ! load proc pointer
1800 ldn [%g5 + P_UTRAPS], %g5 ! are there utraps?
1801
1802 cmp %g3, T_SOFTWARE_TRAP
1803 bne,a,pt %icc, 1f
1804 nop
1805
1806 brz,pt %g5, 3f ! if p_utraps == NULL goto trap()
1807 rdpr %tt, %g3 ! delay - get actual hw trap type
1808
1809 sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1810 ba,pt %icc, 2f
1811 smul %g1, CPTRSIZE, %g2
1812 1:
1813 brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap()
1814 nop
1815
1816 cmp %g3, T_UNIMP_INSTR
1817 bne,a,pt %icc, 2f
1818 nop
1819
1820 mov 1, %g1
1821 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1822 rdpr %tpc, %g1 ! ld trapping instruction using
1823 lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault
1824 st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1825
1826 sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction
1827 andcc %g1, %g4, %g4 ! and instruction with mask
1828 bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP
1829 nop ! fall thru to setup
1830 2:
1831 ldn [%g5 + %g2], %g5
1832 brnz,a,pt %g5, .setup_v9utrap
1833 nop
1834 3:
1835 set trap, %g1
1836 ba,pt %xcc, sys_trap
1837 sub %g0, 1, %g4
1838 /* NOTREACHED */
1839
1840 /*
1841 * Register Inputs:
1842 * %g5 user trap handler
1843 */
1844 .setup_v9utrap:
1845 set trap, %g1 ! setup in case we go
1846 mov T_FLUSH_PCB, %g3 ! through sys_trap on
1847 sub %g0, 1, %g4 ! the save instruction below
1848
1849 /*
1850 * If the DTrace pid provider is single stepping a copied-out
1851 * instruction, t->t_dtrace_step will be set. In that case we need
1852 * to abort the single-stepping (since execution of the instruction
1853 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1854 */
1855 save %sp, -SA(MINFRAME64), %sp ! window for trap handler
1856 CPU_ADDR(%g1, %g4) ! load CPU struct addr
1857 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
1858 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
1859 rdpr %tnpc, %l7 ! arg1 == tnpc
1860 brz,pt %g2, 1f
1861 rdpr %tpc, %l6 ! arg0 == tpc
1862
1863 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
1864 ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step)
1865 brz,pt %g2, 1f
1866 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
1867 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1868 1:
1869 rdpr %tstate, %g2 ! cwp for trap handler
1870 rdpr %cwp, %g4
1871 bclr TSTATE_CWP_MASK, %g2
1872 wrpr %g2, %g4, %tstate
1873
1874 ldn [%g1 + T_PROCP], %g4 ! load proc pointer
1875 ldn [%g4 + P_AS], %g4 ! load as pointer
1876 ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit
1877 cmp %l7, %g4 ! check for single-step set
1878 bne,pt %xcc, 4f
1879 nop
1880 ldn [%g1 + T_LWP], %g1 ! load klwp pointer
1881 ld [%g1 + PCB_STEP], %g4 ! load single-step flag
1882 cmp %g4, STEP_ACTIVE ! step flags set in pcb?
1883 bne,pt %icc, 4f
1884 nop
1885 stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb
1886 mov %l7, %g4 ! on entry to precise user trap
1887 add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc
1888 ! at time of trap
1889 wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS,
1890 ! %g4 == userlimit
1891 FAST_TRAP_DONE
1892 /* NOTREACHED */
1893 4:
1894 wrpr %g0, %g5, %tnpc ! trap handler address
1895 FAST_TRAP_DONE_CHK_INTR
1896 /* NOTREACHED */
1897
1898 .fp_exception:
1899 CPU_ADDR(%g1, %g4)
1900 stx %fsr, [%g1 + CPU_TMP1]
1901 ldx [%g1 + CPU_TMP1], %g2
1902
1903 /*
1904 * Cheetah takes unfinished_FPop trap for certain range of operands
1905 * to the "fitos" instruction. Instead of going through the slow
1906 * software emulation path, we try to simulate the "fitos" instruction
1907 * via "fitod" and "fdtos" provided the following conditions are met:
1908 *
1909 * fpu_exists is set (if DEBUG)
1910 * not in privileged mode
1911 * ftt is unfinished_FPop
1912 * NXM IEEE trap is not enabled
1913 * instruction at %tpc is "fitos"
1914 *
1915 * Usage:
1916 * %g1 per cpu address
1917 * %g2 %fsr
1918 * %g6 user instruction
1919 *
1920 * Note that we can take a memory access related trap while trying
1921 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1922 * flag to catch those traps and let the SFMMU code deal with page
1923 * fault and data access exception.
1924 */
1925 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1926 sethi %hi(fpu_exists), %g7
1927 ld [%g7 + %lo(fpu_exists)], %g7
1928 brz,pn %g7, .fp_exception_cont
1929 nop
1930 #endif
1931 rdpr %tstate, %g7 ! branch if in privileged mode
1932 btst TSTATE_PRIV, %g7
1933 bnz,pn %xcc, .fp_exception_cont
1934 srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr
1935 and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1936 cmp %g7, FTT_UNFIN
1937 set FSR_TEM_NX, %g5
1938 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop
1939 andcc %g2, %g5, %g0
1940 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled
1941 rdpr %tpc, %g5 ! get faulting PC
1942
1943 or %g0, 1, %g7
1944 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
1945 lda [%g5]ASI_USER, %g6 ! get user's instruction
1946 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
1947
1948 set FITOS_INSTR_MASK, %g7
1949 and %g6, %g7, %g7
1950 set FITOS_INSTR, %g5
1951 cmp %g7, %g5
1952 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR
1953 nop
1954
1955 /*
1956 * This is unfinished FPops trap for "fitos" instruction. We
1957 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1958 * sequence.
1959 *
1960 * We need a temporary FP register to do the conversion. Since
1961 * both source and destination operands for the "fitos" instruction
1962 * have to be within %f0-%f31, we use an FP register from the upper
1963 * half to guarantee that it won't collide with the source or the
1964 * dest operand. However, we do have to save and restore its value.
1965 *
1966 * We use %d62 as a temporary FP register for the conversion and
1967 * branch to appropriate instruction within the conversion tables
1968 * based upon the rs2 and rd values.
1969 */
1970
1971 std %d62, [%g1 + CPU_TMP1] ! save original value
1972
1973 srl %g6, FITOS_RS2_SHIFT, %g7
1974 and %g7, FITOS_REG_MASK, %g7
1975 set _fitos_fitod_table, %g4
1976 sllx %g7, 2, %g7
1977 jmp %g4 + %g7
1978 ba,pt %xcc, _fitos_fitod_done
1979 .empty
1980
1981 _fitos_fitod_table:
1982 fitod %f0, %d62
1983 fitod %f1, %d62
1984 fitod %f2, %d62
1985 fitod %f3, %d62
1986 fitod %f4, %d62
1987 fitod %f5, %d62
1988 fitod %f6, %d62
1989 fitod %f7, %d62
1990 fitod %f8, %d62
1991 fitod %f9, %d62
1992 fitod %f10, %d62
1993 fitod %f11, %d62
1994 fitod %f12, %d62
1995 fitod %f13, %d62
1996 fitod %f14, %d62
1997 fitod %f15, %d62
1998 fitod %f16, %d62
1999 fitod %f17, %d62
2000 fitod %f18, %d62
2001 fitod %f19, %d62
2002 fitod %f20, %d62
2003 fitod %f21, %d62
2004 fitod %f22, %d62
2005 fitod %f23, %d62
2006 fitod %f24, %d62
2007 fitod %f25, %d62
2008 fitod %f26, %d62
2009 fitod %f27, %d62
2010 fitod %f28, %d62
2011 fitod %f29, %d62
2012 fitod %f30, %d62
2013 fitod %f31, %d62
2014 _fitos_fitod_done:
2015
2016 /*
2017 * Now convert data back into single precision
2018 */
2019 srl %g6, FITOS_RD_SHIFT, %g7
2020 and %g7, FITOS_REG_MASK, %g7
2021 set _fitos_fdtos_table, %g4
2022 sllx %g7, 2, %g7
2023 jmp %g4 + %g7
2024 ba,pt %xcc, _fitos_fdtos_done
2025 .empty
2026
2027 _fitos_fdtos_table:
2028 fdtos %d62, %f0
2029 fdtos %d62, %f1
2030 fdtos %d62, %f2
2031 fdtos %d62, %f3
2032 fdtos %d62, %f4
2033 fdtos %d62, %f5
2034 fdtos %d62, %f6
2035 fdtos %d62, %f7
2036 fdtos %d62, %f8
2037 fdtos %d62, %f9
2038 fdtos %d62, %f10
2039 fdtos %d62, %f11
2040 fdtos %d62, %f12
2041 fdtos %d62, %f13
2042 fdtos %d62, %f14
2043 fdtos %d62, %f15
2044 fdtos %d62, %f16
2045 fdtos %d62, %f17
2046 fdtos %d62, %f18
2047 fdtos %d62, %f19
2048 fdtos %d62, %f20
2049 fdtos %d62, %f21
2050 fdtos %d62, %f22
2051 fdtos %d62, %f23
2052 fdtos %d62, %f24
2053 fdtos %d62, %f25
2054 fdtos %d62, %f26
2055 fdtos %d62, %f27
2056 fdtos %d62, %f28
2057 fdtos %d62, %f29
2058 fdtos %d62, %f30
2059 fdtos %d62, %f31
2060 _fitos_fdtos_done:
2061
2062 ldd [%g1 + CPU_TMP1], %d62 ! restore %d62
2063
2064 #if DEBUG
2065 /*
2066 * Update FPop_unfinished trap kstat
2067 */
2068 set fpustat+FPUSTAT_UNFIN_KSTAT, %g7
2069 ldx [%g7], %g5
2070 1:
2071 add %g5, 1, %g6
2072
2073 casxa [%g7] ASI_N, %g5, %g6
2074 cmp %g5, %g6
2075 bne,a,pn %xcc, 1b
2076 or %g0, %g6, %g5
2077
2078 /*
2079 * Update fpu_sim_fitos kstat
2080 */
2081 set fpuinfo+FPUINFO_FITOS_KSTAT, %g7
2082 ldx [%g7], %g5
2083 1:
2084 add %g5, 1, %g6
2085
2086 casxa [%g7] ASI_N, %g5, %g6
2087 cmp %g5, %g6
2088 bne,a,pn %xcc, 1b
2089 or %g0, %g6, %g5
2090 #endif /* DEBUG */
2091
2092 FAST_TRAP_DONE
2093
2094 .fp_exception_cont:
2095 /*
2096 * Let _fp_exception deal with simulating FPop instruction.
2097 * Note that we need to pass %fsr in %g2 (already read above).
2098 */
2099
2100 set _fp_exception, %g1
2101 ba,pt %xcc, sys_trap
2102 sub %g0, 1, %g4
2103
2104 .global opl_cleanw_patch
2105 opl_cleanw_patch:
2106 .clean_windows:
2107 set trap, %g1
2108 mov T_FLUSH_PCB, %g3
2109 sub %g0, 1, %g4
2110 save
2111 flushw
2112 restore
2113 wrpr %g0, %g0, %cleanwin ! no clean windows
2114
2115 CPU_ADDR(%g4, %g5)
2116 ldn [%g4 + CPU_MPCB], %g4
2117 brz,a,pn %g4, 1f
2118 nop
2119 ld [%g4 + MPCB_WSTATE], %g5
2120 add %g5, WSTATE_CLEAN_OFFSET, %g5
2121 wrpr %g0, %g5, %wstate
2122 1: FAST_TRAP_DONE
2123
2124 /*
2125 * .spill_clean: clean the previous window, restore the wstate, and
2126 * "done".
2127 *
2128 * Entry: %g7 contains new wstate
2129 */
2130 .spill_clean:
2131 sethi %hi(nwin_minus_one), %g5
2132 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
2133 rdpr %cwp, %g6 ! %g6 = %cwp
2134 deccc %g6 ! %g6--
2135 movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1
2136 wrpr %g6, %cwp
2137 TT_TRACE_L(trace_win)
2138 clr %l0
2139 clr %l1
2140 clr %l2
2141 clr %l3
2142 clr %l4
2143 clr %l5
2144 clr %l6
2145 clr %l7
2146 wrpr %g0, %g7, %wstate
2147 saved
2148 retry ! restores correct %cwp
2149
2150 .fix_alignment:
2151 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2152 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
2153 ldn [%g1 + T_PROCP], %g1
2154 mov 1, %g2
2155 stb %g2, [%g1 + P_FIXALIGNMENT]
2156 FAST_TRAP_DONE
2157
2158 #define STDF_REG(REG, ADDR, TMP) \
2159 sll REG, 3, REG ;\
2160 mark1: set start1, TMP ;\
2161 jmp REG + TMP ;\
2162 nop ;\
2163 start1: ba,pt %xcc, done1 ;\
2164 std %f0, [ADDR + CPU_TMP1] ;\
2165 ba,pt %xcc, done1 ;\
2166 std %f32, [ADDR + CPU_TMP1] ;\
2167 ba,pt %xcc, done1 ;\
2168 std %f2, [ADDR + CPU_TMP1] ;\
2169 ba,pt %xcc, done1 ;\
2170 std %f34, [ADDR + CPU_TMP1] ;\
2171 ba,pt %xcc, done1 ;\
2172 std %f4, [ADDR + CPU_TMP1] ;\
2173 ba,pt %xcc, done1 ;\
2174 std %f36, [ADDR + CPU_TMP1] ;\
2175 ba,pt %xcc, done1 ;\
2176 std %f6, [ADDR + CPU_TMP1] ;\
2177 ba,pt %xcc, done1 ;\
2178 std %f38, [ADDR + CPU_TMP1] ;\
2179 ba,pt %xcc, done1 ;\
2180 std %f8, [ADDR + CPU_TMP1] ;\
2181 ba,pt %xcc, done1 ;\
2182 std %f40, [ADDR + CPU_TMP1] ;\
2183 ba,pt %xcc, done1 ;\
2184 std %f10, [ADDR + CPU_TMP1] ;\
2185 ba,pt %xcc, done1 ;\
2186 std %f42, [ADDR + CPU_TMP1] ;\
2187 ba,pt %xcc, done1 ;\
2188 std %f12, [ADDR + CPU_TMP1] ;\
2189 ba,pt %xcc, done1 ;\
2190 std %f44, [ADDR + CPU_TMP1] ;\
2191 ba,pt %xcc, done1 ;\
2192 std %f14, [ADDR + CPU_TMP1] ;\
2193 ba,pt %xcc, done1 ;\
2194 std %f46, [ADDR + CPU_TMP1] ;\
2195 ba,pt %xcc, done1 ;\
2196 std %f16, [ADDR + CPU_TMP1] ;\
2197 ba,pt %xcc, done1 ;\
2198 std %f48, [ADDR + CPU_TMP1] ;\
2199 ba,pt %xcc, done1 ;\
2200 std %f18, [ADDR + CPU_TMP1] ;\
2201 ba,pt %xcc, done1 ;\
2202 std %f50, [ADDR + CPU_TMP1] ;\
2203 ba,pt %xcc, done1 ;\
2204 std %f20, [ADDR + CPU_TMP1] ;\
2205 ba,pt %xcc, done1 ;\
2206 std %f52, [ADDR + CPU_TMP1] ;\
2207 ba,pt %xcc, done1 ;\
2208 std %f22, [ADDR + CPU_TMP1] ;\
2209 ba,pt %xcc, done1 ;\
2210 std %f54, [ADDR + CPU_TMP1] ;\
2211 ba,pt %xcc, done1 ;\
2212 std %f24, [ADDR + CPU_TMP1] ;\
2213 ba,pt %xcc, done1 ;\
2214 std %f56, [ADDR + CPU_TMP1] ;\
2215 ba,pt %xcc, done1 ;\
2216 std %f26, [ADDR + CPU_TMP1] ;\
2217 ba,pt %xcc, done1 ;\
2218 std %f58, [ADDR + CPU_TMP1] ;\
2219 ba,pt %xcc, done1 ;\
2220 std %f28, [ADDR + CPU_TMP1] ;\
2221 ba,pt %xcc, done1 ;\
2222 std %f60, [ADDR + CPU_TMP1] ;\
2223 ba,pt %xcc, done1 ;\
2224 std %f30, [ADDR + CPU_TMP1] ;\
2225 ba,pt %xcc, done1 ;\
2226 std %f62, [ADDR + CPU_TMP1] ;\
2227 done1:
2228
2229 #define LDDF_REG(REG, ADDR, TMP) \
2230 sll REG, 3, REG ;\
2231 mark2: set start2, TMP ;\
2232 jmp REG + TMP ;\
2233 nop ;\
2234 start2: ba,pt %xcc, done2 ;\
2235 ldd [ADDR + CPU_TMP1], %f0 ;\
2236 ba,pt %xcc, done2 ;\
2237 ldd [ADDR + CPU_TMP1], %f32 ;\
2238 ba,pt %xcc, done2 ;\
2239 ldd [ADDR + CPU_TMP1], %f2 ;\
2240 ba,pt %xcc, done2 ;\
2241 ldd [ADDR + CPU_TMP1], %f34 ;\
2242 ba,pt %xcc, done2 ;\
2243 ldd [ADDR + CPU_TMP1], %f4 ;\
2244 ba,pt %xcc, done2 ;\
2245 ldd [ADDR + CPU_TMP1], %f36 ;\
2246 ba,pt %xcc, done2 ;\
2247 ldd [ADDR + CPU_TMP1], %f6 ;\
2248 ba,pt %xcc, done2 ;\
2249 ldd [ADDR + CPU_TMP1], %f38 ;\
2250 ba,pt %xcc, done2 ;\
2251 ldd [ADDR + CPU_TMP1], %f8 ;\
2252 ba,pt %xcc, done2 ;\
2253 ldd [ADDR + CPU_TMP1], %f40 ;\
2254 ba,pt %xcc, done2 ;\
2255 ldd [ADDR + CPU_TMP1], %f10 ;\
2256 ba,pt %xcc, done2 ;\
2257 ldd [ADDR + CPU_TMP1], %f42 ;\
2258 ba,pt %xcc, done2 ;\
2259 ldd [ADDR + CPU_TMP1], %f12 ;\
2260 ba,pt %xcc, done2 ;\
2261 ldd [ADDR + CPU_TMP1], %f44 ;\
2262 ba,pt %xcc, done2 ;\
2263 ldd [ADDR + CPU_TMP1], %f14 ;\
2264 ba,pt %xcc, done2 ;\
2265 ldd [ADDR + CPU_TMP1], %f46 ;\
2266 ba,pt %xcc, done2 ;\
2267 ldd [ADDR + CPU_TMP1], %f16 ;\
2268 ba,pt %xcc, done2 ;\
2269 ldd [ADDR + CPU_TMP1], %f48 ;\
2270 ba,pt %xcc, done2 ;\
2271 ldd [ADDR + CPU_TMP1], %f18 ;\
2272 ba,pt %xcc, done2 ;\
2273 ldd [ADDR + CPU_TMP1], %f50 ;\
2274 ba,pt %xcc, done2 ;\
2275 ldd [ADDR + CPU_TMP1], %f20 ;\
2276 ba,pt %xcc, done2 ;\
2277 ldd [ADDR + CPU_TMP1], %f52 ;\
2278 ba,pt %xcc, done2 ;\
2279 ldd [ADDR + CPU_TMP1], %f22 ;\
2280 ba,pt %xcc, done2 ;\
2281 ldd [ADDR + CPU_TMP1], %f54 ;\
2282 ba,pt %xcc, done2 ;\
2283 ldd [ADDR + CPU_TMP1], %f24 ;\
2284 ba,pt %xcc, done2 ;\
2285 ldd [ADDR + CPU_TMP1], %f56 ;\
2286 ba,pt %xcc, done2 ;\
2287 ldd [ADDR + CPU_TMP1], %f26 ;\
2288 ba,pt %xcc, done2 ;\
2289 ldd [ADDR + CPU_TMP1], %f58 ;\
2290 ba,pt %xcc, done2 ;\
2291 ldd [ADDR + CPU_TMP1], %f28 ;\
2292 ba,pt %xcc, done2 ;\
2293 ldd [ADDR + CPU_TMP1], %f60 ;\
2294 ba,pt %xcc, done2 ;\
2295 ldd [ADDR + CPU_TMP1], %f30 ;\
2296 ba,pt %xcc, done2 ;\
2297 ldd [ADDR + CPU_TMP1], %f62 ;\
2298 done2:
2299
2300 .lddf_exception_not_aligned:
2301 /*
2302 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2303 */
2304 ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
2305
2306 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2307 sethi %hi(fpu_exists), %g2 ! check fpu_exists
2308 ld [%g2 + %lo(fpu_exists)], %g2
2309 brz,a,pn %g2, 4f
2310 nop
2311 #endif
2312 CPU_ADDR(%g1, %g4)
2313 or %g0, 1, %g4
2314 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2315
2316 rdpr %tpc, %g2
2317 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction
2318 srl %g6, 23, %g1 ! using ldda or not?
2319 and %g1, 1, %g1
2320 brz,a,pt %g1, 2f ! check for ldda instruction
2321 nop
2322 srl %g6, 13, %g1 ! check immflag
2323 and %g1, 1, %g1
2324 rdpr %tstate, %g2 ! %tstate in %g2
2325 brnz,a,pn %g1, 1f
2326 srl %g2, 31, %g1 ! get asi from %tstate
2327 srl %g6, 5, %g1 ! get asi from instruction
2328 and %g1, 0xFF, %g1 ! imm_asi field
2329 1:
2330 cmp %g1, ASI_P ! primary address space
2331 be,a,pt %icc, 2f
2332 nop
2333 cmp %g1, ASI_PNF ! primary no fault address space
2334 be,a,pt %icc, 2f
2335 nop
2336 cmp %g1, ASI_S ! secondary address space
2337 be,a,pt %icc, 2f
2338 nop
2339 cmp %g1, ASI_SNF ! secondary no fault address space
2340 bne,a,pn %icc, 3f
2341 nop
2342 2:
2343 lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data
2344 add %g5, 4, %g5 ! increment misaligned data address
2345 lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data
2346
2347 sllx %g7, 32, %g7
2348 or %g5, %g7, %g5 ! combine data
2349 CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis
2350 stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1
2351
2352 srl %g6, 25, %g3 ! %g6 has the instruction
2353 and %g3, 0x1F, %g3 ! %g3 has rd
2354 LDDF_REG(%g3, %g7, %g4)
2355
2356 CPU_ADDR(%g1, %g4)
2357 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2358 FAST_TRAP_DONE
2359 3:
2360 CPU_ADDR(%g1, %g4)
2361 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2362 4:
2363 set T_USER, %g3 ! trap type in %g3
2364 or %g3, T_LDDF_ALIGN, %g3
2365 mov %g5, %g2 ! misaligned vaddr in %g2
2366 set fpu_trap, %g1 ! goto C for the little and
2367 ba,pt %xcc, sys_trap ! no fault little asi's
2368 sub %g0, 1, %g4
2369
2370 .stdf_exception_not_aligned:
2371 /*
2372 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2373 */
2374 ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
2375
2376 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2377 sethi %hi(fpu_exists), %g7 ! check fpu_exists
2378 ld [%g7 + %lo(fpu_exists)], %g3
2379 brz,a,pn %g3, 4f
2380 nop
2381 #endif
2382 CPU_ADDR(%g1, %g4)
2383 or %g0, 1, %g4
2384 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2385
2386 rdpr %tpc, %g2
2387 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction
2388
2389 srl %g6, 23, %g1 ! using stda or not?
2390 and %g1, 1, %g1
2391 brz,a,pt %g1, 2f ! check for stda instruction
2392 nop
2393 srl %g6, 13, %g1 ! check immflag
2394 and %g1, 1, %g1
2395 rdpr %tstate, %g2 ! %tstate in %g2
2396 brnz,a,pn %g1, 1f
2397 srl %g2, 31, %g1 ! get asi from %tstate
2398 srl %g6, 5, %g1 ! get asi from instruction
2399 and %g1, 0xFF, %g1 ! imm_asi field
2400 1:
2401 cmp %g1, ASI_P ! primary address space
2402 be,a,pt %icc, 2f
2403 nop
2404 cmp %g1, ASI_S ! secondary address space
2405 bne,a,pn %icc, 3f
2406 nop
2407 2:
2408 srl %g6, 25, %g6
2409 and %g6, 0x1F, %g6 ! %g6 has rd
2410 CPU_ADDR(%g7, %g1)
2411 STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP)
2412
2413 ldx [%g7 + CPU_TMP1], %g6
2414 srlx %g6, 32, %g7
2415 stuwa %g7, [%g5]ASI_USER ! first half
2416 add %g5, 4, %g5 ! increment misaligned data address
2417 stuwa %g6, [%g5]ASI_USER ! second half
2418
2419 CPU_ADDR(%g1, %g4)
2420 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2421 FAST_TRAP_DONE
2422 3:
2423 CPU_ADDR(%g1, %g4)
2424 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2425 4:
2426 set T_USER, %g3 ! trap type in %g3
2427 or %g3, T_STDF_ALIGN, %g3
2428 mov %g5, %g2 ! misaligned vaddr in %g2
2429 set fpu_trap, %g1 ! goto C for the little and
2430 ba,pt %xcc, sys_trap ! nofault little asi's
2431 sub %g0, 1, %g4
2432
2433 #ifdef DEBUG_USER_TRAPTRACECTL
2434
2435 .traptrace_freeze:
2436 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2437 TT_TRACE_L(trace_win)
2438 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2439 set trap_freeze, %g1
2440 mov 1, %g2
2441 st %g2, [%g1]
2442 FAST_TRAP_DONE
2443
2444 .traptrace_unfreeze:
2445 set trap_freeze, %g1
2446 st %g0, [%g1]
2447 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
2448 TT_TRACE_L(trace_win)
2449 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
2450 FAST_TRAP_DONE
2451
2452 #endif /* DEBUG_USER_TRAPTRACECTL */
2453
2454 .getcc:
2455 CPU_ADDR(%g1, %g2)
2456 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2457 stx %o1, [%g1 + CPU_TMP2] ! save %o1
2458 rdpr %tstate, %g3 ! get tstate
2459 srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2460 set PSR_ICC, %g2
2461 and %o0, %g2, %o0 ! mask out the rest
2462 srl %o0, PSR_ICC_SHIFT, %o0 ! right justify
2463 rdpr %pstate, %o1
2464 wrpr %o1, PSTATE_AG, %pstate ! get into normal globals
2465 mov %o0, %g1 ! move ccr to normal %g1
2466 wrpr %g0, %o1, %pstate ! back into alternate globals
2467 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2468 ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
2469 FAST_TRAP_DONE
2470
2471 .setcc:
2472 CPU_ADDR(%g1, %g2)
2473 stx %o0, [%g1 + CPU_TMP1] ! save %o0
2474 stx %o1, [%g1 + CPU_TMP2] ! save %o1
2475 rdpr %pstate, %o0
2476 wrpr %o0, PSTATE_AG, %pstate ! get into normal globals
2477 mov %g1, %o1
2478 wrpr %g0, %o0, %pstate ! back to alternates
2479 sll %o1, PSR_ICC_SHIFT, %g2
2480 set PSR_ICC, %g3
2481 and %g2, %g3, %g2 ! mask out rest
2482 sllx %g2, PSR_TSTATE_CC_SHIFT, %g2
2483 rdpr %tstate, %g3 ! get tstate
2484 srl %g3, 0, %g3 ! clear upper word
2485 or %g3, %g2, %g3 ! or in new bits
2486 wrpr %g3, %tstate
2487 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
2488 ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
2489 FAST_TRAP_DONE
2490
2491 /*
2492 * getpsr(void)
2493 * Note that the xcc part of the ccr is not provided.
2494 * The V8 code shows why the V9 trap is not faster:
2495 * #define GETPSR_TRAP() \
2496 * mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2497 */
2498
2499 .type .getpsr, #function
2500 .getpsr:
2501 rdpr %tstate, %g1 ! get tstate
2502 srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
2503 set PSR_ICC, %g2
2504 and %o0, %g2, %o0 ! mask out the rest
2505
2506 rd %fprs, %g1 ! get fprs
2507 and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower
2508 sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef
2509 or %o0, %g2, %o0 ! or result into psr.ef
2510
2511 set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef
2512 or %o0, %g2, %o0 ! or psr.impl/ver
2513 FAST_TRAP_DONE
2514 SET_SIZE(.getpsr)
2515
2516 /*
2517 * setpsr(newpsr)
2518 * Note that there is no support for ccr.xcc in the V9 code.
2519 */
2520
2521 .type .setpsr, #function
2522 .setpsr:
2523 rdpr %tstate, %g1 ! get tstate
2524 ! setx TSTATE_V8_UBITS, %g2
2525 or %g0, CCR_ICC, %g3
2526 sllx %g3, TSTATE_CCR_SHIFT, %g2
2527
2528 andn %g1, %g2, %g1 ! zero current user bits
2529 set PSR_ICC, %g2
2530 and %g2, %o0, %g2 ! clear all but psr.icc bits
2531 sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc
2532 wrpr %g1, %g3, %tstate ! write tstate
2533
2534 set PSR_EF, %g2
2535 and %g2, %o0, %g2 ! clear all but fp enable bit
2536 srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef
2537 wr %g0, %g4, %fprs ! write fprs
2538
2539 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1
2540 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2541 ldn [%g2 + T_LWP], %g3 ! load klwp pointer
2542 ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer
2543 stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs
2544 srlx %g4, 2, %g4 ! shift fef value to bit 0
2545 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en
2546 FAST_TRAP_DONE
2547 SET_SIZE(.setpsr)
2548
2549 /*
2550 * getlgrp
2551 * get home lgrpid on which the calling thread is currently executing.
2552 */
2553 .type .getlgrp, #function
2554 .getlgrp:
2555 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2556 ld [%g1 + CPU_ID], %o0 ! load cpu_id
2557 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2558 ldn [%g2 + T_LPL], %g2 ! load lpl pointer
2559 ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
2560 sra %g1, 0, %o1
2561 FAST_TRAP_DONE
2562 SET_SIZE(.getlgrp)
2563
2564 /*
2565 * Entry for old 4.x trap (trap 0).
2566 */
2567 ENTRY_NP(syscall_trap_4x)
2568 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2569 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2570 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2571 ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr
2572 brz,pn %g2, 1f ! has it been set?
2573 st %l0, [%g1 + CPU_TMP1] ! delay - save some locals
2574 st %l1, [%g1 + CPU_TMP2]
2575 rdpr %tnpc, %l1 ! save old tnpc
2576 wrpr %g0, %g2, %tnpc ! setup tnpc
2577
2578 rdpr %pstate, %l0
2579 wrpr %l0, PSTATE_AG, %pstate ! switch to normal globals
2580 mov %l1, %g6 ! pass tnpc to user code in %g6
2581 wrpr %l0, %g0, %pstate ! switch back to alternate globals
2582
2583 ! Note that %g1 still contains CPU struct addr
2584 ld [%g1 + CPU_TMP2], %l1 ! restore locals
2585 ld [%g1 + CPU_TMP1], %l0
2586 FAST_TRAP_DONE_CHK_INTR
2587 1:
2588 mov %g1, %l0
2589 st %l1, [%g1 + CPU_TMP2]
2590 rdpr %pstate, %l1
2591 wrpr %l1, PSTATE_AG, %pstate
2592 !
2593 ! check for old syscall mmap which is the only different one which
2594 ! must be the same. Others are handled in the compatibility library.
2595 !
2596 cmp %g1, OSYS_mmap ! compare to old 4.x mmap
2597 movz %icc, SYS_mmap, %g1
2598 wrpr %g0, %l1, %pstate
2599 ld [%l0 + CPU_TMP2], %l1 ! restore locals
2600 ld [%l0 + CPU_TMP1], %l0
2601 SYSCALL(syscall_trap32)
2602 SET_SIZE(syscall_trap_4x)
2603
2604 /*
2605 * Handler for software trap 9.
2606 * Set trap0 emulation address for old 4.x system call trap.
2607 * XXX - this should be a system call.
2608 */
2609 ENTRY_NP(set_trap0_addr)
2610 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
2611 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
2612 ldn [%g2 + T_LWP], %g2 ! load klwp pointer
2613 st %l0, [%g1 + CPU_TMP1] ! save some locals
2614 st %l1, [%g1 + CPU_TMP2]
2615 rdpr %pstate, %l0
2616 wrpr %l0, PSTATE_AG, %pstate
2617 mov %g1, %l1
2618 wrpr %g0, %l0, %pstate
2619 andn %l1, 3, %l1 ! force alignment
2620 st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr
2621 ld [%g1 + CPU_TMP1], %l0 ! restore locals
2622 ld [%g1 + CPU_TMP2], %l1
2623 FAST_TRAP_DONE
2624 SET_SIZE(set_trap0_addr)
2625
2626 /*
2627 * mmu_trap_tl1
2628 * trap handler for unexpected mmu traps.
2629 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2630 * case we go to fpu_trap or a user trap from the window handler, in which
2631 * case we go save the state on the pcb. Otherwise, we go to ptl1_panic.
2632 */
2633 .type mmu_trap_tl1, #function
2634 mmu_trap_tl1:
2635 #ifdef TRAPTRACE
2636 TRACE_PTR(%g5, %g6)
2637 GET_TRACE_TICK(%g6, %g7)
2638 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2639 rdpr %tl, %g6
2640 stha %g6, [%g5 + TRAP_ENT_TL]%asi
2641 rdpr %tt, %g6
2642 stha %g6, [%g5 + TRAP_ENT_TT]%asi
2643 rdpr %tstate, %g6
2644 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi
2645 stna %sp, [%g5 + TRAP_ENT_SP]%asi
2646 stna %g0, [%g5 + TRAP_ENT_TR]%asi
2647 rdpr %tpc, %g6
2648 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2649 set MMU_SFAR, %g6
2650 ldxa [%g6]ASI_DMMU, %g6
2651 stxa %g6, [%g5 + TRAP_ENT_F1]%asi
2652 CPU_PADDR(%g7, %g6);
2653 add %g7, CPU_TL1_HDLR, %g7
2654 lda [%g7]ASI_MEM, %g6
2655 stxa %g6, [%g5 + TRAP_ENT_F2]%asi
2656 set 0xdeadbeef, %g6
2657 stna %g6, [%g5 + TRAP_ENT_F3]%asi
2658 stna %g6, [%g5 + TRAP_ENT_F4]%asi
2659 TRACE_NEXT(%g5, %g6, %g7)
2660 #endif /* TRAPTRACE */
2661
2662 GET_CPU_IMPL(%g5)
2663 cmp %g5, PANTHER_IMPL
2664 bne mmu_trap_tl1_4
2665 nop
2666 rdpr %tt, %g5
2667 cmp %g5, T_DATA_EXCEPTION
2668 bne mmu_trap_tl1_4
2669 nop
2670 wr %g0, ASI_DMMU, %asi
2671 ldxa [MMU_SFSR]%asi, %g5
2672 mov 1, %g6
2673 sllx %g6, PN_SFSR_PARITY_SHIFT, %g6
2674 andcc %g5, %g6, %g0
2675 bz mmu_trap_tl1_4
2676
2677 /*
2678 * We are running on a Panther and have hit a DTLB parity error.
2679 */
2680 ldxa [MMU_TAG_ACCESS]%asi, %g2
2681 mov %g5, %g3
2682 ba,pt %xcc, .mmu_exception_is_tlb_parity
2683 mov T_DATA_EXCEPTION, %g1
2684
2685 mmu_trap_tl1_4:
2686 CPU_PADDR(%g7, %g6);
2687 add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA)
2688 /*
2689 * AM is cleared on trap, so addresses are 64 bit
2690 */
2691 lda [%g7]ASI_MEM, %g6
2692 brz,a,pt %g6, 1f
2693 nop
2694 /*
2695 * We are going to update cpu_m.tl1_hdlr using physical address.
2696 * Flush the D$ line, so that stale data won't be accessed later.
2697 */
2698 CPU_ADDR(%g6, %g5)
2699 add %g6, CPU_TL1_HDLR, %g6 ! %g6 = &cpu_m.tl1_hdlr (VA)
2700 GET_CPU_IMPL(%g5)
2701 cmp %g5, CHEETAH_IMPL
2702 bl,pt %icc, 3f
2703 cmp %g5, SPITFIRE_IMPL
2704 stxa %g0, [%g7]ASI_DC_INVAL
2705 membar #Sync
2706 ba,pt %xcc, 2f
2707 nop
2708 3:
2709 bl,pt %icc, 2f
2710 sethi %hi(dcache_line_mask), %g5
2711 ld [%g5 + %lo(dcache_line_mask)], %g5
2712 and %g6, %g5, %g5
2713 stxa %g0, [%g5]ASI_DC_TAG
2714 membar #Sync
2715 2:
2716 sta %g0, [%g7]ASI_MEM
2717 SWITCH_GLOBALS ! back to mmu globals
2718 ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults
2719 1:
2720 rdpr %tt, %g5
2721 rdpr %tl, %g7
2722 sub %g7, 1, %g6
2723 wrpr %g6, %tl
2724 rdpr %tt, %g6
2725 wrpr %g7, %tl
2726 and %g6, WTRAP_TTMASK, %g6
2727 cmp %g6, WTRAP_TYPE
2728 bne,a,pn %xcc, ptl1_panic
2729 mov PTL1_BAD_MMUTRAP, %g1
2730 rdpr %tpc, %g7
2731 /* tpc should be in the trap table */
2732 set trap_table, %g6
2733 cmp %g7, %g6
2734 blt,a,pn %xcc, ptl1_panic
2735 mov PTL1_BAD_MMUTRAP, %g1
2736 set etrap_table, %g6
2737 cmp %g7, %g6
2738 bge,a,pn %xcc, ptl1_panic
2739 mov PTL1_BAD_MMUTRAP, %g1
2740 cmp %g5, T_ALIGNMENT
2741 move %icc, MMU_SFAR, %g6
2742 movne %icc, MMU_TAG_ACCESS, %g6
2743 ldxa [%g6]ASI_DMMU, %g6
2744 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */
2745 add %g7, WTRAP_FAULTOFF, %g7
2746 wrpr %g0, %g7, %tnpc
2747 done
2748 SET_SIZE(mmu_trap_tl1)
2749
2750 /*
2751 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These
2752 * traps are valid only when kmdb is loaded. When the debugger is active,
2753 * the code below is rewritten to transfer control to the appropriate
2754 * debugger entry points.
2755 */
2756 .global kmdb_trap
2757 .align 8
2758 kmdb_trap:
2759 ba,a trap_table0
2760 jmp %g1 + 0
2761 nop
2762
2763 .global kmdb_trap_tl1
2764 .align 8
2765 kmdb_trap_tl1:
2766 ba,a trap_table0
2767 jmp %g1 + 0
2768 nop
2769
2770 /*
2771 * This entry is copied from OBP's trap table during boot.
2772 */
2773 .global obp_bpt
2774 .align 8
2775 obp_bpt:
2776 NOT
2777
2778 /*
2779 * if kernel, set PCONTEXT to 0 for debuggers
2780 * if user, clear nucleus page sizes
2781 */
2782 .global kctx_obp_bpt
2783 kctx_obp_bpt:
2784 set obp_bpt, %g2
2785 1:
2786 #ifndef _OPL
2787 mov MMU_PCONTEXT, %g1
2788 ldxa [%g1]ASI_DMMU, %g1
2789 srlx %g1, CTXREG_NEXT_SHIFT, %g3
2790 brz,pt %g3, 3f ! nucleus pgsz is 0, no problem
2791 sllx %g3, CTXREG_NEXT_SHIFT, %g3
2792 set CTXREG_CTX_MASK, %g4 ! check Pcontext
2793 btst %g4, %g1
2794 bz,a,pt %xcc, 2f
2795 clr %g3 ! kernel: PCONTEXT=0
2796 xor %g3, %g1, %g3 ! user: clr N_pgsz0/1 bits
2797 2:
2798 set DEMAP_ALL_TYPE, %g1
2799 stxa %g0, [%g1]ASI_DTLB_DEMAP
2800 stxa %g0, [%g1]ASI_ITLB_DEMAP
2801 mov MMU_PCONTEXT, %g1
2802 stxa %g3, [%g1]ASI_DMMU
2803 membar #Sync
2804 sethi %hi(FLUSH_ADDR), %g1
2805 flush %g1 ! flush required by immu
2806 #endif /* _OPL */
2807 3:
2808 jmp %g2
2809 nop
2810
2811
2812 #ifdef TRAPTRACE
2813 /*
2814 * TRAPTRACE support.
2815 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2816 * Return is done by "jmp %g7 + 4".
2817 */
2818
2819 trace_gen:
2820 TRACE_PTR(%g3, %g6)
2821 GET_TRACE_TICK(%g6, %g4)
2822 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
2823 rdpr %tl, %g6
2824 stha %g6, [%g3 + TRAP_ENT_TL]%asi
2825 rdpr %tt, %g6
2826 stha %g6, [%g3 + TRAP_ENT_TT]%asi
2827 rdpr %tstate, %g6
2828 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2829 stna %sp, [%g3 + TRAP_ENT_SP]%asi
2830 rdpr %tpc, %g6
2831 stna %g6, [%g3 + TRAP_ENT_TPC]%asi
2832 TRACE_NEXT(%g3, %g4, %g5)
2833 jmp %g7 + 4
2834 nop
2835
2836 trace_win:
2837 TRACE_WIN_INFO(0, %l0, %l1, %l2)
2838 ! Keep the locals as clean as possible, caller cleans %l4
2839 clr %l2
2840 clr %l1
2841 jmp %l4 + 4
2842 clr %l0
2843
2844 /*
2845 * Trace a tsb hit
2846 * g1 = tsbe pointer (in/clobbered)
2847 * g2 = tag access register (in)
2848 * g3 - g4 = scratch (clobbered)
2849 * g5 = tsbe data (in)
2850 * g6 = scratch (clobbered)
2851 * g7 = pc we jumped here from (in)
2852 */
2853
2854 ! Do not disturb %g5, it will be used after the trace
2855 ALTENTRY(trace_tsbhit)
2856 TRACE_TSBHIT(0)
2857 jmp %g7 + 4
2858 nop
2859
2860 /*
2861 * Trace a TSB miss
2862 *
2863 * g1 = tsb8k pointer (in)
2864 * g2 = tag access register (in)
2865 * g3 = tsb4m pointer (in)
2866 * g4 = tsbe tag (in/clobbered)
2867 * g5 - g6 = scratch (clobbered)
2868 * g7 = pc we jumped here from (in)
2869 */
2870 .global trace_tsbmiss
2871 trace_tsbmiss:
2872 membar #Sync
2873 sethi %hi(FLUSH_ADDR), %g6
2874 flush %g6
2875 TRACE_PTR(%g5, %g6)
2876 stxa %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access
2877 stxa %g4, [%g5 + TRAP_ENT_F1]%asi ! tsb tag
2878 GET_TRACE_TICK(%g6, %g4)
2879 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
2880 rdpr %tnpc, %g6
2881 stxa %g6, [%g5 + TRAP_ENT_F2]%asi
2882 stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer
2883 srlx %g1, 32, %g6
2884 stna %g6, [%g5 + TRAP_ENT_F4]%asi ! huh?
2885 rdpr %tpc, %g6
2886 stna %g6, [%g5 + TRAP_ENT_TPC]%asi
2887 rdpr %tl, %g6
2888 stha %g6, [%g5 + TRAP_ENT_TL]%asi
2889 rdpr %tt, %g6
2890 or %g6, TT_MMU_MISS, %g4
2891 stha %g4, [%g5 + TRAP_ENT_TT]%asi
2892 cmp %g6, FAST_IMMU_MISS_TT
2893 be,a %icc, 1f
2894 ldxa [%g0]ASI_IMMU, %g6
2895 ldxa [%g0]ASI_DMMU, %g6
2896 1: stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target
2897 stxa %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer
2898 TRACE_NEXT(%g5, %g4, %g6)
2899 jmp %g7 + 4
2900 nop
2901
2902 /*
2903 * g2 = tag access register (in)
2904 * g3 = ctx number (in)
2905 */
2906 trace_dataprot:
2907 membar #Sync
2908 sethi %hi(FLUSH_ADDR), %g6
2909 flush %g6
2910 TRACE_PTR(%g1, %g6)
2911 GET_TRACE_TICK(%g6, %g5)
2912 stxa %g6, [%g1 + TRAP_ENT_TICK]%asi
2913 rdpr %tpc, %g6
2914 stna %g6, [%g1 + TRAP_ENT_TPC]%asi
2915 rdpr %tstate, %g6
2916 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi
2917 stxa %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg
2918 stxa %g0, [%g1 + TRAP_ENT_TR]%asi
2919 stxa %g0, [%g1 + TRAP_ENT_F1]%asi
2920 stxa %g0, [%g1 + TRAP_ENT_F2]%asi
2921 stxa %g0, [%g1 + TRAP_ENT_F3]%asi
2922 stxa %g0, [%g1 + TRAP_ENT_F4]%asi
2923 rdpr %tl, %g6
2924 stha %g6, [%g1 + TRAP_ENT_TL]%asi
2925 rdpr %tt, %g6
2926 stha %g6, [%g1 + TRAP_ENT_TT]%asi
2927 TRACE_NEXT(%g1, %g4, %g5)
2928 jmp %g7 + 4
2929 nop
2930
2931 #endif /* TRAPTRACE */
2932
2933 .align 32
2934 .global pil15_epilogue
2935 pil15_epilogue:
2936 ba pil_interrupt_common
2937 nop
2938 .align 32
2939
2940 /*
2941 * fast_trap_done, fast_trap_done_chk_intr:
2942 *
2943 * Due to the design of UltraSPARC pipeline, pending interrupts are not
2944 * taken immediately after a RETRY or DONE instruction which causes IE to
2945 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2946 * to execute first before taking any interrupts. If that instruction
2947 * results in other traps, and if the corresponding trap handler runs
2948 * entirely at TL=1 with interrupts disabled, then pending interrupts
2949 * won't be taken until after yet another instruction following the %tpc
2950 * or %tnpc.
2951 *
2952 * A malicious user program can use this feature to block out interrupts
2953 * for extended durations, which can result in send_mondo_timeout kernel
2954 * panic.
2955 *
2956 * This problem is addressed by servicing any pending interrupts via
2957 * sys_trap before returning back to the user mode from a fast trap
2958 * handler. The "done" instruction within a fast trap handler, which
2959 * runs entirely at TL=1 with interrupts disabled, is replaced with the
2960 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2961 * entry point.
2962 *
2963 * We check for any pending interrupts here and force a sys_trap to
2964 * service those interrupts, if any. To minimize overhead, pending
2965 * interrupts are checked if the %tpc happens to be at 16K boundary,
2966 * which allows a malicious program to execute at most 4K consecutive
2967 * instructions before we service any pending interrupts. If a worst
2968 * case fast trap handler takes about 2 usec, then interrupts will be
2969 * blocked for at most 8 msec, less than a clock tick.
2970 *
2971 * For the cases where we don't know if the %tpc will cross a 16K
2972 * boundary, we can't use the above optimization and always process
2973 * any pending interrupts via fast_frap_done_chk_intr entry point.
2974 *
2975 * Entry Conditions:
2976 * %pstate am:0 priv:1 ie:0
2977 * globals are AG (not normal globals)
2978 */
2979
2980 .global fast_trap_done, fast_trap_done_chk_intr
2981 fast_trap_done:
2982 rdpr %tpc, %g5
2983 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff
2984 andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc
2985 bz,a,pn %icc, 1f ! branch if zero (lower 32 bits only)
2986 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
2987 done
2988
2989 ALTENTRY(fast_trap_done_check_interrupts)
2990 fast_trap_done_chk_intr:
2991 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
2992
2993 1: rd SOFTINT, %g6
2994 and %g5, IRSR_BUSY, %g5
2995 orcc %g5, %g6, %g0
2996 bnz,pn %xcc, 2f ! branch if any pending intr
2997 nop
2998 done
2999
3000 2:
3001 /*
3002 * We get here if there are any pending interrupts.
3003 * Adjust %tpc/%tnpc as we'll be resuming via "retry"
3004 * instruction.
3005 */
3006 rdpr %tnpc, %g5
3007 wrpr %g0, %g5, %tpc
3008 add %g5, 4, %g5
3009 wrpr %g0, %g5, %tnpc
3010
3011 /*
3012 * Force a dummy sys_trap call so that interrupts can be serviced.
3013 */
3014 set fast_trap_dummy_call, %g1
3015 ba,pt %xcc, sys_trap
3016 mov -1, %g4
3017
3018 fast_trap_dummy_call:
3019 retl
3020 nop
3021
3022 /*
3023 * Currently the brand syscall interposition code is not enabled by
3024 * default. Instead, when a branded zone is first booted the brand
3025 * infrastructure will patch the trap table so that the syscall
3026 * entry points are redirected to syscall_wrapper32 and syscall_wrapper
3027 * for ILP32 and LP64 syscalls respectively. this is done in
3028 * brand_plat_interposition_enable(). Note that the syscall wrappers
3029 * below do not collect any trap trace data since the syscall hot patch
3030 * points are reached after trap trace data has already been collected.
3031 */
3032 #define BRAND_CALLBACK(callback_id) \
3033 CPU_ADDR(%g2, %g1) /* load CPU struct addr to %g2 */ ;\
3034 ldn [%g2 + CPU_THREAD], %g3 /* load thread pointer */ ;\
3035 ldn [%g3 + T_PROCP], %g3 /* get proc pointer */ ;\
3036 ldn [%g3 + P_BRAND], %g3 /* get brand pointer */ ;\
3037 brz %g3, 1f /* No brand? No callback. */ ;\
3038 nop ;\
3039 ldn [%g3 + B_MACHOPS], %g3 /* get machops list */ ;\
3040 ldn [%g3 + (callback_id << 3)], %g3 ;\
3041 brz %g3, 1f ;\
3042 /* \
3043 * This isn't pretty. We want a low-latency way for the callback \
3044 * routine to decline to do anything. We just pass in an address \
3045 * the routine can directly jmp back to, pretending that nothing \
3046 * has happened. \
3047 * \
3048 * %g1: return address (where the brand handler jumps back to) \
3049 * %g2: address of CPU structure \
3050 * %g3: address of brand handler (where we will jump to) \
3051 */ \
3052 mov %pc, %g1 ;\
3053 add %g1, 16, %g1 ;\
3054 jmp %g3 ;\
3055 nop ;\
3056 1:
3057
3058 ENTRY_NP(syscall_wrapper32)
3059 BRAND_CALLBACK(BRAND_CB_SYSCALL32)
3060 SYSCALL_NOTT(syscall_trap32)
3061 SET_SIZE(syscall_wrapper32)
3062
3063 ENTRY_NP(syscall_wrapper)
3064 BRAND_CALLBACK(BRAND_CB_SYSCALL)
3065 SYSCALL_NOTT(syscall_trap)
3066 SET_SIZE(syscall_wrapper)
3067
3068 #endif /* lint */