1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #include "assym.h"
  28 #include <sys/asm_linkage.h>
  29 #include <sys/privregs.h>
  30 #include <sys/sun4asi.h>
  31 #include <sys/machasi.h>
  32 #include <sys/hypervisor_api.h>
  33 #include <sys/machtrap.h>
  34 #include <sys/machthread.h>
  35 #include <sys/machbrand.h>
  36 #include <sys/pcb.h>
  37 #include <sys/pte.h>
  38 #include <sys/mmu.h>
  39 #include <sys/machpcb.h>
  40 #include <sys/async.h>
  41 #include <sys/intreg.h>
  42 #include <sys/scb.h>
  43 #include <sys/psr_compat.h>
  44 #include <sys/syscall.h>
  45 #include <sys/machparam.h>
  46 #include <sys/traptrace.h>
  47 #include <vm/hat_sfmmu.h>
  48 #include <sys/archsystm.h>
  49 #include <sys/utrap.h>
  50 #include <sys/clock.h>
  51 #include <sys/intr.h>
  52 #include <sys/fpu/fpu_simulator.h>
  53 #include <vm/seg_spt.h>
  54 
  55 /*
  56  * WARNING: If you add a fast trap handler which can be invoked by a
  57  * non-privileged user, you may have to use the FAST_TRAP_DONE macro
  58  * instead of "done" instruction to return back to the user mode. See
  59  * comments for the "fast_trap_done" entry point for more information.
  60  *
  61  * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
  62  * cases where you always want to process any pending interrupts before
  63  * returning back to the user mode.
  64  */
  65 #define FAST_TRAP_DONE          \
  66         ba,a    fast_trap_done
  67 
  68 #define FAST_TRAP_DONE_CHK_INTR \
  69         ba,a    fast_trap_done_chk_intr
  70 
  71 /*
  72  * SPARC V9 Trap Table
  73  *
  74  * Most of the trap handlers are made from common building
  75  * blocks, and some are instantiated multiple times within
  76  * the trap table. So, I build a bunch of macros, then
  77  * populate the table using only the macros.
  78  *
  79  * Many macros branch to sys_trap.  Its calling convention is:
  80  *      %g1             kernel trap handler
  81  *      %g2, %g3        args for above
  82  *      %g4             desire %pil
  83  */
  84 
  85 #ifdef  TRAPTRACE
  86 
  87 /*
  88  * Tracing macro. Adds two instructions if TRAPTRACE is defined.
  89  */
  90 #define TT_TRACE(label)         \
  91         ba      label           ;\
  92         rd      %pc, %g7
  93 #define TT_TRACE_INS    2
  94 
  95 #define TT_TRACE_L(label)       \
  96         ba      label           ;\
  97         rd      %pc, %l4        ;\
  98         clr     %l4
  99 #define TT_TRACE_L_INS  3
 100 
 101 #else
 102 
 103 #define TT_TRACE(label)
 104 #define TT_TRACE_INS    0
 105 
 106 #define TT_TRACE_L(label)
 107 #define TT_TRACE_L_INS  0
 108 
 109 #endif
 110 
 111 /*
 112  * This first set are funneled to trap() with %tt as the type.
 113  * Trap will then either panic or send the user a signal.
 114  */
 115 /*
 116  * NOT is used for traps that just shouldn't happen.
 117  * It comes in both single and quadruple flavors.
 118  */
 119         .global trap
 120 #define NOT                     \
 121         TT_TRACE(trace_gen)     ;\
 122         set     trap, %g1       ;\
 123         rdpr    %tt, %g3        ;\
 124         ba,pt   %xcc, sys_trap  ;\
 125         sub     %g0, 1, %g4     ;\
 126         .align  32
 127 #define NOT4    NOT; NOT; NOT; NOT
 128 
 129 #define NOTP                            \
 130         TT_TRACE(trace_gen)             ;\
 131         ba,pt   %xcc, ptl1_panic        ;\
 132           mov   PTL1_BAD_TRAP, %g1      ;\
 133         .align  32
 134 #define NOTP4   NOTP; NOTP; NOTP; NOTP
 135 
 136 
 137 /*
 138  * BAD is used for trap vectors we don't have a kernel
 139  * handler for.
 140  * It also comes in single and quadruple versions.
 141  */
 142 #define BAD     NOT
 143 #define BAD4    NOT4
 144 
 145 #define DONE                    \
 146         done;                   \
 147         .align  32
 148 
 149 /*
 150  * TRAP vectors to the trap() function.
 151  * It's main use is for user errors.
 152  */
 153         .global trap
 154 #define TRAP(arg)               \
 155         TT_TRACE(trace_gen)     ;\
 156         set     trap, %g1       ;\
 157         mov     arg, %g3        ;\
 158         ba,pt   %xcc, sys_trap  ;\
 159         sub     %g0, 1, %g4     ;\
 160         .align  32
 161 
 162 /*
 163  * SYSCALL is used for unsupported syscall interfaces (with 'which'
 164  * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
 165  * 'which' set to 'syscall_trap32').
 166  *
 167  * The SYSCALL_TRAP* macros are used for syscall entry points.
 168  * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
 169  * is used to support ILP32.  Each macro can only be used once
 170  * since they each define a symbol.  The symbols are used as hot patch
 171  * points by the brand infrastructure to dynamically enable and disable
 172  * brand syscall interposition.  See the comments around BRAND_CALLBACK
 173  * and brand_plat_interposition_enable() for more information.
 174  */
 175 #define SYSCALL_NOTT(which)             \
 176         set     (which), %g1            ;\
 177         ba,pt   %xcc, user_trap         ;\
 178         sub     %g0, 1, %g4             ;\
 179         .align  32
 180 
 181 #define SYSCALL(which)                  \
 182         TT_TRACE(trace_gen)             ;\
 183         SYSCALL_NOTT(which)
 184 
 185 #define SYSCALL_TRAP32                          \
 186         TT_TRACE(trace_gen)                     ;\
 187         ALTENTRY(syscall_trap32_patch_point)    \
 188         SYSCALL_NOTT(syscall_trap32)
 189 
 190 #define SYSCALL_TRAP                            \
 191         TT_TRACE(trace_gen)                     ;\
 192         ALTENTRY(syscall_trap_patch_point)      \
 193         SYSCALL_NOTT(syscall_trap)
 194 
 195 /*
 196  * GOTO just jumps to a label.
 197  * It's used for things that can be fixed without going thru sys_trap.
 198  */
 199 #define GOTO(label)             \
 200         .global label           ;\
 201         ba,a    label           ;\
 202         .empty                  ;\
 203         .align  32
 204 
 205 /*
 206  * GOTO_TT just jumps to a label.
 207  * correctable ECC error traps at  level 0 and 1 will use this macro.
 208  * It's used for things that can be fixed without going thru sys_trap.
 209  */
 210 #define GOTO_TT(label, ttlabel)         \
 211         .global label           ;\
 212         TT_TRACE(ttlabel)       ;\
 213         ba,a    label           ;\
 214         .empty                  ;\
 215         .align  32
 216 
 217 /*
 218  * Privileged traps
 219  * Takes breakpoint if privileged, calls trap() if not.
 220  */
 221 #define PRIV(label)                     \
 222         rdpr    %tstate, %g1            ;\
 223         btst    TSTATE_PRIV, %g1        ;\
 224         bnz     label                   ;\
 225         rdpr    %tt, %g3                ;\
 226         set     trap, %g1               ;\
 227         ba,pt   %xcc, sys_trap          ;\
 228         sub     %g0, 1, %g4             ;\
 229         .align  32
 230 
 231 
 232 /*
 233  * DTrace traps.
 234  */
 235 #define DTRACE_PID                      \
 236         .global dtrace_pid_probe                                ;\
 237         set     dtrace_pid_probe, %g1                           ;\
 238         ba,pt   %xcc, user_trap                                 ;\
 239         sub     %g0, 1, %g4                                     ;\
 240         .align  32
 241 
 242 #define DTRACE_RETURN                   \
 243         .global dtrace_return_probe                             ;\
 244         set     dtrace_return_probe, %g1                        ;\
 245         ba,pt   %xcc, user_trap                                 ;\
 246         sub     %g0, 1, %g4                                     ;\
 247         .align  32
 248 
 249 /*
 250  * REGISTER WINDOW MANAGEMENT MACROS
 251  */
 252 
 253 /*
 254  * various convenient units of padding
 255  */
 256 #define SKIP(n) .skip 4*(n)
 257 
 258 /*
 259  * CLEAN_WINDOW is the simple handler for cleaning a register window.
 260  */
 261 #define CLEAN_WINDOW                                            \
 262         TT_TRACE_L(trace_win)                                   ;\
 263         rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin       ;\
 264         clr %l0; clr %l1; clr %l2; clr %l3                      ;\
 265         clr %l4; clr %l5; clr %l6; clr %l7                      ;\
 266         clr %o0; clr %o1; clr %o2; clr %o3                      ;\
 267         clr %o4; clr %o5; clr %o6; clr %o7                      ;\
 268         retry; .align 128
 269 
 270 /*
 271  * If we get an unresolved tlb miss while in a window handler, the fault
 272  * handler will resume execution at the last instruction of the window
 273  * hander, instead of delivering the fault to the kernel.  Spill handlers
 274  * use this to spill windows into the wbuf.
 275  *
 276  * The mixed handler works by checking %sp, and branching to the correct
 277  * handler.  This is done by branching back to label 1: for 32b frames,
 278  * or label 2: for 64b frames; which implies the handler order is: 32b,
 279  * 64b, mixed.  The 1: and 2: labels are offset into the routines to
 280  * allow the branchs' delay slots to contain useful instructions.
 281  */
 282 
 283 /*
 284  * SPILL_32bit spills a 32-bit-wide kernel register window.  It
 285  * assumes that the kernel context and the nucleus context are the
 286  * same.  The stack pointer is required to be eight-byte aligned even
 287  * though this code only needs it to be four-byte aligned.
 288  */
 289 #define SPILL_32bit(tail)                                       \
 290         srl     %sp, 0, %sp                                     ;\
 291 1:      st      %l0, [%sp + 0]                                  ;\
 292         st      %l1, [%sp + 4]                                  ;\
 293         st      %l2, [%sp + 8]                                  ;\
 294         st      %l3, [%sp + 12]                                 ;\
 295         st      %l4, [%sp + 16]                                 ;\
 296         st      %l5, [%sp + 20]                                 ;\
 297         st      %l6, [%sp + 24]                                 ;\
 298         st      %l7, [%sp + 28]                                 ;\
 299         st      %i0, [%sp + 32]                                 ;\
 300         st      %i1, [%sp + 36]                                 ;\
 301         st      %i2, [%sp + 40]                                 ;\
 302         st      %i3, [%sp + 44]                                 ;\
 303         st      %i4, [%sp + 48]                                 ;\
 304         st      %i5, [%sp + 52]                                 ;\
 305         st      %i6, [%sp + 56]                                 ;\
 306         st      %i7, [%sp + 60]                                 ;\
 307         TT_TRACE_L(trace_win)                                   ;\
 308         saved                                                   ;\
 309         retry                                                   ;\
 310         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 311         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 312         .empty
 313 
 314 /*
 315  * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
 316  * wide address space via the designated asi.  It is used to spill
 317  * non-kernel windows.  The stack pointer is required to be eight-byte
 318  * aligned even though this code only needs it to be four-byte
 319  * aligned.
 320  */
 321 #define SPILL_32bit_asi(asi_num, tail)                          \
 322         srl     %sp, 0, %sp                                     ;\
 323 1:      sta     %l0, [%sp + %g0]asi_num                         ;\
 324         mov     4, %g1                                          ;\
 325         sta     %l1, [%sp + %g1]asi_num                         ;\
 326         mov     8, %g2                                          ;\
 327         sta     %l2, [%sp + %g2]asi_num                         ;\
 328         mov     12, %g3                                         ;\
 329         sta     %l3, [%sp + %g3]asi_num                         ;\
 330         add     %sp, 16, %g4                                    ;\
 331         sta     %l4, [%g4 + %g0]asi_num                         ;\
 332         sta     %l5, [%g4 + %g1]asi_num                         ;\
 333         sta     %l6, [%g4 + %g2]asi_num                         ;\
 334         sta     %l7, [%g4 + %g3]asi_num                         ;\
 335         add     %g4, 16, %g4                                    ;\
 336         sta     %i0, [%g4 + %g0]asi_num                         ;\
 337         sta     %i1, [%g4 + %g1]asi_num                         ;\
 338         sta     %i2, [%g4 + %g2]asi_num                         ;\
 339         sta     %i3, [%g4 + %g3]asi_num                         ;\
 340         add     %g4, 16, %g4                                    ;\
 341         sta     %i4, [%g4 + %g0]asi_num                         ;\
 342         sta     %i5, [%g4 + %g1]asi_num                         ;\
 343         sta     %i6, [%g4 + %g2]asi_num                         ;\
 344         sta     %i7, [%g4 + %g3]asi_num                         ;\
 345         TT_TRACE_L(trace_win)                                   ;\
 346         saved                                                   ;\
 347         retry                                                   ;\
 348         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 349         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 350         .empty
 351 
 352 #define SPILL_32bit_tt1(asi_num, tail)                          \
 353         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 354         .empty                                                  ;\
 355         .align 128
 356 
 357 
 358 /*
 359  * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
 360  * that the kernel context and the nucleus context are the same.  The
 361  * stack pointer is required to be eight-byte aligned even though this
 362  * code only needs it to be four-byte aligned.
 363  */
 364 #define FILL_32bit(tail)                                        \
 365         srl     %sp, 0, %sp                                     ;\
 366 1:      TT_TRACE_L(trace_win)                                   ;\
 367         ld      [%sp + 0], %l0                                  ;\
 368         ld      [%sp + 4], %l1                                  ;\
 369         ld      [%sp + 8], %l2                                  ;\
 370         ld      [%sp + 12], %l3                                 ;\
 371         ld      [%sp + 16], %l4                                 ;\
 372         ld      [%sp + 20], %l5                                 ;\
 373         ld      [%sp + 24], %l6                                 ;\
 374         ld      [%sp + 28], %l7                                 ;\
 375         ld      [%sp + 32], %i0                                 ;\
 376         ld      [%sp + 36], %i1                                 ;\
 377         ld      [%sp + 40], %i2                                 ;\
 378         ld      [%sp + 44], %i3                                 ;\
 379         ld      [%sp + 48], %i4                                 ;\
 380         ld      [%sp + 52], %i5                                 ;\
 381         ld      [%sp + 56], %i6                                 ;\
 382         ld      [%sp + 60], %i7                                 ;\
 383         restored                                                ;\
 384         retry                                                   ;\
 385         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 386         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 387         .empty
 388 
 389 /*
 390  * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
 391  * wide address space via the designated asi.  It is used to fill
 392  * non-kernel windows.  The stack pointer is required to be eight-byte
 393  * aligned even though this code only needs it to be four-byte
 394  * aligned.
 395  */
 396 #define FILL_32bit_asi(asi_num, tail)                           \
 397         srl     %sp, 0, %sp                                     ;\
 398 1:      TT_TRACE_L(trace_win)                                   ;\
 399         mov     4, %g1                                          ;\
 400         lda     [%sp + %g0]asi_num, %l0                         ;\
 401         mov     8, %g2                                          ;\
 402         lda     [%sp + %g1]asi_num, %l1                         ;\
 403         mov     12, %g3                                         ;\
 404         lda     [%sp + %g2]asi_num, %l2                         ;\
 405         lda     [%sp + %g3]asi_num, %l3                         ;\
 406         add     %sp, 16, %g4                                    ;\
 407         lda     [%g4 + %g0]asi_num, %l4                         ;\
 408         lda     [%g4 + %g1]asi_num, %l5                         ;\
 409         lda     [%g4 + %g2]asi_num, %l6                         ;\
 410         lda     [%g4 + %g3]asi_num, %l7                         ;\
 411         add     %g4, 16, %g4                                    ;\
 412         lda     [%g4 + %g0]asi_num, %i0                         ;\
 413         lda     [%g4 + %g1]asi_num, %i1                         ;\
 414         lda     [%g4 + %g2]asi_num, %i2                         ;\
 415         lda     [%g4 + %g3]asi_num, %i3                         ;\
 416         add     %g4, 16, %g4                                    ;\
 417         lda     [%g4 + %g0]asi_num, %i4                         ;\
 418         lda     [%g4 + %g1]asi_num, %i5                         ;\
 419         lda     [%g4 + %g2]asi_num, %i6                         ;\
 420         lda     [%g4 + %g3]asi_num, %i7                         ;\
 421         restored                                                ;\
 422         retry                                                   ;\
 423         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 424         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 425         .empty
 426 
 427 
 428 /*
 429  * SPILL_64bit spills a 64-bit-wide kernel register window.  It
 430  * assumes that the kernel context and the nucleus context are the
 431  * same.  The stack pointer is required to be eight-byte aligned.
 432  */
 433 #define SPILL_64bit(tail)                                       \
 434 2:      stx     %l0, [%sp + V9BIAS64 + 0]                       ;\
 435         stx     %l1, [%sp + V9BIAS64 + 8]                       ;\
 436         stx     %l2, [%sp + V9BIAS64 + 16]                      ;\
 437         stx     %l3, [%sp + V9BIAS64 + 24]                      ;\
 438         stx     %l4, [%sp + V9BIAS64 + 32]                      ;\
 439         stx     %l5, [%sp + V9BIAS64 + 40]                      ;\
 440         stx     %l6, [%sp + V9BIAS64 + 48]                      ;\
 441         stx     %l7, [%sp + V9BIAS64 + 56]                      ;\
 442         stx     %i0, [%sp + V9BIAS64 + 64]                      ;\
 443         stx     %i1, [%sp + V9BIAS64 + 72]                      ;\
 444         stx     %i2, [%sp + V9BIAS64 + 80]                      ;\
 445         stx     %i3, [%sp + V9BIAS64 + 88]                      ;\
 446         stx     %i4, [%sp + V9BIAS64 + 96]                      ;\
 447         stx     %i5, [%sp + V9BIAS64 + 104]                     ;\
 448         stx     %i6, [%sp + V9BIAS64 + 112]                     ;\
 449         stx     %i7, [%sp + V9BIAS64 + 120]                     ;\
 450         TT_TRACE_L(trace_win)                                   ;\
 451         saved                                                   ;\
 452         retry                                                   ;\
 453         SKIP(31-18-TT_TRACE_L_INS)                              ;\
 454         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 455         .empty
 456 
 457 #define SPILL_64bit_ktt1(tail)                          \
 458         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 459         .empty                                                  ;\
 460         .align 128
 461 
 462 #define SPILL_mixed_ktt1(tail)                          \
 463         btst    1, %sp                                          ;\
 464         bz,a,pt %xcc, fault_32bit_/**/tail                      ;\
 465         srl     %sp, 0, %sp                                     ;\
 466         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 467         .empty                                                  ;\
 468         .align 128
 469 
 470 /*
 471  * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
 472  * wide address space via the designated asi.  It is used to spill
 473  * non-kernel windows.  The stack pointer is required to be eight-byte
 474  * aligned.
 475  */
 476 #define SPILL_64bit_asi(asi_num, tail)                          \
 477         mov     0 + V9BIAS64, %g1                               ;\
 478 2:      stxa    %l0, [%sp + %g1]asi_num                         ;\
 479         mov     8 + V9BIAS64, %g2                               ;\
 480         stxa    %l1, [%sp + %g2]asi_num                         ;\
 481         mov     16 + V9BIAS64, %g3                              ;\
 482         stxa    %l2, [%sp + %g3]asi_num                         ;\
 483         mov     24 + V9BIAS64, %g4                              ;\
 484         stxa    %l3, [%sp + %g4]asi_num                         ;\
 485         add     %sp, 32, %g5                                    ;\
 486         stxa    %l4, [%g5 + %g1]asi_num                         ;\
 487         stxa    %l5, [%g5 + %g2]asi_num                         ;\
 488         stxa    %l6, [%g5 + %g3]asi_num                         ;\
 489         stxa    %l7, [%g5 + %g4]asi_num                         ;\
 490         add     %g5, 32, %g5                                    ;\
 491         stxa    %i0, [%g5 + %g1]asi_num                         ;\
 492         stxa    %i1, [%g5 + %g2]asi_num                         ;\
 493         stxa    %i2, [%g5 + %g3]asi_num                         ;\
 494         stxa    %i3, [%g5 + %g4]asi_num                         ;\
 495         add     %g5, 32, %g5                                    ;\
 496         stxa    %i4, [%g5 + %g1]asi_num                         ;\
 497         stxa    %i5, [%g5 + %g2]asi_num                         ;\
 498         stxa    %i6, [%g5 + %g3]asi_num                         ;\
 499         stxa    %i7, [%g5 + %g4]asi_num                         ;\
 500         TT_TRACE_L(trace_win)                                   ;\
 501         saved                                                   ;\
 502         retry                                                   ;\
 503         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 504         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 505         .empty
 506 
 507 #define SPILL_64bit_tt1(asi_num, tail)                          \
 508         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 509         .empty                                                  ;\
 510         .align 128
 511 
 512 /*
 513  * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
 514  * that the kernel context and the nucleus context are the same.  The
 515  * stack pointer is required to be eight-byte aligned.
 516  */
 517 #define FILL_64bit(tail)                                        \
 518 2:      TT_TRACE_L(trace_win)                                   ;\
 519         ldx     [%sp + V9BIAS64 + 0], %l0                       ;\
 520         ldx     [%sp + V9BIAS64 + 8], %l1                       ;\
 521         ldx     [%sp + V9BIAS64 + 16], %l2                      ;\
 522         ldx     [%sp + V9BIAS64 + 24], %l3                      ;\
 523         ldx     [%sp + V9BIAS64 + 32], %l4                      ;\
 524         ldx     [%sp + V9BIAS64 + 40], %l5                      ;\
 525         ldx     [%sp + V9BIAS64 + 48], %l6                      ;\
 526         ldx     [%sp + V9BIAS64 + 56], %l7                      ;\
 527         ldx     [%sp + V9BIAS64 + 64], %i0                      ;\
 528         ldx     [%sp + V9BIAS64 + 72], %i1                      ;\
 529         ldx     [%sp + V9BIAS64 + 80], %i2                      ;\
 530         ldx     [%sp + V9BIAS64 + 88], %i3                      ;\
 531         ldx     [%sp + V9BIAS64 + 96], %i4                      ;\
 532         ldx     [%sp + V9BIAS64 + 104], %i5                     ;\
 533         ldx     [%sp + V9BIAS64 + 112], %i6                     ;\
 534         ldx     [%sp + V9BIAS64 + 120], %i7                     ;\
 535         restored                                                ;\
 536         retry                                                   ;\
 537         SKIP(31-18-TT_TRACE_L_INS)                              ;\
 538         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 539         .empty
 540 
 541 /*
 542  * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
 543  * wide address space via the designated asi.  It is used to fill
 544  * non-kernel windows.  The stack pointer is required to be eight-byte
 545  * aligned.
 546  */
 547 #define FILL_64bit_asi(asi_num, tail)                           \
 548         mov     V9BIAS64 + 0, %g1                               ;\
 549 2:      TT_TRACE_L(trace_win)                                   ;\
 550         ldxa    [%sp + %g1]asi_num, %l0                         ;\
 551         mov     V9BIAS64 + 8, %g2                               ;\
 552         ldxa    [%sp + %g2]asi_num, %l1                         ;\
 553         mov     V9BIAS64 + 16, %g3                              ;\
 554         ldxa    [%sp + %g3]asi_num, %l2                         ;\
 555         mov     V9BIAS64 + 24, %g4                              ;\
 556         ldxa    [%sp + %g4]asi_num, %l3                         ;\
 557         add     %sp, 32, %g5                                    ;\
 558         ldxa    [%g5 + %g1]asi_num, %l4                         ;\
 559         ldxa    [%g5 + %g2]asi_num, %l5                         ;\
 560         ldxa    [%g5 + %g3]asi_num, %l6                         ;\
 561         ldxa    [%g5 + %g4]asi_num, %l7                         ;\
 562         add     %g5, 32, %g5                                    ;\
 563         ldxa    [%g5 + %g1]asi_num, %i0                         ;\
 564         ldxa    [%g5 + %g2]asi_num, %i1                         ;\
 565         ldxa    [%g5 + %g3]asi_num, %i2                         ;\
 566         ldxa    [%g5 + %g4]asi_num, %i3                         ;\
 567         add     %g5, 32, %g5                                    ;\
 568         ldxa    [%g5 + %g1]asi_num, %i4                         ;\
 569         ldxa    [%g5 + %g2]asi_num, %i5                         ;\
 570         ldxa    [%g5 + %g3]asi_num, %i6                         ;\
 571         ldxa    [%g5 + %g4]asi_num, %i7                         ;\
 572         restored                                                ;\
 573         retry                                                   ;\
 574         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 575         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 576         .empty
 577 
 578 
 579 /*
 580  * SPILL_mixed spills either size window, depending on
 581  * whether %sp is even or odd, to a 32-bit address space.
 582  * This may only be used in conjunction with SPILL_32bit/
 583  * FILL_64bit.
 584  * Clear upper 32 bits of %sp if it is odd.
 585  * We won't need to clear them in 64 bit kernel.
 586  */
 587 #define SPILL_mixed                                             \
 588         btst    1, %sp                                          ;\
 589         bz,a,pt %xcc, 1b                                        ;\
 590         srl     %sp, 0, %sp                                     ;\
 591         ba,pt   %xcc, 2b                                        ;\
 592         nop                                                     ;\
 593         .align  128
 594 
 595 /*
 596  * FILL_mixed(ASI) fills either size window, depending on
 597  * whether %sp is even or odd, from a 32-bit address space.
 598  * This may only be used in conjunction with FILL_32bit/
 599  * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
 600  * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
 601  * attention should be paid to the instructions that belong
 602  * in the delay slots of the branches depending on the type
 603  * of fill handler being branched to.
 604  * Clear upper 32 bits of %sp if it is odd.
 605  * We won't need to clear them in 64 bit kernel.
 606  */
 607 #define FILL_mixed                                              \
 608         btst    1, %sp                                          ;\
 609         bz,a,pt %xcc, 1b                                        ;\
 610         srl     %sp, 0, %sp                                     ;\
 611         ba,pt   %xcc, 2b                                        ;\
 612         nop                                                     ;\
 613         .align  128
 614 
 615 
 616 /*
 617  * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
 618  * respectively, into the address space via the designated asi.  The
 619  * unbiased stack pointer is required to be eight-byte aligned (even for
 620  * the 32-bit case even though this code does not require such strict
 621  * alignment).
 622  *
 623  * With SPARC v9 the spill trap takes precedence over the cleanwin trap
 624  * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
 625  * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
 626  * window may contain kernel data so in user_rtt we set wstate to call
 627  * these spill handlers on the first user spill trap.  These handler then
 628  * spill the appropriate window but also back up a window and clean the
 629  * window that didn't get a cleanwin trap.
 630  */
 631 #define SPILL_32clean(asi_num, tail)                            \
 632         srl     %sp, 0, %sp                                     ;\
 633         sta     %l0, [%sp + %g0]asi_num                         ;\
 634         mov     4, %g1                                          ;\
 635         sta     %l1, [%sp + %g1]asi_num                         ;\
 636         mov     8, %g2                                          ;\
 637         sta     %l2, [%sp + %g2]asi_num                         ;\
 638         mov     12, %g3                                         ;\
 639         sta     %l3, [%sp + %g3]asi_num                         ;\
 640         add     %sp, 16, %g4                                    ;\
 641         sta     %l4, [%g4 + %g0]asi_num                         ;\
 642         sta     %l5, [%g4 + %g1]asi_num                         ;\
 643         sta     %l6, [%g4 + %g2]asi_num                         ;\
 644         sta     %l7, [%g4 + %g3]asi_num                         ;\
 645         add     %g4, 16, %g4                                    ;\
 646         sta     %i0, [%g4 + %g0]asi_num                         ;\
 647         sta     %i1, [%g4 + %g1]asi_num                         ;\
 648         sta     %i2, [%g4 + %g2]asi_num                         ;\
 649         sta     %i3, [%g4 + %g3]asi_num                         ;\
 650         add     %g4, 16, %g4                                    ;\
 651         sta     %i4, [%g4 + %g0]asi_num                         ;\
 652         sta     %i5, [%g4 + %g1]asi_num                         ;\
 653         sta     %i6, [%g4 + %g2]asi_num                         ;\
 654         sta     %i7, [%g4 + %g3]asi_num                         ;\
 655         TT_TRACE_L(trace_win)                                   ;\
 656         b       .spill_clean                                    ;\
 657           mov   WSTATE_USER32, %g7                              ;\
 658         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 659         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 660         .empty
 661 
 662 #define SPILL_64clean(asi_num, tail)                            \
 663         mov     0 + V9BIAS64, %g1                               ;\
 664         stxa    %l0, [%sp + %g1]asi_num                         ;\
 665         mov     8 + V9BIAS64, %g2                               ;\
 666         stxa    %l1, [%sp + %g2]asi_num                         ;\
 667         mov     16 + V9BIAS64, %g3                              ;\
 668         stxa    %l2, [%sp + %g3]asi_num                         ;\
 669         mov     24 + V9BIAS64, %g4                              ;\
 670         stxa    %l3, [%sp + %g4]asi_num                         ;\
 671         add     %sp, 32, %g5                                    ;\
 672         stxa    %l4, [%g5 + %g1]asi_num                         ;\
 673         stxa    %l5, [%g5 + %g2]asi_num                         ;\
 674         stxa    %l6, [%g5 + %g3]asi_num                         ;\
 675         stxa    %l7, [%g5 + %g4]asi_num                         ;\
 676         add     %g5, 32, %g5                                    ;\
 677         stxa    %i0, [%g5 + %g1]asi_num                         ;\
 678         stxa    %i1, [%g5 + %g2]asi_num                         ;\
 679         stxa    %i2, [%g5 + %g3]asi_num                         ;\
 680         stxa    %i3, [%g5 + %g4]asi_num                         ;\
 681         add     %g5, 32, %g5                                    ;\
 682         stxa    %i4, [%g5 + %g1]asi_num                         ;\
 683         stxa    %i5, [%g5 + %g2]asi_num                         ;\
 684         stxa    %i6, [%g5 + %g3]asi_num                         ;\
 685         stxa    %i7, [%g5 + %g4]asi_num                         ;\
 686         TT_TRACE_L(trace_win)                                   ;\
 687         b       .spill_clean                                    ;\
 688           mov   WSTATE_USER64, %g7                              ;\
 689         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 690         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 691         .empty
 692 
 693 
 694 /*
 695  * Floating point disabled.
 696  */
 697 #define FP_DISABLED_TRAP                \
 698         TT_TRACE(trace_gen)             ;\
 699         ba,pt   %xcc,.fp_disabled       ;\
 700         nop                             ;\
 701         .align  32
 702 
 703 /*
 704  * Floating point exceptions.
 705  */
 706 #define FP_IEEE_TRAP                    \
 707         TT_TRACE(trace_gen)             ;\
 708         ba,pt   %xcc,.fp_ieee_exception ;\
 709         nop                             ;\
 710         .align  32
 711 
 712 #define FP_TRAP                         \
 713         TT_TRACE(trace_gen)             ;\
 714         ba,pt   %xcc,.fp_exception      ;\
 715         nop                             ;\
 716         .align  32
 717 
 718 /*
 719  * ECACHE_ECC error traps at level 0 and level 1
 720  */
 721 #define ECACHE_ECC(table_name)          \
 722         .global table_name              ;\
 723 table_name:                             ;\
 724         membar  #Sync                   ;\
 725         set     trap, %g1               ;\
 726         rdpr    %tt, %g3                ;\
 727         ba,pt   %xcc, sys_trap          ;\
 728         sub     %g0, 1, %g4             ;\
 729         .align  32
 730 
 731 /*
 732  * illegal instruction trap
 733  */
 734 #define ILLTRAP_INSTR                     \
 735         membar  #Sync                     ;\
 736         TT_TRACE(trace_gen)               ;\
 737         or      %g0, P_UTRAP4, %g2        ;\
 738         or      %g0, T_UNIMP_INSTR, %g3   ;\
 739         sethi   %hi(.check_v9utrap), %g4  ;\
 740         jmp     %g4 + %lo(.check_v9utrap) ;\
 741         nop                               ;\
 742         .align  32
 743 
 744 /*
 745  * tag overflow trap
 746  */
 747 #define TAG_OVERFLOW                      \
 748         TT_TRACE(trace_gen)               ;\
 749         or      %g0, P_UTRAP10, %g2       ;\
 750         or      %g0, T_TAG_OVERFLOW, %g3  ;\
 751         sethi   %hi(.check_v9utrap), %g4  ;\
 752         jmp     %g4 + %lo(.check_v9utrap) ;\
 753         nop                               ;\
 754         .align  32
 755 
 756 /*
 757  * divide by zero trap
 758  */
 759 #define DIV_BY_ZERO                       \
 760         TT_TRACE(trace_gen)               ;\
 761         or      %g0, P_UTRAP11, %g2       ;\
 762         or      %g0, T_IDIV0, %g3         ;\
 763         sethi   %hi(.check_v9utrap), %g4  ;\
 764         jmp     %g4 + %lo(.check_v9utrap) ;\
 765         nop                               ;\
 766         .align  32
 767 
 768 /*
 769  * trap instruction for V9 user trap handlers
 770  */
 771 #define TRAP_INSTR                        \
 772         TT_TRACE(trace_gen)               ;\
 773         or      %g0, T_SOFTWARE_TRAP, %g3 ;\
 774         sethi   %hi(.check_v9utrap), %g4  ;\
 775         jmp     %g4 + %lo(.check_v9utrap) ;\
 776         nop                               ;\
 777         .align  32
 778 #define TRP4    TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
 779 
 780 /*
 781  * LEVEL_INTERRUPT is for level N interrupts.
 782  * VECTOR_INTERRUPT is for the vector trap.
 783  */
 784 #define LEVEL_INTERRUPT(level)          \
 785         .global tt_pil/**/level         ;\
 786 tt_pil/**/level:                        ;\
 787         ba,pt   %xcc, pil_interrupt     ;\
 788         mov     level, %g4              ;\
 789         .align  32
 790 
 791 #define LEVEL14_INTERRUPT                       \
 792         ba      pil14_interrupt                 ;\
 793         mov     PIL_14, %g4                     ;\
 794         .align  32
 795 
 796 #define        LEVEL15_INTERRUPT                       \
 797        ba      pil15_interrupt                 ;\
 798        mov     PIL_15, %g4                     ;\
 799        .align  32
 800 
 801 #define CPU_MONDO                       \
 802         ba,a,pt %xcc, cpu_mondo         ;\
 803         .align  32
 804 
 805 #define DEV_MONDO                       \
 806         ba,a,pt %xcc, dev_mondo         ;\
 807         .align  32
 808 
 809 /*
 810  * We take over the rtba after we set our trap table and
 811  * fault status area. The watchdog reset trap is now handled by the OS.
 812  */
 813 #define WATCHDOG_RESET                  \
 814         mov     PTL1_BAD_WATCHDOG, %g1  ;\
 815         ba,a,pt %xcc, .watchdog_trap    ;\
 816         .align  32
 817 
 818 /*
 819  * RED is for traps that use the red mode handler.
 820  * We should never see these either.
 821  */
 822 #define RED                     \
 823         mov     PTL1_BAD_RED, %g1       ;\
 824         ba,a,pt %xcc, .watchdog_trap    ;\
 825         .align  32
 826 
 827 
 828 /*
 829  * MMU Trap Handlers.
 830  */
 831 
 832 /*
 833  * synthesize for trap(): SFSR in %g3
 834  */
 835 #define IMMU_EXCEPTION                                                  \
 836         MMU_FAULT_STATUS_AREA(%g3)                                      ;\
 837         rdpr    %tpc, %g2                                               ;\
 838         ldx     [%g3 + MMFSA_I_TYPE], %g1                               ;\
 839         ldx     [%g3 + MMFSA_I_CTX], %g3                                ;\
 840         sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
 841         or      %g3, %g1, %g3                                           ;\
 842         ba,pt   %xcc, .mmu_exception_end                                ;\
 843         mov     T_INSTR_EXCEPTION, %g1                                  ;\
 844         .align  32
 845 
 846 /*
 847  * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3
 848  */
 849 #define DMMU_EXCEPTION                                                  \
 850         ba,a,pt %xcc, .dmmu_exception                                   ;\
 851         .align  32
 852 
 853 /*
 854  * synthesize for trap(): SFAR in %g2, SFSR in %g3
 855  */
 856 #define DMMU_EXC_AG_PRIV                                                \
 857         MMU_FAULT_STATUS_AREA(%g3)                                      ;\
 858         ldx     [%g3 + MMFSA_D_ADDR], %g2                               ;\
 859         /* Fault type not available in MMU fault status area */         ;\
 860         mov     MMFSA_F_PRVACT, %g1                                     ;\
 861         ldx     [%g3 + MMFSA_D_CTX], %g3                                ;\
 862         sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
 863         ba,pt   %xcc, .mmu_priv_exception                               ;\
 864         or      %g3, %g1, %g3                                           ;\
 865         .align  32
 866 
 867 /*
 868  * synthesize for trap(): SFAR in %g2, SFSR in %g3
 869  */
 870 #define DMMU_EXC_AG_NOT_ALIGNED                                         \
 871         MMU_FAULT_STATUS_AREA(%g3)                                      ;\
 872         ldx     [%g3 + MMFSA_D_ADDR], %g2                               ;\
 873         /* Fault type not available in MMU fault status area */         ;\
 874         mov     MMFSA_F_UNALIGN, %g1                                    ;\
 875         ldx     [%g3 + MMFSA_D_CTX], %g3                                ;\
 876         sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
 877         ba,pt   %xcc, .mmu_exception_not_aligned                        ;\
 878         or      %g3, %g1, %g3                   /* SFSR */              ;\
 879         .align  32
 880 /*
 881  * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
 882  */
 883 
 884 /*
 885  * synthesize for trap(): SFAR in %g2, SFSR in %g3
 886  */
 887 #define DMMU_EXC_LDDF_NOT_ALIGNED                                       \
 888         ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned                        ;\
 889         .align  32
 890 /*
 891  * synthesize for trap(): SFAR in %g2, SFSR in %g3
 892  */
 893 #define DMMU_EXC_STDF_NOT_ALIGNED                                       \
 894         ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned                        ;\
 895         .align  32
 896 
 897 #if defined(cscope)
 898 /*
 899  * Define labels to direct cscope quickly to labels that
 900  * are generated by macro expansion of DTLB_MISS().
 901  */
 902         .global tt0_dtlbmiss
 903 tt0_dtlbmiss:
 904         .global tt1_dtlbmiss
 905 tt1_dtlbmiss:
 906         nop
 907 #endif
 908 
 909 /*
 910  * Data miss handler (must be exactly 32 instructions)
 911  *
 912  * This handler is invoked only if the hypervisor has been instructed
 913  * not to do any TSB walk.
 914  *
 915  * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss
 916  * handler.
 917  *
 918  * User TLB miss handling depends upon whether a user process has one or
 919  * two TSBs. User TSB information (physical base and size code) is kept
 920  * in two dedicated scratchpad registers. Absence of a user TSB (primarily
 921  * second TSB) is indicated by a negative value (-1) in that register.
 922  */
 923 
 924 /*
 925  * synthesize for miss handler: pseudo-tag access in %g2 (with context "type"
 926  * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 927  */
 928 #define DTLB_MISS(table_name)                                           ;\
 929         .global table_name/**/_dtlbmiss                                 ;\
 930 table_name/**/_dtlbmiss:                                                ;\
 931         GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
 932         cmp     %g3, INVALID_CONTEXT                                    ;\
 933         ble,pn  %xcc, sfmmu_kdtlb_miss                                  ;\
 934           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
 935         mov     SCRATCHPAD_UTSBREG2, %g1                                ;\
 936         ldxa    [%g1]ASI_SCRATCHPAD, %g1        /* get 2nd tsbreg */    ;\
 937         brgez,pn %g1, sfmmu_udtlb_slowpath      /* branch if 2 TSBs */  ;\
 938           nop                                                           ;\
 939         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)    /* 11 instr */          ;\
 940         ba,pt   %xcc, sfmmu_udtlb_fastpath      /* no 4M TSB, miss */   ;\
 941           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
 942         .align 128
 943 
 944 
 945 #if defined(cscope)
 946 /*
 947  * Define labels to direct cscope quickly to labels that
 948  * are generated by macro expansion of ITLB_MISS().
 949  */
 950         .global tt0_itlbmiss
 951 tt0_itlbmiss:
 952         .global tt1_itlbmiss
 953 tt1_itlbmiss:
 954         nop
 955 #endif
 956 
 957 /*
 958  * Instruction miss handler.
 959  *
 960  * This handler is invoked only if the hypervisor has been instructed
 961  * not to do any TSB walk.
 962  *
 963  * ldda instructions will have their ASI patched
 964  * by sfmmu_patch_ktsb at runtime.
 965  * MUST be EXACTLY 32 instructions or we'll break.
 966  */
 967 
 968 /*
 969  * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
 970  * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 971  */
 972 #define ITLB_MISS(table_name)                                            \
 973         .global table_name/**/_itlbmiss                                 ;\
 974 table_name/**/_itlbmiss:                                                ;\
 975         GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
 976         cmp     %g3, INVALID_CONTEXT                                    ;\
 977         ble,pn  %xcc, sfmmu_kitlb_miss                                  ;\
 978           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
 979         mov     SCRATCHPAD_UTSBREG2, %g1                                ;\
 980         ldxa    [%g1]ASI_SCRATCHPAD, %g1        /* get 2nd tsbreg */    ;\
 981         brgez,pn %g1, sfmmu_uitlb_slowpath      /* branch if 2 TSBs */  ;\
 982           nop                                                           ;\
 983         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)    /* 11 instr */          ;\
 984         ba,pt   %xcc, sfmmu_uitlb_fastpath      /* no 4M TSB, miss */   ;\
 985           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
 986         .align 128
 987 
 988 #define DTSB_MISS \
 989         GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu)
 990 
 991 #define ITSB_MISS \
 992         GOTO_TT(sfmmu_slow_immu_miss,trace_immu)
 993 
 994 /*
 995  * This macro is the first level handler for fast protection faults.
 996  * It first demaps the tlb entry which generated the fault and then
 997  * attempts to set the modify bit on the hash.  It needs to be
 998  * exactly 32 instructions.
 999  */
1000 /*
1001  * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1002  * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1003  */
1004 #define DTLB_PROT                                                        \
1005         GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
1006         /*                                                              ;\
1007          *   g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\
1008          *   g3 = ctx type (0, 1, or 2)                                 ;\
1009          */                                                             ;\
1010         TT_TRACE(trace_dataprot)        /* 2 instr ifdef TRAPTRACE */   ;\
1011                                         /* clobbers g1 and g6 XXXQ? */  ;\
1012         brnz,pt %g3, sfmmu_uprot_trap           /* user trap */         ;\
1013           nop                                                           ;\
1014         ba,a,pt %xcc, sfmmu_kprot_trap          /* kernel trap */       ;\
1015         .align 128
1016 
1017 #define DMMU_EXCEPTION_TL1                                              ;\
1018         ba,a,pt %xcc, mmu_trap_tl1                                      ;\
1019         .align 32
1020 
1021 #define MISALIGN_ADDR_TL1                                               ;\
1022         ba,a,pt %xcc, mmu_trap_tl1                                      ;\
1023         .align 32
1024 
1025 /*
1026  * Trace a tsb hit
1027  * g1 = tsbe pointer (in/clobbered)
1028  * g2 = tag access register (in)
1029  * g3 - g4 = scratch (clobbered)
1030  * g5 = tsbe data (in)
1031  * g6 = scratch (clobbered)
1032  * g7 = pc we jumped here from (in)
1033  * ttextra = value to OR in to trap type (%tt) (in)
1034  */
1035 #ifdef TRAPTRACE
1036 #define TRACE_TSBHIT(ttextra)                                            \
1037         membar  #Sync                                                   ;\
1038         sethi   %hi(FLUSH_ADDR), %g6                                    ;\
1039         flush   %g6                                                     ;\
1040         TRACE_PTR(%g3, %g6)                                             ;\
1041         GET_TRACE_TICK(%g6, %g4)                                        ;\
1042         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi                          ;\
1043         stna    %g2, [%g3 + TRAP_ENT_SP]%asi    /* tag access */        ;\
1044         stna    %g5, [%g3 + TRAP_ENT_F1]%asi    /* tsb data */          ;\
1045         rdpr    %tnpc, %g6                                              ;\
1046         stna    %g6, [%g3 + TRAP_ENT_F2]%asi                            ;\
1047         stna    %g1, [%g3 + TRAP_ENT_F3]%asi    /* tsb pointer */       ;\
1048         stna    %g0, [%g3 + TRAP_ENT_F4]%asi                            ;\
1049         rdpr    %tpc, %g6                                               ;\
1050         stna    %g6, [%g3 + TRAP_ENT_TPC]%asi                           ;\
1051         TRACE_SAVE_TL_GL_REGS(%g3, %g6)                                 ;\
1052         rdpr    %tt, %g6                                                ;\
1053         or      %g6, (ttextra), %g1                                     ;\
1054         stha    %g1, [%g3 + TRAP_ENT_TT]%asi                            ;\
1055         MMU_FAULT_STATUS_AREA(%g4)                                      ;\
1056         mov     MMFSA_D_ADDR, %g1                                       ;\
1057         cmp     %g6, FAST_IMMU_MISS_TT                                  ;\
1058         move    %xcc, MMFSA_I_ADDR, %g1                                 ;\
1059         cmp     %g6, T_INSTR_MMU_MISS                                   ;\
1060         move    %xcc, MMFSA_I_ADDR, %g1                                 ;\
1061         ldx     [%g4 + %g1], %g1                                        ;\
1062         stxa    %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */       ;\
1063         mov     MMFSA_D_CTX, %g1                                        ;\
1064         cmp     %g6, FAST_IMMU_MISS_TT                                  ;\
1065         move    %xcc, MMFSA_I_CTX, %g1                                  ;\
1066         cmp     %g6, T_INSTR_MMU_MISS                                   ;\
1067         move    %xcc, MMFSA_I_CTX, %g1                                  ;\
1068         ldx     [%g4 + %g1], %g1                                        ;\
1069         stna    %g1, [%g3 + TRAP_ENT_TR]%asi                            ;\
1070         TRACE_NEXT(%g3, %g4, %g6)
1071 #else
1072 #define TRACE_TSBHIT(ttextra)
1073 #endif
1074 
1075 
1076 /*
1077  * =======================================================================
1078  *              SPARC V9 TRAP TABLE
1079  *
1080  * The trap table is divided into two halves: the first half is used when
1081  * taking traps when TL=0; the second half is used when taking traps from
1082  * TL>0. Note that handlers in the second half of the table might not be able
1083  * to make the same assumptions as handlers in the first half of the table.
1084  *
1085  * Worst case trap nesting so far:
1086  *
1087  *      at TL=0 client issues software trap requesting service
1088  *      at TL=1 nucleus wants a register window
1089  *      at TL=2 register window clean/spill/fill takes a TLB miss
1090  *      at TL=3 processing TLB miss
1091  *      at TL=4 handle asynchronous error
1092  *
1093  * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1094  *
1095  * =======================================================================
1096  */
1097         .section ".text"
1098         .align  4
1099         .global trap_table, scb, trap_table0, trap_table1, etrap_table
1100         .type   trap_table, #object
1101         .type   trap_table0, #object
1102         .type   trap_table1, #object
1103         .type   scb, #object
1104 trap_table:
1105 scb:
1106 trap_table0:
1107         /* hardware traps */
1108         NOT;                            /* 000  reserved */
1109         RED;                            /* 001  power on reset */
1110         WATCHDOG_RESET;                 /* 002  watchdog reset */
1111         RED;                            /* 003  externally initiated reset */
1112         RED;                            /* 004  software initiated reset */
1113         RED;                            /* 005  red mode exception */
1114         NOT; NOT;                       /* 006 - 007 reserved */
1115         IMMU_EXCEPTION;                 /* 008  instruction access exception */
1116         ITSB_MISS;                      /* 009  instruction access MMU miss */
1117         NOT;                            /* 00A  reserved */
1118         NOT; NOT4;                      /* 00B - 00F reserved */
1119         ILLTRAP_INSTR;                  /* 010  illegal instruction */
1120         TRAP(T_PRIV_INSTR);             /* 011  privileged opcode */
1121         TRAP(T_UNIMP_LDD);              /* 012  unimplemented LDD */
1122         TRAP(T_UNIMP_STD);              /* 013  unimplemented STD */
1123         NOT4; NOT4; NOT4;               /* 014 - 01F reserved */
1124         FP_DISABLED_TRAP;               /* 020  fp disabled */
1125         FP_IEEE_TRAP;                   /* 021  fp exception ieee 754 */
1126         FP_TRAP;                        /* 022  fp exception other */
1127         TAG_OVERFLOW;                   /* 023  tag overflow */
1128         CLEAN_WINDOW;                   /* 024 - 027 clean window */
1129         DIV_BY_ZERO;                    /* 028  division by zero */
1130         NOT;                            /* 029  internal processor error */
1131         NOT; NOT; NOT4;                 /* 02A - 02F reserved */
1132         DMMU_EXCEPTION;                 /* 030  data access exception */
1133         DTSB_MISS;                      /* 031  data access MMU miss */
1134         NOT;                            /* 032  reserved */
1135         NOT;                            /* 033  data access protection */
1136         DMMU_EXC_AG_NOT_ALIGNED;        /* 034  mem address not aligned */
1137         DMMU_EXC_LDDF_NOT_ALIGNED;      /* 035  LDDF mem address not aligned */
1138         DMMU_EXC_STDF_NOT_ALIGNED;      /* 036  STDF mem address not aligned */
1139         DMMU_EXC_AG_PRIV;               /* 037  privileged action */
1140         NOT;                            /* 038  LDQF mem address not aligned */
1141         NOT;                            /* 039  STQF mem address not aligned */
1142         NOT; NOT; NOT4;                 /* 03A - 03F reserved */
1143         NOT;                            /* 040  async data error */
1144         LEVEL_INTERRUPT(1);             /* 041  interrupt level 1 */
1145         LEVEL_INTERRUPT(2);             /* 042  interrupt level 2 */
1146         LEVEL_INTERRUPT(3);             /* 043  interrupt level 3 */
1147         LEVEL_INTERRUPT(4);             /* 044  interrupt level 4 */
1148         LEVEL_INTERRUPT(5);             /* 045  interrupt level 5 */
1149         LEVEL_INTERRUPT(6);             /* 046  interrupt level 6 */
1150         LEVEL_INTERRUPT(7);             /* 047  interrupt level 7 */
1151         LEVEL_INTERRUPT(8);             /* 048  interrupt level 8 */
1152         LEVEL_INTERRUPT(9);             /* 049  interrupt level 9 */
1153         LEVEL_INTERRUPT(10);            /* 04A  interrupt level 10 */
1154         LEVEL_INTERRUPT(11);            /* 04B  interrupt level 11 */
1155         LEVEL_INTERRUPT(12);            /* 04C  interrupt level 12 */
1156         LEVEL_INTERRUPT(13);            /* 04D  interrupt level 13 */
1157         LEVEL14_INTERRUPT;              /* 04E  interrupt level 14 */
1158         LEVEL15_INTERRUPT;              /* 04F  interrupt level 15 */
1159         NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F reserved */
1160         NOT;                            /* 060  interrupt vector */
1161         GOTO(kmdb_trap);                /* 061  PA watchpoint */
1162         GOTO(kmdb_trap);                /* 062  VA watchpoint */
1163         NOT;                            /* 063  reserved */
1164         ITLB_MISS(tt0);                 /* 064  instruction access MMU miss */
1165         DTLB_MISS(tt0);                 /* 068  data access MMU miss */
1166         DTLB_PROT;                      /* 06C  data access protection */
1167         NOT;                            /* 070  reserved */
1168         NOT;                            /* 071  reserved */
1169         NOT;                            /* 072  reserved */
1170         NOT;                            /* 073  reserved */
1171         NOT4; NOT4                      /* 074 - 07B reserved */
1172         CPU_MONDO;                      /* 07C  cpu_mondo */
1173         DEV_MONDO;                      /* 07D  dev_mondo */
1174         GOTO_TT(resumable_error, trace_gen);    /* 07E  resumable error */
1175         GOTO_TT(nonresumable_error, trace_gen); /* 07F  non-reasumable error */
1176         NOT4;                           /* 080  spill 0 normal */
1177         SPILL_32bit_asi(ASI_AIUP,sn0);  /* 084  spill 1 normal */
1178         SPILL_64bit_asi(ASI_AIUP,sn0);  /* 088  spill 2 normal */
1179         SPILL_32clean(ASI_AIUP,sn0);    /* 08C  spill 3 normal */
1180         SPILL_64clean(ASI_AIUP,sn0);    /* 090  spill 4 normal */
1181         SPILL_32bit(not);               /* 094  spill 5 normal */
1182         SPILL_64bit(not);               /* 098  spill 6 normal */
1183         SPILL_mixed;                    /* 09C  spill 7 normal */
1184         NOT4;                           /* 0A0  spill 0 other */
1185         SPILL_32bit_asi(ASI_AIUS,so0);  /* 0A4  spill 1 other */
1186         SPILL_64bit_asi(ASI_AIUS,so0);  /* 0A8  spill 2 other */
1187         SPILL_32bit_asi(ASI_AIUS,so0);  /* 0AC  spill 3 other */
1188         SPILL_64bit_asi(ASI_AIUS,so0);  /* 0B0  spill 4 other */
1189         NOT4;                           /* 0B4  spill 5 other */
1190         NOT4;                           /* 0B8  spill 6 other */
1191         NOT4;                           /* 0BC  spill 7 other */
1192         NOT4;                           /* 0C0  fill 0 normal */
1193         FILL_32bit_asi(ASI_AIUP,fn0);   /* 0C4  fill 1 normal */
1194         FILL_64bit_asi(ASI_AIUP,fn0);   /* 0C8  fill 2 normal */
1195         FILL_32bit_asi(ASI_AIUP,fn0);   /* 0CC  fill 3 normal */
1196         FILL_64bit_asi(ASI_AIUP,fn0);   /* 0D0  fill 4 normal */
1197         FILL_32bit(not);                /* 0D4  fill 5 normal */
1198         FILL_64bit(not);                /* 0D8  fill 6 normal */
1199         FILL_mixed;                     /* 0DC  fill 7 normal */
1200         NOT4;                           /* 0E0  fill 0 other */
1201         NOT4;                           /* 0E4  fill 1 other */
1202         NOT4;                           /* 0E8  fill 2 other */
1203         NOT4;                           /* 0EC  fill 3 other */
1204         NOT4;                           /* 0F0  fill 4 other */
1205         NOT4;                           /* 0F4  fill 5 other */
1206         NOT4;                           /* 0F8  fill 6 other */
1207         NOT4;                           /* 0FC  fill 7 other */
1208         /* user traps */
1209         GOTO(syscall_trap_4x);          /* 100  old system call */
1210         TRAP(T_BREAKPOINT);             /* 101  user breakpoint */
1211         TRAP(T_DIV0);                   /* 102  user divide by zero */
1212         GOTO(.flushw);                  /* 103  flush windows */
1213         GOTO(.clean_windows);           /* 104  clean windows */
1214         BAD;                            /* 105  range check ?? */
1215         GOTO(.fix_alignment);           /* 106  do unaligned references */
1216         BAD;                            /* 107  unused */
1217         SYSCALL_TRAP32;                 /* 108  ILP32 system call on LP64 */
1218         GOTO(set_trap0_addr);           /* 109  set trap0 address */
1219         BAD; BAD; BAD4;                 /* 10A - 10F unused */
1220         TRP4; TRP4; TRP4; TRP4;         /* 110 - 11F V9 user trap handlers */
1221         GOTO(.getcc);                   /* 120  get condition codes */
1222         GOTO(.setcc);                   /* 121  set condition codes */
1223         GOTO(.getpsr);                  /* 122  get psr */
1224         GOTO(.setpsr);                  /* 123  set psr (some fields) */
1225         GOTO(get_timestamp);            /* 124  get timestamp */
1226         GOTO(get_virtime);              /* 125  get lwp virtual time */
1227         PRIV(self_xcall);               /* 126  self xcall */
1228         GOTO(get_hrestime);             /* 127  get hrestime */
1229         BAD;                            /* 128  ST_SETV9STACK */
1230         GOTO(.getlgrp);                 /* 129  get lgrpid */
1231         BAD; BAD; BAD4;                 /* 12A - 12F unused */
1232         BAD4; BAD4;                     /* 130 - 137 unused */
1233         DTRACE_PID;                     /* 138  dtrace pid tracing provider */
1234         BAD;                            /* 139  unused */
1235         DTRACE_RETURN;                  /* 13A  dtrace pid return probe */
1236         BAD; BAD4;                      /* 13B - 13F unused */
1237         SYSCALL_TRAP;                   /* 140  LP64 system call */
1238         SYSCALL(nosys);                 /* 141  unused system call trap */
1239 #ifdef DEBUG_USER_TRAPTRACECTL
1240         GOTO(.traptrace_freeze);        /* 142  freeze traptrace */
1241         GOTO(.traptrace_unfreeze);      /* 143  unfreeze traptrace */
1242 #else
1243         SYSCALL(nosys);                 /* 142  unused system call trap */
1244         SYSCALL(nosys);                 /* 143  unused system call trap */
1245 #endif
1246         BAD4; BAD4; BAD4;               /* 144 - 14F unused */
1247         BAD4; BAD4; BAD4; BAD4;         /* 150 - 15F unused */
1248         BAD4; BAD4; BAD4; BAD4;         /* 160 - 16F unused */
1249         BAD;                            /* 170 - unused */
1250         BAD;                            /* 171 - unused */
1251         BAD; BAD;                       /* 172 - 173 unused */
1252         BAD4; BAD4;                     /* 174 - 17B unused */
1253 #ifdef  PTL1_PANIC_DEBUG
1254         mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1255                                         /* 17C  test ptl1_panic */
1256 #else
1257         BAD;                            /* 17C  unused */
1258 #endif  /* PTL1_PANIC_DEBUG */
1259         PRIV(kmdb_trap);                /* 17D  kmdb enter (L1-A) */
1260         PRIV(kmdb_trap);                /* 17E  kmdb breakpoint */
1261         PRIV(obp_bpt);                  /* 17F  obp breakpoint */
1262         /* reserved */
1263         NOT4; NOT4; NOT4; NOT4;         /* 180 - 18F reserved */
1264         NOT4; NOT4; NOT4; NOT4;         /* 190 - 19F reserved */
1265         NOT4; NOT4; NOT4; NOT4;         /* 1A0 - 1AF reserved */
1266         NOT4; NOT4; NOT4; NOT4;         /* 1B0 - 1BF reserved */
1267         NOT4; NOT4; NOT4; NOT4;         /* 1C0 - 1CF reserved */
1268         NOT4; NOT4; NOT4; NOT4;         /* 1D0 - 1DF reserved */
1269         NOT4; NOT4; NOT4; NOT4;         /* 1E0 - 1EF reserved */
1270         NOT4; NOT4; NOT4; NOT4;         /* 1F0 - 1FF reserved */
1271         .size   trap_table0, (.-trap_table0)
1272 trap_table1:
1273         NOT4; NOT4;                     /* 000 - 007 unused */
1274         NOT;                            /* 008  instruction access exception */
1275         ITSB_MISS;                      /* 009  instruction access MMU miss */
1276         NOT;                            /* 00A  reserved */
1277         NOT; NOT4;                      /* 00B - 00F unused */
1278         NOT4; NOT4; NOT4; NOT4;         /* 010 - 01F unused */
1279         NOT4;                           /* 020 - 023 unused */
1280         CLEAN_WINDOW;                   /* 024 - 027 clean window */
1281         NOT4; NOT4;                     /* 028 - 02F unused */
1282         DMMU_EXCEPTION_TL1;             /* 030  data access exception */
1283         DTSB_MISS;                      /* 031  data access MMU miss */
1284         NOT;                            /* 032  reserved */
1285         NOT;                            /* 033  unused */
1286         MISALIGN_ADDR_TL1;              /* 034  mem address not aligned */
1287         NOT; NOT; NOT; NOT4; NOT4       /* 035 - 03F unused */
1288         NOT4; NOT4; NOT4; NOT4;         /* 040 - 04F unused */
1289         NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F unused */
1290         NOT;                            /* 060  unused */
1291         GOTO(kmdb_trap_tl1);            /* 061  PA watchpoint */
1292         GOTO(kmdb_trap_tl1);            /* 062  VA watchpoint */
1293         NOT;                            /* 063  reserved */
1294         ITLB_MISS(tt1);                 /* 064  instruction access MMU miss */
1295         DTLB_MISS(tt1);                 /* 068  data access MMU miss */
1296         DTLB_PROT;                      /* 06C  data access protection */
1297         NOT;                            /* 070  reserved */
1298         NOT;                            /* 071  reserved */
1299         NOT;                            /* 072  reserved */
1300         NOT;                            /* 073  reserved */
1301         NOT4; NOT4;                     /* 074 - 07B reserved */
1302         NOT;                            /* 07C  reserved */
1303         NOT;                            /* 07D  reserved */
1304         NOT;                            /* 07E  resumable error */
1305         GOTO_TT(nonresumable_error, trace_gen); /* 07F  nonresumable error */
1306         NOTP4;                          /* 080  spill 0 normal */
1307         SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 084  spill 1 normal */
1308         SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 088  spill 2 normal */
1309         SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 08C  spill 3 normal */
1310         SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 090  spill 4 normal */
1311         NOTP4;                          /* 094  spill 5 normal */
1312         SPILL_64bit_ktt1(sk);           /* 098  spill 6 normal */
1313         SPILL_mixed_ktt1(sk);           /* 09C  spill 7 normal */
1314         NOTP4;                          /* 0A0  spill 0 other */
1315         SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0A4  spill 1 other */
1316         SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0A8  spill 2 other */
1317         SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0AC  spill 3 other */
1318         SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0B0  spill 4 other */
1319         NOTP4;                          /* 0B4  spill 5 other */
1320         NOTP4;                          /* 0B8  spill 6 other */
1321         NOTP4;                          /* 0BC  spill 7 other */
1322         NOT4;                           /* 0C0  fill 0 normal */
1323         NOT4;                           /* 0C4  fill 1 normal */
1324         NOT4;                           /* 0C8  fill 2 normal */
1325         NOT4;                           /* 0CC  fill 3 normal */
1326         NOT4;                           /* 0D0  fill 4 normal */
1327         NOT4;                           /* 0D4  fill 5 normal */
1328         NOT4;                           /* 0D8  fill 6 normal */
1329         NOT4;                           /* 0DC  fill 7 normal */
1330         NOT4; NOT4; NOT4; NOT4;         /* 0E0 - 0EF unused */
1331         NOT4; NOT4; NOT4; NOT4;         /* 0F0 - 0FF unused */
1332 /*
1333  * Code running at TL>0 does not use soft traps, so
1334  * we can truncate the table here.
1335  * However:
1336  * sun4v uses (hypervisor) ta instructions at TL > 0, so
1337  * provide a safety net for now.
1338  */
1339         /* soft traps */
1340         BAD4; BAD4; BAD4; BAD4;         /* 100 - 10F unused */
1341         BAD4; BAD4; BAD4; BAD4;         /* 110 - 11F unused */
1342         BAD4; BAD4; BAD4; BAD4;         /* 120 - 12F unused */
1343         BAD4; BAD4; BAD4; BAD4;         /* 130 - 13F unused */
1344         BAD4; BAD4; BAD4; BAD4;         /* 140 - 14F unused */
1345         BAD4; BAD4; BAD4; BAD4;         /* 150 - 15F unused */
1346         BAD4; BAD4; BAD4; BAD4;         /* 160 - 16F unused */
1347         BAD4; BAD4; BAD4; BAD4;         /* 170 - 17F unused */
1348         /* reserved */
1349         NOT4; NOT4; NOT4; NOT4;         /* 180 - 18F reserved */
1350         NOT4; NOT4; NOT4; NOT4;         /* 190 - 19F reserved */
1351         NOT4; NOT4; NOT4; NOT4;         /* 1A0 - 1AF reserved */
1352         NOT4; NOT4; NOT4; NOT4;         /* 1B0 - 1BF reserved */
1353         NOT4; NOT4; NOT4; NOT4;         /* 1C0 - 1CF reserved */
1354         NOT4; NOT4; NOT4; NOT4;         /* 1D0 - 1DF reserved */
1355         NOT4; NOT4; NOT4; NOT4;         /* 1E0 - 1EF reserved */
1356         NOT4; NOT4; NOT4; NOT4;         /* 1F0 - 1FF reserved */
1357 etrap_table:
1358         .size   trap_table1, (.-trap_table1)
1359         .size   trap_table, (.-trap_table)
1360         .size   scb, (.-scb)
1361 
1362 /*
1363  * We get to exec_fault in the case of an instruction miss and tte
1364  * has no execute bit set.  We go to tl0 to handle it.
1365  *
1366  * g1 = tsbe pointer (in/clobbered)
1367  * g2 = tag access register (in)
1368  * g3 - g4 = scratch (clobbered)
1369  * g5 = tsbe data (in)
1370  * g6 = scratch (clobbered)
1371  * g7 = pc we jumped here from (in)
1372  */
1373 /*
1374  * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1375  * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1376  */
1377         ALTENTRY(exec_fault)
1378         TRACE_TSBHIT(TT_MMU_EXEC)
1379         MMU_FAULT_STATUS_AREA(%g4)
1380         ldx     [%g4 + MMFSA_I_ADDR], %g2       /* g2 = address */
1381         ldx     [%g4 + MMFSA_I_CTX], %g3        /* g3 = ctx */
1382         srlx    %g2, MMU_PAGESHIFT, %g2         ! align address to page boundry
1383         cmp     %g3, USER_CONTEXT_TYPE
1384         sllx    %g2, MMU_PAGESHIFT, %g2
1385         movgu   %icc, USER_CONTEXT_TYPE, %g3
1386         or      %g2, %g3, %g2                   /* TAG_ACCESS */
1387         mov     T_INSTR_MMU_MISS, %g3           ! arg2 = traptype
1388         set     trap, %g1
1389         ba,pt   %xcc, sys_trap
1390           mov   -1, %g4
1391 
1392 .mmu_exception_not_aligned:
1393         /* %g2 = sfar, %g3 = sfsr */
1394         rdpr    %tstate, %g1
1395         btst    TSTATE_PRIV, %g1
1396         bnz,pn  %icc, 2f
1397         nop
1398         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1399         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1400         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1401         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1402         brz,pt  %g5, 2f
1403         nop
1404         ldn     [%g5 + P_UTRAP15], %g5                  ! unaligned utrap?
1405         brz,pn  %g5, 2f
1406         nop
1407         btst    1, %sp
1408         bz,pt   %xcc, 1f                                ! 32 bit user program
1409         nop
1410         ba,pt   %xcc, .setup_v9utrap                    ! 64 bit user program
1411         nop
1412 1:
1413         ba,pt   %xcc, .setup_utrap
1414         or      %g2, %g0, %g7
1415 2:
1416         ba,pt   %xcc, .mmu_exception_end
1417         mov     T_ALIGNMENT, %g1
1418 
1419 .mmu_priv_exception:
1420         rdpr    %tstate, %g1
1421         btst    TSTATE_PRIV, %g1
1422         bnz,pn  %icc, 1f
1423         nop
1424         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1425         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1426         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1427         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1428         brz,pt  %g5, 1f
1429         nop
1430         ldn     [%g5 + P_UTRAP16], %g5
1431         brnz,pt %g5, .setup_v9utrap
1432         nop
1433 1:
1434         mov     T_PRIV_INSTR, %g1
1435 
1436 .mmu_exception_end:
1437         CPU_INDEX(%g4, %g5)
1438         set     cpu_core, %g5
1439         sllx    %g4, CPU_CORE_SHIFT, %g4
1440         add     %g4, %g5, %g4
1441         lduh    [%g4 + CPUC_DTRACE_FLAGS], %g5
1442         andcc   %g5, CPU_DTRACE_NOFAULT, %g0
1443         bz      1f
1444         or      %g5, CPU_DTRACE_BADADDR, %g5
1445         stuh    %g5, [%g4 + CPUC_DTRACE_FLAGS]
1446         done
1447 
1448 1:
1449         sllx    %g3, 32, %g3
1450         or      %g3, %g1, %g3
1451         set     trap, %g1
1452         ba,pt   %xcc, sys_trap
1453         sub     %g0, 1, %g4
1454 
1455 .fp_disabled:
1456         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1457         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1458         rdpr    %tstate, %g4
1459         btst    TSTATE_PRIV, %g4
1460         bnz,a,pn %icc, ptl1_panic
1461           mov   PTL1_BAD_FPTRAP, %g1
1462 
1463         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1464         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1465         brz,a,pt %g5, 2f
1466           nop
1467         ldn     [%g5 + P_UTRAP7], %g5                   ! fp_disabled utrap?
1468         brz,a,pn %g5, 2f
1469           nop
1470         btst    1, %sp
1471         bz,a,pt %xcc, 1f                                ! 32 bit user program
1472           nop
1473         ba,a,pt %xcc, .setup_v9utrap                    ! 64 bit user program
1474           nop
1475 1:
1476         ba,pt   %xcc, .setup_utrap
1477           or    %g0, %g0, %g7
1478 2:
1479         set     fp_disabled, %g1
1480         ba,pt   %xcc, sys_trap
1481           sub   %g0, 1, %g4
1482 
1483 .fp_ieee_exception:
1484         rdpr    %tstate, %g1
1485         btst    TSTATE_PRIV, %g1
1486         bnz,a,pn %icc, ptl1_panic
1487           mov   PTL1_BAD_FPTRAP, %g1
1488         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1489         stx     %fsr, [%g1 + CPU_TMP1]
1490         ldx     [%g1 + CPU_TMP1], %g2
1491         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1492         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1493         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1494         brz,a,pt %g5, 1f
1495           nop
1496         ldn     [%g5 + P_UTRAP8], %g5
1497         brnz,a,pt %g5, .setup_v9utrap
1498           nop
1499 1:
1500         set     _fp_ieee_exception, %g1
1501         ba,pt   %xcc, sys_trap
1502           sub   %g0, 1, %g4
1503 
1504 /*
1505  * Register Inputs:
1506  *      %g5             user trap handler
1507  *      %g7             misaligned addr - for alignment traps only
1508  */
1509 .setup_utrap:
1510         set     trap, %g1                       ! setup in case we go
1511         mov     T_FLUSH_PCB, %g3                ! through sys_trap on
1512         sub     %g0, 1, %g4                     ! the save instruction below
1513 
1514         /*
1515          * If the DTrace pid provider is single stepping a copied-out
1516          * instruction, t->t_dtrace_step will be set. In that case we need
1517          * to abort the single-stepping (since execution of the instruction
1518          * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1519          */
1520         save    %sp, -SA(MINFRAME32), %sp       ! window for trap handler
1521         CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
1522         ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
1523         ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
1524         rdpr    %tnpc, %l2                      ! arg1 == tnpc
1525         brz,pt  %g2, 1f
1526         rdpr    %tpc, %l1                       ! arg0 == tpc
1527 
1528         ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
1529         ldn     [%g1 + T_DTRACE_NPC], %l2       ! arg1 = t->t_dtrace_npc (step)
1530         brz,pt  %g2, 1f
1531         st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
1532         stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1533 1:
1534         mov     %g7, %l3                        ! arg2 == misaligned address
1535 
1536         rdpr    %tstate, %g1                    ! cwp for trap handler
1537         rdpr    %cwp, %g4
1538         bclr    TSTATE_CWP_MASK, %g1
1539         wrpr    %g1, %g4, %tstate
1540         wrpr    %g0, %g5, %tnpc                 ! trap handler address
1541         FAST_TRAP_DONE
1542         /* NOTREACHED */
1543 
1544 .check_v9utrap:
1545         rdpr    %tstate, %g1
1546         btst    TSTATE_PRIV, %g1
1547         bnz,a,pn %icc, 3f
1548           nop
1549         CPU_ADDR(%g4, %g1)                              ! load CPU struct addr
1550         ldn     [%g4 + CPU_THREAD], %g5                 ! load thread pointer
1551         ldn     [%g5 + T_PROCP], %g5                    ! load proc pointer
1552         ldn     [%g5 + P_UTRAPS], %g5                   ! are there utraps?
1553 
1554         cmp     %g3, T_SOFTWARE_TRAP
1555         bne,a,pt %icc, 1f
1556           nop
1557 
1558         brz,pt %g5, 3f                  ! if p_utraps == NULL goto trap()
1559           rdpr  %tt, %g3                ! delay - get actual hw trap type
1560 
1561         sub     %g3, 254, %g1           ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1562         ba,pt   %icc, 2f
1563           smul  %g1, CPTRSIZE, %g2
1564 1:
1565         brz,a,pt %g5, 3f                ! if p_utraps == NULL goto trap()
1566           nop
1567 
1568         cmp     %g3, T_UNIMP_INSTR
1569         bne,a,pt %icc, 2f
1570           nop
1571 
1572         mov     1, %g1
1573         st      %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1574         rdpr    %tpc, %g1               ! ld trapping instruction using
1575         lduwa   [%g1]ASI_AIUP, %g1      ! "AS IF USER" ASI which could fault
1576         st      %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1577 
1578         sethi   %hi(0xc1c00000), %g4    ! setup mask for illtrap instruction
1579         andcc   %g1, %g4, %g4           ! and instruction with mask
1580         bnz,a,pt %icc, 3f               ! if %g4 == zero, %g1 is an ILLTRAP
1581           nop                           ! fall thru to setup
1582 2:
1583         ldn     [%g5 + %g2], %g5
1584         brnz,a,pt %g5, .setup_v9utrap
1585           nop
1586 3:
1587         set     trap, %g1
1588         ba,pt   %xcc, sys_trap
1589           sub   %g0, 1, %g4
1590         /* NOTREACHED */
1591 
1592 /*
1593  * Register Inputs:
1594  *      %g5             user trap handler
1595  */
1596 .setup_v9utrap:
1597         set     trap, %g1                       ! setup in case we go
1598         mov     T_FLUSH_PCB, %g3                ! through sys_trap on
1599         sub     %g0, 1, %g4                     ! the save instruction below
1600 
1601         /*
1602          * If the DTrace pid provider is single stepping a copied-out
1603          * instruction, t->t_dtrace_step will be set. In that case we need
1604          * to abort the single-stepping (since execution of the instruction
1605          * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1606          */
1607         save    %sp, -SA(MINFRAME64), %sp       ! window for trap handler
1608         CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
1609         ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
1610         ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
1611         rdpr    %tnpc, %l7                      ! arg1 == tnpc
1612         brz,pt  %g2, 1f
1613         rdpr    %tpc, %l6                       ! arg0 == tpc
1614 
1615         ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
1616         ldn     [%g1 + T_DTRACE_NPC], %l7       ! arg1 == t->t_dtrace_npc (step)
1617         brz,pt  %g2, 1f
1618         st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
1619         stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1620 1:
1621         rdpr    %tstate, %g2                    ! cwp for trap handler
1622         rdpr    %cwp, %g4
1623         bclr    TSTATE_CWP_MASK, %g2
1624         wrpr    %g2, %g4, %tstate
1625 
1626         ldn     [%g1 + T_PROCP], %g4            ! load proc pointer
1627         ldn     [%g4 + P_AS], %g4               ! load as pointer
1628         ldn     [%g4 + A_USERLIMIT], %g4        ! load as userlimit
1629         cmp     %l7, %g4                        ! check for single-step set
1630         bne,pt  %xcc, 4f
1631           nop
1632         ldn     [%g1 + T_LWP], %g1              ! load klwp pointer
1633         ld      [%g1 + PCB_STEP], %g4           ! load single-step flag
1634         cmp     %g4, STEP_ACTIVE                ! step flags set in pcb?
1635         bne,pt  %icc, 4f
1636           nop
1637         stn     %g5, [%g1 + PCB_TRACEPC]        ! save trap handler addr in pcb
1638         mov     %l7, %g4                        ! on entry to precise user trap
1639         add     %l6, 4, %l7                     ! handler, %l6 == pc, %l7 == npc
1640                                                 ! at time of trap
1641         wrpr    %g0, %g4, %tnpc                 ! generate FLTBOUNDS,
1642                                                 ! %g4 == userlimit
1643         FAST_TRAP_DONE
1644         /* NOTREACHED */
1645 4:
1646         wrpr    %g0, %g5, %tnpc                 ! trap handler address
1647         FAST_TRAP_DONE_CHK_INTR
1648         /* NOTREACHED */
1649 
1650 .fp_exception:
1651         CPU_ADDR(%g1, %g4)
1652         stx     %fsr, [%g1 + CPU_TMP1]
1653         ldx     [%g1 + CPU_TMP1], %g2
1654 
1655         /*
1656          * Cheetah takes unfinished_FPop trap for certain range of operands
1657          * to the "fitos" instruction. Instead of going through the slow
1658          * software emulation path, we try to simulate the "fitos" instruction
1659          * via "fitod" and "fdtos" provided the following conditions are met:
1660          *
1661          *      fpu_exists is set (if DEBUG)
1662          *      not in privileged mode
1663          *      ftt is unfinished_FPop
1664          *      NXM IEEE trap is not enabled
1665          *      instruction at %tpc is "fitos"
1666          *
1667          *  Usage:
1668          *      %g1     per cpu address
1669          *      %g2     %fsr
1670          *      %g6     user instruction
1671          *
1672          * Note that we can take a memory access related trap while trying
1673          * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1674          * flag to catch those traps and let the SFMMU code deal with page
1675          * fault and data access exception.
1676          */
1677 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1678         sethi   %hi(fpu_exists), %g7
1679         ld      [%g7 + %lo(fpu_exists)], %g7
1680         brz,pn %g7, .fp_exception_cont
1681           nop
1682 #endif
1683         rdpr    %tstate, %g7                    ! branch if in privileged mode
1684         btst    TSTATE_PRIV, %g7
1685         bnz,pn  %xcc, .fp_exception_cont
1686         srl     %g2, FSR_FTT_SHIFT, %g7         ! extract ftt from %fsr
1687         and     %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1688         cmp     %g7, FTT_UNFIN
1689         set     FSR_TEM_NX, %g5
1690         bne,pn  %xcc, .fp_exception_cont        ! branch if NOT unfinished_FPop
1691           andcc %g2, %g5, %g0
1692         bne,pn  %xcc, .fp_exception_cont        ! branch if FSR_TEM_NX enabled
1693           rdpr  %tpc, %g5                       ! get faulting PC
1694 
1695         or      %g0, 1, %g7
1696         st      %g7, [%g1 + CPU_TL1_HDLR]       ! set tl1_hdlr flag
1697         lda     [%g5]ASI_USER, %g6              ! get user's instruction
1698         st      %g0, [%g1 + CPU_TL1_HDLR]       ! clear tl1_hdlr flag
1699 
1700         set     FITOS_INSTR_MASK, %g7
1701         and     %g6, %g7, %g7
1702         set     FITOS_INSTR, %g5
1703         cmp     %g7, %g5
1704         bne,pn  %xcc, .fp_exception_cont        ! branch if not FITOS_INSTR
1705          nop
1706 
1707         /*
1708          * This is unfinished FPops trap for "fitos" instruction. We
1709          * need to simulate "fitos" via "fitod" and "fdtos" instruction
1710          * sequence.
1711          *
1712          * We need a temporary FP register to do the conversion. Since
1713          * both source and destination operands for the "fitos" instruction
1714          * have to be within %f0-%f31, we use an FP register from the upper
1715          * half to guarantee that it won't collide with the source or the
1716          * dest operand. However, we do have to save and restore its value.
1717          *
1718          * We use %d62 as a temporary FP register for the conversion and
1719          * branch to appropriate instruction within the conversion tables
1720          * based upon the rs2 and rd values.
1721          */
1722 
1723         std     %d62, [%g1 + CPU_TMP1]          ! save original value
1724 
1725         srl     %g6, FITOS_RS2_SHIFT, %g7
1726         and     %g7, FITOS_REG_MASK, %g7
1727         set     _fitos_fitod_table, %g4
1728         sllx    %g7, 2, %g7
1729         jmp     %g4 + %g7
1730           ba,pt %xcc, _fitos_fitod_done
1731         .empty
1732 
1733 _fitos_fitod_table:
1734           fitod %f0, %d62
1735           fitod %f1, %d62
1736           fitod %f2, %d62
1737           fitod %f3, %d62
1738           fitod %f4, %d62
1739           fitod %f5, %d62
1740           fitod %f6, %d62
1741           fitod %f7, %d62
1742           fitod %f8, %d62
1743           fitod %f9, %d62
1744           fitod %f10, %d62
1745           fitod %f11, %d62
1746           fitod %f12, %d62
1747           fitod %f13, %d62
1748           fitod %f14, %d62
1749           fitod %f15, %d62
1750           fitod %f16, %d62
1751           fitod %f17, %d62
1752           fitod %f18, %d62
1753           fitod %f19, %d62
1754           fitod %f20, %d62
1755           fitod %f21, %d62
1756           fitod %f22, %d62
1757           fitod %f23, %d62
1758           fitod %f24, %d62
1759           fitod %f25, %d62
1760           fitod %f26, %d62
1761           fitod %f27, %d62
1762           fitod %f28, %d62
1763           fitod %f29, %d62
1764           fitod %f30, %d62
1765           fitod %f31, %d62
1766 _fitos_fitod_done:
1767 
1768         /*
1769          * Now convert data back into single precision
1770          */
1771         srl     %g6, FITOS_RD_SHIFT, %g7
1772         and     %g7, FITOS_REG_MASK, %g7
1773         set     _fitos_fdtos_table, %g4
1774         sllx    %g7, 2, %g7
1775         jmp     %g4 + %g7
1776           ba,pt %xcc, _fitos_fdtos_done
1777         .empty
1778 
1779 _fitos_fdtos_table:
1780           fdtos %d62, %f0
1781           fdtos %d62, %f1
1782           fdtos %d62, %f2
1783           fdtos %d62, %f3
1784           fdtos %d62, %f4
1785           fdtos %d62, %f5
1786           fdtos %d62, %f6
1787           fdtos %d62, %f7
1788           fdtos %d62, %f8
1789           fdtos %d62, %f9
1790           fdtos %d62, %f10
1791           fdtos %d62, %f11
1792           fdtos %d62, %f12
1793           fdtos %d62, %f13
1794           fdtos %d62, %f14
1795           fdtos %d62, %f15
1796           fdtos %d62, %f16
1797           fdtos %d62, %f17
1798           fdtos %d62, %f18
1799           fdtos %d62, %f19
1800           fdtos %d62, %f20
1801           fdtos %d62, %f21
1802           fdtos %d62, %f22
1803           fdtos %d62, %f23
1804           fdtos %d62, %f24
1805           fdtos %d62, %f25
1806           fdtos %d62, %f26
1807           fdtos %d62, %f27
1808           fdtos %d62, %f28
1809           fdtos %d62, %f29
1810           fdtos %d62, %f30
1811           fdtos %d62, %f31
1812 _fitos_fdtos_done:
1813 
1814         ldd     [%g1 + CPU_TMP1], %d62          ! restore %d62
1815 
1816 #if DEBUG
1817         /*
1818          * Update FPop_unfinished trap kstat
1819          */
1820         set     fpustat+FPUSTAT_UNFIN_KSTAT, %g7
1821         ldx     [%g7], %g5
1822 1:
1823         add     %g5, 1, %g6
1824 
1825         casxa   [%g7] ASI_N, %g5, %g6
1826         cmp     %g5, %g6
1827         bne,a,pn %xcc, 1b
1828           or    %g0, %g6, %g5
1829 
1830         /*
1831          * Update fpu_sim_fitos kstat
1832          */
1833         set     fpuinfo+FPUINFO_FITOS_KSTAT, %g7
1834         ldx     [%g7], %g5
1835 1:
1836         add     %g5, 1, %g6
1837 
1838         casxa   [%g7] ASI_N, %g5, %g6
1839         cmp     %g5, %g6
1840         bne,a,pn %xcc, 1b
1841           or    %g0, %g6, %g5
1842 #endif /* DEBUG */
1843 
1844         FAST_TRAP_DONE
1845 
1846 .fp_exception_cont:
1847         /*
1848          * Let _fp_exception deal with simulating FPop instruction.
1849          * Note that we need to pass %fsr in %g2 (already read above).
1850          */
1851 
1852         set     _fp_exception, %g1
1853         ba,pt   %xcc, sys_trap
1854         sub     %g0, 1, %g4
1855 
1856 
1857 /*
1858  * Register windows
1859  */
1860 .flushw:
1861 .clean_windows:
1862         rdpr    %tnpc, %g1
1863         wrpr    %g1, %tpc
1864         add     %g1, 4, %g1
1865         wrpr    %g1, %tnpc
1866         set     trap, %g1
1867         mov     T_FLUSH_PCB, %g3
1868         ba,pt   %xcc, sys_trap
1869         sub     %g0, 1, %g4
1870 
1871 /*
1872  * .spill_clean: clean the previous window, restore the wstate, and
1873  * "done".
1874  *
1875  * Entry: %g7 contains new wstate
1876  */
1877 .spill_clean:
1878         sethi   %hi(nwin_minus_one), %g5
1879         ld      [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
1880         rdpr    %cwp, %g6                       ! %g6 = %cwp
1881         deccc   %g6                             ! %g6--
1882         movneg  %xcc, %g5, %g6                  ! if (%g6<0) %g6 = nwin-1
1883         wrpr    %g6, %cwp
1884         TT_TRACE_L(trace_win)
1885         clr     %l0
1886         clr     %l1
1887         clr     %l2
1888         clr     %l3
1889         clr     %l4
1890         clr     %l5
1891         clr     %l6
1892         clr     %l7
1893         wrpr    %g0, %g7, %wstate
1894         saved
1895         retry                   ! restores correct %cwp
1896 
1897 .fix_alignment:
1898         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
1899         ldn     [%g1 + CPU_THREAD], %g1 ! load thread pointer
1900         ldn     [%g1 + T_PROCP], %g1
1901         mov     1, %g2
1902         stb     %g2, [%g1 + P_FIXALIGNMENT]
1903         FAST_TRAP_DONE
1904 
1905 #define STDF_REG(REG, ADDR, TMP)                \
1906         sll     REG, 3, REG                     ;\
1907 mark1:  set     start1, TMP                     ;\
1908         jmp     REG + TMP                       ;\
1909           nop                                   ;\
1910 start1: ba,pt   %xcc, done1                     ;\
1911           std   %f0, [ADDR + CPU_TMP1]          ;\
1912         ba,pt   %xcc, done1                     ;\
1913           std   %f32, [ADDR + CPU_TMP1]         ;\
1914         ba,pt   %xcc, done1                     ;\
1915           std   %f2, [ADDR + CPU_TMP1]          ;\
1916         ba,pt   %xcc, done1                     ;\
1917           std   %f34, [ADDR + CPU_TMP1]         ;\
1918         ba,pt   %xcc, done1                     ;\
1919           std   %f4, [ADDR + CPU_TMP1]          ;\
1920         ba,pt   %xcc, done1                     ;\
1921           std   %f36, [ADDR + CPU_TMP1]         ;\
1922         ba,pt   %xcc, done1                     ;\
1923           std   %f6, [ADDR + CPU_TMP1]          ;\
1924         ba,pt   %xcc, done1                     ;\
1925           std   %f38, [ADDR + CPU_TMP1]         ;\
1926         ba,pt   %xcc, done1                     ;\
1927           std   %f8, [ADDR + CPU_TMP1]          ;\
1928         ba,pt   %xcc, done1                     ;\
1929           std   %f40, [ADDR + CPU_TMP1]         ;\
1930         ba,pt   %xcc, done1                     ;\
1931           std   %f10, [ADDR + CPU_TMP1]         ;\
1932         ba,pt   %xcc, done1                     ;\
1933           std   %f42, [ADDR + CPU_TMP1]         ;\
1934         ba,pt   %xcc, done1                     ;\
1935           std   %f12, [ADDR + CPU_TMP1]         ;\
1936         ba,pt   %xcc, done1                     ;\
1937           std   %f44, [ADDR + CPU_TMP1]         ;\
1938         ba,pt   %xcc, done1                     ;\
1939           std   %f14, [ADDR + CPU_TMP1]         ;\
1940         ba,pt   %xcc, done1                     ;\
1941           std   %f46, [ADDR + CPU_TMP1]         ;\
1942         ba,pt   %xcc, done1                     ;\
1943           std   %f16, [ADDR + CPU_TMP1]         ;\
1944         ba,pt   %xcc, done1                     ;\
1945           std   %f48, [ADDR + CPU_TMP1]         ;\
1946         ba,pt   %xcc, done1                     ;\
1947           std   %f18, [ADDR + CPU_TMP1]         ;\
1948         ba,pt   %xcc, done1                     ;\
1949           std   %f50, [ADDR + CPU_TMP1]         ;\
1950         ba,pt   %xcc, done1                     ;\
1951           std   %f20, [ADDR + CPU_TMP1]         ;\
1952         ba,pt   %xcc, done1                     ;\
1953           std   %f52, [ADDR + CPU_TMP1]         ;\
1954         ba,pt   %xcc, done1                     ;\
1955           std   %f22, [ADDR + CPU_TMP1]         ;\
1956         ba,pt   %xcc, done1                     ;\
1957           std   %f54, [ADDR + CPU_TMP1]         ;\
1958         ba,pt   %xcc, done1                     ;\
1959           std   %f24, [ADDR + CPU_TMP1]         ;\
1960         ba,pt   %xcc, done1                     ;\
1961           std   %f56, [ADDR + CPU_TMP1]         ;\
1962         ba,pt   %xcc, done1                     ;\
1963           std   %f26, [ADDR + CPU_TMP1]         ;\
1964         ba,pt   %xcc, done1                     ;\
1965           std   %f58, [ADDR + CPU_TMP1]         ;\
1966         ba,pt   %xcc, done1                     ;\
1967           std   %f28, [ADDR + CPU_TMP1]         ;\
1968         ba,pt   %xcc, done1                     ;\
1969           std   %f60, [ADDR + CPU_TMP1]         ;\
1970         ba,pt   %xcc, done1                     ;\
1971           std   %f30, [ADDR + CPU_TMP1]         ;\
1972         ba,pt   %xcc, done1                     ;\
1973           std   %f62, [ADDR + CPU_TMP1]         ;\
1974 done1:
1975 
1976 #define LDDF_REG(REG, ADDR, TMP)                \
1977         sll     REG, 3, REG                     ;\
1978 mark2:  set     start2, TMP                     ;\
1979         jmp     REG + TMP                       ;\
1980           nop                                   ;\
1981 start2: ba,pt   %xcc, done2                     ;\
1982           ldd   [ADDR + CPU_TMP1], %f0          ;\
1983         ba,pt   %xcc, done2                     ;\
1984           ldd   [ADDR + CPU_TMP1], %f32         ;\
1985         ba,pt   %xcc, done2                     ;\
1986           ldd   [ADDR + CPU_TMP1], %f2          ;\
1987         ba,pt   %xcc, done2                     ;\
1988           ldd   [ADDR + CPU_TMP1], %f34         ;\
1989         ba,pt   %xcc, done2                     ;\
1990           ldd   [ADDR + CPU_TMP1], %f4          ;\
1991         ba,pt   %xcc, done2                     ;\
1992           ldd   [ADDR + CPU_TMP1], %f36         ;\
1993         ba,pt   %xcc, done2                     ;\
1994           ldd   [ADDR + CPU_TMP1], %f6          ;\
1995         ba,pt   %xcc, done2                     ;\
1996           ldd   [ADDR + CPU_TMP1], %f38         ;\
1997         ba,pt   %xcc, done2                     ;\
1998           ldd   [ADDR + CPU_TMP1], %f8          ;\
1999         ba,pt   %xcc, done2                     ;\
2000           ldd   [ADDR + CPU_TMP1], %f40         ;\
2001         ba,pt   %xcc, done2                     ;\
2002           ldd   [ADDR + CPU_TMP1], %f10         ;\
2003         ba,pt   %xcc, done2                     ;\
2004           ldd   [ADDR + CPU_TMP1], %f42         ;\
2005         ba,pt   %xcc, done2                     ;\
2006           ldd   [ADDR + CPU_TMP1], %f12         ;\
2007         ba,pt   %xcc, done2                     ;\
2008           ldd   [ADDR + CPU_TMP1], %f44         ;\
2009         ba,pt   %xcc, done2                     ;\
2010           ldd   [ADDR + CPU_TMP1], %f14         ;\
2011         ba,pt   %xcc, done2                     ;\
2012           ldd   [ADDR + CPU_TMP1], %f46         ;\
2013         ba,pt   %xcc, done2                     ;\
2014           ldd   [ADDR + CPU_TMP1], %f16         ;\
2015         ba,pt   %xcc, done2                     ;\
2016           ldd   [ADDR + CPU_TMP1], %f48         ;\
2017         ba,pt   %xcc, done2                     ;\
2018           ldd   [ADDR + CPU_TMP1], %f18         ;\
2019         ba,pt   %xcc, done2                     ;\
2020           ldd   [ADDR + CPU_TMP1], %f50         ;\
2021         ba,pt   %xcc, done2                     ;\
2022           ldd   [ADDR + CPU_TMP1], %f20         ;\
2023         ba,pt   %xcc, done2                     ;\
2024           ldd   [ADDR + CPU_TMP1], %f52         ;\
2025         ba,pt   %xcc, done2                     ;\
2026           ldd   [ADDR + CPU_TMP1], %f22         ;\
2027         ba,pt   %xcc, done2                     ;\
2028           ldd   [ADDR + CPU_TMP1], %f54         ;\
2029         ba,pt   %xcc, done2                     ;\
2030           ldd   [ADDR + CPU_TMP1], %f24         ;\
2031         ba,pt   %xcc, done2                     ;\
2032           ldd   [ADDR + CPU_TMP1], %f56         ;\
2033         ba,pt   %xcc, done2                     ;\
2034           ldd   [ADDR + CPU_TMP1], %f26         ;\
2035         ba,pt   %xcc, done2                     ;\
2036           ldd   [ADDR + CPU_TMP1], %f58         ;\
2037         ba,pt   %xcc, done2                     ;\
2038           ldd   [ADDR + CPU_TMP1], %f28         ;\
2039         ba,pt   %xcc, done2                     ;\
2040           ldd   [ADDR + CPU_TMP1], %f60         ;\
2041         ba,pt   %xcc, done2                     ;\
2042           ldd   [ADDR + CPU_TMP1], %f30         ;\
2043         ba,pt   %xcc, done2                     ;\
2044           ldd   [ADDR + CPU_TMP1], %f62         ;\
2045 done2:
2046 
2047 .lddf_exception_not_aligned:
2048         /* %g2 = sfar, %g3 = sfsr */
2049         mov     %g2, %g5                ! stash sfar
2050 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2051         sethi   %hi(fpu_exists), %g2    ! check fpu_exists
2052         ld      [%g2 + %lo(fpu_exists)], %g2
2053         brz,a,pn %g2, 4f
2054           nop
2055 #endif
2056         CPU_ADDR(%g1, %g4)
2057         or      %g0, 1, %g4
2058         st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2059 
2060         rdpr    %tpc, %g2
2061         lda     [%g2]ASI_AIUP, %g6      ! get the user's lddf instruction
2062         srl     %g6, 23, %g1            ! using ldda or not?
2063         and     %g1, 1, %g1
2064         brz,a,pt %g1, 2f                ! check for ldda instruction
2065           nop
2066         srl     %g6, 13, %g1            ! check immflag
2067         and     %g1, 1, %g1
2068         rdpr    %tstate, %g2            ! %tstate in %g2
2069         brnz,a,pn %g1, 1f
2070           srl   %g2, 31, %g1            ! get asi from %tstate
2071         srl     %g6, 5, %g1             ! get asi from instruction
2072         and     %g1, 0xFF, %g1          ! imm_asi field
2073 1:
2074         cmp     %g1, ASI_P              ! primary address space
2075         be,a,pt %icc, 2f
2076           nop
2077         cmp     %g1, ASI_PNF            ! primary no fault address space
2078         be,a,pt %icc, 2f
2079           nop
2080         cmp     %g1, ASI_S              ! secondary address space
2081         be,a,pt %icc, 2f
2082           nop
2083         cmp     %g1, ASI_SNF            ! secondary no fault address space
2084         bne,a,pn %icc, 3f
2085           nop
2086 2:
2087         lduwa   [%g5]ASI_USER, %g7      ! get first half of misaligned data
2088         add     %g5, 4, %g5             ! increment misaligned data address
2089         lduwa   [%g5]ASI_USER, %g5      ! get second half of misaligned data
2090 
2091         sllx    %g7, 32, %g7
2092         or      %g5, %g7, %g5           ! combine data
2093         CPU_ADDR(%g7, %g1)              ! save data on a per-cpu basis
2094         stx     %g5, [%g7 + CPU_TMP1]   ! save in cpu_tmp1
2095 
2096         srl     %g6, 25, %g3            ! %g6 has the instruction
2097         and     %g3, 0x1F, %g3          ! %g3 has rd
2098         LDDF_REG(%g3, %g7, %g4)
2099 
2100         CPU_ADDR(%g1, %g4)
2101         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2102         FAST_TRAP_DONE
2103 3:
2104         CPU_ADDR(%g1, %g4)
2105         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2106 4:
2107         set     T_USER, %g3             ! trap type in %g3
2108         or      %g3, T_LDDF_ALIGN, %g3
2109         mov     %g5, %g2                ! misaligned vaddr in %g2
2110         set     fpu_trap, %g1           ! goto C for the little and
2111         ba,pt   %xcc, sys_trap          ! no fault little asi's
2112           sub   %g0, 1, %g4
2113 
2114 .stdf_exception_not_aligned:
2115         /* %g2 = sfar, %g3 = sfsr */
2116         mov     %g2, %g5
2117 
2118 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2119         sethi   %hi(fpu_exists), %g7            ! check fpu_exists
2120         ld      [%g7 + %lo(fpu_exists)], %g3
2121         brz,a,pn %g3, 4f
2122           nop
2123 #endif
2124         CPU_ADDR(%g1, %g4)
2125         or      %g0, 1, %g4
2126         st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2127 
2128         rdpr    %tpc, %g2
2129         lda     [%g2]ASI_AIUP, %g6      ! get the user's stdf instruction
2130 
2131         srl     %g6, 23, %g1            ! using stda or not?
2132         and     %g1, 1, %g1
2133         brz,a,pt %g1, 2f                ! check for stda instruction
2134           nop
2135         srl     %g6, 13, %g1            ! check immflag
2136         and     %g1, 1, %g1
2137         rdpr    %tstate, %g2            ! %tstate in %g2
2138         brnz,a,pn %g1, 1f
2139           srl   %g2, 31, %g1            ! get asi from %tstate
2140         srl     %g6, 5, %g1             ! get asi from instruction
2141         and     %g1, 0xff, %g1          ! imm_asi field
2142 1:
2143         cmp     %g1, ASI_P              ! primary address space
2144         be,a,pt %icc, 2f
2145           nop
2146         cmp     %g1, ASI_S              ! secondary address space
2147         bne,a,pn %icc, 3f
2148           nop
2149 2:
2150         srl     %g6, 25, %g6
2151         and     %g6, 0x1F, %g6          ! %g6 has rd
2152         CPU_ADDR(%g7, %g1)
2153         STDF_REG(%g6, %g7, %g4)         ! STDF_REG(REG, ADDR, TMP)
2154 
2155         ldx     [%g7 + CPU_TMP1], %g6
2156         srlx    %g6, 32, %g7
2157         stuwa   %g7, [%g5]ASI_USER      ! first half
2158         add     %g5, 4, %g5             ! increment misaligned data address
2159         stuwa   %g6, [%g5]ASI_USER      ! second half
2160 
2161         CPU_ADDR(%g1, %g4)
2162         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2163         FAST_TRAP_DONE
2164 3:
2165         CPU_ADDR(%g1, %g4)
2166         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2167 4:
2168         set     T_USER, %g3             ! trap type in %g3
2169         or      %g3, T_STDF_ALIGN, %g3
2170         mov     %g5, %g2                ! misaligned vaddr in %g2
2171         set     fpu_trap, %g1           ! goto C for the little and
2172         ba,pt   %xcc, sys_trap          ! nofault little asi's
2173           sub   %g0, 1, %g4
2174 
2175 #ifdef DEBUG_USER_TRAPTRACECTL
2176 
2177 .traptrace_freeze:
2178         mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
2179         TT_TRACE_L(trace_win)
2180         mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
2181         set     trap_freeze, %g1
2182         mov     1, %g2
2183         st      %g2, [%g1]
2184         FAST_TRAP_DONE
2185 
2186 .traptrace_unfreeze:
2187         set     trap_freeze, %g1
2188         st      %g0, [%g1]
2189         mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
2190         TT_TRACE_L(trace_win)
2191         mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
2192         FAST_TRAP_DONE
2193 
2194 #endif /* DEBUG_USER_TRAPTRACECTL */
2195 
2196 .getcc:
2197         CPU_ADDR(%g1, %g2)
2198         stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
2199         rdpr    %tstate, %g3                    ! get tstate
2200         srlx    %g3, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
2201         set     PSR_ICC, %g2
2202         and     %o0, %g2, %o0                   ! mask out the rest
2203         srl     %o0, PSR_ICC_SHIFT, %o0         ! right justify
2204         wrpr    %g0, 0, %gl
2205         mov     %o0, %g1                        ! move ccr to normal %g1
2206         wrpr    %g0, 1, %gl
2207         ! cannot assume globals retained their values after increasing %gl
2208         CPU_ADDR(%g1, %g2)
2209         ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
2210         FAST_TRAP_DONE
2211 
2212 .setcc:
2213         CPU_ADDR(%g1, %g2)
2214         stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
2215         wrpr    %g0, 0, %gl
2216         mov     %g1, %o0
2217         wrpr    %g0, 1, %gl
2218         ! cannot assume globals retained their values after increasing %gl
2219         CPU_ADDR(%g1, %g2)
2220         sll     %o0, PSR_ICC_SHIFT, %g2
2221         set     PSR_ICC, %g3
2222         and     %g2, %g3, %g2                   ! mask out rest
2223         sllx    %g2, PSR_TSTATE_CC_SHIFT, %g2
2224         rdpr    %tstate, %g3                    ! get tstate
2225         srl     %g3, 0, %g3                     ! clear upper word
2226         or      %g3, %g2, %g3                   ! or in new bits
2227         wrpr    %g3, %tstate
2228         ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
2229         FAST_TRAP_DONE
2230 
2231 /*
2232  * getpsr(void)
2233  * Note that the xcc part of the ccr is not provided.
2234  * The V8 code shows why the V9 trap is not faster:
2235  * #define GETPSR_TRAP() \
2236  *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2237  */
2238 
2239         .type   .getpsr, #function
2240 .getpsr:
2241         rdpr    %tstate, %g1                    ! get tstate
2242         srlx    %g1, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
2243         set     PSR_ICC, %g2
2244         and     %o0, %g2, %o0                   ! mask out the rest
2245 
2246         rd      %fprs, %g1                      ! get fprs
2247         and     %g1, FPRS_FEF, %g2              ! mask out dirty upper/lower
2248         sllx    %g2, PSR_FPRS_FEF_SHIFT, %g2    ! shift fef to V8 psr.ef
2249         or      %o0, %g2, %o0                   ! or result into psr.ef
2250 
2251         set     V9_PSR_IMPLVER, %g2             ! SI assigned impl/ver: 0xef
2252         or      %o0, %g2, %o0                   ! or psr.impl/ver
2253         FAST_TRAP_DONE
2254         SET_SIZE(.getpsr)
2255 
2256 /*
2257  * setpsr(newpsr)
2258  * Note that there is no support for ccr.xcc in the V9 code.
2259  */
2260 
2261         .type   .setpsr, #function
2262 .setpsr:
2263         rdpr    %tstate, %g1                    ! get tstate
2264 !       setx    TSTATE_V8_UBITS, %g2
2265         or      %g0, CCR_ICC, %g3
2266         sllx    %g3, TSTATE_CCR_SHIFT, %g2
2267 
2268         andn    %g1, %g2, %g1                   ! zero current user bits
2269         set     PSR_ICC, %g2
2270         and     %g2, %o0, %g2                   ! clear all but psr.icc bits
2271         sllx    %g2, PSR_TSTATE_CC_SHIFT, %g3   ! shift to tstate.ccr.icc
2272         wrpr    %g1, %g3, %tstate               ! write tstate
2273 
2274         set     PSR_EF, %g2
2275         and     %g2, %o0, %g2                   ! clear all but fp enable bit
2276         srlx    %g2, PSR_FPRS_FEF_SHIFT, %g4    ! shift ef to V9 fprs.fef
2277         wr      %g0, %g4, %fprs                 ! write fprs
2278 
2279         CPU_ADDR(%g1, %g2)                      ! load CPU struct addr to %g1
2280         ldn     [%g1 + CPU_THREAD], %g2         ! load thread pointer
2281         ldn     [%g2 + T_LWP], %g3              ! load klwp pointer
2282         ldn     [%g3 + LWP_FPU], %g2            ! get lwp_fpu pointer
2283         stuw    %g4, [%g2 + FPU_FPRS]           ! write fef value to fpu_fprs
2284         srlx    %g4, 2, %g4                     ! shift fef value to bit 0
2285         stub    %g4, [%g2 + FPU_EN]             ! write fef value to fpu_en
2286         FAST_TRAP_DONE
2287         SET_SIZE(.setpsr)
2288 
2289 /*
2290  * getlgrp
2291  * get home lgrpid on which the calling thread is currently executing.
2292  */
2293         .type   .getlgrp, #function
2294 .getlgrp:
2295         ! Thanks for the incredibly helpful comments
2296         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2297         ld      [%g1 + CPU_ID], %o0     ! load cpu_id
2298         ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
2299         ldn     [%g2 + T_LPL], %g2      ! load lpl pointer
2300         ld      [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
2301         sra     %g1, 0, %o1
2302         FAST_TRAP_DONE
2303         SET_SIZE(.getlgrp)
2304 
2305 /*
2306  * Entry for old 4.x trap (trap 0).
2307  */
2308         ENTRY_NP(syscall_trap_4x)
2309         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2310         ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
2311         ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
2312         ld      [%g2 + PCB_TRAP0], %g2  ! lwp->lwp_pcb.pcb_trap0addr
2313         brz,pn  %g2, 1f                 ! has it been set?
2314         st      %l0, [%g1 + CPU_TMP1]   ! delay - save some locals
2315         st      %l1, [%g1 + CPU_TMP2]
2316         rdpr    %tnpc, %l1              ! save old tnpc
2317         wrpr    %g0, %g2, %tnpc         ! setup tnpc
2318 
2319         mov     %g1, %l0                ! save CPU struct addr
2320         wrpr    %g0, 0, %gl
2321         mov     %l1, %g6                ! pass tnpc to user code in %g6
2322         wrpr    %g0, 1, %gl
2323         ld      [%l0 + CPU_TMP2], %l1   ! restore locals
2324         ld      [%l0 + CPU_TMP1], %l0
2325         FAST_TRAP_DONE_CHK_INTR
2326 1:
2327         !
2328         ! check for old syscall mmap which is the only different one which
2329         ! must be the same.  Others are handled in the compatibility library.
2330         !
2331         mov     %g1, %l0                ! save CPU struct addr
2332         wrpr    %g0, 0, %gl
2333         cmp     %g1, OSYS_mmap          ! compare to old 4.x mmap
2334         movz    %icc, SYS_mmap, %g1
2335         wrpr    %g0, 1, %gl
2336         ld      [%l0 + CPU_TMP1], %l0
2337         SYSCALL(syscall_trap32)
2338         SET_SIZE(syscall_trap_4x)
2339 
2340 /*
2341  * Handler for software trap 9.
2342  * Set trap0 emulation address for old 4.x system call trap.
2343  * XXX - this should be a system call.
2344  */
2345         ENTRY_NP(set_trap0_addr)
2346         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2347         st      %l0, [%g1 + CPU_TMP1]   ! save some locals
2348         st      %l1, [%g1 + CPU_TMP2]
2349         mov     %g1, %l0        ! preserve CPU addr
2350         wrpr    %g0, 0, %gl
2351         mov     %g1, %l1
2352         wrpr    %g0, 1, %gl
2353         ! cannot assume globals retained their values after increasing %gl
2354         ldn     [%l0 + CPU_THREAD], %g2 ! load thread pointer
2355         ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
2356         andn    %l1, 3, %l1             ! force alignment
2357         st      %l1, [%g2 + PCB_TRAP0]  ! lwp->lwp_pcb.pcb_trap0addr
2358         ld      [%l0 + CPU_TMP2], %l1   ! restore locals
2359         ld      [%l0 + CPU_TMP1], %l0
2360         FAST_TRAP_DONE
2361         SET_SIZE(set_trap0_addr)
2362 
2363 /*
2364  * mmu_trap_tl1
2365  * trap handler for unexpected mmu traps.
2366  * simply checks if the trap was a user lddf/stdf alignment trap, in which
2367  * case we go to fpu_trap or a user trap from the window handler, in which
2368  * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
2369  */
2370         .type   mmu_trap_tl1, #function
2371 mmu_trap_tl1:
2372 #ifdef  TRAPTRACE
2373         TRACE_PTR(%g5, %g6)
2374         GET_TRACE_TICK(%g6, %g7)
2375         stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
2376         TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2377         rdpr    %tt, %g6
2378         stha    %g6, [%g5 + TRAP_ENT_TT]%asi
2379         rdpr    %tstate, %g6
2380         stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi
2381         stna    %sp, [%g5 + TRAP_ENT_SP]%asi
2382         stna    %g0, [%g5 + TRAP_ENT_TR]%asi
2383         rdpr    %tpc, %g6
2384         stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
2385         MMU_FAULT_STATUS_AREA(%g6)
2386         ldx     [%g6 + MMFSA_D_ADDR], %g6
2387         stna    %g6, [%g5 + TRAP_ENT_F1]%asi !  MMU fault address
2388         CPU_PADDR(%g7, %g6);
2389         add     %g7, CPU_TL1_HDLR, %g7
2390         lda     [%g7]ASI_MEM, %g6
2391         stna    %g6, [%g5 + TRAP_ENT_F2]%asi
2392         MMU_FAULT_STATUS_AREA(%g6)
2393         ldx     [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant?
2394         ldx     [%g6 + MMFSA_D_CTX], %g6
2395         sllx    %g6, SFSR_CTX_SHIFT, %g6
2396         or      %g6, %g7, %g6
2397         stna    %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type
2398         set     0xdeadbeef, %g6
2399         stna    %g6, [%g5 + TRAP_ENT_F4]%asi
2400         TRACE_NEXT(%g5, %g6, %g7)
2401 #endif /* TRAPTRACE */
2402         CPU_PADDR(%g7, %g6);
2403         add     %g7, CPU_TL1_HDLR, %g7          ! %g7 = &cpu_m.tl1_hdlr (PA)
2404         lda     [%g7]ASI_MEM, %g6
2405         brz,a,pt %g6, 1f
2406           nop
2407         sta     %g0, [%g7]ASI_MEM
2408         ! XXXQ need to setup registers for sfmmu_mmu_trap?
2409         ba,a,pt %xcc, sfmmu_mmu_trap            ! handle page faults
2410 1:
2411         rdpr    %tpc, %g7
2412         /* in user_rtt? */
2413         set     rtt_fill_start, %g6
2414         cmp     %g7, %g6
2415         blu,pn  %xcc, 6f
2416          .empty
2417         set     rtt_fill_end, %g6
2418         cmp     %g7, %g6
2419         bgeu,pn %xcc, 6f
2420          nop
2421         set     fault_rtt_fn1, %g7
2422         ba,a    7f
2423 6:
2424         ! check to see if the trap pc is in a window spill/fill handling
2425         rdpr    %tpc, %g7
2426         /* tpc should be in the trap table */
2427         set     trap_table, %g6
2428         cmp     %g7, %g6
2429         blu,a,pn %xcc, ptl1_panic
2430           mov   PTL1_BAD_MMUTRAP, %g1
2431         set     etrap_table, %g6
2432         cmp     %g7, %g6
2433         bgeu,a,pn %xcc, ptl1_panic
2434           mov   PTL1_BAD_MMUTRAP, %g1
2435         ! pc is inside the trap table, convert to trap type
2436         srl     %g7, 5, %g6             ! XXXQ need #define
2437         and     %g6, 0x1ff, %g6         ! XXXQ need #define
2438         ! and check for a window trap type
2439         and     %g6, WTRAP_TTMASK, %g6
2440         cmp     %g6, WTRAP_TYPE
2441         bne,a,pn %xcc, ptl1_panic
2442           mov   PTL1_BAD_MMUTRAP, %g1
2443         andn    %g7, WTRAP_ALIGN, %g7   /* 128 byte aligned */
2444         add     %g7, WTRAP_FAULTOFF, %g7
2445 
2446 7:
2447         ! Arguments are passed in the global set active after the
2448         ! 'done' instruction. Before switching sets, must save
2449         ! the calculated next pc
2450         wrpr    %g0, %g7, %tnpc
2451         wrpr    %g0, 1, %gl
2452         rdpr    %tt, %g5
2453         MMU_FAULT_STATUS_AREA(%g7)
2454         cmp     %g5, T_ALIGNMENT
2455         be,pn   %xcc, 1f
2456         ldx     [%g7 + MMFSA_D_ADDR], %g6
2457         ldx     [%g7 + MMFSA_D_CTX], %g7
2458         srlx    %g6, MMU_PAGESHIFT, %g6         /* align address */
2459         cmp     %g7, USER_CONTEXT_TYPE
2460         sllx    %g6, MMU_PAGESHIFT, %g6
2461         movgu   %icc, USER_CONTEXT_TYPE, %g7
2462         or      %g6, %g7, %g6                   /* TAG_ACCESS */
2463 1:
2464         done
2465         SET_SIZE(mmu_trap_tl1)
2466 
2467 /*
2468  * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
2469  * traps are valid only when kmdb is loaded.  When the debugger is active,
2470  * the code below is rewritten to transfer control to the appropriate
2471  * debugger entry points.
2472  */
2473         .global kmdb_trap
2474         .align  8
2475 kmdb_trap:
2476         ba,a    trap_table0
2477         jmp     %g1 + 0
2478         nop
2479 
2480         .global kmdb_trap_tl1
2481         .align  8
2482 kmdb_trap_tl1:
2483         ba,a    trap_table0
2484         jmp     %g1 + 0
2485         nop
2486 
2487 /*
2488  * This entry is copied from OBP's trap table during boot.
2489  */
2490         .global obp_bpt
2491         .align  8
2492 obp_bpt:
2493         NOT
2494 
2495 
2496 
2497 #ifdef  TRAPTRACE
2498 /*
2499  * TRAPTRACE support.
2500  * labels here are branched to with "rd %pc, %g7" in the delay slot.
2501  * Return is done by "jmp %g7 + 4".
2502  */
2503 
2504 trace_dmmu:
2505         TRACE_PTR(%g3, %g6)
2506         GET_TRACE_TICK(%g6, %g5)
2507         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
2508         TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2509         rdpr    %tt, %g6
2510         stha    %g6, [%g3 + TRAP_ENT_TT]%asi
2511         rdpr    %tstate, %g6
2512         stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2513         stna    %sp, [%g3 + TRAP_ENT_SP]%asi
2514         rdpr    %tpc, %g6
2515         stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
2516         MMU_FAULT_STATUS_AREA(%g6)
2517         ldx     [%g6 + MMFSA_D_ADDR], %g4
2518         stxa    %g4, [%g3 + TRAP_ENT_TR]%asi
2519         ldx     [%g6 + MMFSA_D_CTX], %g4
2520         stxa    %g4, [%g3 + TRAP_ENT_F1]%asi
2521         ldx     [%g6 + MMFSA_D_TYPE], %g4
2522         stxa    %g4, [%g3 + TRAP_ENT_F2]%asi
2523         stxa    %g6, [%g3 + TRAP_ENT_F3]%asi
2524         stna    %g0, [%g3 + TRAP_ENT_F4]%asi
2525         TRACE_NEXT(%g3, %g4, %g5)
2526         jmp     %g7 + 4
2527         nop
2528 
2529 trace_immu:
2530         TRACE_PTR(%g3, %g6)
2531         GET_TRACE_TICK(%g6, %g5)
2532         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
2533         TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2534         rdpr    %tt, %g6
2535         stha    %g6, [%g3 + TRAP_ENT_TT]%asi
2536         rdpr    %tstate, %g6
2537         stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2538         stna    %sp, [%g3 + TRAP_ENT_SP]%asi
2539         rdpr    %tpc, %g6
2540         stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
2541         MMU_FAULT_STATUS_AREA(%g6)
2542         ldx     [%g6 + MMFSA_I_ADDR], %g4
2543         stxa    %g4, [%g3 + TRAP_ENT_TR]%asi
2544         ldx     [%g6 + MMFSA_I_CTX], %g4
2545         stxa    %g4, [%g3 + TRAP_ENT_F1]%asi
2546         ldx     [%g6 + MMFSA_I_TYPE], %g4
2547         stxa    %g4, [%g3 + TRAP_ENT_F2]%asi
2548         stxa    %g6, [%g3 + TRAP_ENT_F3]%asi
2549         stna    %g0, [%g3 + TRAP_ENT_F4]%asi
2550         TRACE_NEXT(%g3, %g4, %g5)
2551         jmp     %g7 + 4
2552         nop
2553 
2554 trace_gen:
2555         TRACE_PTR(%g3, %g6)
2556         GET_TRACE_TICK(%g6, %g5)
2557         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
2558         TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2559         rdpr    %tt, %g6
2560         stha    %g6, [%g3 + TRAP_ENT_TT]%asi
2561         rdpr    %tstate, %g6
2562         stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2563         stna    %sp, [%g3 + TRAP_ENT_SP]%asi
2564         rdpr    %tpc, %g6
2565         stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
2566         stna    %g0, [%g3 + TRAP_ENT_TR]%asi
2567         stna    %g0, [%g3 + TRAP_ENT_F1]%asi
2568         stna    %g0, [%g3 + TRAP_ENT_F2]%asi
2569         stna    %g0, [%g3 + TRAP_ENT_F3]%asi
2570         stna    %g0, [%g3 + TRAP_ENT_F4]%asi
2571         TRACE_NEXT(%g3, %g4, %g5)
2572         jmp     %g7 + 4
2573         nop
2574 
2575 trace_win:
2576         TRACE_WIN_INFO(0, %l0, %l1, %l2)
2577         ! Keep the locals as clean as possible, caller cleans %l4
2578         clr     %l2
2579         clr     %l1
2580         jmp     %l4 + 4
2581           clr   %l0
2582 
2583 /*
2584  * Trace a tsb hit
2585  * g1 = tsbe pointer (in/clobbered)
2586  * g2 = tag access register (in)
2587  * g3 - g4 = scratch (clobbered)
2588  * g5 = tsbe data (in)
2589  * g6 = scratch (clobbered)
2590  * g7 = pc we jumped here from (in)
2591  */
2592 
2593         ! Do not disturb %g5, it will be used after the trace
2594         ALTENTRY(trace_tsbhit)
2595         TRACE_TSBHIT(0)
2596         jmp     %g7 + 4
2597         nop
2598 
2599 /*
2600  * Trace a TSB miss
2601  *
2602  * g1 = tsb8k pointer (in)
2603  * g2 = tag access register (in)
2604  * g3 = tsb4m pointer (in)
2605  * g4 = tsbe tag (in/clobbered)
2606  * g5 - g6 = scratch (clobbered)
2607  * g7 = pc we jumped here from (in)
2608  */
2609         .global trace_tsbmiss
2610 trace_tsbmiss:
2611         membar  #Sync
2612         sethi   %hi(FLUSH_ADDR), %g6
2613         flush   %g6
2614         TRACE_PTR(%g5, %g6)
2615         stna    %g2, [%g5 + TRAP_ENT_SP]%asi            ! tag access
2616         stna    %g4, [%g5 + TRAP_ENT_F1]%asi            ! XXX? tsb tag
2617         GET_TRACE_TICK(%g6, %g4)
2618         stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
2619         rdpr    %tnpc, %g6
2620         stna    %g6, [%g5 + TRAP_ENT_F2]%asi
2621         stna    %g1, [%g5 + TRAP_ENT_F3]%asi            ! tsb8k pointer
2622         rdpr    %tpc, %g6
2623         stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
2624         TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2625         rdpr    %tt, %g6
2626         or      %g6, TT_MMU_MISS, %g4
2627         stha    %g4, [%g5 + TRAP_ENT_TT]%asi
2628         mov     MMFSA_D_ADDR, %g4
2629         cmp     %g6, FAST_IMMU_MISS_TT
2630         move    %xcc, MMFSA_I_ADDR, %g4
2631         cmp     %g6, T_INSTR_MMU_MISS
2632         move    %xcc, MMFSA_I_ADDR, %g4
2633         MMU_FAULT_STATUS_AREA(%g6)
2634         ldx     [%g6 + %g4], %g6
2635         stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi        ! tag target
2636         cmp     %g4, MMFSA_D_ADDR
2637         move    %xcc, MMFSA_D_CTX, %g4
2638         movne   %xcc, MMFSA_I_CTX, %g4
2639         MMU_FAULT_STATUS_AREA(%g6)
2640         ldx     [%g6 + %g4], %g6
2641         stxa    %g6, [%g5 + TRAP_ENT_F4]%asi            ! context ID
2642         stna    %g3, [%g5 + TRAP_ENT_TR]%asi            ! tsb4m pointer
2643         TRACE_NEXT(%g5, %g4, %g6)
2644         jmp     %g7 + 4
2645         nop
2646 
2647 /*
2648  * g2 = tag access register (in)
2649  * g3 = ctx type (0, 1 or 2) (in) (not used)
2650  */
2651 trace_dataprot:
2652         membar  #Sync
2653         sethi   %hi(FLUSH_ADDR), %g6
2654         flush   %g6
2655         TRACE_PTR(%g1, %g6)
2656         GET_TRACE_TICK(%g6, %g4)
2657         stxa    %g6, [%g1 + TRAP_ENT_TICK]%asi
2658         rdpr    %tpc, %g6
2659         stna    %g6, [%g1 + TRAP_ENT_TPC]%asi
2660         rdpr    %tstate, %g6
2661         stxa    %g6, [%g1 + TRAP_ENT_TSTATE]%asi
2662         stna    %g2, [%g1 + TRAP_ENT_SP]%asi            ! tag access reg
2663         stna    %g0, [%g1 + TRAP_ENT_F1]%asi
2664         stna    %g0, [%g1 + TRAP_ENT_F2]%asi
2665         stna    %g0, [%g1 + TRAP_ENT_F3]%asi
2666         stna    %g0, [%g1 + TRAP_ENT_F4]%asi
2667         TRACE_SAVE_TL_GL_REGS(%g1, %g6)
2668         rdpr    %tt, %g6
2669         stha    %g6, [%g1 + TRAP_ENT_TT]%asi
2670         mov     MMFSA_D_CTX, %g4
2671         cmp     %g6, FAST_IMMU_MISS_TT
2672         move    %xcc, MMFSA_I_CTX, %g4
2673         cmp     %g6, T_INSTR_MMU_MISS
2674         move    %xcc, MMFSA_I_CTX, %g4
2675         MMU_FAULT_STATUS_AREA(%g6)
2676         ldx     [%g6 + %g4], %g6
2677         stxa    %g6, [%g1 + TRAP_ENT_TR]%asi    ! context ID
2678         TRACE_NEXT(%g1, %g4, %g5)
2679         jmp     %g7 + 4
2680         nop
2681 
2682 #endif /* TRAPTRACE */
2683 
2684 /*
2685  * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE
2686  * HV service, which requires the return target to be specified as a VA
2687  * since we are enabling the MMU. We set the target to ptl1_panic.
2688  */
2689 
2690         .type   .watchdog_trap, #function
2691 .watchdog_trap:
2692         mov     1, %o0
2693         setx    ptl1_panic, %g2, %o1
2694         mov     MMU_ENABLE, %o5
2695         ta      FAST_TRAP
2696         done
2697         SET_SIZE(.watchdog_trap)
2698 /*
2699  * synthesize for trap(): SFAR in %g2, SFSR in %g3
2700  */
2701         .type   .dmmu_exc_lddf_not_aligned, #function
2702 .dmmu_exc_lddf_not_aligned:
2703         MMU_FAULT_STATUS_AREA(%g3)
2704         ldx     [%g3 + MMFSA_D_ADDR], %g2
2705         /* Fault type not available in MMU fault status area */
2706         mov     MMFSA_F_UNALIGN, %g1
2707         ldx     [%g3 + MMFSA_D_CTX], %g3
2708         sllx    %g3, SFSR_CTX_SHIFT, %g3
2709         btst    1, %sp
2710         bnz,pt  %xcc, .lddf_exception_not_aligned
2711         or      %g3, %g1, %g3                   /* SFSR */
2712         ba,a,pt %xcc, .mmu_exception_not_aligned
2713         SET_SIZE(.dmmu_exc_lddf_not_aligned)
2714 
2715 /*
2716  * synthesize for trap(): SFAR in %g2, SFSR in %g3
2717  */
2718         .type   .dmmu_exc_stdf_not_aligned, #function
2719 .dmmu_exc_stdf_not_aligned:
2720         MMU_FAULT_STATUS_AREA(%g3)
2721         ldx     [%g3 + MMFSA_D_ADDR], %g2
2722         /* Fault type not available in MMU fault status area */
2723         mov     MMFSA_F_UNALIGN, %g1
2724         ldx     [%g3 + MMFSA_D_CTX], %g3
2725         sllx    %g3, SFSR_CTX_SHIFT, %g3
2726         btst    1, %sp
2727         bnz,pt  %xcc, .stdf_exception_not_aligned
2728         or      %g3, %g1, %g3                   /* SFSR */
2729         ba,a,pt %xcc, .mmu_exception_not_aligned
2730         SET_SIZE(.dmmu_exc_stdf_not_aligned)
2731 
2732         .type   .dmmu_exception, #function
2733 .dmmu_exception:
2734         MMU_FAULT_STATUS_AREA(%g3)
2735         ldx     [%g3 + MMFSA_D_ADDR], %g2
2736         ldx     [%g3 + MMFSA_D_TYPE], %g1
2737         ldx     [%g3 + MMFSA_D_CTX], %g4
2738         srlx    %g2, MMU_PAGESHIFT, %g2         /* align address */
2739         sllx    %g2, MMU_PAGESHIFT, %g2
2740         sllx    %g4, SFSR_CTX_SHIFT, %g3
2741         or      %g3, %g1, %g3                   /* SFSR */
2742         cmp     %g4, USER_CONTEXT_TYPE
2743         movgeu  %icc, USER_CONTEXT_TYPE, %g4
2744         or      %g2, %g4, %g2                   /* TAG_ACCESS */
2745         ba,pt   %xcc, .mmu_exception_end
2746         mov     T_DATA_EXCEPTION, %g1
2747         SET_SIZE(.dmmu_exception)
2748 
2749         .align 32
2750         .global pil15_epilogue
2751 pil15_epilogue:
2752         ba      pil_interrupt_common
2753         nop
2754         .align 32
2755 
2756 /*
2757  * fast_trap_done, fast_trap_done_chk_intr:
2758  *
2759  * Due to the design of UltraSPARC pipeline, pending interrupts are not
2760  * taken immediately after a RETRY or DONE instruction which causes IE to
2761  * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2762  * to execute first before taking any interrupts. If that instruction
2763  * results in other traps, and if the corresponding trap handler runs
2764  * entirely at TL=1 with interrupts disabled, then pending interrupts
2765  * won't be taken until after yet another instruction following the %tpc
2766  * or %tnpc.
2767  *
2768  * A malicious user program can use this feature to block out interrupts
2769  * for extended durations, which can result in send_mondo_timeout kernel
2770  * panic.
2771  *
2772  * This problem is addressed by servicing any pending interrupts via
2773  * sys_trap before returning back to the user mode from a fast trap
2774  * handler. The "done" instruction within a fast trap handler, which
2775  * runs entirely at TL=1 with interrupts disabled, is replaced with the
2776  * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2777  * entry point.
2778  *
2779  * We check for any pending interrupts here and force a sys_trap to
2780  * service those interrupts, if any. To minimize overhead, pending
2781  * interrupts are checked if the %tpc happens to be at 16K boundary,
2782  * which allows a malicious program to execute at most 4K consecutive
2783  * instructions before we service any pending interrupts. If a worst
2784  * case fast trap handler takes about 2 usec, then interrupts will be
2785  * blocked for at most 8 msec, less than a clock tick.
2786  *
2787  * For the cases where we don't know if the %tpc will cross a 16K
2788  * boundary, we can't use the above optimization and always process
2789  * any pending interrupts via fast_frap_done_chk_intr entry point.
2790  *
2791  * Entry Conditions:
2792  *      %pstate         am:0 priv:1 ie:0
2793  *                      globals are AG (not normal globals)
2794  */
2795 
2796         .global fast_trap_done, fast_trap_done_chk_intr
2797 fast_trap_done:
2798         rdpr    %tpc, %g5
2799         sethi   %hi(0xffffc000), %g6    ! 1's complement of 0x3fff
2800         andncc  %g5, %g6, %g0           ! check lower 14 bits of %tpc
2801         bz,pn   %icc, 1f                ! branch if zero (lower 32 bits only)
2802         nop
2803         done
2804 
2805 fast_trap_done_chk_intr:
2806 1:      rd      SOFTINT, %g6
2807         brnz,pn %g6, 2f         ! branch if any pending intr
2808         nop
2809         done
2810 
2811 2:
2812         /*
2813          * We get here if there are any pending interrupts.
2814          * Adjust %tpc/%tnpc as we'll be resuming via "retry"
2815          * instruction.
2816          */
2817         rdpr    %tnpc, %g5
2818         wrpr    %g0, %g5, %tpc
2819         add     %g5, 4, %g5
2820         wrpr    %g0, %g5, %tnpc
2821 
2822         /*
2823          * Force a dummy sys_trap call so that interrupts can be serviced.
2824          */
2825         set     fast_trap_dummy_call, %g1
2826         ba,pt   %xcc, sys_trap
2827           mov   -1, %g4
2828 
2829 fast_trap_dummy_call:
2830         retl
2831         nop
2832 
2833 /*
2834  * Currently the brand syscall interposition code is not enabled by
2835  * default.  Instead, when a branded zone is first booted the brand
2836  * infrastructure will patch the trap table so that the syscall
2837  * entry points are redirected to syscall_wrapper32 and syscall_wrapper
2838  * for ILP32 and LP64 syscalls respectively.  this is done in
2839  * brand_plat_interposition_enable().  Note that the syscall wrappers
2840  * below do not collect any trap trace data since the syscall hot patch
2841  * points are reached after trap trace data has already been collected.
2842  */
2843 #define BRAND_CALLBACK(callback_id)                                         \
2844         CPU_ADDR(%g2, %g1)              /* load CPU struct addr to %g2  */ ;\
2845         ldn     [%g2 + CPU_THREAD], %g3 /* load thread pointer          */ ;\
2846         ldn     [%g3 + T_PROCP], %g3    /* get proc pointer             */ ;\
2847         ldn     [%g3 + P_BRAND], %g3    /* get brand pointer            */ ;\
2848         brz     %g3, 1f                 /* No brand?  No callback.      */ ;\
2849         nop                                                                ;\
2850         ldn     [%g3 + B_MACHOPS], %g3  /* get machops list             */ ;\
2851         ldn     [%g3 + (callback_id << 3)], %g3                      ;\
2852         brz     %g3, 1f                                                    ;\
2853         /*                                                                  \
2854          * This isn't pretty.  We want a low-latency way for the callback   \
2855          * routine to decline to do anything.  We just pass in an address   \
2856          * the routine can directly jmp back to, pretending that nothing    \
2857          * has happened.                                                    \
2858          *                                                                  \
2859          * %g1: return address (where the brand handler jumps back to)      \
2860          * %g2: address of CPU structure                                    \
2861          * %g3: address of brand handler (where we will jump to)            \
2862          */                                                                 \
2863         mov     %pc, %g1                                                   ;\
2864         add     %g1, 16, %g1                                               ;\
2865         jmp     %g3                                                        ;\
2866         nop                                                                ;\
2867 1:
2868 
2869         ENTRY_NP(syscall_wrapper32)
2870         BRAND_CALLBACK(BRAND_CB_SYSCALL32)
2871         SYSCALL_NOTT(syscall_trap32)
2872         SET_SIZE(syscall_wrapper32)
2873 
2874         ENTRY_NP(syscall_wrapper)
2875         BRAND_CALLBACK(BRAND_CB_SYSCALL)
2876         SYSCALL_NOTT(syscall_trap)
2877         SET_SIZE(syscall_wrapper)
2878