de-linting of .s files

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25  
  26 #if !defined(lint)
  27 #include "assym.h"
  28 #endif /* !lint */
  29 #include <sys/asm_linkage.h>
  30 #include <sys/privregs.h>
  31 #include <sys/sun4asi.h>
  32 #include <sys/spitregs.h>
  33 #include <sys/cheetahregs.h>
  34 #include <sys/machtrap.h>
  35 #include <sys/machthread.h>
  36 #include <sys/machbrand.h>
  37 #include <sys/pcb.h>
  38 #include <sys/pte.h>
  39 #include <sys/mmu.h>
  40 #include <sys/machpcb.h>
  41 #include <sys/async.h>
  42 #include <sys/intreg.h>
  43 #include <sys/scb.h>
  44 #include <sys/psr_compat.h>
  45 #include <sys/syscall.h>
  46 #include <sys/machparam.h>
  47 #include <sys/traptrace.h>
  48 #include <vm/hat_sfmmu.h>
  49 #include <sys/archsystm.h>
  50 #include <sys/utrap.h>
  51 #include <sys/clock.h>
  52 #include <sys/intr.h>
  53 #include <sys/fpu/fpu_simulator.h>
  54 #include <vm/seg_spt.h>
  55 
  56 /*
  57  * WARNING: If you add a fast trap handler which can be invoked by a
  58  * non-privileged user, you may have to use the FAST_TRAP_DONE macro
  59  * instead of "done" instruction to return back to the user mode. See
  60  * comments for the "fast_trap_done" entry point for more information.
  61  *
  62  * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
  63  * cases where you always want to process any pending interrupts before
  64  * returning back to the user mode.
  65  */
  66 #define FAST_TRAP_DONE          \
  67         ba,a    fast_trap_done
  68 
  69 #define FAST_TRAP_DONE_CHK_INTR \
  70         ba,a    fast_trap_done_chk_intr
  71 
  72 /*
  73  * SPARC V9 Trap Table
  74  *
  75  * Most of the trap handlers are made from common building
  76  * blocks, and some are instantiated multiple times within
  77  * the trap table. So, I build a bunch of macros, then
  78  * populate the table using only the macros.
  79  *
  80  * Many macros branch to sys_trap.  Its calling convention is:
  81  *      %g1             kernel trap handler
  82  *      %g2, %g3        args for above
  83  *      %g4             desire %pil
  84  */
  85 
  86 #ifdef  TRAPTRACE
  87 
  88 /*
  89  * Tracing macro. Adds two instructions if TRAPTRACE is defined.
  90  */
  91 #define TT_TRACE(label)         \
  92         ba      label           ;\
  93         rd      %pc, %g7
  94 #define TT_TRACE_INS    2
  95 
  96 #define TT_TRACE_L(label)       \
  97         ba      label           ;\
  98         rd      %pc, %l4        ;\
  99         clr     %l4
 100 #define TT_TRACE_L_INS  3
 101 
 102 #else
 103 
 104 #define TT_TRACE(label)
 105 #define TT_TRACE_INS    0
 106 
 107 #define TT_TRACE_L(label)
 108 #define TT_TRACE_L_INS  0
 109 
 110 #endif
 111 
 112 /*
 113  * This first set are funneled to trap() with %tt as the type.
 114  * Trap will then either panic or send the user a signal.
 115  */
 116 /*
 117  * NOT is used for traps that just shouldn't happen.
 118  * It comes in both single and quadruple flavors.
 119  */
 120 #if !defined(lint)
 121         .global trap
 122 #endif /* !lint */
 123 #define NOT                     \
 124         TT_TRACE(trace_gen)     ;\
 125         set     trap, %g1       ;\
 126         rdpr    %tt, %g3        ;\
 127         ba,pt   %xcc, sys_trap  ;\
 128         sub     %g0, 1, %g4     ;\
 129         .align  32
 130 #define NOT4    NOT; NOT; NOT; NOT
 131 /*
 132  * RED is for traps that use the red mode handler.
 133  * We should never see these either.
 134  */
 135 #define RED     NOT
 136 /*
 137  * BAD is used for trap vectors we don't have a kernel
 138  * handler for.
 139  * It also comes in single and quadruple versions.
 140  */
 141 #define BAD     NOT
 142 #define BAD4    NOT4
 143 
 144 #define DONE                    \
 145         done;                   \
 146         .align  32
 147 
 148 /*
 149  * TRAP vectors to the trap() function.
 150  * It's main use is for user errors.
 151  */
 152 #if !defined(lint)
 153         .global trap
 154 #endif /* !lint */
 155 #define TRAP(arg)               \
 156         TT_TRACE(trace_gen)     ;\
 157         set     trap, %g1       ;\
 158         mov     arg, %g3        ;\
 159         ba,pt   %xcc, sys_trap  ;\
 160         sub     %g0, 1, %g4     ;\
 161         .align  32
 162 
 163 /*
 164  * SYSCALL is used for unsupported syscall interfaces (with 'which'
 165  * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
 166  * 'which' set to 'syscall_trap32').
 167  *
 168  * The SYSCALL_TRAP* macros are used for syscall entry points.
 169  * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
 170  * is used to support ILP32.  Each macro can only be used once
 171  * since they each define a symbol.  The symbols are used as hot patch
 172  * points by the brand infrastructure to dynamically enable and disable
 173  * brand syscall interposition.  See the comments around BRAND_CALLBACK
 174  * and brand_plat_interposition_enable() for more information.
 175  */
 176 #define SYSCALL_NOTT(which)             \
 177         set     (which), %g1            ;\
 178         ba,pt   %xcc, sys_trap          ;\
 179         sub     %g0, 1, %g4             ;\
 180         .align  32
 181 
 182 #define SYSCALL(which)                  \
 183         TT_TRACE(trace_gen)             ;\
 184         SYSCALL_NOTT(which)
 185 
 186 #define SYSCALL_TRAP32                          \
 187         TT_TRACE(trace_gen)                     ;\
 188         ALTENTRY(syscall_trap32_patch_point)    \
 189         SYSCALL_NOTT(syscall_trap32)
 190 
 191 #define SYSCALL_TRAP                            \
 192         TT_TRACE(trace_gen)                     ;\
 193         ALTENTRY(syscall_trap_patch_point)      \
 194         SYSCALL_NOTT(syscall_trap)
 195 
 196 #define FLUSHW(h_name)                  \
 197         .global h_name                  ;\
 198 h_name:                                 ;\
 199         set     trap, %g1               ;\
 200         mov     T_FLUSHW, %g3           ;\
 201         sub     %g0, 1, %g4             ;\
 202         save                            ;\
 203         flushw                          ;\
 204         restore                         ;\
 205         FAST_TRAP_DONE                  ;\
 206         .align  32
 207 
 208 /*
 209  * GOTO just jumps to a label.
 210  * It's used for things that can be fixed without going thru sys_trap.
 211  */
 212 #define GOTO(label)             \
 213         .global label           ;\
 214         ba,a    label           ;\
 215         .empty                  ;\
 216         .align  32
 217 
 218 /*
 219  * GOTO_TT just jumps to a label.
 220  * correctable ECC error traps at  level 0 and 1 will use this macro.
 221  * It's used for things that can be fixed without going thru sys_trap.
 222  */
 223 #define GOTO_TT(label, ttlabel)         \
 224         .global label           ;\
 225         TT_TRACE(ttlabel)       ;\
 226         ba,a    label           ;\
 227         .empty                  ;\
 228         .align  32
 229 
 230 /*
 231  * Privileged traps
 232  * Takes breakpoint if privileged, calls trap() if not.
 233  */
 234 #define PRIV(label)                     \
 235         rdpr    %tstate, %g1            ;\
 236         btst    TSTATE_PRIV, %g1        ;\
 237         bnz     label                   ;\
 238         rdpr    %tt, %g3                ;\
 239         set     trap, %g1               ;\
 240         ba,pt   %xcc, sys_trap          ;\
 241         sub     %g0, 1, %g4             ;\
 242         .align  32
 243 
 244 
 245 /*
 246  * DTrace traps.
 247  */
 248 #define DTRACE_PID                      \
 249         .global dtrace_pid_probe                                ;\
 250         set     dtrace_pid_probe, %g1                           ;\
 251         ba,pt   %xcc, user_trap                                 ;\
 252         sub     %g0, 1, %g4                                     ;\
 253         .align  32
 254 
 255 #define DTRACE_RETURN                   \
 256         .global dtrace_return_probe                             ;\
 257         set     dtrace_return_probe, %g1                        ;\
 258         ba,pt   %xcc, user_trap                                 ;\
 259         sub     %g0, 1, %g4                                     ;\
 260         .align  32
 261 
 262 /*
 263  * REGISTER WINDOW MANAGEMENT MACROS
 264  */
 265 
 266 /*
 267  * various convenient units of padding
 268  */
 269 #define SKIP(n) .skip 4*(n)
 270 
 271 /*
 272  * CLEAN_WINDOW is the simple handler for cleaning a register window.
 273  */
 274 #define CLEAN_WINDOW                                            \
 275         TT_TRACE_L(trace_win)                                   ;\
 276         rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin       ;\
 277         clr %l0; clr %l1; clr %l2; clr %l3                      ;\
 278         clr %l4; clr %l5; clr %l6; clr %l7                      ;\
 279         clr %o0; clr %o1; clr %o2; clr %o3                      ;\
 280         clr %o4; clr %o5; clr %o6; clr %o7                      ;\
 281         retry; .align 128
 282 
 283 #if !defined(lint)
 284 
 285 /*
 286  * If we get an unresolved tlb miss while in a window handler, the fault
 287  * handler will resume execution at the last instruction of the window
 288  * hander, instead of delivering the fault to the kernel.  Spill handlers
 289  * use this to spill windows into the wbuf.
 290  *
 291  * The mixed handler works by checking %sp, and branching to the correct
 292  * handler.  This is done by branching back to label 1: for 32b frames,
 293  * or label 2: for 64b frames; which implies the handler order is: 32b,
 294  * 64b, mixed.  The 1: and 2: labels are offset into the routines to
 295  * allow the branchs' delay slots to contain useful instructions.
 296  */
 297 
 298 /*
 299  * SPILL_32bit spills a 32-bit-wide kernel register window.  It
 300  * assumes that the kernel context and the nucleus context are the
 301  * same.  The stack pointer is required to be eight-byte aligned even
 302  * though this code only needs it to be four-byte aligned.
 303  */
 304 #define SPILL_32bit(tail)                                       \
 305         srl     %sp, 0, %sp                                     ;\
 306 1:      st      %l0, [%sp + 0]                                  ;\
 307         st      %l1, [%sp + 4]                                  ;\
 308         st      %l2, [%sp + 8]                                  ;\
 309         st      %l3, [%sp + 12]                                 ;\
 310         st      %l4, [%sp + 16]                                 ;\
 311         st      %l5, [%sp + 20]                                 ;\
 312         st      %l6, [%sp + 24]                                 ;\
 313         st      %l7, [%sp + 28]                                 ;\
 314         st      %i0, [%sp + 32]                                 ;\
 315         st      %i1, [%sp + 36]                                 ;\
 316         st      %i2, [%sp + 40]                                 ;\
 317         st      %i3, [%sp + 44]                                 ;\
 318         st      %i4, [%sp + 48]                                 ;\
 319         st      %i5, [%sp + 52]                                 ;\
 320         st      %i6, [%sp + 56]                                 ;\
 321         st      %i7, [%sp + 60]                                 ;\
 322         TT_TRACE_L(trace_win)                                   ;\
 323         saved                                                   ;\
 324         retry                                                   ;\
 325         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 326         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 327         .empty
 328 
 329 /*
 330  * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
 331  * wide address space via the designated asi.  It is used to spill
 332  * non-kernel windows.  The stack pointer is required to be eight-byte
 333  * aligned even though this code only needs it to be four-byte
 334  * aligned.
 335  */
 336 #define SPILL_32bit_asi(asi_num, tail)                          \
 337         srl     %sp, 0, %sp                                     ;\
 338 1:      sta     %l0, [%sp + %g0]asi_num                         ;\
 339         mov     4, %g1                                          ;\
 340         sta     %l1, [%sp + %g1]asi_num                         ;\
 341         mov     8, %g2                                          ;\
 342         sta     %l2, [%sp + %g2]asi_num                         ;\
 343         mov     12, %g3                                         ;\
 344         sta     %l3, [%sp + %g3]asi_num                         ;\
 345         add     %sp, 16, %g4                                    ;\
 346         sta     %l4, [%g4 + %g0]asi_num                         ;\
 347         sta     %l5, [%g4 + %g1]asi_num                         ;\
 348         sta     %l6, [%g4 + %g2]asi_num                         ;\
 349         sta     %l7, [%g4 + %g3]asi_num                         ;\
 350         add     %g4, 16, %g4                                    ;\
 351         sta     %i0, [%g4 + %g0]asi_num                         ;\
 352         sta     %i1, [%g4 + %g1]asi_num                         ;\
 353         sta     %i2, [%g4 + %g2]asi_num                         ;\
 354         sta     %i3, [%g4 + %g3]asi_num                         ;\
 355         add     %g4, 16, %g4                                    ;\
 356         sta     %i4, [%g4 + %g0]asi_num                         ;\
 357         sta     %i5, [%g4 + %g1]asi_num                         ;\
 358         sta     %i6, [%g4 + %g2]asi_num                         ;\
 359         sta     %i7, [%g4 + %g3]asi_num                         ;\
 360         TT_TRACE_L(trace_win)                                   ;\
 361         saved                                                   ;\
 362         retry                                                   ;\
 363         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 364         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 365         .empty
 366 
 367 /*
 368  * SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit
 369  * wide address space via the designated asi.  It is used to spill
 370  * windows at tl>1 where performance isn't the primary concern and
 371  * where we don't want to use unnecessary registers.  The stack
 372  * pointer is required to be eight-byte aligned even though this code
 373  * only needs it to be four-byte aligned.
 374  */
 375 #define SPILL_32bit_tt1(asi_num, tail)                          \
 376         mov     asi_num, %asi                                   ;\
 377 1:      srl     %sp, 0, %sp                                     ;\
 378         sta     %l0, [%sp + 0]%asi                              ;\
 379         sta     %l1, [%sp + 4]%asi                              ;\
 380         sta     %l2, [%sp + 8]%asi                              ;\
 381         sta     %l3, [%sp + 12]%asi                             ;\
 382         sta     %l4, [%sp + 16]%asi                             ;\
 383         sta     %l5, [%sp + 20]%asi                             ;\
 384         sta     %l6, [%sp + 24]%asi                             ;\
 385         sta     %l7, [%sp + 28]%asi                             ;\
 386         sta     %i0, [%sp + 32]%asi                             ;\
 387         sta     %i1, [%sp + 36]%asi                             ;\
 388         sta     %i2, [%sp + 40]%asi                             ;\
 389         sta     %i3, [%sp + 44]%asi                             ;\
 390         sta     %i4, [%sp + 48]%asi                             ;\
 391         sta     %i5, [%sp + 52]%asi                             ;\
 392         sta     %i6, [%sp + 56]%asi                             ;\
 393         sta     %i7, [%sp + 60]%asi                             ;\
 394         TT_TRACE_L(trace_win)                                   ;\
 395         saved                                                   ;\
 396         retry                                                   ;\
 397         SKIP(31-20-TT_TRACE_L_INS)                              ;\
 398         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 399         .empty
 400 
 401 
 402 /*
 403  * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
 404  * that the kernel context and the nucleus context are the same.  The
 405  * stack pointer is required to be eight-byte aligned even though this
 406  * code only needs it to be four-byte aligned.
 407  */
 408 #define FILL_32bit(tail)                                        \
 409         srl     %sp, 0, %sp                                     ;\
 410 1:      TT_TRACE_L(trace_win)                                   ;\
 411         ld      [%sp + 0], %l0                                  ;\
 412         ld      [%sp + 4], %l1                                  ;\
 413         ld      [%sp + 8], %l2                                  ;\
 414         ld      [%sp + 12], %l3                                 ;\
 415         ld      [%sp + 16], %l4                                 ;\
 416         ld      [%sp + 20], %l5                                 ;\
 417         ld      [%sp + 24], %l6                                 ;\
 418         ld      [%sp + 28], %l7                                 ;\
 419         ld      [%sp + 32], %i0                                 ;\
 420         ld      [%sp + 36], %i1                                 ;\
 421         ld      [%sp + 40], %i2                                 ;\
 422         ld      [%sp + 44], %i3                                 ;\
 423         ld      [%sp + 48], %i4                                 ;\
 424         ld      [%sp + 52], %i5                                 ;\
 425         ld      [%sp + 56], %i6                                 ;\
 426         ld      [%sp + 60], %i7                                 ;\
 427         restored                                                ;\
 428         retry                                                   ;\
 429         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 430         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 431         .empty
 432 
 433 /*
 434  * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
 435  * wide address space via the designated asi.  It is used to fill
 436  * non-kernel windows.  The stack pointer is required to be eight-byte
 437  * aligned even though this code only needs it to be four-byte
 438  * aligned.
 439  */
 440 #define FILL_32bit_asi(asi_num, tail)                           \
 441         srl     %sp, 0, %sp                                     ;\
 442 1:      TT_TRACE_L(trace_win)                                   ;\
 443         mov     4, %g1                                          ;\
 444         lda     [%sp + %g0]asi_num, %l0                         ;\
 445         mov     8, %g2                                          ;\
 446         lda     [%sp + %g1]asi_num, %l1                         ;\
 447         mov     12, %g3                                         ;\
 448         lda     [%sp + %g2]asi_num, %l2                         ;\
 449         lda     [%sp + %g3]asi_num, %l3                         ;\
 450         add     %sp, 16, %g4                                    ;\
 451         lda     [%g4 + %g0]asi_num, %l4                         ;\
 452         lda     [%g4 + %g1]asi_num, %l5                         ;\
 453         lda     [%g4 + %g2]asi_num, %l6                         ;\
 454         lda     [%g4 + %g3]asi_num, %l7                         ;\
 455         add     %g4, 16, %g4                                    ;\
 456         lda     [%g4 + %g0]asi_num, %i0                         ;\
 457         lda     [%g4 + %g1]asi_num, %i1                         ;\
 458         lda     [%g4 + %g2]asi_num, %i2                         ;\
 459         lda     [%g4 + %g3]asi_num, %i3                         ;\
 460         add     %g4, 16, %g4                                    ;\
 461         lda     [%g4 + %g0]asi_num, %i4                         ;\
 462         lda     [%g4 + %g1]asi_num, %i5                         ;\
 463         lda     [%g4 + %g2]asi_num, %i6                         ;\
 464         lda     [%g4 + %g3]asi_num, %i7                         ;\
 465         restored                                                ;\
 466         retry                                                   ;\
 467         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 468         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 469         .empty
 470 
 471 /*
 472  * FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit
 473  * wide address space via the designated asi.  It is used to fill
 474  * windows at tl>1 where performance isn't the primary concern and
 475  * where we don't want to use unnecessary registers.  The stack
 476  * pointer is required to be eight-byte aligned even though this code
 477  * only needs it to be four-byte aligned.
 478  */
 479 #define FILL_32bit_tt1(asi_num, tail)                           \
 480         mov     asi_num, %asi                                   ;\
 481 1:      srl     %sp, 0, %sp                                     ;\
 482         TT_TRACE_L(trace_win)                                   ;\
 483         lda     [%sp + 0]%asi, %l0                              ;\
 484         lda     [%sp + 4]%asi, %l1                              ;\
 485         lda     [%sp + 8]%asi, %l2                              ;\
 486         lda     [%sp + 12]%asi, %l3                             ;\
 487         lda     [%sp + 16]%asi, %l4                             ;\
 488         lda     [%sp + 20]%asi, %l5                             ;\
 489         lda     [%sp + 24]%asi, %l6                             ;\
 490         lda     [%sp + 28]%asi, %l7                             ;\
 491         lda     [%sp + 32]%asi, %i0                             ;\
 492         lda     [%sp + 36]%asi, %i1                             ;\
 493         lda     [%sp + 40]%asi, %i2                             ;\
 494         lda     [%sp + 44]%asi, %i3                             ;\
 495         lda     [%sp + 48]%asi, %i4                             ;\
 496         lda     [%sp + 52]%asi, %i5                             ;\
 497         lda     [%sp + 56]%asi, %i6                             ;\
 498         lda     [%sp + 60]%asi, %i7                             ;\
 499         restored                                                ;\
 500         retry                                                   ;\
 501         SKIP(31-20-TT_TRACE_L_INS)                              ;\
 502         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 503         .empty
 504 
 505 
 506 /*
 507  * SPILL_64bit spills a 64-bit-wide kernel register window.  It
 508  * assumes that the kernel context and the nucleus context are the
 509  * same.  The stack pointer is required to be eight-byte aligned.
 510  */
 511 #define SPILL_64bit(tail)                                       \
 512 2:      stx     %l0, [%sp + V9BIAS64 + 0]                       ;\
 513         stx     %l1, [%sp + V9BIAS64 + 8]                       ;\
 514         stx     %l2, [%sp + V9BIAS64 + 16]                      ;\
 515         stx     %l3, [%sp + V9BIAS64 + 24]                      ;\
 516         stx     %l4, [%sp + V9BIAS64 + 32]                      ;\
 517         stx     %l5, [%sp + V9BIAS64 + 40]                      ;\
 518         stx     %l6, [%sp + V9BIAS64 + 48]                      ;\
 519         stx     %l7, [%sp + V9BIAS64 + 56]                      ;\
 520         stx     %i0, [%sp + V9BIAS64 + 64]                      ;\
 521         stx     %i1, [%sp + V9BIAS64 + 72]                      ;\
 522         stx     %i2, [%sp + V9BIAS64 + 80]                      ;\
 523         stx     %i3, [%sp + V9BIAS64 + 88]                      ;\
 524         stx     %i4, [%sp + V9BIAS64 + 96]                      ;\
 525         stx     %i5, [%sp + V9BIAS64 + 104]                     ;\
 526         stx     %i6, [%sp + V9BIAS64 + 112]                     ;\
 527         stx     %i7, [%sp + V9BIAS64 + 120]                     ;\
 528         TT_TRACE_L(trace_win)                                   ;\
 529         saved                                                   ;\
 530         retry                                                   ;\
 531         SKIP(31-18-TT_TRACE_L_INS)                              ;\
 532         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 533         .empty
 534 
 535 /*
 536  * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
 537  * wide address space via the designated asi.  It is used to spill
 538  * non-kernel windows.  The stack pointer is required to be eight-byte
 539  * aligned.
 540  */
 541 #define SPILL_64bit_asi(asi_num, tail)                          \
 542         mov     0 + V9BIAS64, %g1                               ;\
 543 2:      stxa    %l0, [%sp + %g1]asi_num                         ;\
 544         mov     8 + V9BIAS64, %g2                               ;\
 545         stxa    %l1, [%sp + %g2]asi_num                         ;\
 546         mov     16 + V9BIAS64, %g3                              ;\
 547         stxa    %l2, [%sp + %g3]asi_num                         ;\
 548         mov     24 + V9BIAS64, %g4                              ;\
 549         stxa    %l3, [%sp + %g4]asi_num                         ;\
 550         add     %sp, 32, %g5                                    ;\
 551         stxa    %l4, [%g5 + %g1]asi_num                         ;\
 552         stxa    %l5, [%g5 + %g2]asi_num                         ;\
 553         stxa    %l6, [%g5 + %g3]asi_num                         ;\
 554         stxa    %l7, [%g5 + %g4]asi_num                         ;\
 555         add     %g5, 32, %g5                                    ;\
 556         stxa    %i0, [%g5 + %g1]asi_num                         ;\
 557         stxa    %i1, [%g5 + %g2]asi_num                         ;\
 558         stxa    %i2, [%g5 + %g3]asi_num                         ;\
 559         stxa    %i3, [%g5 + %g4]asi_num                         ;\
 560         add     %g5, 32, %g5                                    ;\
 561         stxa    %i4, [%g5 + %g1]asi_num                         ;\
 562         stxa    %i5, [%g5 + %g2]asi_num                         ;\
 563         stxa    %i6, [%g5 + %g3]asi_num                         ;\
 564         stxa    %i7, [%g5 + %g4]asi_num                         ;\
 565         TT_TRACE_L(trace_win)                                   ;\
 566         saved                                                   ;\
 567         retry                                                   ;\
 568         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 569         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 570         .empty
 571 
 572 /*
 573  * SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit
 574  * wide address space via the designated asi.  It is used to spill
 575  * windows at tl>1 where performance isn't the primary concern and
 576  * where we don't want to use unnecessary registers.  The stack
 577  * pointer is required to be eight-byte aligned.
 578  */
 579 #define SPILL_64bit_tt1(asi_num, tail)                          \
 580         mov     asi_num, %asi                                   ;\
 581 2:      stxa    %l0, [%sp + V9BIAS64 + 0]%asi                   ;\
 582         stxa    %l1, [%sp + V9BIAS64 + 8]%asi                   ;\
 583         stxa    %l2, [%sp + V9BIAS64 + 16]%asi                  ;\
 584         stxa    %l3, [%sp + V9BIAS64 + 24]%asi                  ;\
 585         stxa    %l4, [%sp + V9BIAS64 + 32]%asi                  ;\
 586         stxa    %l5, [%sp + V9BIAS64 + 40]%asi                  ;\
 587         stxa    %l6, [%sp + V9BIAS64 + 48]%asi                  ;\
 588         stxa    %l7, [%sp + V9BIAS64 + 56]%asi                  ;\
 589         stxa    %i0, [%sp + V9BIAS64 + 64]%asi                  ;\
 590         stxa    %i1, [%sp + V9BIAS64 + 72]%asi                  ;\
 591         stxa    %i2, [%sp + V9BIAS64 + 80]%asi                  ;\
 592         stxa    %i3, [%sp + V9BIAS64 + 88]%asi                  ;\
 593         stxa    %i4, [%sp + V9BIAS64 + 96]%asi                  ;\
 594         stxa    %i5, [%sp + V9BIAS64 + 104]%asi                 ;\
 595         stxa    %i6, [%sp + V9BIAS64 + 112]%asi                 ;\
 596         stxa    %i7, [%sp + V9BIAS64 + 120]%asi                 ;\
 597         TT_TRACE_L(trace_win)                                   ;\
 598         saved                                                   ;\
 599         retry                                                   ;\
 600         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 601         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 602         .empty
 603 
 604 
 605 /*
 606  * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
 607  * that the kernel context and the nucleus context are the same.  The
 608  * stack pointer is required to be eight-byte aligned.
 609  */
 610 #define FILL_64bit(tail)                                        \
 611 2:      TT_TRACE_L(trace_win)                                   ;\
 612         ldx     [%sp + V9BIAS64 + 0], %l0                       ;\
 613         ldx     [%sp + V9BIAS64 + 8], %l1                       ;\
 614         ldx     [%sp + V9BIAS64 + 16], %l2                      ;\
 615         ldx     [%sp + V9BIAS64 + 24], %l3                      ;\
 616         ldx     [%sp + V9BIAS64 + 32], %l4                      ;\
 617         ldx     [%sp + V9BIAS64 + 40], %l5                      ;\
 618         ldx     [%sp + V9BIAS64 + 48], %l6                      ;\
 619         ldx     [%sp + V9BIAS64 + 56], %l7                      ;\
 620         ldx     [%sp + V9BIAS64 + 64], %i0                      ;\
 621         ldx     [%sp + V9BIAS64 + 72], %i1                      ;\
 622         ldx     [%sp + V9BIAS64 + 80], %i2                      ;\
 623         ldx     [%sp + V9BIAS64 + 88], %i3                      ;\
 624         ldx     [%sp + V9BIAS64 + 96], %i4                      ;\
 625         ldx     [%sp + V9BIAS64 + 104], %i5                     ;\
 626         ldx     [%sp + V9BIAS64 + 112], %i6                     ;\
 627         ldx     [%sp + V9BIAS64 + 120], %i7                     ;\
 628         restored                                                ;\
 629         retry                                                   ;\
 630         SKIP(31-18-TT_TRACE_L_INS)                              ;\
 631         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 632         .empty
 633 
 634 /*
 635  * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
 636  * wide address space via the designated asi.  It is used to fill
 637  * non-kernel windows.  The stack pointer is required to be eight-byte
 638  * aligned.
 639  */
 640 #define FILL_64bit_asi(asi_num, tail)                           \
 641         mov     V9BIAS64 + 0, %g1                               ;\
 642 2:      TT_TRACE_L(trace_win)                                   ;\
 643         ldxa    [%sp + %g1]asi_num, %l0                         ;\
 644         mov     V9BIAS64 + 8, %g2                               ;\
 645         ldxa    [%sp + %g2]asi_num, %l1                         ;\
 646         mov     V9BIAS64 + 16, %g3                              ;\
 647         ldxa    [%sp + %g3]asi_num, %l2                         ;\
 648         mov     V9BIAS64 + 24, %g4                              ;\
 649         ldxa    [%sp + %g4]asi_num, %l3                         ;\
 650         add     %sp, 32, %g5                                    ;\
 651         ldxa    [%g5 + %g1]asi_num, %l4                         ;\
 652         ldxa    [%g5 + %g2]asi_num, %l5                         ;\
 653         ldxa    [%g5 + %g3]asi_num, %l6                         ;\
 654         ldxa    [%g5 + %g4]asi_num, %l7                         ;\
 655         add     %g5, 32, %g5                                    ;\
 656         ldxa    [%g5 + %g1]asi_num, %i0                         ;\
 657         ldxa    [%g5 + %g2]asi_num, %i1                         ;\
 658         ldxa    [%g5 + %g3]asi_num, %i2                         ;\
 659         ldxa    [%g5 + %g4]asi_num, %i3                         ;\
 660         add     %g5, 32, %g5                                    ;\
 661         ldxa    [%g5 + %g1]asi_num, %i4                         ;\
 662         ldxa    [%g5 + %g2]asi_num, %i5                         ;\
 663         ldxa    [%g5 + %g3]asi_num, %i6                         ;\
 664         ldxa    [%g5 + %g4]asi_num, %i7                         ;\
 665         restored                                                ;\
 666         retry                                                   ;\
 667         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 668         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 669         .empty
 670 
 671 /*
 672  * FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit
 673  * wide address space via the designated asi.  It is used to fill
 674  * windows at tl>1 where performance isn't the primary concern and
 675  * where we don't want to use unnecessary registers.  The stack
 676  * pointer is required to be eight-byte aligned.
 677  */
 678 #define FILL_64bit_tt1(asi_num, tail)                           \
 679         mov     asi_num, %asi                                   ;\
 680         TT_TRACE_L(trace_win)                                   ;\
 681         ldxa    [%sp + V9BIAS64 + 0]%asi, %l0                   ;\
 682         ldxa    [%sp + V9BIAS64 + 8]%asi, %l1                   ;\
 683         ldxa    [%sp + V9BIAS64 + 16]%asi, %l2                  ;\
 684         ldxa    [%sp + V9BIAS64 + 24]%asi, %l3                  ;\
 685         ldxa    [%sp + V9BIAS64 + 32]%asi, %l4                  ;\
 686         ldxa    [%sp + V9BIAS64 + 40]%asi, %l5                  ;\
 687         ldxa    [%sp + V9BIAS64 + 48]%asi, %l6                  ;\
 688         ldxa    [%sp + V9BIAS64 + 56]%asi, %l7                  ;\
 689         ldxa    [%sp + V9BIAS64 + 64]%asi, %i0                  ;\
 690         ldxa    [%sp + V9BIAS64 + 72]%asi, %i1                  ;\
 691         ldxa    [%sp + V9BIAS64 + 80]%asi, %i2                  ;\
 692         ldxa    [%sp + V9BIAS64 + 88]%asi, %i3                  ;\
 693         ldxa    [%sp + V9BIAS64 + 96]%asi, %i4                  ;\
 694         ldxa    [%sp + V9BIAS64 + 104]%asi, %i5                 ;\
 695         ldxa    [%sp + V9BIAS64 + 112]%asi, %i6                 ;\
 696         ldxa    [%sp + V9BIAS64 + 120]%asi, %i7                 ;\
 697         restored                                                ;\
 698         retry                                                   ;\
 699         SKIP(31-19-TT_TRACE_L_INS)                              ;\
 700         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 701         .empty
 702 
 703 #endif /* !lint */
 704 
 705 /*
 706  * SPILL_mixed spills either size window, depending on
 707  * whether %sp is even or odd, to a 32-bit address space.
 708  * This may only be used in conjunction with SPILL_32bit/
 709  * SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be
 710  * needed for use with SPILL_{32,64}bit_{tt1,asi}.  Particular
 711  * attention should be paid to the instructions that belong
 712  * in the delay slots of the branches depending on the type
 713  * of spill handler being branched to.
 714  * Clear upper 32 bits of %sp if it is odd.
 715  * We won't need to clear them in 64 bit kernel.
 716  */
 717 #define SPILL_mixed                                             \
 718         btst    1, %sp                                          ;\
 719         bz,a,pt %xcc, 1b                                        ;\
 720         srl     %sp, 0, %sp                                     ;\
 721         ba,pt   %xcc, 2b                                        ;\
 722         nop                                                     ;\
 723         .align  128
 724 
 725 /*
 726  * FILL_mixed(ASI) fills either size window, depending on
 727  * whether %sp is even or odd, from a 32-bit address space.
 728  * This may only be used in conjunction with FILL_32bit/
 729  * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
 730  * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
 731  * attention should be paid to the instructions that belong
 732  * in the delay slots of the branches depending on the type
 733  * of fill handler being branched to.
 734  * Clear upper 32 bits of %sp if it is odd.
 735  * We won't need to clear them in 64 bit kernel.
 736  */
 737 #define FILL_mixed                                              \
 738         btst    1, %sp                                          ;\
 739         bz,a,pt %xcc, 1b                                        ;\
 740         srl     %sp, 0, %sp                                     ;\
 741         ba,pt   %xcc, 2b                                        ;\
 742         nop                                                     ;\
 743         .align  128
 744 
 745 
 746 /*
 747  * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
 748  * respectively, into the address space via the designated asi.  The
 749  * unbiased stack pointer is required to be eight-byte aligned (even for
 750  * the 32-bit case even though this code does not require such strict
 751  * alignment).
 752  *
 753  * With SPARC v9 the spill trap takes precedence over the cleanwin trap
 754  * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
 755  * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
 756  * window may contain kernel data so in user_rtt we set wstate to call
 757  * these spill handlers on the first user spill trap.  These handler then
 758  * spill the appropriate window but also back up a window and clean the
 759  * window that didn't get a cleanwin trap.
 760  */
 761 #define SPILL_32clean(asi_num, tail)                            \
 762         srl     %sp, 0, %sp                                     ;\
 763         sta     %l0, [%sp + %g0]asi_num                         ;\
 764         mov     4, %g1                                          ;\
 765         sta     %l1, [%sp + %g1]asi_num                         ;\
 766         mov     8, %g2                                          ;\
 767         sta     %l2, [%sp + %g2]asi_num                         ;\
 768         mov     12, %g3                                         ;\
 769         sta     %l3, [%sp + %g3]asi_num                         ;\
 770         add     %sp, 16, %g4                                    ;\
 771         sta     %l4, [%g4 + %g0]asi_num                         ;\
 772         sta     %l5, [%g4 + %g1]asi_num                         ;\
 773         sta     %l6, [%g4 + %g2]asi_num                         ;\
 774         sta     %l7, [%g4 + %g3]asi_num                         ;\
 775         add     %g4, 16, %g4                                    ;\
 776         sta     %i0, [%g4 + %g0]asi_num                         ;\
 777         sta     %i1, [%g4 + %g1]asi_num                         ;\
 778         sta     %i2, [%g4 + %g2]asi_num                         ;\
 779         sta     %i3, [%g4 + %g3]asi_num                         ;\
 780         add     %g4, 16, %g4                                    ;\
 781         sta     %i4, [%g4 + %g0]asi_num                         ;\
 782         sta     %i5, [%g4 + %g1]asi_num                         ;\
 783         sta     %i6, [%g4 + %g2]asi_num                         ;\
 784         sta     %i7, [%g4 + %g3]asi_num                         ;\
 785         TT_TRACE_L(trace_win)                                   ;\
 786         b       .spill_clean                                    ;\
 787           mov   WSTATE_USER32, %g7                              ;\
 788         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 789         ba,a,pt %xcc, fault_32bit_/**/tail                      ;\
 790         .empty
 791 
 792 #define SPILL_64clean(asi_num, tail)                            \
 793         mov     0 + V9BIAS64, %g1                               ;\
 794         stxa    %l0, [%sp + %g1]asi_num                         ;\
 795         mov     8 + V9BIAS64, %g2                               ;\
 796         stxa    %l1, [%sp + %g2]asi_num                         ;\
 797         mov     16 + V9BIAS64, %g3                              ;\
 798         stxa    %l2, [%sp + %g3]asi_num                         ;\
 799         mov     24 + V9BIAS64, %g4                              ;\
 800         stxa    %l3, [%sp + %g4]asi_num                         ;\
 801         add     %sp, 32, %g5                                    ;\
 802         stxa    %l4, [%g5 + %g1]asi_num                         ;\
 803         stxa    %l5, [%g5 + %g2]asi_num                         ;\
 804         stxa    %l6, [%g5 + %g3]asi_num                         ;\
 805         stxa    %l7, [%g5 + %g4]asi_num                         ;\
 806         add     %g5, 32, %g5                                    ;\
 807         stxa    %i0, [%g5 + %g1]asi_num                         ;\
 808         stxa    %i1, [%g5 + %g2]asi_num                         ;\
 809         stxa    %i2, [%g5 + %g3]asi_num                         ;\
 810         stxa    %i3, [%g5 + %g4]asi_num                         ;\
 811         add     %g5, 32, %g5                                    ;\
 812         stxa    %i4, [%g5 + %g1]asi_num                         ;\
 813         stxa    %i5, [%g5 + %g2]asi_num                         ;\
 814         stxa    %i6, [%g5 + %g3]asi_num                         ;\
 815         stxa    %i7, [%g5 + %g4]asi_num                         ;\
 816         TT_TRACE_L(trace_win)                                   ;\
 817         b       .spill_clean                                    ;\
 818           mov   WSTATE_USER64, %g7                              ;\
 819         SKIP(31-25-TT_TRACE_L_INS)                              ;\
 820         ba,a,pt %xcc, fault_64bit_/**/tail                      ;\
 821         .empty
 822 
 823 
 824 /*
 825  * Floating point disabled.
 826  */
 827 #define FP_DISABLED_TRAP                \
 828         TT_TRACE(trace_gen)             ;\
 829         ba,pt   %xcc,.fp_disabled       ;\
 830         nop                             ;\
 831         .align  32
 832 
 833 /*
 834  * Floating point exceptions.
 835  */
 836 #define FP_IEEE_TRAP                    \
 837         TT_TRACE(trace_gen)             ;\
 838         ba,pt   %xcc,.fp_ieee_exception ;\
 839         nop                             ;\
 840         .align  32
 841 
 842 #define FP_TRAP                         \
 843         TT_TRACE(trace_gen)             ;\
 844         ba,pt   %xcc,.fp_exception      ;\
 845         nop                             ;\
 846         .align  32
 847 
 848 #if !defined(lint)
 849 /*
 850  * asynchronous traps at level 0 and level 1
 851  *
 852  * The first instruction must be a membar for UltraSPARC-III
 853  * to stop RED state entry if the store queue has many
 854  * pending bad stores (PRM, Chapter 11).
 855  */
 856 #define ASYNC_TRAP(ttype, ttlabel, table_name)\
 857         .global table_name      ;\
 858 table_name:                     ;\
 859         membar  #Sync           ;\
 860         TT_TRACE(ttlabel)       ;\
 861         ba      async_err       ;\
 862         mov     ttype, %g5      ;\
 863         .align  32
 864 
 865 /*
 866  * Defaults to BAD entry, but establishes label to be used for
 867  * architecture-specific overwrite of trap table entry.
 868  */
 869 #define LABELED_BAD(table_name)         \
 870         .global table_name              ;\
 871 table_name:                             ;\
 872         BAD
 873 
 874 #endif /* !lint */
 875 
 876 /*
 877  * illegal instruction trap
 878  */
 879 #define ILLTRAP_INSTR                     \
 880         membar  #Sync                     ;\
 881         TT_TRACE(trace_gen)               ;\
 882         or      %g0, P_UTRAP4, %g2        ;\
 883         or      %g0, T_UNIMP_INSTR, %g3   ;\
 884         sethi   %hi(.check_v9utrap), %g4  ;\
 885         jmp     %g4 + %lo(.check_v9utrap) ;\
 886         nop                               ;\
 887         .align  32
 888 
 889 /*
 890  * tag overflow trap
 891  */
 892 #define TAG_OVERFLOW                      \
 893         TT_TRACE(trace_gen)               ;\
 894         or      %g0, P_UTRAP10, %g2       ;\
 895         or      %g0, T_TAG_OVERFLOW, %g3  ;\
 896         sethi   %hi(.check_v9utrap), %g4  ;\
 897         jmp     %g4 + %lo(.check_v9utrap) ;\
 898         nop                               ;\
 899         .align  32
 900 
 901 /*
 902  * divide by zero trap
 903  */
 904 #define DIV_BY_ZERO                       \
 905         TT_TRACE(trace_gen)               ;\
 906         or      %g0, P_UTRAP11, %g2       ;\
 907         or      %g0, T_IDIV0, %g3         ;\
 908         sethi   %hi(.check_v9utrap), %g4  ;\
 909         jmp     %g4 + %lo(.check_v9utrap) ;\
 910         nop                               ;\
 911         .align  32
 912 
 913 /*
 914  * trap instruction for V9 user trap handlers
 915  */
 916 #define TRAP_INSTR                        \
 917         TT_TRACE(trace_gen)               ;\
 918         or      %g0, T_SOFTWARE_TRAP, %g3 ;\
 919         sethi   %hi(.check_v9utrap), %g4  ;\
 920         jmp     %g4 + %lo(.check_v9utrap) ;\
 921         nop                               ;\
 922         .align  32
 923 #define TRP4    TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
 924 
 925 /*
 926  * LEVEL_INTERRUPT is for level N interrupts.
 927  * VECTOR_INTERRUPT is for the vector trap.
 928  */
 929 #define LEVEL_INTERRUPT(level)          \
 930         .global tt_pil/**/level         ;\
 931 tt_pil/**/level:                        ;\
 932         ba,pt   %xcc, pil_interrupt     ;\
 933         mov     level, %g4              ;\
 934         .align  32
 935 
 936 #define LEVEL14_INTERRUPT                       \
 937         ba      pil14_interrupt                 ;\
 938         mov     PIL_14, %g4                     ;\
 939         .align  32
 940 
 941 #define        LEVEL15_INTERRUPT                       \
 942        ba      pil15_interrupt                 ;\
 943        mov     PIL_15, %g4                     ;\
 944        .align  32
 945 
 946 #define VECTOR_INTERRUPT                                \
 947         ldxa    [%g0]ASI_INTR_RECEIVE_STATUS, %g1       ;\
 948         btst    IRSR_BUSY, %g1                          ;\
 949         bnz,pt  %xcc, vec_interrupt                     ;\
 950         nop                                             ;\
 951         ba,a,pt %xcc, vec_intr_spurious                 ;\
 952         .empty                                          ;\
 953         .align  32
 954 
 955 /*
 956  * MMU Trap Handlers.
 957  */
 958 #define SWITCH_GLOBALS  /* mmu->alt, alt->mmu */                  \
 959         rdpr    %pstate, %g5                                            ;\
 960         wrpr    %g5, PSTATE_MG | PSTATE_AG, %pstate
 961 
 962 #define IMMU_EXCEPTION                                                  \
 963         membar  #Sync                                                   ;\
 964         SWITCH_GLOBALS                                                  ;\
 965         wr      %g0, ASI_IMMU, %asi                                     ;\
 966         rdpr    %tpc, %g2                                               ;\
 967         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
 968         ba,pt   %xcc, .mmu_exception_end                                ;\
 969         mov     T_INSTR_EXCEPTION, %g1                                  ;\
 970         .align  32
 971 
 972 #define DMMU_EXCEPTION                                                  \
 973         SWITCH_GLOBALS                                                  ;\
 974         wr      %g0, ASI_DMMU, %asi                                     ;\
 975         ldxa    [MMU_TAG_ACCESS]%asi, %g2                               ;\
 976         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
 977         ba,pt   %xcc, .mmu_exception_end                                ;\
 978         mov     T_DATA_EXCEPTION, %g1                                   ;\
 979         .align  32
 980 
 981 #define DMMU_EXC_AG_PRIV                                                \
 982         wr      %g0, ASI_DMMU, %asi                                     ;\
 983         ldxa    [MMU_SFAR]%asi, %g2                                     ;\
 984         ba,pt   %xcc, .mmu_priv_exception                               ;\
 985         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
 986         .align  32
 987 
 988 #define DMMU_EXC_AG_NOT_ALIGNED                                         \
 989         wr      %g0, ASI_DMMU, %asi                                     ;\
 990         ldxa    [MMU_SFAR]%asi, %g2                                     ;\
 991         ba,pt   %xcc, .mmu_exception_not_aligned                        ;\
 992         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
 993         .align  32
 994 
 995 /*
 996  * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
 997  */
 998 #define DMMU_EXC_LDDF_NOT_ALIGNED                                       \
 999         btst    1, %sp                                                  ;\
1000         bnz,pt  %xcc, .lddf_exception_not_aligned                       ;\
1001         wr      %g0, ASI_DMMU, %asi                                     ;\
1002         ldxa    [MMU_SFAR]%asi, %g2                                     ;\
1003         ba,pt   %xcc, .mmu_exception_not_aligned                        ;\
1004         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
1005         .align  32
1006 
1007 #define DMMU_EXC_STDF_NOT_ALIGNED                                       \
1008         btst    1, %sp                                                  ;\
1009         bnz,pt  %xcc, .stdf_exception_not_aligned                       ;\
1010         wr      %g0, ASI_DMMU, %asi                                     ;\
1011         ldxa    [MMU_SFAR]%asi, %g2                                     ;\
1012         ba,pt   %xcc, .mmu_exception_not_aligned                        ;\
1013         ldxa    [MMU_SFSR]%asi, %g3                                     ;\
1014         .align  32
1015 
1016 /*
1017  * Flush the TLB using either the primary, secondary, or nucleus flush
1018  * operation based on whether the ctx from the tag access register matches
1019  * the primary or secondary context (flush the nucleus if neither matches).
1020  *
1021  * Requires a membar #Sync before next ld/st.
1022  * exits with:
1023  * g2 = tag access register
1024  * g3 = ctx number
1025  */
1026 #if TAGACC_CTX_MASK != CTXREG_CTX_MASK
1027 #error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
1028 #endif
1029 #define DTLB_DEMAP_ENTRY                                                \
1030         mov     MMU_TAG_ACCESS, %g1                                     ;\
1031         mov     MMU_PCONTEXT, %g5                                       ;\
1032         ldxa    [%g1]ASI_DMMU, %g2                                      ;\
1033         sethi   %hi(TAGACC_CTX_MASK), %g4                               ;\
1034         or      %g4, %lo(TAGACC_CTX_MASK), %g4                          ;\
1035         and     %g2, %g4, %g3                   /* g3 = ctx */          ;\
1036         ldxa    [%g5]ASI_DMMU, %g6              /* g6 = primary ctx */  ;\
1037         and     %g6, %g4, %g6                   /* &= CTXREG_CTX_MASK */ ;\
1038         cmp     %g3, %g6                                                ;\
1039         be,pt   %xcc, 1f                                                ;\
1040         andn    %g2, %g4, %g1                   /* ctx = primary */     ;\
1041         mov     MMU_SCONTEXT, %g5                                       ;\
1042         ldxa    [%g5]ASI_DMMU, %g6              /* g6 = secondary ctx */ ;\
1043         and     %g6, %g4, %g6                   /* &= CTXREG_CTX_MASK */ ;\
1044         cmp     %g3, %g6                                                ;\
1045         be,a,pt %xcc, 1f                                                ;\
1046           or    %g1, DEMAP_SECOND, %g1                                  ;\
1047         or      %g1, DEMAP_NUCLEUS, %g1                                 ;\
1048 1:      stxa    %g0, [%g1]ASI_DTLB_DEMAP        /* MMU_DEMAP_PAGE */    ;\
1049         membar  #Sync
1050 
1051 #if defined(cscope)
1052 /*
1053  * Define labels to direct cscope quickly to labels that
1054  * are generated by macro expansion of DTLB_MISS().
1055  */
1056         .global tt0_dtlbmiss
1057 tt0_dtlbmiss:
1058         .global tt1_dtlbmiss
1059 tt1_dtlbmiss:
1060         nop
1061 #endif
1062 
1063 /*
1064  * Needs to be exactly 32 instructions
1065  *
1066  * UTLB NOTE: If we don't hit on the 8k pointer then we branch
1067  * to a special 4M tsb handler. It would be nice if that handler
1068  * could live in this file but currently it seems better to allow
1069  * it to fall thru to sfmmu_tsb_miss.
1070  */
1071 #ifdef UTSB_PHYS
1072 #define DTLB_MISS(table_name)                                           ;\
1073         .global table_name/**/_dtlbmiss                                 ;\
1074 table_name/**/_dtlbmiss:                                                ;\
1075         mov     MMU_TAG_ACCESS, %g6             /* select tag acc */    ;\
1076         ldxa    [%g0]ASI_DMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1077         ldxa    [%g6]ASI_DMMU, %g2              /* g2 = tag access */   ;\
1078         sllx    %g2, TAGACC_CTX_LSHIFT, %g3                             ;\
1079         srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctx */          ;\
1080         cmp     %g3, INVALID_CONTEXT                                    ;\
1081         ble,pn  %xcc, sfmmu_kdtlb_miss                                  ;\
1082           srax  %g2, PREDISM_BASESHIFT, %g6  /* g6 > 0 ISM predicted */ ;\
1083         brgz,pn %g6, sfmmu_udtlb_slowpath_ismpred                       ;\
1084           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
1085         ldda    [%g1]ASI_QUAD_LDD_PHYS, %g4     /* g4 = tag, %g5 data */;\
1086         cmp     %g4, %g7                                                ;\
1087         bne,pn %xcc, sfmmu_udtlb_slowpath_noismpred                     ;\
1088           nop                                                           ;\
1089         TT_TRACE(trace_tsbhit)          /* 2 instr ifdef TRAPTRACE */   ;\
1090         stxa    %g5, [%g0]ASI_DTLB_IN   /* trapstat expects TTE */      ;\
1091         retry                           /* in %g5 */                    ;\
1092         unimp   0                                                       ;\
1093         unimp   0                                                       ;\
1094         unimp   0                                                       ;\
1095         unimp   0                                                       ;\
1096         unimp   0                                                       ;\
1097         unimp   0                                                       ;\
1098         unimp   0                                                       ;\
1099         unimp   0                                                       ;\
1100         unimp   0                                                       ;\
1101         unimp   0                                                       ;\
1102         unimp   0                                                       ;\
1103         unimp   0                                                       ;\
1104         unimp   0                                                       ;\
1105         unimp   0                                                       ;\
1106         .align 128
1107         
1108 #else /* UTSB_PHYS */
1109 #define DTLB_MISS(table_name)                                           ;\
1110         .global table_name/**/_dtlbmiss                                 ;\
1111 table_name/**/_dtlbmiss:                                                ;\
1112         mov     MMU_TAG_ACCESS, %g6             /* select tag acc */    ;\
1113         ldxa    [%g0]ASI_DMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1114         ldxa    [%g6]ASI_DMMU, %g2              /* g2 = tag access */   ;\
1115         sllx    %g2, TAGACC_CTX_LSHIFT, %g3                             ;\
1116         srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctx */          ;\
1117         cmp     %g3, INVALID_CONTEXT                                    ;\
1118         ble,pn  %xcc, sfmmu_kdtlb_miss                                  ;\
1119           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
1120         brlz,pn %g1, sfmmu_udtlb_slowpath                               ;\
1121           nop                                                           ;\
1122         ldda    [%g1]ASI_NQUAD_LD, %g4  /* g4 = tag, %g5 data */        ;\
1123         cmp     %g4, %g7                                                ;\
1124         bne,pn  %xcc, sfmmu_tsb_miss_tt         /* no 4M TSB, miss */   ;\
1125           mov   -1, %g3         /* set 4M tsbe ptr to -1 */             ;\
1126         TT_TRACE(trace_tsbhit)          /* 2 instr ifdef TRAPTRACE */   ;\
1127         stxa    %g5, [%g0]ASI_DTLB_IN   /* trapstat expects TTE */      ;\
1128         retry                           /* in %g5 */                    ;\
1129         unimp   0                                                       ;\
1130         unimp   0                                                       ;\
1131         unimp   0                                                       ;\
1132         unimp   0                                                       ;\
1133         unimp   0                                                       ;\
1134         unimp   0                                                       ;\
1135         unimp   0                                                       ;\
1136         unimp   0                                                       ;\
1137         unimp   0                                                       ;\
1138         unimp   0                                                       ;\
1139         unimp   0                                                       ;\
1140         unimp   0                                                       ;\
1141         unimp   0                                                       ;\
1142         unimp   0                                                       ;\
1143         .align 128
1144 #endif /* UTSB_PHYS */
1145 
1146 #if defined(cscope)
1147 /*
1148  * Define labels to direct cscope quickly to labels that
1149  * are generated by macro expansion of ITLB_MISS().
1150  */
1151         .global tt0_itlbmiss
1152 tt0_itlbmiss:
1153         .global tt1_itlbmiss
1154 tt1_itlbmiss:
1155         nop
1156 #endif
1157 
1158 /*
1159  * Instruction miss handler.
1160  * ldda instructions will have their ASI patched
1161  * by sfmmu_patch_ktsb at runtime.
1162  * MUST be EXACTLY 32 instructions or we'll break.
1163  */
1164 #ifdef UTSB_PHYS
1165 #define ITLB_MISS(table_name)                                            \
1166         .global table_name/**/_itlbmiss                                 ;\
1167 table_name/**/_itlbmiss:                                                ;\
1168         mov     MMU_TAG_ACCESS, %g6             /* select tag acc */    ;\
1169         ldxa    [%g0]ASI_IMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1170         ldxa    [%g6]ASI_IMMU, %g2              /* g2 = tag access */   ;\
1171         sllx    %g2, TAGACC_CTX_LSHIFT, %g3                             ;\
1172         srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctx */          ;\
1173         cmp     %g3, INVALID_CONTEXT                                    ;\
1174         ble,pn  %xcc, sfmmu_kitlb_miss                                  ;\
1175           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
1176         ldda    [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */   ;\
1177         cmp     %g4, %g7                                                ;\
1178         bne,pn  %xcc, sfmmu_uitlb_slowpath                              ;\
1179           andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */       ;\
1180         bz,pn   %icc, exec_fault                                        ;\
1181           nop                                                           ;\
1182         TT_TRACE(trace_tsbhit)          /* 2 instr ifdef TRAPTRACE */   ;\
1183         stxa    %g5, [%g0]ASI_ITLB_IN   /* trapstat expects %g5 */      ;\
1184         retry                                                           ;\
1185         unimp   0                                                       ;\
1186         unimp   0                                                       ;\
1187         unimp   0                                                       ;\
1188         unimp   0                                                       ;\
1189         unimp   0                                                       ;\
1190         unimp   0                                                       ;\
1191         unimp   0                                                       ;\
1192         unimp   0                                                       ;\
1193         unimp   0                                                       ;\
1194         unimp   0                                                       ;\
1195         unimp   0                                                       ;\
1196         unimp   0                                                       ;\
1197         unimp   0                                                       ;\
1198         unimp   0                                                       ;\
1199         .align 128 
1200 
1201 #else /* UTSB_PHYS */
1202 #define ITLB_MISS(table_name)                                            \
1203         .global table_name/**/_itlbmiss                                 ;\
1204 table_name/**/_itlbmiss:                                                ;\
1205         mov     MMU_TAG_ACCESS, %g6             /* select tag acc */    ;\
1206         ldxa    [%g0]ASI_IMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1207         ldxa    [%g6]ASI_IMMU, %g2              /* g2 = tag access */   ;\
1208         sllx    %g2, TAGACC_CTX_LSHIFT, %g3                             ;\
1209         srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctx */          ;\
1210         cmp     %g3, INVALID_CONTEXT                                    ;\
1211         ble,pn  %xcc, sfmmu_kitlb_miss                                  ;\
1212           srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
1213         brlz,pn %g1, sfmmu_uitlb_slowpath       /* if >1 TSB branch */       ;\
1214           nop                                                           ;\
1215         ldda    [%g1]ASI_NQUAD_LD, %g4  /* g4 = tag, g5 = data */       ;\
1216         cmp     %g4, %g7                                                ;\
1217         bne,pn  %xcc, sfmmu_tsb_miss_tt /* br if 8k ptr miss */         ;\
1218           mov   -1, %g3         /* set 4M TSB ptr to -1 */              ;\
1219         andcc   %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */       ;\
1220         bz,pn   %icc, exec_fault                                        ;\
1221           nop                                                           ;\
1222         TT_TRACE(trace_tsbhit)          /* 2 instr ifdef TRAPTRACE */   ;\
1223         stxa    %g5, [%g0]ASI_ITLB_IN   /* trapstat expects %g5 */      ;\
1224         retry                                                           ;\
1225         unimp   0                                                       ;\
1226         unimp   0                                                       ;\
1227         unimp   0                                                       ;\
1228         unimp   0                                                       ;\
1229         unimp   0                                                       ;\
1230         unimp   0                                                       ;\
1231         unimp   0                                                       ;\
1232         unimp   0                                                       ;\
1233         unimp   0                                                       ;\
1234         unimp   0                                                       ;\
1235         unimp   0                                                       ;\
1236         .align 128
1237 #endif /* UTSB_PHYS */
1238 
1239 
1240 /*
1241  * This macro is the first level handler for fast protection faults.
1242  * It first demaps the tlb entry which generated the fault and then
1243  * attempts to set the modify bit on the hash.  It needs to be
1244  * exactly 32 instructions.
1245  */
1246 #define DTLB_PROT                                                        \
1247         DTLB_DEMAP_ENTRY                /* 20 instructions */           ;\
1248         /*                                                              ;\
1249          * At this point:                                               ;\
1250          *   g1 = ????                                                  ;\
1251          *   g2 = tag access register                                   ;\
1252          *   g3 = ctx number                                            ;\
1253          *   g4 = ????                                                  ;\
1254          */                                                             ;\
1255         TT_TRACE(trace_dataprot)        /* 2 instr ifdef TRAPTRACE */   ;\
1256                                         /* clobbers g1 and g6 */        ;\
1257         ldxa    [%g0]ASI_DMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1258         brnz,pt %g3, sfmmu_uprot_trap           /* user trap */         ;\
1259           nop                                                           ;\
1260         ba,a,pt %xcc, sfmmu_kprot_trap          /* kernel trap */       ;\
1261         unimp   0                                                       ;\
1262         unimp   0                                                       ;\
1263         unimp   0                                                       ;\
1264         unimp   0                                                       ;\
1265         unimp   0                                                       ;\
1266         unimp   0                                                       ;\
1267         .align 128
1268 
1269 #define DMMU_EXCEPTION_TL1                                              ;\
1270         SWITCH_GLOBALS                                                  ;\
1271         ba,a,pt %xcc, mmu_trap_tl1                                      ;\
1272           nop                                                           ;\
1273         .align 32
1274 
1275 #define MISALIGN_ADDR_TL1                                               ;\
1276         ba,a,pt %xcc, mmu_trap_tl1                                      ;\
1277           nop                                                           ;\
1278         .align 32
1279 
1280 /*
1281  * Trace a tsb hit
1282  * g1 = tsbe pointer (in/clobbered)
1283  * g2 = tag access register (in)
1284  * g3 - g4 = scratch (clobbered)
1285  * g5 = tsbe data (in)
1286  * g6 = scratch (clobbered)
1287  * g7 = pc we jumped here from (in)
1288  * ttextra = value to OR in to trap type (%tt) (in)
1289  */
1290 #ifdef TRAPTRACE
1291 #define TRACE_TSBHIT(ttextra)                                            \
1292         membar  #Sync                                                   ;\
1293         sethi   %hi(FLUSH_ADDR), %g6                                    ;\
1294         flush   %g6                                                     ;\
1295         TRACE_PTR(%g3, %g6)                                             ;\
1296         GET_TRACE_TICK(%g6, %g4)                                        ;\
1297         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi                          ;\
1298         stxa    %g2, [%g3 + TRAP_ENT_SP]%asi    /* tag access */        ;\
1299         stxa    %g5, [%g3 + TRAP_ENT_F1]%asi    /* tsb data */          ;\
1300         rdpr    %tnpc, %g6                                              ;\
1301         stxa    %g6, [%g3 + TRAP_ENT_F2]%asi                            ;\
1302         stxa    %g1, [%g3 + TRAP_ENT_F3]%asi    /* tsb pointer */       ;\
1303         stxa    %g0, [%g3 + TRAP_ENT_F4]%asi                            ;\
1304         rdpr    %tpc, %g6                                               ;\
1305         stxa    %g6, [%g3 + TRAP_ENT_TPC]%asi                           ;\
1306         rdpr    %tl, %g6                                                ;\
1307         stha    %g6, [%g3 + TRAP_ENT_TL]%asi                            ;\
1308         rdpr    %tt, %g6                                                ;\
1309         or      %g6, (ttextra), %g6                                     ;\
1310         stha    %g6, [%g3 + TRAP_ENT_TT]%asi                            ;\
1311         ldxa    [%g0]ASI_IMMU, %g1              /* tag target */        ;\
1312         ldxa    [%g0]ASI_DMMU, %g4                                      ;\
1313         cmp     %g6, FAST_IMMU_MISS_TT                                  ;\
1314         movne   %icc, %g4, %g1                                          ;\
1315         stxa    %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */          ;\
1316         stxa    %g0, [%g3 + TRAP_ENT_TR]%asi                            ;\
1317         TRACE_NEXT(%g3, %g4, %g6)
1318 #else
1319 #define TRACE_TSBHIT(ttextra)
1320 #endif
1321 
1322 #if defined(lint)
1323 
1324 struct scb      trap_table;
1325 struct scb      scb;            /* trap_table/scb are the same object */
1326 
1327 #else /* lint */
1328 
1329 /*
1330  * =======================================================================
1331  *              SPARC V9 TRAP TABLE
1332  *
1333  * The trap table is divided into two halves: the first half is used when
1334  * taking traps when TL=0; the second half is used when taking traps from
1335  * TL>0. Note that handlers in the second half of the table might not be able
1336  * to make the same assumptions as handlers in the first half of the table.
1337  *
1338  * Worst case trap nesting so far:
1339  *
1340  *      at TL=0 client issues software trap requesting service
1341  *      at TL=1 nucleus wants a register window
1342  *      at TL=2 register window clean/spill/fill takes a TLB miss
1343  *      at TL=3 processing TLB miss
1344  *      at TL=4 handle asynchronous error
1345  *
1346  * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1347  *
1348  * =======================================================================
1349  */
1350         .section ".text"
1351         .align  4
1352         .global trap_table, scb, trap_table0, trap_table1, etrap_table
1353         .type   trap_table, #object
1354         .type   scb, #object
1355 trap_table:
1356 scb:
1357 trap_table0:
1358         /* hardware traps */
1359         NOT;                            /* 000  reserved */
1360         RED;                            /* 001  power on reset */
1361         RED;                            /* 002  watchdog reset */
1362         RED;                            /* 003  externally initiated reset */
1363         RED;                            /* 004  software initiated reset */
1364         RED;                            /* 005  red mode exception */
1365         NOT; NOT;                       /* 006 - 007 reserved */
1366         IMMU_EXCEPTION;                 /* 008  instruction access exception */
1367         NOT;                            /* 009  instruction access MMU miss */
1368         ASYNC_TRAP(T_INSTR_ERROR, trace_gen, tt0_iae);
1369                                         /* 00A  instruction access error */
1370         NOT; NOT4;                      /* 00B - 00F reserved */
1371         ILLTRAP_INSTR;                  /* 010  illegal instruction */
1372         TRAP(T_PRIV_INSTR);             /* 011  privileged opcode */
1373         NOT;                            /* 012  unimplemented LDD */
1374         NOT;                            /* 013  unimplemented STD */
1375         NOT4; NOT4; NOT4;               /* 014 - 01F reserved */
1376         FP_DISABLED_TRAP;               /* 020  fp disabled */
1377         FP_IEEE_TRAP;                   /* 021  fp exception ieee 754 */
1378         FP_TRAP;                        /* 022  fp exception other */
1379         TAG_OVERFLOW;                   /* 023  tag overflow */
1380         CLEAN_WINDOW;                   /* 024 - 027 clean window */
1381         DIV_BY_ZERO;                    /* 028  division by zero */
1382         NOT;                            /* 029  internal processor error */
1383         NOT; NOT; NOT4;                 /* 02A - 02F reserved */
1384         DMMU_EXCEPTION;                 /* 030  data access exception */
1385         NOT;                            /* 031  data access MMU miss */
1386         ASYNC_TRAP(T_DATA_ERROR, trace_gen, tt0_dae);
1387                                         /* 032  data access error */
1388         NOT;                            /* 033  data access protection */
1389         DMMU_EXC_AG_NOT_ALIGNED;        /* 034  mem address not aligned */
1390         DMMU_EXC_LDDF_NOT_ALIGNED;      /* 035  LDDF mem address not aligned */
1391         DMMU_EXC_STDF_NOT_ALIGNED;      /* 036  STDF mem address not aligned */
1392         DMMU_EXC_AG_PRIV;               /* 037  privileged action */
1393         NOT;                            /* 038  LDQF mem address not aligned */
1394         NOT;                            /* 039  STQF mem address not aligned */
1395         NOT; NOT; NOT4;                 /* 03A - 03F reserved */
1396         LABELED_BAD(tt0_asdat);         /* 040  async data error */
1397         LEVEL_INTERRUPT(1);             /* 041  interrupt level 1 */
1398         LEVEL_INTERRUPT(2);             /* 042  interrupt level 2 */
1399         LEVEL_INTERRUPT(3);             /* 043  interrupt level 3 */
1400         LEVEL_INTERRUPT(4);             /* 044  interrupt level 4 */
1401         LEVEL_INTERRUPT(5);             /* 045  interrupt level 5 */
1402         LEVEL_INTERRUPT(6);             /* 046  interrupt level 6 */
1403         LEVEL_INTERRUPT(7);             /* 047  interrupt level 7 */
1404         LEVEL_INTERRUPT(8);             /* 048  interrupt level 8 */
1405         LEVEL_INTERRUPT(9);             /* 049  interrupt level 9 */
1406         LEVEL_INTERRUPT(10);            /* 04A  interrupt level 10 */
1407         LEVEL_INTERRUPT(11);            /* 04B  interrupt level 11 */
1408         LEVEL_INTERRUPT(12);            /* 04C  interrupt level 12 */
1409         LEVEL_INTERRUPT(13);            /* 04D  interrupt level 13 */
1410         LEVEL14_INTERRUPT;              /* 04E  interrupt level 14 */
1411         LEVEL15_INTERRUPT;              /* 04F  interrupt level 15 */
1412         NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F reserved */
1413         VECTOR_INTERRUPT;               /* 060  interrupt vector */
1414         GOTO(kmdb_trap);                /* 061  PA watchpoint */
1415         GOTO(kmdb_trap);                /* 062  VA watchpoint */
1416         GOTO_TT(ce_err, trace_gen);     /* 063  corrected ECC error */
1417         ITLB_MISS(tt0);                 /* 064  instruction access MMU miss */
1418         DTLB_MISS(tt0);                 /* 068  data access MMU miss */
1419         DTLB_PROT;                      /* 06C  data access protection */
1420         LABELED_BAD(tt0_fecc);          /* 070  fast ecache ECC error */
1421         LABELED_BAD(tt0_dperr);         /* 071  Cheetah+ dcache parity error */
1422         LABELED_BAD(tt0_iperr);         /* 072  Cheetah+ icache parity error */
1423         NOT;                            /* 073  reserved */
1424         NOT4; NOT4; NOT4;               /* 074 - 07F reserved */
1425         NOT4;                           /* 080  spill 0 normal */
1426         SPILL_32bit_asi(ASI_AIUP,sn0);  /* 084  spill 1 normal */
1427         SPILL_64bit_asi(ASI_AIUP,sn0);  /* 088  spill 2 normal */
1428         SPILL_32clean(ASI_AIUP,sn0);    /* 08C  spill 3 normal */
1429         SPILL_64clean(ASI_AIUP,sn0);    /* 090  spill 4 normal */
1430         SPILL_32bit(not);               /* 094  spill 5 normal */
1431         SPILL_64bit(not);               /* 098  spill 6 normal */
1432         SPILL_mixed;                    /* 09C  spill 7 normal */
1433         NOT4;                           /* 0A0  spill 0 other */
1434         SPILL_32bit_asi(ASI_AIUS,so0);  /* 0A4  spill 1 other */
1435         SPILL_64bit_asi(ASI_AIUS,so0);  /* 0A8  spill 2 other */
1436         SPILL_32bit_asi(ASI_AIUS,so0);  /* 0AC  spill 3 other */
1437         SPILL_64bit_asi(ASI_AIUS,so0);  /* 0B0  spill 4 other */
1438         NOT4;                           /* 0B4  spill 5 other */
1439         NOT4;                           /* 0B8  spill 6 other */
1440         NOT4;                           /* 0BC  spill 7 other */
1441         NOT4;                           /* 0C0  fill 0 normal */
1442         FILL_32bit_asi(ASI_AIUP,fn0);   /* 0C4  fill 1 normal */
1443         FILL_64bit_asi(ASI_AIUP,fn0);   /* 0C8  fill 2 normal */
1444         FILL_32bit_asi(ASI_AIUP,fn0);   /* 0CC  fill 3 normal */
1445         FILL_64bit_asi(ASI_AIUP,fn0);   /* 0D0  fill 4 normal */
1446         FILL_32bit(not);                /* 0D4  fill 5 normal */
1447         FILL_64bit(not);                /* 0D8  fill 6 normal */
1448         FILL_mixed;                     /* 0DC  fill 7 normal */
1449         NOT4;                           /* 0E0  fill 0 other */
1450         NOT4;                           /* 0E4  fill 1 other */
1451         NOT4;                           /* 0E8  fill 2 other */
1452         NOT4;                           /* 0EC  fill 3 other */
1453         NOT4;                           /* 0F0  fill 4 other */
1454         NOT4;                           /* 0F4  fill 5 other */
1455         NOT4;                           /* 0F8  fill 6 other */
1456         NOT4;                           /* 0FC  fill 7 other */
1457         /* user traps */
1458         GOTO(syscall_trap_4x);          /* 100  old system call */
1459         TRAP(T_BREAKPOINT);             /* 101  user breakpoint */
1460         TRAP(T_DIV0);                   /* 102  user divide by zero */
1461         FLUSHW(tt0_flushw);             /* 103  flush windows */
1462         GOTO(.clean_windows);           /* 104  clean windows */
1463         BAD;                            /* 105  range check ?? */
1464         GOTO(.fix_alignment);           /* 106  do unaligned references */
1465         BAD;                            /* 107  unused */
1466         SYSCALL_TRAP32;                 /* 108  ILP32 system call on LP64 */
1467         GOTO(set_trap0_addr);           /* 109  set trap0 address */
1468         BAD; BAD; BAD4;                 /* 10A - 10F unused */
1469         TRP4; TRP4; TRP4; TRP4;         /* 110 - 11F V9 user trap handlers */
1470         GOTO(.getcc);                   /* 120  get condition codes */
1471         GOTO(.setcc);                   /* 121  set condition codes */
1472         GOTO(.getpsr);                  /* 122  get psr */
1473         GOTO(.setpsr);                  /* 123  set psr (some fields) */
1474         GOTO(get_timestamp);            /* 124  get timestamp */
1475         GOTO(get_virtime);              /* 125  get lwp virtual time */
1476         PRIV(self_xcall);               /* 126  self xcall */
1477         GOTO(get_hrestime);             /* 127  get hrestime */
1478         BAD;                            /* 128  ST_SETV9STACK */
1479         GOTO(.getlgrp);                 /* 129  get lgrpid */
1480         BAD; BAD; BAD4;                 /* 12A - 12F unused */
1481         BAD4; BAD4;                     /* 130 - 137 unused */
1482         DTRACE_PID;                     /* 138  dtrace pid tracing provider */
1483         BAD;                            /* 139  unused */
1484         DTRACE_RETURN;                  /* 13A  dtrace pid return probe */
1485         BAD; BAD4;                      /* 13B - 13F unused */
1486         SYSCALL_TRAP;                   /* 140  LP64 system call */
1487         SYSCALL(nosys);                 /* 141  unused system call trap */
1488 #ifdef DEBUG_USER_TRAPTRACECTL
1489         GOTO(.traptrace_freeze);        /* 142  freeze traptrace */
1490         GOTO(.traptrace_unfreeze);      /* 143  unfreeze traptrace */
1491 #else
1492         SYSCALL(nosys);                 /* 142  unused system call trap */
1493         SYSCALL(nosys);                 /* 143  unused system call trap */
1494 #endif
1495         BAD4; BAD4; BAD4;               /* 144 - 14F unused */
1496         BAD4; BAD4; BAD4; BAD4;         /* 150 - 15F unused */
1497         BAD4; BAD4; BAD4; BAD4;         /* 160 - 16F unused */
1498         BAD;                            /* 170 - unused */
1499         BAD;                            /* 171 - unused */
1500         BAD; BAD;                       /* 172 - 173 unused */
1501         BAD4; BAD4;                     /* 174 - 17B unused */
1502 #ifdef  PTL1_PANIC_DEBUG
1503         mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1504                                         /* 17C  test ptl1_panic */
1505 #else
1506         BAD;                            /* 17C  unused */
1507 #endif  /* PTL1_PANIC_DEBUG */
1508         PRIV(kmdb_trap);                /* 17D  kmdb enter (L1-A) */
1509         PRIV(kmdb_trap);                /* 17E  kmdb breakpoint */
1510         PRIV(kctx_obp_bpt);             /* 17F  obp breakpoint */
1511         /* reserved */
1512         NOT4; NOT4; NOT4; NOT4;         /* 180 - 18F reserved */
1513         NOT4; NOT4; NOT4; NOT4;         /* 190 - 19F reserved */
1514         NOT4; NOT4; NOT4; NOT4;         /* 1A0 - 1AF reserved */
1515         NOT4; NOT4; NOT4; NOT4;         /* 1B0 - 1BF reserved */
1516         NOT4; NOT4; NOT4; NOT4;         /* 1C0 - 1CF reserved */
1517         NOT4; NOT4; NOT4; NOT4;         /* 1D0 - 1DF reserved */
1518         NOT4; NOT4; NOT4; NOT4;         /* 1E0 - 1EF reserved */
1519         NOT4; NOT4; NOT4; NOT4;         /* 1F0 - 1FF reserved */
1520 trap_table1:
1521         NOT4; NOT4; NOT; NOT;           /* 000 - 009 unused */
1522         ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen, tt1_iae);
1523                                         /* 00A  instruction access error */
1524         NOT; NOT4;                      /* 00B - 00F unused */
1525         NOT4; NOT4; NOT4; NOT4;         /* 010 - 01F unused */
1526         NOT4;                           /* 020 - 023 unused */
1527         CLEAN_WINDOW;                   /* 024 - 027 clean window */
1528         NOT4; NOT4;                     /* 028 - 02F unused */
1529         DMMU_EXCEPTION_TL1;             /* 030  data access exception */
1530         NOT;                            /* 031 unused */
1531         ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen, tt1_dae);
1532                                         /* 032  data access error */
1533         NOT;                            /* 033  unused */
1534         MISALIGN_ADDR_TL1;              /* 034  mem address not aligned */
1535         NOT; NOT; NOT; NOT4; NOT4       /* 035 - 03F unused */
1536         LABELED_BAD(tt1_asdat);         /* 040  async data error */
1537         NOT; NOT; NOT;                  /* 041 - 043 unused */
1538         NOT4; NOT4; NOT4;               /* 044 - 04F unused */
1539         NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F unused */
1540         NOT;                            /* 060  unused */
1541         GOTO(kmdb_trap_tl1);            /* 061  PA watchpoint */
1542         GOTO(kmdb_trap_tl1);            /* 062  VA watchpoint */
1543         GOTO_TT(ce_err_tl1, trace_gen); /* 063  corrected ECC error */
1544         ITLB_MISS(tt1);                 /* 064  instruction access MMU miss */
1545         DTLB_MISS(tt1);                 /* 068  data access MMU miss */
1546         DTLB_PROT;                      /* 06C  data access protection */
1547         LABELED_BAD(tt1_fecc);          /* 070  fast ecache ECC error */
1548         LABELED_BAD(tt1_dperr);         /* 071  Cheetah+ dcache parity error */
1549         LABELED_BAD(tt1_iperr);         /* 072  Cheetah+ icache parity error */
1550         NOT;                            /* 073  reserved */
1551         NOT4; NOT4; NOT4;               /* 074 - 07F reserved */
1552         NOT4;                           /* 080  spill 0 normal */
1553         SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 084  spill 1 normal */
1554         SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 088  spill 2 normal */
1555         SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 08C  spill 3 normal */
1556         SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 090  spill 4 normal */
1557         SPILL_32bit(not);               /* 094  spill 5 normal */
1558         SPILL_64bit(not);               /* 098  spill 6 normal */
1559         SPILL_mixed;                    /* 09C  spill 7 normal */
1560         NOT4;                           /* 0A0  spill 0 other */
1561         SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0A4  spill 1 other */
1562         SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0A8  spill 2 other */
1563         SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0AC  spill 3 other */
1564         SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0B0  spill 4 other */
1565         NOT4;                           /* 0B4  spill 5 other */
1566         NOT4;                           /* 0B8  spill 6 other */
1567         NOT4;                           /* 0BC  spill 7 other */
1568         NOT4;                           /* 0C0  fill 0 normal */
1569         FILL_32bit_tt1(ASI_AIUP,fn1);   /* 0C4  fill 1 normal */
1570         FILL_64bit_tt1(ASI_AIUP,fn1);   /* 0C8  fill 2 normal */
1571         FILL_32bit_tt1(ASI_AIUP,fn1);   /* 0CC  fill 3 normal */
1572         FILL_64bit_tt1(ASI_AIUP,fn1);   /* 0D0  fill 4 normal */
1573         FILL_32bit(not);                /* 0D4  fill 5 normal */
1574         FILL_64bit(not);                /* 0D8  fill 6 normal */
1575         FILL_mixed;                     /* 0DC  fill 7 normal */
1576         NOT4; NOT4; NOT4; NOT4;         /* 0E0 - 0EF unused */
1577         NOT4; NOT4; NOT4; NOT4;         /* 0F0 - 0FF unused */
1578         LABELED_BAD(tt1_swtrap0);       /* 100  fast ecache ECC error (cont) */
1579         LABELED_BAD(tt1_swtrap1);       /* 101  Ch+ D$ parity error (cont) */
1580         LABELED_BAD(tt1_swtrap2);       /* 102  Ch+ I$ parity error (cont) */
1581         NOT;                            /* 103  reserved */
1582 /*
1583  * We only reserve the above four special case soft traps for code running
1584  * at TL>0, so we can truncate the trap table here.
1585  */
1586 etrap_table:
1587         .size   trap_table, (.-trap_table)
1588         .size   scb, (.-scb)
1589 
1590 /*
1591  * We get to exec_fault in the case of an instruction miss and tte
1592  * has no execute bit set.  We go to tl0 to handle it.
1593  *
1594  * g1 = tsbe pointer (in/clobbered)
1595  * g2 = tag access register (in)
1596  * g3 - g4 = scratch (clobbered)
1597  * g5 = tsbe data (in)
1598  * g6 = scratch (clobbered)
1599  */
1600         ALTENTRY(exec_fault)
1601         TRACE_TSBHIT(0x200)
1602         SWITCH_GLOBALS
1603         mov     MMU_TAG_ACCESS, %g4
1604         ldxa    [%g4]ASI_IMMU, %g2                      ! arg1 = addr
1605         mov     T_INSTR_MMU_MISS, %g3                   ! arg2 = traptype
1606         set     trap, %g1
1607         ba,pt   %xcc, sys_trap
1608           mov   -1, %g4
1609 
1610 .mmu_exception_not_aligned:
1611         rdpr    %tstate, %g1
1612         btst    TSTATE_PRIV, %g1
1613         bnz,pn  %icc, 2f
1614         nop
1615         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1616         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1617         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1618         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1619         brz,pt  %g5, 2f
1620         nop
1621         ldn     [%g5 + P_UTRAP15], %g5                  ! unaligned utrap?
1622         brz,pn  %g5, 2f
1623         nop
1624         btst    1, %sp
1625         bz,pt   %xcc, 1f                                ! 32 bit user program
1626         nop
1627         ba,pt   %xcc, .setup_v9utrap                    ! 64 bit user program
1628         nop
1629 1:
1630         ba,pt   %xcc, .setup_utrap
1631         or      %g2, %g0, %g7
1632 2:
1633         ba,pt   %xcc, .mmu_exception_end
1634         mov     T_ALIGNMENT, %g1
1635 
1636 .mmu_priv_exception:
1637         rdpr    %tstate, %g1
1638         btst    TSTATE_PRIV, %g1
1639         bnz,pn  %icc, 1f
1640         nop
1641         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1642         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1643         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1644         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1645         brz,pt  %g5, 1f
1646         nop
1647         ldn     [%g5 + P_UTRAP16], %g5
1648         brnz,pt %g5, .setup_v9utrap
1649         nop
1650 1:
1651         mov     T_PRIV_INSTR, %g1
1652 
1653 .mmu_exception_end:
1654         CPU_INDEX(%g4, %g5)
1655         set     cpu_core, %g5
1656         sllx    %g4, CPU_CORE_SHIFT, %g4
1657         add     %g4, %g5, %g4
1658         lduh    [%g4 + CPUC_DTRACE_FLAGS], %g5
1659         andcc   %g5, CPU_DTRACE_NOFAULT, %g0
1660         bz      %xcc, .mmu_exception_tlb_chk
1661         or      %g5, CPU_DTRACE_BADADDR, %g5
1662         stuh    %g5, [%g4 + CPUC_DTRACE_FLAGS]
1663         done
1664 
1665 .mmu_exception_tlb_chk:
1666         GET_CPU_IMPL(%g5)                       ! check SFSR.FT to see if this
1667         cmp     %g5, PANTHER_IMPL               ! is a TLB parity error. But
1668         bne     2f                              ! we only do this check while
1669         mov     1, %g4                          ! running on Panther CPUs
1670         sllx    %g4, PN_SFSR_PARITY_SHIFT, %g4  ! since US-I/II use the same
1671         andcc   %g3, %g4, %g0                   ! bit for something else which
1672         bz      2f                              ! will be handled later.
1673         nop
1674 .mmu_exception_is_tlb_parity:
1675         .weak itlb_parity_trap
1676         .weak dtlb_parity_trap
1677         set     itlb_parity_trap, %g4
1678         cmp     %g1, T_INSTR_EXCEPTION          ! branch to the itlb or
1679         be      3f                              ! dtlb parity handler
1680         nop                                     ! if this trap is due
1681         set     dtlb_parity_trap, %g4
1682         cmp     %g1, T_DATA_EXCEPTION           ! to a IMMU exception
1683         be      3f                              ! or DMMU exception.
1684         nop
1685 2:
1686         sllx    %g3, 32, %g3
1687         or      %g3, %g1, %g3
1688         set     trap, %g1
1689         ba,pt   %xcc, sys_trap
1690         sub     %g0, 1, %g4
1691 3:
1692         jmp     %g4                             ! off to the appropriate
1693         nop                                     ! TLB parity handler
1694 
1695 .fp_disabled:
1696         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1697         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1698 #ifdef SF_ERRATA_30 /* call causes fp-disabled */
1699         brz,a,pn %g1, 2f
1700           nop
1701 #endif
1702         rdpr    %tstate, %g4
1703         btst    TSTATE_PRIV, %g4
1704 #ifdef SF_ERRATA_30 /* call causes fp-disabled */
1705         bnz,pn %icc, 2f
1706           nop
1707 #else
1708         bnz,a,pn %icc, ptl1_panic
1709           mov   PTL1_BAD_FPTRAP, %g1
1710 #endif
1711         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1712         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1713         brz,a,pt %g5, 2f
1714           nop
1715         ldn     [%g5 + P_UTRAP7], %g5                   ! fp_disabled utrap?
1716         brz,a,pn %g5, 2f
1717           nop
1718         btst    1, %sp
1719         bz,a,pt %xcc, 1f                                ! 32 bit user program
1720           nop
1721         ba,a,pt %xcc, .setup_v9utrap                    ! 64 bit user program
1722           nop
1723 1:
1724         ba,pt   %xcc, .setup_utrap
1725           or    %g0, %g0, %g7
1726 2:
1727         set     fp_disabled, %g1
1728         ba,pt   %xcc, sys_trap
1729           sub   %g0, 1, %g4
1730 
1731 .fp_ieee_exception:
1732         rdpr    %tstate, %g1
1733         btst    TSTATE_PRIV, %g1
1734         bnz,a,pn %icc, ptl1_panic
1735           mov   PTL1_BAD_FPTRAP, %g1
1736         CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
1737         stx     %fsr, [%g1 + CPU_TMP1]
1738         ldx     [%g1 + CPU_TMP1], %g2
1739         ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
1740         ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
1741         ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
1742         brz,a,pt %g5, 1f
1743           nop
1744         ldn     [%g5 + P_UTRAP8], %g5
1745         brnz,a,pt %g5, .setup_v9utrap
1746           nop
1747 1:
1748         set     _fp_ieee_exception, %g1
1749         ba,pt   %xcc, sys_trap
1750           sub   %g0, 1, %g4
1751 
1752 /*
1753  * Register Inputs:
1754  *      %g5             user trap handler
1755  *      %g7             misaligned addr - for alignment traps only
1756  */
1757 .setup_utrap:
1758         set     trap, %g1                       ! setup in case we go
1759         mov     T_FLUSH_PCB, %g3                ! through sys_trap on
1760         sub     %g0, 1, %g4                     ! the save instruction below
1761 
1762         /*
1763          * If the DTrace pid provider is single stepping a copied-out
1764          * instruction, t->t_dtrace_step will be set. In that case we need
1765          * to abort the single-stepping (since execution of the instruction
1766          * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1767          */
1768         save    %sp, -SA(MINFRAME32), %sp       ! window for trap handler
1769         CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
1770         ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
1771         ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
1772         rdpr    %tnpc, %l2                      ! arg1 == tnpc
1773         brz,pt  %g2, 1f
1774         rdpr    %tpc, %l1                       ! arg0 == tpc
1775 
1776         ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
1777         ldn     [%g1 + T_DTRACE_NPC], %l2       ! arg1 = t->t_dtrace_npc (step)
1778         brz,pt  %g2, 1f
1779         st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
1780         stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1781 1:
1782         mov     %g7, %l3                        ! arg2 == misaligned address
1783 
1784         rdpr    %tstate, %g1                    ! cwp for trap handler
1785         rdpr    %cwp, %g4
1786         bclr    TSTATE_CWP_MASK, %g1
1787         wrpr    %g1, %g4, %tstate
1788         wrpr    %g0, %g5, %tnpc                 ! trap handler address
1789         FAST_TRAP_DONE
1790         /* NOTREACHED */
1791 
1792 .check_v9utrap:
1793         rdpr    %tstate, %g1
1794         btst    TSTATE_PRIV, %g1
1795         bnz,a,pn %icc, 3f
1796           nop
1797         CPU_ADDR(%g4, %g1)                              ! load CPU struct addr
1798         ldn     [%g4 + CPU_THREAD], %g5                 ! load thread pointer
1799         ldn     [%g5 + T_PROCP], %g5                    ! load proc pointer
1800         ldn     [%g5 + P_UTRAPS], %g5                   ! are there utraps?
1801 
1802         cmp     %g3, T_SOFTWARE_TRAP
1803         bne,a,pt %icc, 1f
1804           nop
1805 
1806         brz,pt %g5, 3f                  ! if p_utraps == NULL goto trap()
1807           rdpr  %tt, %g3                ! delay - get actual hw trap type
1808 
1809         sub     %g3, 254, %g1           ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1810         ba,pt   %icc, 2f
1811           smul  %g1, CPTRSIZE, %g2
1812 1:
1813         brz,a,pt %g5, 3f                ! if p_utraps == NULL goto trap()
1814           nop
1815 
1816         cmp     %g3, T_UNIMP_INSTR
1817         bne,a,pt %icc, 2f
1818           nop
1819 
1820         mov     1, %g1
1821         st      %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1822         rdpr    %tpc, %g1               ! ld trapping instruction using
1823         lduwa   [%g1]ASI_AIUP, %g1      ! "AS IF USER" ASI which could fault
1824         st      %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1825 
1826         sethi   %hi(0xc1c00000), %g4    ! setup mask for illtrap instruction
1827         andcc   %g1, %g4, %g4           ! and instruction with mask
1828         bnz,a,pt %icc, 3f               ! if %g4 == zero, %g1 is an ILLTRAP
1829           nop                           ! fall thru to setup
1830 2:
1831         ldn     [%g5 + %g2], %g5
1832         brnz,a,pt %g5, .setup_v9utrap
1833           nop
1834 3:
1835         set     trap, %g1
1836         ba,pt   %xcc, sys_trap
1837           sub   %g0, 1, %g4
1838         /* NOTREACHED */
1839 
1840 /*
1841  * Register Inputs:
1842  *      %g5             user trap handler
1843  */
1844 .setup_v9utrap:
1845         set     trap, %g1                       ! setup in case we go
1846         mov     T_FLUSH_PCB, %g3                ! through sys_trap on
1847         sub     %g0, 1, %g4                     ! the save instruction below
1848 
1849         /*
1850          * If the DTrace pid provider is single stepping a copied-out
1851          * instruction, t->t_dtrace_step will be set. In that case we need
1852          * to abort the single-stepping (since execution of the instruction
1853          * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1854          */
1855         save    %sp, -SA(MINFRAME64), %sp       ! window for trap handler
1856         CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
1857         ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
1858         ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
1859         rdpr    %tnpc, %l7                      ! arg1 == tnpc
1860         brz,pt  %g2, 1f
1861         rdpr    %tpc, %l6                       ! arg0 == tpc
1862 
1863         ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
1864         ldn     [%g1 + T_DTRACE_NPC], %l7       ! arg1 == t->t_dtrace_npc (step)
1865         brz,pt  %g2, 1f
1866         st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
1867         stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1868 1:
1869         rdpr    %tstate, %g2                    ! cwp for trap handler
1870         rdpr    %cwp, %g4
1871         bclr    TSTATE_CWP_MASK, %g2
1872         wrpr    %g2, %g4, %tstate
1873 
1874         ldn     [%g1 + T_PROCP], %g4            ! load proc pointer
1875         ldn     [%g4 + P_AS], %g4               ! load as pointer
1876         ldn     [%g4 + A_USERLIMIT], %g4        ! load as userlimit
1877         cmp     %l7, %g4                        ! check for single-step set
1878         bne,pt  %xcc, 4f
1879           nop
1880         ldn     [%g1 + T_LWP], %g1              ! load klwp pointer
1881         ld      [%g1 + PCB_STEP], %g4           ! load single-step flag
1882         cmp     %g4, STEP_ACTIVE                ! step flags set in pcb?
1883         bne,pt  %icc, 4f
1884           nop
1885         stn     %g5, [%g1 + PCB_TRACEPC]        ! save trap handler addr in pcb
1886         mov     %l7, %g4                        ! on entry to precise user trap
1887         add     %l6, 4, %l7                     ! handler, %l6 == pc, %l7 == npc
1888                                                 ! at time of trap
1889         wrpr    %g0, %g4, %tnpc                 ! generate FLTBOUNDS,
1890                                                 ! %g4 == userlimit
1891         FAST_TRAP_DONE
1892         /* NOTREACHED */
1893 4:
1894         wrpr    %g0, %g5, %tnpc                 ! trap handler address
1895         FAST_TRAP_DONE_CHK_INTR
1896         /* NOTREACHED */
1897 
1898 .fp_exception:
1899         CPU_ADDR(%g1, %g4)
1900         stx     %fsr, [%g1 + CPU_TMP1]
1901         ldx     [%g1 + CPU_TMP1], %g2
1902 
1903         /*
1904          * Cheetah takes unfinished_FPop trap for certain range of operands
1905          * to the "fitos" instruction. Instead of going through the slow
1906          * software emulation path, we try to simulate the "fitos" instruction
1907          * via "fitod" and "fdtos" provided the following conditions are met:
1908          *
1909          *      fpu_exists is set (if DEBUG)
1910          *      not in privileged mode
1911          *      ftt is unfinished_FPop
1912          *      NXM IEEE trap is not enabled
1913          *      instruction at %tpc is "fitos"
1914          *
1915          *  Usage:
1916          *      %g1     per cpu address
1917          *      %g2     %fsr
1918          *      %g6     user instruction
1919          *
1920          * Note that we can take a memory access related trap while trying
1921          * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1922          * flag to catch those traps and let the SFMMU code deal with page
1923          * fault and data access exception.
1924          */
1925 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1926         sethi   %hi(fpu_exists), %g7
1927         ld      [%g7 + %lo(fpu_exists)], %g7
1928         brz,pn %g7, .fp_exception_cont
1929           nop
1930 #endif
1931         rdpr    %tstate, %g7                    ! branch if in privileged mode
1932         btst    TSTATE_PRIV, %g7
1933         bnz,pn  %xcc, .fp_exception_cont
1934         srl     %g2, FSR_FTT_SHIFT, %g7         ! extract ftt from %fsr
1935         and     %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1936         cmp     %g7, FTT_UNFIN
1937         set     FSR_TEM_NX, %g5
1938         bne,pn  %xcc, .fp_exception_cont        ! branch if NOT unfinished_FPop
1939           andcc %g2, %g5, %g0
1940         bne,pn  %xcc, .fp_exception_cont        ! branch if FSR_TEM_NX enabled
1941           rdpr  %tpc, %g5                       ! get faulting PC
1942 
1943         or      %g0, 1, %g7
1944         st      %g7, [%g1 + CPU_TL1_HDLR]       ! set tl1_hdlr flag
1945         lda     [%g5]ASI_USER, %g6              ! get user's instruction
1946         st      %g0, [%g1 + CPU_TL1_HDLR]       ! clear tl1_hdlr flag
1947 
1948         set     FITOS_INSTR_MASK, %g7
1949         and     %g6, %g7, %g7
1950         set     FITOS_INSTR, %g5
1951         cmp     %g7, %g5
1952         bne,pn  %xcc, .fp_exception_cont        ! branch if not FITOS_INSTR
1953          nop
1954 
1955         /*
1956          * This is unfinished FPops trap for "fitos" instruction. We
1957          * need to simulate "fitos" via "fitod" and "fdtos" instruction
1958          * sequence.
1959          *
1960          * We need a temporary FP register to do the conversion. Since
1961          * both source and destination operands for the "fitos" instruction
1962          * have to be within %f0-%f31, we use an FP register from the upper
1963          * half to guarantee that it won't collide with the source or the
1964          * dest operand. However, we do have to save and restore its value.
1965          *
1966          * We use %d62 as a temporary FP register for the conversion and
1967          * branch to appropriate instruction within the conversion tables
1968          * based upon the rs2 and rd values.
1969          */
1970 
1971         std     %d62, [%g1 + CPU_TMP1]          ! save original value
1972 
1973         srl     %g6, FITOS_RS2_SHIFT, %g7
1974         and     %g7, FITOS_REG_MASK, %g7
1975         set     _fitos_fitod_table, %g4
1976         sllx    %g7, 2, %g7
1977         jmp     %g4 + %g7
1978           ba,pt %xcc, _fitos_fitod_done
1979         .empty
1980 
1981 _fitos_fitod_table:
1982           fitod %f0, %d62
1983           fitod %f1, %d62
1984           fitod %f2, %d62
1985           fitod %f3, %d62
1986           fitod %f4, %d62
1987           fitod %f5, %d62
1988           fitod %f6, %d62
1989           fitod %f7, %d62
1990           fitod %f8, %d62
1991           fitod %f9, %d62
1992           fitod %f10, %d62
1993           fitod %f11, %d62
1994           fitod %f12, %d62
1995           fitod %f13, %d62
1996           fitod %f14, %d62
1997           fitod %f15, %d62
1998           fitod %f16, %d62
1999           fitod %f17, %d62
2000           fitod %f18, %d62
2001           fitod %f19, %d62
2002           fitod %f20, %d62
2003           fitod %f21, %d62
2004           fitod %f22, %d62
2005           fitod %f23, %d62
2006           fitod %f24, %d62
2007           fitod %f25, %d62
2008           fitod %f26, %d62
2009           fitod %f27, %d62
2010           fitod %f28, %d62
2011           fitod %f29, %d62
2012           fitod %f30, %d62
2013           fitod %f31, %d62
2014 _fitos_fitod_done:
2015 
2016         /*
2017          * Now convert data back into single precision
2018          */
2019         srl     %g6, FITOS_RD_SHIFT, %g7
2020         and     %g7, FITOS_REG_MASK, %g7
2021         set     _fitos_fdtos_table, %g4
2022         sllx    %g7, 2, %g7
2023         jmp     %g4 + %g7
2024           ba,pt %xcc, _fitos_fdtos_done
2025         .empty
2026 
2027 _fitos_fdtos_table:
2028           fdtos %d62, %f0
2029           fdtos %d62, %f1
2030           fdtos %d62, %f2
2031           fdtos %d62, %f3
2032           fdtos %d62, %f4
2033           fdtos %d62, %f5
2034           fdtos %d62, %f6
2035           fdtos %d62, %f7
2036           fdtos %d62, %f8
2037           fdtos %d62, %f9
2038           fdtos %d62, %f10
2039           fdtos %d62, %f11
2040           fdtos %d62, %f12
2041           fdtos %d62, %f13
2042           fdtos %d62, %f14
2043           fdtos %d62, %f15
2044           fdtos %d62, %f16
2045           fdtos %d62, %f17
2046           fdtos %d62, %f18
2047           fdtos %d62, %f19
2048           fdtos %d62, %f20
2049           fdtos %d62, %f21
2050           fdtos %d62, %f22
2051           fdtos %d62, %f23
2052           fdtos %d62, %f24
2053           fdtos %d62, %f25
2054           fdtos %d62, %f26
2055           fdtos %d62, %f27
2056           fdtos %d62, %f28
2057           fdtos %d62, %f29
2058           fdtos %d62, %f30
2059           fdtos %d62, %f31
2060 _fitos_fdtos_done:
2061 
2062         ldd     [%g1 + CPU_TMP1], %d62          ! restore %d62
2063 
2064 #if DEBUG
2065         /*
2066          * Update FPop_unfinished trap kstat
2067          */
2068         set     fpustat+FPUSTAT_UNFIN_KSTAT, %g7
2069         ldx     [%g7], %g5
2070 1:
2071         add     %g5, 1, %g6
2072 
2073         casxa   [%g7] ASI_N, %g5, %g6
2074         cmp     %g5, %g6
2075         bne,a,pn %xcc, 1b
2076           or    %g0, %g6, %g5
2077 
2078         /*
2079          * Update fpu_sim_fitos kstat
2080          */
2081         set     fpuinfo+FPUINFO_FITOS_KSTAT, %g7
2082         ldx     [%g7], %g5
2083 1:
2084         add     %g5, 1, %g6
2085 
2086         casxa   [%g7] ASI_N, %g5, %g6
2087         cmp     %g5, %g6
2088         bne,a,pn %xcc, 1b
2089           or    %g0, %g6, %g5
2090 #endif /* DEBUG */
2091 
2092         FAST_TRAP_DONE
2093 
2094 .fp_exception_cont:
2095         /*
2096          * Let _fp_exception deal with simulating FPop instruction.
2097          * Note that we need to pass %fsr in %g2 (already read above).
2098          */
2099 
2100         set     _fp_exception, %g1
2101         ba,pt   %xcc, sys_trap
2102         sub     %g0, 1, %g4
2103 
2104         .global opl_cleanw_patch
2105 opl_cleanw_patch:
2106 .clean_windows:
2107         set     trap, %g1
2108         mov     T_FLUSH_PCB, %g3
2109         sub     %g0, 1, %g4
2110         save
2111         flushw
2112         restore
2113         wrpr    %g0, %g0, %cleanwin     ! no clean windows
2114 
2115         CPU_ADDR(%g4, %g5)
2116         ldn     [%g4 + CPU_MPCB], %g4
2117         brz,a,pn %g4, 1f
2118           nop
2119         ld      [%g4 + MPCB_WSTATE], %g5
2120         add     %g5, WSTATE_CLEAN_OFFSET, %g5
2121         wrpr    %g0, %g5, %wstate
2122 1:      FAST_TRAP_DONE
2123 
2124 /*
2125  * .spill_clean: clean the previous window, restore the wstate, and
2126  * "done".
2127  *
2128  * Entry: %g7 contains new wstate
2129  */
2130 .spill_clean:
2131         sethi   %hi(nwin_minus_one), %g5
2132         ld      [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
2133         rdpr    %cwp, %g6                       ! %g6 = %cwp
2134         deccc   %g6                             ! %g6--
2135         movneg  %xcc, %g5, %g6                  ! if (%g6<0) %g6 = nwin-1
2136         wrpr    %g6, %cwp
2137         TT_TRACE_L(trace_win)
2138         clr     %l0
2139         clr     %l1
2140         clr     %l2
2141         clr     %l3
2142         clr     %l4
2143         clr     %l5
2144         clr     %l6
2145         clr     %l7
2146         wrpr    %g0, %g7, %wstate
2147         saved
2148         retry                   ! restores correct %cwp
2149 
2150 .fix_alignment:
2151         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2152         ldn     [%g1 + CPU_THREAD], %g1 ! load thread pointer
2153         ldn     [%g1 + T_PROCP], %g1
2154         mov     1, %g2
2155         stb     %g2, [%g1 + P_FIXALIGNMENT]
2156         FAST_TRAP_DONE
2157 
2158 #define STDF_REG(REG, ADDR, TMP)                \
2159         sll     REG, 3, REG                     ;\
2160 mark1:  set     start1, TMP                     ;\
2161         jmp     REG + TMP                       ;\
2162           nop                                   ;\
2163 start1: ba,pt   %xcc, done1                     ;\
2164           std   %f0, [ADDR + CPU_TMP1]          ;\
2165         ba,pt   %xcc, done1                     ;\
2166           std   %f32, [ADDR + CPU_TMP1]         ;\
2167         ba,pt   %xcc, done1                     ;\
2168           std   %f2, [ADDR + CPU_TMP1]          ;\
2169         ba,pt   %xcc, done1                     ;\
2170           std   %f34, [ADDR + CPU_TMP1]         ;\
2171         ba,pt   %xcc, done1                     ;\
2172           std   %f4, [ADDR + CPU_TMP1]          ;\
2173         ba,pt   %xcc, done1                     ;\
2174           std   %f36, [ADDR + CPU_TMP1]         ;\
2175         ba,pt   %xcc, done1                     ;\
2176           std   %f6, [ADDR + CPU_TMP1]          ;\
2177         ba,pt   %xcc, done1                     ;\
2178           std   %f38, [ADDR + CPU_TMP1]         ;\
2179         ba,pt   %xcc, done1                     ;\
2180           std   %f8, [ADDR + CPU_TMP1]          ;\
2181         ba,pt   %xcc, done1                     ;\
2182           std   %f40, [ADDR + CPU_TMP1]         ;\
2183         ba,pt   %xcc, done1                     ;\
2184           std   %f10, [ADDR + CPU_TMP1]         ;\
2185         ba,pt   %xcc, done1                     ;\
2186           std   %f42, [ADDR + CPU_TMP1]         ;\
2187         ba,pt   %xcc, done1                     ;\
2188           std   %f12, [ADDR + CPU_TMP1]         ;\
2189         ba,pt   %xcc, done1                     ;\
2190           std   %f44, [ADDR + CPU_TMP1]         ;\
2191         ba,pt   %xcc, done1                     ;\
2192           std   %f14, [ADDR + CPU_TMP1]         ;\
2193         ba,pt   %xcc, done1                     ;\
2194           std   %f46, [ADDR + CPU_TMP1]         ;\
2195         ba,pt   %xcc, done1                     ;\
2196           std   %f16, [ADDR + CPU_TMP1]         ;\
2197         ba,pt   %xcc, done1                     ;\
2198           std   %f48, [ADDR + CPU_TMP1]         ;\
2199         ba,pt   %xcc, done1                     ;\
2200           std   %f18, [ADDR + CPU_TMP1]         ;\
2201         ba,pt   %xcc, done1                     ;\
2202           std   %f50, [ADDR + CPU_TMP1]         ;\
2203         ba,pt   %xcc, done1                     ;\
2204           std   %f20, [ADDR + CPU_TMP1]         ;\
2205         ba,pt   %xcc, done1                     ;\
2206           std   %f52, [ADDR + CPU_TMP1]         ;\
2207         ba,pt   %xcc, done1                     ;\
2208           std   %f22, [ADDR + CPU_TMP1]         ;\
2209         ba,pt   %xcc, done1                     ;\
2210           std   %f54, [ADDR + CPU_TMP1]         ;\
2211         ba,pt   %xcc, done1                     ;\
2212           std   %f24, [ADDR + CPU_TMP1]         ;\
2213         ba,pt   %xcc, done1                     ;\
2214           std   %f56, [ADDR + CPU_TMP1]         ;\
2215         ba,pt   %xcc, done1                     ;\
2216           std   %f26, [ADDR + CPU_TMP1]         ;\
2217         ba,pt   %xcc, done1                     ;\
2218           std   %f58, [ADDR + CPU_TMP1]         ;\
2219         ba,pt   %xcc, done1                     ;\
2220           std   %f28, [ADDR + CPU_TMP1]         ;\
2221         ba,pt   %xcc, done1                     ;\
2222           std   %f60, [ADDR + CPU_TMP1]         ;\
2223         ba,pt   %xcc, done1                     ;\
2224           std   %f30, [ADDR + CPU_TMP1]         ;\
2225         ba,pt   %xcc, done1                     ;\
2226           std   %f62, [ADDR + CPU_TMP1]         ;\
2227 done1:
2228 
2229 #define LDDF_REG(REG, ADDR, TMP)                \
2230         sll     REG, 3, REG                     ;\
2231 mark2:  set     start2, TMP                     ;\
2232         jmp     REG + TMP                       ;\
2233           nop                                   ;\
2234 start2: ba,pt   %xcc, done2                     ;\
2235           ldd   [ADDR + CPU_TMP1], %f0          ;\
2236         ba,pt   %xcc, done2                     ;\
2237           ldd   [ADDR + CPU_TMP1], %f32         ;\
2238         ba,pt   %xcc, done2                     ;\
2239           ldd   [ADDR + CPU_TMP1], %f2          ;\
2240         ba,pt   %xcc, done2                     ;\
2241           ldd   [ADDR + CPU_TMP1], %f34         ;\
2242         ba,pt   %xcc, done2                     ;\
2243           ldd   [ADDR + CPU_TMP1], %f4          ;\
2244         ba,pt   %xcc, done2                     ;\
2245           ldd   [ADDR + CPU_TMP1], %f36         ;\
2246         ba,pt   %xcc, done2                     ;\
2247           ldd   [ADDR + CPU_TMP1], %f6          ;\
2248         ba,pt   %xcc, done2                     ;\
2249           ldd   [ADDR + CPU_TMP1], %f38         ;\
2250         ba,pt   %xcc, done2                     ;\
2251           ldd   [ADDR + CPU_TMP1], %f8          ;\
2252         ba,pt   %xcc, done2                     ;\
2253           ldd   [ADDR + CPU_TMP1], %f40         ;\
2254         ba,pt   %xcc, done2                     ;\
2255           ldd   [ADDR + CPU_TMP1], %f10         ;\
2256         ba,pt   %xcc, done2                     ;\
2257           ldd   [ADDR + CPU_TMP1], %f42         ;\
2258         ba,pt   %xcc, done2                     ;\
2259           ldd   [ADDR + CPU_TMP1], %f12         ;\
2260         ba,pt   %xcc, done2                     ;\
2261           ldd   [ADDR + CPU_TMP1], %f44         ;\
2262         ba,pt   %xcc, done2                     ;\
2263           ldd   [ADDR + CPU_TMP1], %f14         ;\
2264         ba,pt   %xcc, done2                     ;\
2265           ldd   [ADDR + CPU_TMP1], %f46         ;\
2266         ba,pt   %xcc, done2                     ;\
2267           ldd   [ADDR + CPU_TMP1], %f16         ;\
2268         ba,pt   %xcc, done2                     ;\
2269           ldd   [ADDR + CPU_TMP1], %f48         ;\
2270         ba,pt   %xcc, done2                     ;\
2271           ldd   [ADDR + CPU_TMP1], %f18         ;\
2272         ba,pt   %xcc, done2                     ;\
2273           ldd   [ADDR + CPU_TMP1], %f50         ;\
2274         ba,pt   %xcc, done2                     ;\
2275           ldd   [ADDR + CPU_TMP1], %f20         ;\
2276         ba,pt   %xcc, done2                     ;\
2277           ldd   [ADDR + CPU_TMP1], %f52         ;\
2278         ba,pt   %xcc, done2                     ;\
2279           ldd   [ADDR + CPU_TMP1], %f22         ;\
2280         ba,pt   %xcc, done2                     ;\
2281           ldd   [ADDR + CPU_TMP1], %f54         ;\
2282         ba,pt   %xcc, done2                     ;\
2283           ldd   [ADDR + CPU_TMP1], %f24         ;\
2284         ba,pt   %xcc, done2                     ;\
2285           ldd   [ADDR + CPU_TMP1], %f56         ;\
2286         ba,pt   %xcc, done2                     ;\
2287           ldd   [ADDR + CPU_TMP1], %f26         ;\
2288         ba,pt   %xcc, done2                     ;\
2289           ldd   [ADDR + CPU_TMP1], %f58         ;\
2290         ba,pt   %xcc, done2                     ;\
2291           ldd   [ADDR + CPU_TMP1], %f28         ;\
2292         ba,pt   %xcc, done2                     ;\
2293           ldd   [ADDR + CPU_TMP1], %f60         ;\
2294         ba,pt   %xcc, done2                     ;\
2295           ldd   [ADDR + CPU_TMP1], %f30         ;\
2296         ba,pt   %xcc, done2                     ;\
2297           ldd   [ADDR + CPU_TMP1], %f62         ;\
2298 done2:
2299 
2300 .lddf_exception_not_aligned:
2301         /*
2302          * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2303          */
2304         ldxa    [MMU_SFAR]%asi, %g5     ! misaligned vaddr in %g5
2305 
2306 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2307         sethi   %hi(fpu_exists), %g2            ! check fpu_exists
2308         ld      [%g2 + %lo(fpu_exists)], %g2
2309         brz,a,pn %g2, 4f
2310           nop
2311 #endif
2312         CPU_ADDR(%g1, %g4)
2313         or      %g0, 1, %g4
2314         st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2315 
2316         rdpr    %tpc, %g2
2317         lda     [%g2]ASI_AIUP, %g6      ! get the user's lddf instruction
2318         srl     %g6, 23, %g1            ! using ldda or not?
2319         and     %g1, 1, %g1
2320         brz,a,pt %g1, 2f                ! check for ldda instruction
2321           nop
2322         srl     %g6, 13, %g1            ! check immflag
2323         and     %g1, 1, %g1
2324         rdpr    %tstate, %g2            ! %tstate in %g2
2325         brnz,a,pn %g1, 1f
2326           srl   %g2, 31, %g1            ! get asi from %tstate
2327         srl     %g6, 5, %g1             ! get asi from instruction
2328         and     %g1, 0xFF, %g1          ! imm_asi field
2329 1:
2330         cmp     %g1, ASI_P              ! primary address space
2331         be,a,pt %icc, 2f
2332           nop
2333         cmp     %g1, ASI_PNF            ! primary no fault address space
2334         be,a,pt %icc, 2f
2335           nop
2336         cmp     %g1, ASI_S              ! secondary address space
2337         be,a,pt %icc, 2f
2338           nop
2339         cmp     %g1, ASI_SNF            ! secondary no fault address space
2340         bne,a,pn %icc, 3f
2341           nop
2342 2:
2343         lduwa   [%g5]ASI_USER, %g7      ! get first half of misaligned data
2344         add     %g5, 4, %g5             ! increment misaligned data address
2345         lduwa   [%g5]ASI_USER, %g5      ! get second half of misaligned data
2346 
2347         sllx    %g7, 32, %g7
2348         or      %g5, %g7, %g5           ! combine data
2349         CPU_ADDR(%g7, %g1)              ! save data on a per-cpu basis
2350         stx     %g5, [%g7 + CPU_TMP1]   ! save in cpu_tmp1
2351 
2352         srl     %g6, 25, %g3            ! %g6 has the instruction
2353         and     %g3, 0x1F, %g3          ! %g3 has rd
2354         LDDF_REG(%g3, %g7, %g4)
2355 
2356         CPU_ADDR(%g1, %g4)
2357         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2358         FAST_TRAP_DONE
2359 3:
2360         CPU_ADDR(%g1, %g4)
2361         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2362 4:
2363         set     T_USER, %g3             ! trap type in %g3
2364         or      %g3, T_LDDF_ALIGN, %g3
2365         mov     %g5, %g2                ! misaligned vaddr in %g2
2366         set     fpu_trap, %g1           ! goto C for the little and
2367         ba,pt   %xcc, sys_trap          ! no fault little asi's
2368           sub   %g0, 1, %g4
2369 
2370 .stdf_exception_not_aligned:
2371         /*
2372          * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2373          */
2374         ldxa    [MMU_SFAR]%asi, %g5     ! misaligned vaddr in %g5
2375 
2376 #if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2377         sethi   %hi(fpu_exists), %g7            ! check fpu_exists
2378         ld      [%g7 + %lo(fpu_exists)], %g3
2379         brz,a,pn %g3, 4f
2380           nop
2381 #endif
2382         CPU_ADDR(%g1, %g4)
2383         or      %g0, 1, %g4
2384         st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2385 
2386         rdpr    %tpc, %g2
2387         lda     [%g2]ASI_AIUP, %g6      ! get the user's stdf instruction
2388 
2389         srl     %g6, 23, %g1            ! using stda or not?
2390         and     %g1, 1, %g1
2391         brz,a,pt %g1, 2f                ! check for stda instruction
2392           nop
2393         srl     %g6, 13, %g1            ! check immflag
2394         and     %g1, 1, %g1
2395         rdpr    %tstate, %g2            ! %tstate in %g2
2396         brnz,a,pn %g1, 1f
2397           srl   %g2, 31, %g1            ! get asi from %tstate
2398         srl     %g6, 5, %g1             ! get asi from instruction
2399         and     %g1, 0xFF, %g1          ! imm_asi field
2400 1:
2401         cmp     %g1, ASI_P              ! primary address space
2402         be,a,pt %icc, 2f
2403           nop
2404         cmp     %g1, ASI_S              ! secondary address space
2405         bne,a,pn %icc, 3f
2406           nop
2407 2:
2408         srl     %g6, 25, %g6
2409         and     %g6, 0x1F, %g6          ! %g6 has rd
2410         CPU_ADDR(%g7, %g1)
2411         STDF_REG(%g6, %g7, %g4)         ! STDF_REG(REG, ADDR, TMP)
2412 
2413         ldx     [%g7 + CPU_TMP1], %g6
2414         srlx    %g6, 32, %g7
2415         stuwa   %g7, [%g5]ASI_USER      ! first half
2416         add     %g5, 4, %g5             ! increment misaligned data address
2417         stuwa   %g6, [%g5]ASI_USER      ! second half
2418 
2419         CPU_ADDR(%g1, %g4)
2420         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2421         FAST_TRAP_DONE
2422 3:
2423         CPU_ADDR(%g1, %g4)
2424         st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2425 4:
2426         set     T_USER, %g3             ! trap type in %g3
2427         or      %g3, T_STDF_ALIGN, %g3
2428         mov     %g5, %g2                ! misaligned vaddr in %g2
2429         set     fpu_trap, %g1           ! goto C for the little and
2430         ba,pt   %xcc, sys_trap          ! nofault little asi's
2431           sub   %g0, 1, %g4
2432 
2433 #ifdef DEBUG_USER_TRAPTRACECTL
2434 
2435 .traptrace_freeze:
2436         mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
2437         TT_TRACE_L(trace_win)
2438         mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
2439         set     trap_freeze, %g1
2440         mov     1, %g2
2441         st      %g2, [%g1]
2442         FAST_TRAP_DONE
2443 
2444 .traptrace_unfreeze:
2445         set     trap_freeze, %g1
2446         st      %g0, [%g1]
2447         mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
2448         TT_TRACE_L(trace_win)
2449         mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
2450         FAST_TRAP_DONE
2451 
2452 #endif /* DEBUG_USER_TRAPTRACECTL */
2453 
2454 .getcc:
2455         CPU_ADDR(%g1, %g2)
2456         stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
2457         stx     %o1, [%g1 + CPU_TMP2]           ! save %o1
2458         rdpr    %tstate, %g3                    ! get tstate
2459         srlx    %g3, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
2460         set     PSR_ICC, %g2
2461         and     %o0, %g2, %o0                   ! mask out the rest
2462         srl     %o0, PSR_ICC_SHIFT, %o0         ! right justify
2463         rdpr    %pstate, %o1
2464         wrpr    %o1, PSTATE_AG, %pstate         ! get into normal globals
2465         mov     %o0, %g1                        ! move ccr to normal %g1
2466         wrpr    %g0, %o1, %pstate               ! back into alternate globals
2467         ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
2468         ldx     [%g1 + CPU_TMP2], %o1           ! restore %o1
2469         FAST_TRAP_DONE
2470 
2471 .setcc:
2472         CPU_ADDR(%g1, %g2)
2473         stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
2474         stx     %o1, [%g1 + CPU_TMP2]           ! save %o1
2475         rdpr    %pstate, %o0
2476         wrpr    %o0, PSTATE_AG, %pstate         ! get into normal globals
2477         mov     %g1, %o1
2478         wrpr    %g0, %o0, %pstate               ! back to alternates
2479         sll     %o1, PSR_ICC_SHIFT, %g2
2480         set     PSR_ICC, %g3
2481         and     %g2, %g3, %g2                   ! mask out rest
2482         sllx    %g2, PSR_TSTATE_CC_SHIFT, %g2
2483         rdpr    %tstate, %g3                    ! get tstate
2484         srl     %g3, 0, %g3                     ! clear upper word
2485         or      %g3, %g2, %g3                   ! or in new bits
2486         wrpr    %g3, %tstate
2487         ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
2488         ldx     [%g1 + CPU_TMP2], %o1           ! restore %o1
2489         FAST_TRAP_DONE
2490 
2491 /*
2492  * getpsr(void)
2493  * Note that the xcc part of the ccr is not provided.
2494  * The V8 code shows why the V9 trap is not faster:
2495  * #define GETPSR_TRAP() \
2496  *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2497  */
2498 
2499         .type   .getpsr, #function
2500 .getpsr:
2501         rdpr    %tstate, %g1                    ! get tstate
2502         srlx    %g1, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
2503         set     PSR_ICC, %g2
2504         and     %o0, %g2, %o0                   ! mask out the rest
2505 
2506         rd      %fprs, %g1                      ! get fprs
2507         and     %g1, FPRS_FEF, %g2              ! mask out dirty upper/lower
2508         sllx    %g2, PSR_FPRS_FEF_SHIFT, %g2    ! shift fef to V8 psr.ef
2509         or      %o0, %g2, %o0                   ! or result into psr.ef
2510 
2511         set     V9_PSR_IMPLVER, %g2             ! SI assigned impl/ver: 0xef
2512         or      %o0, %g2, %o0                   ! or psr.impl/ver
2513         FAST_TRAP_DONE
2514         SET_SIZE(.getpsr)
2515 
2516 /*
2517  * setpsr(newpsr)
2518  * Note that there is no support for ccr.xcc in the V9 code.
2519  */
2520 
2521         .type   .setpsr, #function
2522 .setpsr:
2523         rdpr    %tstate, %g1                    ! get tstate
2524 !       setx    TSTATE_V8_UBITS, %g2
2525         or      %g0, CCR_ICC, %g3
2526         sllx    %g3, TSTATE_CCR_SHIFT, %g2
2527 
2528         andn    %g1, %g2, %g1                   ! zero current user bits
2529         set     PSR_ICC, %g2
2530         and     %g2, %o0, %g2                   ! clear all but psr.icc bits
2531         sllx    %g2, PSR_TSTATE_CC_SHIFT, %g3   ! shift to tstate.ccr.icc
2532         wrpr    %g1, %g3, %tstate               ! write tstate
2533 
2534         set     PSR_EF, %g2
2535         and     %g2, %o0, %g2                   ! clear all but fp enable bit
2536         srlx    %g2, PSR_FPRS_FEF_SHIFT, %g4    ! shift ef to V9 fprs.fef
2537         wr      %g0, %g4, %fprs                 ! write fprs
2538 
2539         CPU_ADDR(%g1, %g2)                      ! load CPU struct addr to %g1
2540         ldn     [%g1 + CPU_THREAD], %g2         ! load thread pointer
2541         ldn     [%g2 + T_LWP], %g3              ! load klwp pointer
2542         ldn     [%g3 + LWP_FPU], %g2            ! get lwp_fpu pointer
2543         stuw    %g4, [%g2 + FPU_FPRS]           ! write fef value to fpu_fprs
2544         srlx    %g4, 2, %g4                     ! shift fef value to bit 0
2545         stub    %g4, [%g2 + FPU_EN]             ! write fef value to fpu_en
2546         FAST_TRAP_DONE
2547         SET_SIZE(.setpsr)
2548 
2549 /*
2550  * getlgrp
2551  * get home lgrpid on which the calling thread is currently executing.
2552  */
2553         .type   .getlgrp, #function
2554 .getlgrp:
2555         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2556         ld      [%g1 + CPU_ID], %o0     ! load cpu_id
2557         ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
2558         ldn     [%g2 + T_LPL], %g2      ! load lpl pointer
2559         ld      [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
2560         sra     %g1, 0, %o1
2561         FAST_TRAP_DONE
2562         SET_SIZE(.getlgrp)
2563 
2564 /*
2565  * Entry for old 4.x trap (trap 0).
2566  */
2567         ENTRY_NP(syscall_trap_4x)
2568         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2569         ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
2570         ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
2571         ld      [%g2 + PCB_TRAP0], %g2  ! lwp->lwp_pcb.pcb_trap0addr
2572         brz,pn  %g2, 1f                 ! has it been set?
2573         st      %l0, [%g1 + CPU_TMP1]   ! delay - save some locals
2574         st      %l1, [%g1 + CPU_TMP2]
2575         rdpr    %tnpc, %l1              ! save old tnpc
2576         wrpr    %g0, %g2, %tnpc         ! setup tnpc
2577 
2578         rdpr    %pstate, %l0
2579         wrpr    %l0, PSTATE_AG, %pstate ! switch to normal globals
2580         mov     %l1, %g6                ! pass tnpc to user code in %g6
2581         wrpr    %l0, %g0, %pstate       ! switch back to alternate globals
2582 
2583         ! Note that %g1 still contains CPU struct addr
2584         ld      [%g1 + CPU_TMP2], %l1   ! restore locals
2585         ld      [%g1 + CPU_TMP1], %l0
2586         FAST_TRAP_DONE_CHK_INTR
2587 1:
2588         mov     %g1, %l0
2589         st      %l1, [%g1 + CPU_TMP2]
2590         rdpr    %pstate, %l1
2591         wrpr    %l1, PSTATE_AG, %pstate
2592         !
2593         ! check for old syscall mmap which is the only different one which
2594         ! must be the same.  Others are handled in the compatibility library.
2595         !
2596         cmp     %g1, OSYS_mmap  ! compare to old 4.x mmap
2597         movz    %icc, SYS_mmap, %g1
2598         wrpr    %g0, %l1, %pstate
2599         ld      [%l0 + CPU_TMP2], %l1   ! restore locals
2600         ld      [%l0 + CPU_TMP1], %l0
2601         SYSCALL(syscall_trap32)
2602         SET_SIZE(syscall_trap_4x)
2603 
2604 /*
2605  * Handler for software trap 9.
2606  * Set trap0 emulation address for old 4.x system call trap.
2607  * XXX - this should be a system call.
2608  */
2609         ENTRY_NP(set_trap0_addr)
2610         CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
2611         ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
2612         ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
2613         st      %l0, [%g1 + CPU_TMP1]   ! save some locals
2614         st      %l1, [%g1 + CPU_TMP2]
2615         rdpr    %pstate, %l0
2616         wrpr    %l0, PSTATE_AG, %pstate
2617         mov     %g1, %l1
2618         wrpr    %g0, %l0, %pstate
2619         andn    %l1, 3, %l1             ! force alignment
2620         st      %l1, [%g2 + PCB_TRAP0]  ! lwp->lwp_pcb.pcb_trap0addr
2621         ld      [%g1 + CPU_TMP1], %l0   ! restore locals
2622         ld      [%g1 + CPU_TMP2], %l1
2623         FAST_TRAP_DONE
2624         SET_SIZE(set_trap0_addr)
2625 
2626 /*
2627  * mmu_trap_tl1
2628  * trap handler for unexpected mmu traps.
2629  * simply checks if the trap was a user lddf/stdf alignment trap, in which
2630  * case we go to fpu_trap or a user trap from the window handler, in which
2631  * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
2632  */
2633         .type   mmu_trap_tl1, #function
2634 mmu_trap_tl1:
2635 #ifdef  TRAPTRACE
2636         TRACE_PTR(%g5, %g6)
2637         GET_TRACE_TICK(%g6, %g7)
2638         stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
2639         rdpr    %tl, %g6
2640         stha    %g6, [%g5 + TRAP_ENT_TL]%asi
2641         rdpr    %tt, %g6
2642         stha    %g6, [%g5 + TRAP_ENT_TT]%asi
2643         rdpr    %tstate, %g6
2644         stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi
2645         stna    %sp, [%g5 + TRAP_ENT_SP]%asi
2646         stna    %g0, [%g5 + TRAP_ENT_TR]%asi
2647         rdpr    %tpc, %g6
2648         stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
2649         set     MMU_SFAR, %g6
2650         ldxa    [%g6]ASI_DMMU, %g6
2651         stxa    %g6, [%g5 + TRAP_ENT_F1]%asi
2652         CPU_PADDR(%g7, %g6);
2653         add     %g7, CPU_TL1_HDLR, %g7
2654         lda     [%g7]ASI_MEM, %g6
2655         stxa    %g6, [%g5 + TRAP_ENT_F2]%asi
2656         set     0xdeadbeef, %g6
2657         stna    %g6, [%g5 + TRAP_ENT_F3]%asi
2658         stna    %g6, [%g5 + TRAP_ENT_F4]%asi
2659         TRACE_NEXT(%g5, %g6, %g7)
2660 #endif /* TRAPTRACE */
2661 
2662         GET_CPU_IMPL(%g5)
2663         cmp     %g5, PANTHER_IMPL
2664         bne     mmu_trap_tl1_4
2665           nop
2666         rdpr    %tt, %g5
2667         cmp     %g5, T_DATA_EXCEPTION
2668         bne     mmu_trap_tl1_4
2669           nop
2670         wr      %g0, ASI_DMMU, %asi
2671         ldxa    [MMU_SFSR]%asi, %g5
2672         mov     1, %g6
2673         sllx    %g6, PN_SFSR_PARITY_SHIFT, %g6
2674         andcc   %g5, %g6, %g0
2675         bz      mmu_trap_tl1_4
2676 
2677         /*
2678          * We are running on a Panther and have hit a DTLB parity error.
2679          */
2680         ldxa    [MMU_TAG_ACCESS]%asi, %g2
2681         mov     %g5, %g3
2682         ba,pt   %xcc, .mmu_exception_is_tlb_parity
2683         mov     T_DATA_EXCEPTION, %g1
2684 
2685 mmu_trap_tl1_4:
2686         CPU_PADDR(%g7, %g6);
2687         add     %g7, CPU_TL1_HDLR, %g7          ! %g7 = &cpu_m.tl1_hdlr (PA)
2688         /*
2689          * AM is cleared on trap, so addresses are 64 bit
2690          */
2691         lda     [%g7]ASI_MEM, %g6
2692         brz,a,pt %g6, 1f
2693           nop
2694         /*
2695          * We are going to update cpu_m.tl1_hdlr using physical address.
2696          * Flush the D$ line, so that stale data won't be accessed later.
2697          */
2698         CPU_ADDR(%g6, %g5)
2699         add     %g6, CPU_TL1_HDLR, %g6          ! %g6 = &cpu_m.tl1_hdlr (VA)
2700         GET_CPU_IMPL(%g5)
2701         cmp     %g5, CHEETAH_IMPL
2702         bl,pt   %icc, 3f
2703          cmp    %g5, SPITFIRE_IMPL
2704         stxa    %g0, [%g7]ASI_DC_INVAL
2705         membar  #Sync
2706         ba,pt   %xcc, 2f
2707          nop
2708 3:
2709         bl,pt   %icc, 2f
2710          sethi  %hi(dcache_line_mask), %g5
2711         ld      [%g5 + %lo(dcache_line_mask)], %g5
2712         and     %g6, %g5, %g5
2713         stxa    %g0, [%g5]ASI_DC_TAG
2714         membar  #Sync
2715 2:
2716         sta     %g0, [%g7]ASI_MEM
2717         SWITCH_GLOBALS                          ! back to mmu globals
2718         ba,a,pt %xcc, sfmmu_mmu_trap            ! handle page faults
2719 1:
2720         rdpr    %tt, %g5
2721         rdpr    %tl, %g7
2722         sub     %g7, 1, %g6
2723         wrpr    %g6, %tl
2724         rdpr    %tt, %g6
2725         wrpr    %g7, %tl
2726         and     %g6, WTRAP_TTMASK, %g6
2727         cmp     %g6, WTRAP_TYPE
2728         bne,a,pn %xcc, ptl1_panic
2729         mov     PTL1_BAD_MMUTRAP, %g1
2730         rdpr    %tpc, %g7
2731         /* tpc should be in the trap table */
2732         set     trap_table, %g6
2733         cmp     %g7, %g6
2734         blt,a,pn %xcc, ptl1_panic
2735           mov   PTL1_BAD_MMUTRAP, %g1
2736         set     etrap_table, %g6
2737         cmp     %g7, %g6
2738         bge,a,pn %xcc, ptl1_panic
2739           mov   PTL1_BAD_MMUTRAP, %g1
2740         cmp     %g5, T_ALIGNMENT
2741         move    %icc, MMU_SFAR, %g6
2742         movne   %icc, MMU_TAG_ACCESS, %g6
2743         ldxa    [%g6]ASI_DMMU, %g6
2744         andn    %g7, WTRAP_ALIGN, %g7   /* 128 byte aligned */
2745         add     %g7, WTRAP_FAULTOFF, %g7
2746         wrpr    %g0, %g7, %tnpc
2747         done
2748         SET_SIZE(mmu_trap_tl1)
2749 
2750 /*
2751  * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
2752  * traps are valid only when kmdb is loaded.  When the debugger is active,
2753  * the code below is rewritten to transfer control to the appropriate
2754  * debugger entry points.
2755  */
2756         .global kmdb_trap
2757         .align  8
2758 kmdb_trap:
2759         ba,a    trap_table0
2760         jmp     %g1 + 0
2761         nop
2762 
2763         .global kmdb_trap_tl1
2764         .align  8
2765 kmdb_trap_tl1:
2766         ba,a    trap_table0
2767         jmp     %g1 + 0
2768         nop
2769 
2770 /*
2771  * This entry is copied from OBP's trap table during boot.
2772  */
2773         .global obp_bpt
2774         .align  8
2775 obp_bpt:
2776         NOT
2777 
2778 /*
2779  * if kernel, set PCONTEXT to 0 for debuggers
2780  * if user, clear nucleus page sizes
2781  */
2782         .global kctx_obp_bpt
2783 kctx_obp_bpt:
2784         set     obp_bpt, %g2
2785 1:
2786 #ifndef _OPL
2787         mov     MMU_PCONTEXT, %g1
2788         ldxa    [%g1]ASI_DMMU, %g1
2789         srlx    %g1, CTXREG_NEXT_SHIFT, %g3
2790         brz,pt  %g3, 3f                 ! nucleus pgsz is 0, no problem
2791           sllx  %g3, CTXREG_NEXT_SHIFT, %g3
2792         set     CTXREG_CTX_MASK, %g4    ! check Pcontext
2793         btst    %g4, %g1
2794         bz,a,pt %xcc, 2f
2795           clr   %g3                     ! kernel:  PCONTEXT=0
2796         xor     %g3, %g1, %g3           ! user: clr N_pgsz0/1 bits
2797 2:
2798         set     DEMAP_ALL_TYPE, %g1
2799         stxa    %g0, [%g1]ASI_DTLB_DEMAP
2800         stxa    %g0, [%g1]ASI_ITLB_DEMAP
2801         mov     MMU_PCONTEXT, %g1
2802         stxa    %g3, [%g1]ASI_DMMU
2803         membar  #Sync
2804         sethi   %hi(FLUSH_ADDR), %g1
2805         flush   %g1                     ! flush required by immu
2806 #endif /* _OPL */
2807 3:
2808         jmp     %g2
2809           nop
2810 
2811 
2812 #ifdef  TRAPTRACE
2813 /*
2814  * TRAPTRACE support.
2815  * labels here are branched to with "rd %pc, %g7" in the delay slot.
2816  * Return is done by "jmp %g7 + 4".
2817  */
2818 
2819 trace_gen:
2820         TRACE_PTR(%g3, %g6)
2821         GET_TRACE_TICK(%g6, %g4)
2822         stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
2823         rdpr    %tl, %g6
2824         stha    %g6, [%g3 + TRAP_ENT_TL]%asi
2825         rdpr    %tt, %g6
2826         stha    %g6, [%g3 + TRAP_ENT_TT]%asi
2827         rdpr    %tstate, %g6
2828         stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
2829         stna    %sp, [%g3 + TRAP_ENT_SP]%asi
2830         rdpr    %tpc, %g6
2831         stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
2832         TRACE_NEXT(%g3, %g4, %g5)
2833         jmp     %g7 + 4
2834         nop
2835 
2836 trace_win:
2837         TRACE_WIN_INFO(0, %l0, %l1, %l2)
2838         ! Keep the locals as clean as possible, caller cleans %l4
2839         clr     %l2
2840         clr     %l1
2841         jmp     %l4 + 4
2842           clr   %l0
2843 
2844 /*
2845  * Trace a tsb hit
2846  * g1 = tsbe pointer (in/clobbered)
2847  * g2 = tag access register (in)
2848  * g3 - g4 = scratch (clobbered)
2849  * g5 = tsbe data (in)
2850  * g6 = scratch (clobbered)
2851  * g7 = pc we jumped here from (in)
2852  */
2853 
2854         ! Do not disturb %g5, it will be used after the trace
2855         ALTENTRY(trace_tsbhit)
2856         TRACE_TSBHIT(0)
2857         jmp     %g7 + 4
2858         nop
2859 
2860 /*
2861  * Trace a TSB miss
2862  *
2863  * g1 = tsb8k pointer (in)
2864  * g2 = tag access register (in)
2865  * g3 = tsb4m pointer (in)
2866  * g4 = tsbe tag (in/clobbered)
2867  * g5 - g6 = scratch (clobbered)
2868  * g7 = pc we jumped here from (in)
2869  */
2870         .global trace_tsbmiss
2871 trace_tsbmiss:
2872         membar  #Sync
2873         sethi   %hi(FLUSH_ADDR), %g6
2874         flush   %g6
2875         TRACE_PTR(%g5, %g6)
2876         stxa    %g2, [%g5 + TRAP_ENT_SP]%asi            ! tag access
2877         stxa    %g4, [%g5 + TRAP_ENT_F1]%asi            ! tsb tag
2878         GET_TRACE_TICK(%g6, %g4)
2879         stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
2880         rdpr    %tnpc, %g6
2881         stxa    %g6, [%g5 + TRAP_ENT_F2]%asi
2882         stna    %g1, [%g5 + TRAP_ENT_F3]%asi            ! tsb8k pointer
2883         srlx    %g1, 32, %g6
2884         stna    %g6, [%g5 + TRAP_ENT_F4]%asi            ! huh?
2885         rdpr    %tpc, %g6
2886         stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
2887         rdpr    %tl, %g6
2888         stha    %g6, [%g5 + TRAP_ENT_TL]%asi
2889         rdpr    %tt, %g6
2890         or      %g6, TT_MMU_MISS, %g4
2891         stha    %g4, [%g5 + TRAP_ENT_TT]%asi
2892         cmp     %g6, FAST_IMMU_MISS_TT
2893         be,a    %icc, 1f
2894           ldxa  [%g0]ASI_IMMU, %g6
2895         ldxa    [%g0]ASI_DMMU, %g6
2896 1:      stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi        ! tag target
2897         stxa    %g3, [%g5 + TRAP_ENT_TR]%asi            ! tsb4m pointer
2898         TRACE_NEXT(%g5, %g4, %g6)
2899         jmp     %g7 + 4
2900         nop
2901 
2902 /*
2903  * g2 = tag access register (in)
2904  * g3 = ctx number (in)
2905  */
2906 trace_dataprot:
2907         membar  #Sync
2908         sethi   %hi(FLUSH_ADDR), %g6
2909         flush   %g6
2910         TRACE_PTR(%g1, %g6)
2911         GET_TRACE_TICK(%g6, %g5)
2912         stxa    %g6, [%g1 + TRAP_ENT_TICK]%asi
2913         rdpr    %tpc, %g6
2914         stna    %g6, [%g1 + TRAP_ENT_TPC]%asi
2915         rdpr    %tstate, %g6
2916         stxa    %g6, [%g1 + TRAP_ENT_TSTATE]%asi
2917         stxa    %g2, [%g1 + TRAP_ENT_SP]%asi            ! tag access reg
2918         stxa    %g0, [%g1 + TRAP_ENT_TR]%asi
2919         stxa    %g0, [%g1 + TRAP_ENT_F1]%asi
2920         stxa    %g0, [%g1 + TRAP_ENT_F2]%asi
2921         stxa    %g0, [%g1 + TRAP_ENT_F3]%asi
2922         stxa    %g0, [%g1 + TRAP_ENT_F4]%asi
2923         rdpr    %tl, %g6
2924         stha    %g6, [%g1 + TRAP_ENT_TL]%asi
2925         rdpr    %tt, %g6
2926         stha    %g6, [%g1 + TRAP_ENT_TT]%asi
2927         TRACE_NEXT(%g1, %g4, %g5)
2928         jmp     %g7 + 4
2929         nop
2930 
2931 #endif /* TRAPTRACE */
2932 
2933        .align  32
2934        .global pil15_epilogue
2935 pil15_epilogue:
2936        ba      pil_interrupt_common
2937        nop
2938        .align  32
2939 
2940 /*
2941  * fast_trap_done, fast_trap_done_chk_intr:
2942  *
2943  * Due to the design of UltraSPARC pipeline, pending interrupts are not
2944  * taken immediately after a RETRY or DONE instruction which causes IE to
2945  * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2946  * to execute first before taking any interrupts. If that instruction
2947  * results in other traps, and if the corresponding trap handler runs
2948  * entirely at TL=1 with interrupts disabled, then pending interrupts
2949  * won't be taken until after yet another instruction following the %tpc
2950  * or %tnpc.
2951  *
2952  * A malicious user program can use this feature to block out interrupts
2953  * for extended durations, which can result in send_mondo_timeout kernel
2954  * panic.
2955  *
2956  * This problem is addressed by servicing any pending interrupts via
2957  * sys_trap before returning back to the user mode from a fast trap
2958  * handler. The "done" instruction within a fast trap handler, which
2959  * runs entirely at TL=1 with interrupts disabled, is replaced with the
2960  * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2961  * entry point.
2962  *
2963  * We check for any pending interrupts here and force a sys_trap to
2964  * service those interrupts, if any. To minimize overhead, pending
2965  * interrupts are checked if the %tpc happens to be at 16K boundary,
2966  * which allows a malicious program to execute at most 4K consecutive
2967  * instructions before we service any pending interrupts. If a worst
2968  * case fast trap handler takes about 2 usec, then interrupts will be
2969  * blocked for at most 8 msec, less than a clock tick.
2970  *
2971  * For the cases where we don't know if the %tpc will cross a 16K
2972  * boundary, we can't use the above optimization and always process
2973  * any pending interrupts via fast_frap_done_chk_intr entry point.
2974  *
2975  * Entry Conditions:
2976  *      %pstate         am:0 priv:1 ie:0
2977  *                      globals are AG (not normal globals)
2978  */
2979 
2980         .global fast_trap_done, fast_trap_done_chk_intr
2981 fast_trap_done:
2982         rdpr    %tpc, %g5
2983         sethi   %hi(0xffffc000), %g6    ! 1's complement of 0x3fff
2984         andncc  %g5, %g6, %g0           ! check lower 14 bits of %tpc
2985         bz,a,pn %icc, 1f                ! branch if zero (lower 32 bits only)
2986           ldxa  [%g0]ASI_INTR_RECEIVE_STATUS, %g5
2987         done
2988 
2989         ALTENTRY(fast_trap_done_check_interrupts)
2990 fast_trap_done_chk_intr:
2991         ldxa    [%g0]ASI_INTR_RECEIVE_STATUS, %g5
2992 
2993 1:      rd      SOFTINT, %g6
2994         and     %g5, IRSR_BUSY, %g5
2995         orcc    %g5, %g6, %g0
2996         bnz,pn  %xcc, 2f                ! branch if any pending intr
2997         nop
2998         done
2999 
3000 2:
3001         /*
3002          * We get here if there are any pending interrupts.
3003          * Adjust %tpc/%tnpc as we'll be resuming via "retry"
3004          * instruction.
3005          */
3006         rdpr    %tnpc, %g5
3007         wrpr    %g0, %g5, %tpc
3008         add     %g5, 4, %g5
3009         wrpr    %g0, %g5, %tnpc
3010 
3011         /*
3012          * Force a dummy sys_trap call so that interrupts can be serviced.
3013          */
3014         set     fast_trap_dummy_call, %g1
3015         ba,pt   %xcc, sys_trap
3016           mov   -1, %g4
3017 
3018 fast_trap_dummy_call:
3019         retl
3020         nop
3021 
3022 /*
3023  * Currently the brand syscall interposition code is not enabled by
3024  * default.  Instead, when a branded zone is first booted the brand
3025  * infrastructure will patch the trap table so that the syscall
3026  * entry points are redirected to syscall_wrapper32 and syscall_wrapper
3027  * for ILP32 and LP64 syscalls respectively.  this is done in
3028  * brand_plat_interposition_enable().  Note that the syscall wrappers
3029  * below do not collect any trap trace data since the syscall hot patch
3030  * points are reached after trap trace data has already been collected.
3031  */
3032 #define BRAND_CALLBACK(callback_id)                                         \
3033         CPU_ADDR(%g2, %g1)              /* load CPU struct addr to %g2  */ ;\
3034         ldn     [%g2 + CPU_THREAD], %g3 /* load thread pointer          */ ;\
3035         ldn     [%g3 + T_PROCP], %g3    /* get proc pointer             */ ;\
3036         ldn     [%g3 + P_BRAND], %g3    /* get brand pointer            */ ;\
3037         brz     %g3, 1f                 /* No brand?  No callback.      */ ;\
3038         nop                                                                ;\
3039         ldn     [%g3 + B_MACHOPS], %g3  /* get machops list             */ ;\
3040         ldn     [%g3 + (callback_id << 3)], %g3                      ;\
3041         brz     %g3, 1f                                                    ;\
3042         /*                                                                  \
3043          * This isn't pretty.  We want a low-latency way for the callback   \
3044          * routine to decline to do anything.  We just pass in an address   \
3045          * the routine can directly jmp back to, pretending that nothing    \
3046          * has happened.                                                    \
3047          *                                                                  \
3048          * %g1: return address (where the brand handler jumps back to)      \
3049          * %g2: address of CPU structure                                    \
3050          * %g3: address of brand handler (where we will jump to)            \
3051          */                                                                 \
3052         mov     %pc, %g1                                                   ;\
3053         add     %g1, 16, %g1                                               ;\
3054         jmp     %g3                                                        ;\
3055         nop                                                                ;\
3056 1:
3057 
3058         ENTRY_NP(syscall_wrapper32)
3059         BRAND_CALLBACK(BRAND_CB_SYSCALL32)
3060         SYSCALL_NOTT(syscall_trap32)
3061         SET_SIZE(syscall_wrapper32)
3062 
3063         ENTRY_NP(syscall_wrapper)
3064         BRAND_CALLBACK(BRAND_CB_SYSCALL)
3065         SYSCALL_NOTT(syscall_trap)
3066         SET_SIZE(syscall_wrapper)
3067 
3068 #endif  /* lint */
--- EOF ---