1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  * Copyright (c) 2016 by Delphix. All rights reserved.
  25  */
  26 
  27 /*
  28  * SFMMU primitives.  These primitives should only be used by sfmmu
  29  * routines.
  30  */
  31 
  32 #include "assym.h"
  33 
  34 #include <sys/asm_linkage.h>
  35 #include <sys/machtrap.h>
  36 #include <sys/machasi.h>
  37 #include <sys/sun4asi.h>
  38 #include <sys/pte.h>
  39 #include <sys/mmu.h>
  40 #include <vm/hat_sfmmu.h>
  41 #include <vm/seg_spt.h>
  42 #include <sys/machparam.h>
  43 #include <sys/privregs.h>
  44 #include <sys/scb.h>
  45 #include <sys/intreg.h>
  46 #include <sys/machthread.h>
  47 #include <sys/intr.h>
  48 #include <sys/clock.h>
  49 #include <sys/trapstat.h>
  50 
  51 #ifdef TRAPTRACE
  52 #include <sys/traptrace.h>
  53 
  54 /*
  55  * Tracing macro. Adds two instructions if TRAPTRACE is defined.
  56  */
  57 #define TT_TRACE(label)         \
  58         ba      label           ;\
  59         rd      %pc, %g7
  60 #else
  61 
  62 #define TT_TRACE(label)
  63 
  64 #endif /* TRAPTRACE */
  65 
  66 #if (TTE_SUSPEND_SHIFT > 0)
  67 #define TTE_SUSPEND_INT_SHIFT(reg)                              \
  68         sllx    reg, TTE_SUSPEND_SHIFT, reg
  69 #else
  70 #define TTE_SUSPEND_INT_SHIFT(reg)
  71 #endif
  72 
  73 /*
  74  * Assumes TSBE_TAG is 0
  75  * Assumes TSBE_INTHI is 0
  76  * Assumes TSBREG.split is 0
  77  */
  78 
  79 #if TSBE_TAG != 0
  80 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
  81 #endif
  82 
  83 #if TSBTAG_INTHI != 0
  84 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
  85 #endif
  86 
  87 /*
  88  * The following code assumes the tsb is not split.
  89  *
  90  * With TSBs no longer shared between processes, it's no longer
  91  * necessary to hash the context bits into the tsb index to get
  92  * tsb coloring; the new implementation treats the TSB as a
  93  * direct-mapped, virtually-addressed cache.
  94  *
  95  * In:
  96  *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
  97  *    tsbbase = base address of TSB (clobbered)
  98  *    tagacc = tag access register (clobbered)
  99  *    szc = size code of TSB (ro)
 100  *    tmp = scratch reg
 101  * Out:
 102  *    tsbbase = pointer to entry in TSB
 103  */
 104 #define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)            \
 105         mov     TSB_ENTRIES(0), tmp     /* nentries in TSB size 0 */    ;\
 106         srlx    tagacc, vpshift, tagacc                                 ;\
 107         sllx    tmp, szc, tmp           /* tmp = nentries in TSB */     ;\
 108         sub     tmp, 1, tmp             /* mask = nentries - 1 */       ;\
 109         and     tagacc, tmp, tmp        /* tsbent = virtpage & mask */      ;\
 110         sllx    tmp, TSB_ENTRY_SHIFT, tmp       /* entry num --> ptr */      ;\
 111         add     tsbbase, tmp, tsbbase   /* add entry offset to TSB base */
 112 
 113 /*
 114  * When the kpm TSB is used it is assumed that it is direct mapped
 115  * using (vaddr>>vpshift)%tsbsz as the index.
 116  *
 117  * Note that, for now, the kpm TSB and kernel TSB are the same for
 118  * each mapping size.  However that need not always be the case.  If
 119  * the trap handlers are updated to search a different TSB for kpm
 120  * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
 121  * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
 122  *
 123  * In:
 124  *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
 125  *    vaddr = virtual address (clobbered)
 126  *    tsbp, szc, tmp = scratch
 127  * Out:
 128  *    tsbp = pointer to entry in TSB
 129  */
 130 #define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)            \
 131         cmp     vpshift, MMU_PAGESHIFT                                  ;\
 132         bne,pn  %icc, 1f                /* branch if large case */      ;\
 133           sethi %hi(kpmsm_tsbsz), szc                                   ;\
 134         sethi   %hi(kpmsm_tsbbase), tsbp                                ;\
 135         ld      [szc + %lo(kpmsm_tsbsz)], szc                           ;\
 136         ldx     [tsbp + %lo(kpmsm_tsbbase)], tsbp                       ;\
 137         ba,pt   %icc, 2f                                                ;\
 138           nop                                                           ;\
 139 1:      sethi   %hi(kpm_tsbsz), szc                                     ;\
 140         sethi   %hi(kpm_tsbbase), tsbp                                  ;\
 141         ld      [szc + %lo(kpm_tsbsz)], szc                             ;\
 142         ldx     [tsbp + %lo(kpm_tsbbase)], tsbp                         ;\
 143 2:      GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
 144 
 145 /*
 146  * Lock the TSBE at virtual address tsbep.
 147  *
 148  * tsbep = TSBE va (ro)
 149  * tmp1, tmp2 = scratch registers (clobbered)
 150  * label = label to jump to if we fail to lock the tsb entry
 151  * %asi = ASI to use for TSB access
 152  *
 153  * NOTE that we flush the TSB using fast VIS instructions that
 154  * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
 155  * not be treated as a locked entry or we'll get stuck spinning on
 156  * an entry that isn't locked but really invalid.
 157  */
 158 
 159 #if defined(UTSB_PHYS)
 160 
 161 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        \
 162         lda     [tsbep]ASI_MEM, tmp1                                    ;\
 163         sethi   %hi(TSBTAG_LOCKED), tmp2                                ;\
 164         cmp     tmp1, tmp2                                              ;\
 165         be,a,pn %icc, label             /* if locked ignore */          ;\
 166           nop                                                           ;\
 167         casa    [tsbep]ASI_MEM, tmp1, tmp2                              ;\
 168         cmp     tmp1, tmp2                                              ;\
 169         bne,a,pn %icc, label            /* didn't lock so ignore */     ;\
 170           nop                                                           ;\
 171         /* tsbe lock acquired */                                        ;\
 172         membar #StoreStore
 173 
 174 #else /* UTSB_PHYS */
 175 
 176 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        \
 177         lda     [tsbep]%asi, tmp1                                       ;\
 178         sethi   %hi(TSBTAG_LOCKED), tmp2                                ;\
 179         cmp     tmp1, tmp2                                              ;\
 180         be,a,pn %icc, label             /* if locked ignore */          ;\
 181           nop                                                           ;\
 182         casa    [tsbep]%asi, tmp1, tmp2                                 ;\
 183         cmp     tmp1, tmp2                                              ;\
 184         bne,a,pn %icc, label            /* didn't lock so ignore */     ;\
 185           nop                                                           ;\
 186         /* tsbe lock acquired */                                        ;\
 187         membar #StoreStore
 188 
 189 #endif /* UTSB_PHYS */
 190 
 191 /*
 192  * Atomically write TSBE at virtual address tsbep.
 193  *
 194  * tsbep = TSBE va (ro)
 195  * tte = TSBE TTE (ro)
 196  * tagtarget = TSBE tag (ro)
 197  * %asi = ASI to use for TSB access
 198  */
 199 
 200 #if defined(UTSB_PHYS)
 201 
 202 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)            \
 203         add     tsbep, TSBE_TTE, tmp1                                   ;\
 204         stxa    tte, [tmp1]ASI_MEM              /* write tte data */    ;\
 205         membar #StoreStore                                              ;\
 206         add     tsbep, TSBE_TAG, tmp1                                   ;\
 207         stxa    tagtarget, [tmp1]ASI_MEM        /* write tte tag & unlock */
 208 
 209 #else /* UTSB_PHYS */
 210 
 211 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)             \
 212         stxa    tte, [tsbep + TSBE_TTE]%asi     /* write tte data */    ;\
 213         membar #StoreStore                                              ;\
 214         stxa    tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
 215 
 216 #endif /* UTSB_PHYS */
 217 
 218 /*
 219  * Load an entry into the TSB at TL > 0.
 220  *
 221  * tsbep = pointer to the TSBE to load as va (ro)
 222  * tte = value of the TTE retrieved and loaded (wo)
 223  * tagtarget = tag target register.  To get TSBE tag to load,
 224  *   we need to mask off the context and leave only the va (clobbered)
 225  * ttepa = pointer to the TTE to retrieve/load as pa (ro)
 226  * tmp1, tmp2 = scratch registers
 227  * label = label to jump to if we fail to lock the tsb entry
 228  * %asi = ASI to use for TSB access
 229  */
 230 
 231 #if defined(UTSB_PHYS)
 232 
 233 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
 234         TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        ;\
 235         /*                                                              ;\
 236          * I don't need to update the TSB then check for the valid tte. ;\
 237          * TSB invalidate will spin till the entry is unlocked. Note,   ;\
 238          * we always invalidate the hash table before we unload the TSB.;\
 239          */                                                             ;\
 240         sllx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 241         ldxa    [ttepa]ASI_MEM, tte                                     ;\
 242         srlx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 243         sethi   %hi(TSBTAG_INVALID), tmp2                               ;\
 244         add     tsbep, TSBE_TAG, tmp1                                   ;\
 245         brgez,a,pn tte, label                                           ;\
 246          sta    tmp2, [tmp1]ASI_MEM                     /* unlock */    ;\
 247         TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)            ;\
 248 label:
 249 
 250 #else /* UTSB_PHYS */
 251 
 252 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
 253         TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        ;\
 254         /*                                                              ;\
 255          * I don't need to update the TSB then check for the valid tte. ;\
 256          * TSB invalidate will spin till the entry is unlocked. Note,   ;\
 257          * we always invalidate the hash table before we unload the TSB.;\
 258          */                                                             ;\
 259         sllx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 260         ldxa    [ttepa]ASI_MEM, tte                                     ;\
 261         srlx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 262         sethi   %hi(TSBTAG_INVALID), tmp2                               ;\
 263         brgez,a,pn tte, label                                           ;\
 264          sta    tmp2, [tsbep + TSBE_TAG]%asi            /* unlock */    ;\
 265         TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)            ;\
 266 label:
 267 
 268 #endif /* UTSB_PHYS */
 269 
 270 /*
 271  * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
 272  *   for ITLB synthesis.
 273  *
 274  * tsbep = pointer to the TSBE to load as va (ro)
 275  * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
 276  *   with exec_perm turned off and exec_synth turned on
 277  * tagtarget = tag target register.  To get TSBE tag to load,
 278  *   we need to mask off the context and leave only the va (clobbered)
 279  * ttepa = pointer to the TTE to retrieve/load as pa (ro)
 280  * tmp1, tmp2 = scratch registers
 281  * label = label to use for branch (text)
 282  * %asi = ASI to use for TSB access
 283  */
 284 
 285 #define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
 286         TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        ;\
 287         /*                                                              ;\
 288          * I don't need to update the TSB then check for the valid tte. ;\
 289          * TSB invalidate will spin till the entry is unlocked. Note,   ;\
 290          * we always invalidate the hash table before we unload the TSB.;\
 291          * Or in 4M pfn offset to TTE and set the exec_perm bit to 0    ;\
 292          * and exec_synth bit to 1.                                     ;\
 293          */                                                             ;\
 294         sllx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 295         mov     tte, tmp1                                               ;\
 296         ldxa    [ttepa]ASI_MEM, tte                                     ;\
 297         srlx    tagtarget, TTARGET_VA_SHIFT, tagtarget                  ;\
 298         sethi   %hi(TSBTAG_INVALID), tmp2                               ;\
 299         brgez,a,pn tte, label                                           ;\
 300          sta    tmp2, [tsbep + TSBE_TAG]%asi            /* unlock */    ;\
 301         or      tte, tmp1, tte                                          ;\
 302         andn    tte, TTE_EXECPRM_INT, tte                               ;\
 303         or      tte, TTE_E_SYNTH_INT, tte                               ;\
 304         TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)            ;\
 305 label:
 306 
 307 /*
 308  * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
 309  *
 310  * tte = value of the TTE, used to get tte_size bits (ro)
 311  * tagaccess = tag access register, used to get 4M pfn bits (ro)
 312  * pfn = 4M pfn bits shifted to offset for tte (out)
 313  * tmp1 = scratch register
 314  * label = label to use for branch (text)
 315  */
 316 
 317 #define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)                 \
 318         /*                                                              ;\
 319          * Get 4M bits from tagaccess for 32M, 256M pagesizes.          ;\
 320          * Return them, shifted, in pfn.                                ;\
 321          */                                                             ;\
 322         srlx    tagaccess, MMU_PAGESHIFT4M, tagaccess                   ;\
 323         srlx    tte, TTE_SZ_SHFT, tmp           /* isolate the */       ;\
 324         andcc   tmp, TTE_SZ_BITS, %g0           /* tte_size bits */     ;\
 325         bz,a,pt %icc, label/**/f                /* if 0, is */          ;\
 326           and   tagaccess, 0x7, tagaccess       /* 32M page size */     ;\
 327         and     tagaccess, 0x3f, tagaccess /* else 256M page size */    ;\
 328 label:                                                                  ;\
 329         sllx    tagaccess, MMU_PAGESHIFT4M, pfn
 330 
 331 /*
 332  * Add 4M TTE size code to a tte for a Panther 32M/256M page,
 333  * for ITLB synthesis.
 334  *
 335  * tte = value of the TTE, used to get tte_size bits (rw)
 336  * tmp1 = scratch register
 337  */
 338 
 339 #define SET_TTE4M_PN(tte, tmp)                                          \
 340         /*                                                              ;\
 341          * Set 4M pagesize tte bits.                                    ;\
 342          */                                                             ;\
 343         set     TTE4M, tmp                                              ;\
 344         sllx    tmp, TTE_SZ_SHFT, tmp                                   ;\
 345         or      tte, tmp, tte
 346 
 347 /*
 348  * Load an entry into the TSB at TL=0.
 349  *
 350  * tsbep = pointer to the TSBE to load as va (ro)
 351  * tteva = pointer to the TTE to load as va (ro)
 352  * tagtarget = TSBE tag to load (which contains no context), synthesized
 353  * to match va of MMU tag target register only (ro)
 354  * tmp1, tmp2 = scratch registers (clobbered)
 355  * label = label to use for branches (text)
 356  * %asi = ASI to use for TSB access
 357  */
 358 
 359 #if defined(UTSB_PHYS)
 360 
 361 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)          \
 362         /* can't rd tteva after locking tsb because it can tlb miss */  ;\
 363         ldx     [tteva], tteva                  /* load tte */          ;\
 364         TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        ;\
 365         sethi   %hi(TSBTAG_INVALID), tmp2                               ;\
 366         add     tsbep, TSBE_TAG, tmp1                                   ;\
 367         brgez,a,pn tteva, label                                         ;\
 368          sta    tmp2, [tmp1]ASI_MEM                     /* unlock */    ;\
 369         TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)          ;\
 370 label:
 371 
 372 #else /* UTSB_PHYS */
 373 
 374 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)          \
 375         /* can't rd tteva after locking tsb because it can tlb miss */  ;\
 376         ldx     [tteva], tteva                  /* load tte */          ;\
 377         TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)                        ;\
 378         sethi   %hi(TSBTAG_INVALID), tmp2                               ;\
 379         brgez,a,pn tteva, label                                         ;\
 380          sta    tmp2, [tsbep + TSBE_TAG]%asi            /* unlock */    ;\
 381         TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)          ;\
 382 label:
 383 
 384 #endif /* UTSB_PHYS */
 385 
 386 /*
 387  * Invalidate a TSB entry in the TSB.
 388  *
 389  * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
 390  *       about this earlier to ensure this is true.  Thus when we are
 391  *       directly referencing tsbep below, we are referencing the tte_tag
 392  *       field of the TSBE.  If this  offset ever changes, the code below
 393  *       will need to be modified.
 394  *
 395  * tsbep = pointer to TSBE as va (ro)
 396  * tag = invalidation is done if this matches the TSBE tag (ro)
 397  * tmp1 - tmp3 = scratch registers (clobbered)
 398  * label = label name to use for branches (text)
 399  * %asi = ASI to use for TSB access
 400  */
 401 
 402 #if defined(UTSB_PHYS)
 403 
 404 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)             \
 405         lda     [tsbep]ASI_MEM, tmp1    /* tmp1 = tsbe tag */           ;\
 406         sethi   %hi(TSBTAG_LOCKED), tmp2                                ;\
 407 label/**/1:                                                             ;\
 408         cmp     tmp1, tmp2              /* see if tsbe is locked, if */ ;\
 409         be,a,pn %icc, label/**/1        /* so, loop until unlocked */   ;\
 410           lda   [tsbep]ASI_MEM, tmp1    /* reloading value each time */ ;\
 411         ldxa    [tsbep]ASI_MEM, tmp3    /* tmp3 = tsbe tag */           ;\
 412         cmp     tag, tmp3               /* compare tags */              ;\
 413         bne,pt  %xcc, label/**/2        /* if different, do nothing */  ;\
 414           sethi %hi(TSBTAG_INVALID), tmp3                               ;\
 415         casa    [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\
 416         cmp     tmp1, tmp3              /* if not successful */         ;\
 417         bne,a,pn %icc, label/**/1       /* start over from the top */   ;\
 418           lda   [tsbep]ASI_MEM, tmp1    /* reloading tsbe tag */        ;\
 419 label/**/2:
 420 
 421 #else /* UTSB_PHYS */
 422 
 423 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)             \
 424         lda     [tsbep]%asi, tmp1       /* tmp1 = tsbe tag */           ;\
 425         sethi   %hi(TSBTAG_LOCKED), tmp2                                ;\
 426 label/**/1:                                                             ;\
 427         cmp     tmp1, tmp2              /* see if tsbe is locked, if */ ;\
 428         be,a,pn %icc, label/**/1        /* so, loop until unlocked */   ;\
 429           lda   [tsbep]%asi, tmp1       /* reloading value each time */ ;\
 430         ldxa    [tsbep]%asi, tmp3       /* tmp3 = tsbe tag */           ;\
 431         cmp     tag, tmp3               /* compare tags */              ;\
 432         bne,pt  %xcc, label/**/2        /* if different, do nothing */  ;\
 433           sethi %hi(TSBTAG_INVALID), tmp3                               ;\
 434         casa    [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */    ;\
 435         cmp     tmp1, tmp3              /* if not successful */         ;\
 436         bne,a,pn %icc, label/**/1       /* start over from the top */   ;\
 437           lda   [tsbep]%asi, tmp1       /* reloading tsbe tag */        ;\
 438 label/**/2:
 439 
 440 #endif /* UTSB_PHYS */
 441 
 442 #if TSB_SOFTSZ_MASK < TSB_SZ_MASK
 443 #error  - TSB_SOFTSZ_MASK too small
 444 #endif
 445 
 446 
 447 /*
 448  * An implementation of setx which will be hot patched at run time.
 449  * since it is being hot patched, there is no value passed in.
 450  * Thus, essentially we are implementing
 451  *      setx value, tmp, dest
 452  * where value is RUNTIME_PATCH (aka 0) in this case.
 453  */
 454 #define RUNTIME_PATCH_SETX(dest, tmp)                                   \
 455         sethi   %hh(RUNTIME_PATCH), tmp                                 ;\
 456         sethi   %lm(RUNTIME_PATCH), dest                                ;\
 457         or      tmp, %hm(RUNTIME_PATCH), tmp                            ;\
 458         or      dest, %lo(RUNTIME_PATCH), dest                          ;\
 459         sllx    tmp, 32, tmp                                            ;\
 460         nop                             /* for perf reasons */          ;\
 461         or      tmp, dest, dest         /* contents of patched value */
 462 
 463 
 464         .seg    ".data"
 465         .global sfmmu_panic1
 466 sfmmu_panic1:
 467         .asciz  "sfmmu_asm: interrupts already disabled"
 468 
 469         .global sfmmu_panic3
 470 sfmmu_panic3:
 471         .asciz  "sfmmu_asm: sfmmu_vatopfn called for user"
 472 
 473         .global sfmmu_panic4
 474 sfmmu_panic4:
 475         .asciz  "sfmmu_asm: 4M tsb pointer mis-match"
 476 
 477         .global sfmmu_panic5
 478 sfmmu_panic5:
 479         .asciz  "sfmmu_asm: no unlocked TTEs in TLB 0"
 480 
 481         .global sfmmu_panic6
 482 sfmmu_panic6:
 483         .asciz  "sfmmu_asm: interrupts not disabled"
 484 
 485         .global sfmmu_panic7
 486 sfmmu_panic7:
 487         .asciz  "sfmmu_asm: kernel as"
 488 
 489         .global sfmmu_panic8
 490 sfmmu_panic8:
 491         .asciz  "sfmmu_asm: gnum is zero"
 492 
 493         .global sfmmu_panic9
 494 sfmmu_panic9:
 495         .asciz  "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
 496 
 497         .global sfmmu_panic10
 498 sfmmu_panic10:
 499         .asciz  "sfmmu_asm: valid SCD with no 3rd scd TSB"
 500 
 501         .global sfmmu_panic11
 502 sfmmu_panic11:
 503         .asciz  "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform"
 504 
 505         ENTRY(sfmmu_disable_intrs)
 506         rdpr    %pstate, %o0
 507 #ifdef DEBUG
 508         PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
 509 #endif /* DEBUG */
 510         retl
 511           wrpr   %o0, PSTATE_IE, %pstate
 512         SET_SIZE(sfmmu_disable_intrs)
 513 
 514         ENTRY(sfmmu_enable_intrs)
 515         retl
 516           wrpr    %g0, %o0, %pstate
 517         SET_SIZE(sfmmu_enable_intrs)
 518 
 519 /*
 520  * This routine is called both by resume() and sfmmu_get_ctx() to
 521  * allocate a new context for the process on a MMU.
 522  * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
 523  * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
 524  * is the case when sfmmu_alloc_ctx is called from resume().
 525  *
 526  * The caller must disable interrupts before entering this routine.
 527  * To reduce ctx switch overhead, the code contains both 'fast path' and
 528  * 'slow path' code. The fast path code covers the common case where only
 529  * a quick check is needed and the real ctx allocation is not required.
 530  * It can be done without holding the per-process (PP) lock.
 531  * The 'slow path' code must be protected by the PP Lock and performs ctx
 532  * allocation.
 533  * Hardware context register and HAT mmu cnum are updated accordingly.
 534  *
 535  * %o0 - sfmmup
 536  * %o1 - allocflag
 537  * %o2 - CPU
 538  * %o3 - sfmmu private/shared flag
 539  *
 540  * ret - 0: no ctx is allocated
 541  *       1: a ctx is allocated
 542  */
 543         ENTRY_NP(sfmmu_alloc_ctx)
 544 
 545 #ifdef DEBUG
 546         sethi   %hi(ksfmmup), %g1
 547         ldx     [%g1 + %lo(ksfmmup)], %g1
 548         cmp     %g1, %o0
 549         bne,pt   %xcc, 0f
 550           nop
 551 
 552         sethi   %hi(panicstr), %g1              ! if kernel as, panic
 553         ldx     [%g1 + %lo(panicstr)], %g1
 554         tst     %g1
 555         bnz,pn  %icc, 7f
 556           nop
 557 
 558         sethi   %hi(sfmmu_panic7), %o0
 559         call    panic
 560           or    %o0, %lo(sfmmu_panic7), %o0
 561 
 562 7:
 563         retl
 564           mov   %g0, %o0                        ! %o0 = ret = 0
 565 
 566 0:
 567         PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
 568 #endif /* DEBUG */
 569 
 570         mov     %o3, %g1                        ! save sfmmu pri/sh flag in %g1
 571 
 572         ! load global mmu_ctxp info
 573         ldx     [%o2 + CPU_MMU_CTXP], %o3               ! %o3 = mmu_ctx_t ptr
 574 
 575 #ifdef sun4v
 576         /* During suspend on sun4v, context domains can be temporary removed */
 577         brz,a,pn       %o3, 0f
 578           nop
 579 #endif
 580 
 581         lduw    [%o2 + CPU_MMU_IDX], %g2                ! %g2 = mmu index
 582 
 583         ! load global mmu_ctxp gnum
 584         ldx     [%o3 + MMU_CTX_GNUM], %o4               ! %o4 = mmu_ctxp->gnum
 585 
 586 #ifdef DEBUG
 587         cmp     %o4, %g0                ! mmu_ctxp->gnum should never be 0
 588         bne,pt  %xcc, 3f
 589           nop
 590 
 591         sethi   %hi(panicstr), %g1      ! test if panicstr is already set
 592         ldx     [%g1 + %lo(panicstr)], %g1
 593         tst     %g1
 594         bnz,pn  %icc, 1f
 595           nop
 596 
 597         sethi   %hi(sfmmu_panic8), %o0
 598         call    panic
 599           or    %o0, %lo(sfmmu_panic8), %o0
 600 1:
 601         retl
 602           mov   %g0, %o0                        ! %o0 = ret = 0
 603 3:
 604 #endif
 605 
 606         ! load HAT sfmmu_ctxs[mmuid] gnum, cnum
 607 
 608         sllx    %g2, SFMMU_MMU_CTX_SHIFT, %g2
 609         add     %o0, %g2, %g2           ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
 610 
 611         /*
 612          * %g5 = sfmmu gnum returned
 613          * %g6 = sfmmu cnum returned
 614          * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
 615          * %g4 = scratch
 616          *
 617          * Fast path code, do a quick check.
 618          */
 619         SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
 620 
 621         cmp     %g6, INVALID_CONTEXT            ! hat cnum == INVALID ??
 622         bne,pt  %icc, 1f                        ! valid hat cnum, check gnum
 623           nop
 624 
 625         ! cnum == INVALID, check allocflag
 626         mov     %g0, %g4        ! %g4 = ret = 0
 627         brz,pt  %o1, 8f         ! allocflag == 0, skip ctx allocation, bail
 628           mov   %g6, %o1
 629 
 630         ! (invalid HAT cnum) && (allocflag == 1)
 631         ba,pt   %icc, 2f
 632           nop
 633 #ifdef sun4v
 634 0:
 635         set     INVALID_CONTEXT, %o1
 636         membar  #LoadStore|#StoreStore
 637         ba,pt   %icc, 8f
 638           mov   %g0, %g4                ! %g4 = ret = 0
 639 #endif
 640 1:
 641         ! valid HAT cnum, check gnum
 642         cmp     %g5, %o4
 643         mov     1, %g4                          !%g4 = ret = 1
 644         be,a,pt %icc, 8f                        ! gnum unchanged, go to done
 645           mov   %g6, %o1
 646 
 647 2:
 648         /*
 649          * Grab per process (PP) sfmmu_ctx_lock spinlock,
 650          * followed by the 'slow path' code.
 651          */
 652         ldstub  [%o0 + SFMMU_CTX_LOCK], %g3     ! %g3 = per process (PP) lock
 653 3:
 654         brz     %g3, 5f
 655           nop
 656 4:
 657         brnz,a,pt       %g3, 4b                         ! spin if lock is 1
 658           ldub  [%o0 + SFMMU_CTX_LOCK], %g3
 659         ba      %xcc, 3b                                ! retry the lock
 660           ldstub        [%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
 661 
 662 5:
 663         membar  #LoadLoad
 664         /*
 665          * %g5 = sfmmu gnum returned
 666          * %g6 = sfmmu cnum returned
 667          * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
 668          * %g4 = scratch
 669          */
 670         SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
 671 
 672         cmp     %g6, INVALID_CONTEXT            ! hat cnum == INVALID ??
 673         bne,pt  %icc, 1f                        ! valid hat cnum, check gnum
 674           nop
 675 
 676         ! cnum == INVALID, check allocflag
 677         mov     %g0, %g4        ! %g4 = ret = 0
 678         brz,pt  %o1, 2f         ! allocflag == 0, called from resume, set hw
 679           mov   %g6, %o1
 680 
 681         ! (invalid HAT cnum) && (allocflag == 1)
 682         ba,pt   %icc, 6f
 683           nop
 684 1:
 685         ! valid HAT cnum, check gnum
 686         cmp     %g5, %o4
 687         mov     1, %g4                          ! %g4 = ret  = 1
 688         be,a,pt %icc, 2f                        ! gnum unchanged, go to done
 689           mov   %g6, %o1
 690 
 691         ba,pt   %icc, 6f
 692           nop
 693 2:
 694         membar  #LoadStore|#StoreStore
 695         ba,pt %icc, 8f
 696           clrb  [%o0 + SFMMU_CTX_LOCK]
 697 6:
 698         /*
 699          * We get here if we do not have a valid context, or
 700          * the HAT gnum does not match global gnum. We hold
 701          * sfmmu_ctx_lock spinlock. Allocate that context.
 702          *
 703          * %o3 = mmu_ctxp
 704          */
 705         add     %o3, MMU_CTX_CNUM, %g3
 706         ld      [%o3 + MMU_CTX_NCTXS], %g4
 707 
 708         /*
 709          * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
 710          * %g3 = mmu cnum address
 711          * %g4 = mmu nctxs
 712          *
 713          * %o0 = sfmmup
 714          * %o1 = mmu current cnum value (used as new cnum)
 715          * %o4 = mmu gnum
 716          *
 717          * %o5 = scratch
 718          */
 719         ld      [%g3], %o1
 720 0:
 721         cmp     %o1, %g4
 722         bl,a,pt %icc, 1f
 723           add   %o1, 1, %o5             ! %o5 = mmu_ctxp->cnum + 1
 724 
 725         /*
 726          * cnum reachs max, bail, so wrap around can be performed later.
 727          */
 728         set     INVALID_CONTEXT, %o1
 729         mov     %g0, %g4                ! %g4 = ret = 0
 730 
 731         membar  #LoadStore|#StoreStore
 732         ba,pt   %icc, 8f
 733           clrb  [%o0 + SFMMU_CTX_LOCK]
 734 1:
 735         ! %g3 = addr of mmu_ctxp->cnum
 736         ! %o5 = mmu_ctxp->cnum + 1
 737         cas     [%g3], %o1, %o5
 738         cmp     %o1, %o5
 739         bne,a,pn %xcc, 0b       ! cas failed
 740           ld    [%g3], %o1
 741 
 742 #ifdef DEBUG
 743         set     MAX_SFMMU_CTX_VAL, %o5
 744         cmp     %o1, %o5
 745         ble,pt %icc, 2f
 746           nop
 747 
 748         sethi   %hi(sfmmu_panic9), %o0
 749         call    panic
 750           or    %o0, %lo(sfmmu_panic9), %o0
 751 2:
 752 #endif
 753         ! update hat gnum and cnum
 754         sllx    %o4, SFMMU_MMU_GNUM_RSHIFT, %o4
 755         or      %o4, %o1, %o4
 756         stx     %o4, [%g2 + SFMMU_CTXS]
 757 
 758         membar  #LoadStore|#StoreStore
 759         clrb    [%o0 + SFMMU_CTX_LOCK]
 760 
 761         mov     1, %g4                  ! %g4 = ret = 1
 762 8:
 763         /*
 764          * program the secondary context register
 765          *
 766          * %o1 = cnum
 767          * %g1 = sfmmu private/shared flag (0:private,  1:shared)
 768          */
 769 
 770         /*
 771          * When we come here and context is invalid, we want to set both
 772          * private and shared ctx regs to INVALID. In order to
 773          * do so, we set the sfmmu priv/shared flag to 'private' regardless
 774          * so that private ctx reg will be set to invalid.
 775          * Note that on sun4v values written to private context register are
 776          * automatically written to corresponding shared context register as
 777          * well. On sun4u SET_SECCTX() will invalidate shared context register
 778          * when it sets a private secondary context register.
 779          */
 780 
 781         cmp     %o1, INVALID_CONTEXT
 782         be,a,pn %icc, 9f
 783           clr   %g1
 784 9:
 785 
 786 #ifdef  sun4u
 787         ldub    [%o0 + SFMMU_CEXT], %o2
 788         sll     %o2, CTXREG_EXT_SHIFT, %o2
 789         or      %o1, %o2, %o1
 790 #endif /* sun4u */
 791 
 792         SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
 793 
 794         retl
 795           mov   %g4, %o0                        ! %o0 = ret
 796 
 797         SET_SIZE(sfmmu_alloc_ctx)
 798 
 799         ENTRY_NP(sfmmu_modifytte)
 800         ldx     [%o2], %g3                      /* current */
 801         ldx     [%o0], %g1                      /* original */
 802 2:
 803         ldx     [%o1], %g2                      /* modified */
 804         cmp     %g2, %g3                        /* is modified = current? */
 805         be,a,pt %xcc,1f                         /* yes, don't write */
 806         stx     %g3, [%o0]                      /* update new original */
 807         casx    [%o2], %g1, %g2
 808         cmp     %g1, %g2
 809         be,pt   %xcc, 1f                        /* cas succeeded - return */
 810           nop
 811         ldx     [%o2], %g3                      /* new current */
 812         stx     %g3, [%o0]                      /* save as new original */
 813         ba,pt   %xcc, 2b
 814           mov   %g3, %g1
 815 1:      retl
 816         membar  #StoreLoad
 817         SET_SIZE(sfmmu_modifytte)
 818 
 819         ENTRY_NP(sfmmu_modifytte_try)
 820         ldx     [%o1], %g2                      /* modified */
 821         ldx     [%o2], %g3                      /* current */
 822         ldx     [%o0], %g1                      /* original */
 823         cmp     %g3, %g2                        /* is modified = current? */
 824         be,a,pn %xcc,1f                         /* yes, don't write */
 825         mov     0, %o1                          /* as if cas failed. */
 826 
 827         casx    [%o2], %g1, %g2
 828         membar  #StoreLoad
 829         cmp     %g1, %g2
 830         movne   %xcc, -1, %o1                   /* cas failed. */
 831         move    %xcc, 1, %o1                    /* cas succeeded. */
 832 1:
 833         stx     %g2, [%o0]                      /* report "current" value */
 834         retl
 835         mov     %o1, %o0
 836         SET_SIZE(sfmmu_modifytte_try)
 837 
 838         ENTRY_NP(sfmmu_copytte)
 839         ldx     [%o0], %g1
 840         retl
 841         stx     %g1, [%o1]
 842         SET_SIZE(sfmmu_copytte)
 843 
 844 
 845         /*
 846          * Calculate a TSB entry pointer for the given TSB, va, pagesize.
 847          * %o0 = TSB base address (in), pointer to TSB entry (out)
 848          * %o1 = vaddr (in)
 849          * %o2 = vpshift (in)
 850          * %o3 = tsb size code (in)
 851          * %o4 = scratch register
 852          */
 853         ENTRY_NP(sfmmu_get_tsbe)
 854         GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
 855         retl
 856         nop
 857         SET_SIZE(sfmmu_get_tsbe)
 858 
 859         /*
 860          * Return a TSB tag for the given va.
 861          * %o0 = va (in/clobbered)
 862          * %o0 = va shifted to be in tsb tag format (with no context) (out)
 863          */
 864         ENTRY_NP(sfmmu_make_tsbtag)
 865         retl
 866         srln    %o0, TTARGET_VA_SHIFT, %o0
 867         SET_SIZE(sfmmu_make_tsbtag)
 868 
 869 /*
 870  * Other sfmmu primitives
 871  */
 872 
 873 
 874 #define I_SIZE          4
 875 
 876         ENTRY_NP(sfmmu_fix_ktlb_traptable)
 877         /*
 878          * %o0 = start of patch area
 879          * %o1 = size code of TSB to patch
 880          * %o3 = scratch
 881          */
 882         /* fix sll */
 883         ld      [%o0], %o3                      /* get sll */
 884         sub     %o3, %o1, %o3                   /* decrease shift by tsb szc */
 885         st      %o3, [%o0]                      /* write sll */
 886         flush   %o0
 887         /* fix srl */
 888         add     %o0, I_SIZE, %o0                /* goto next instr. */
 889         ld      [%o0], %o3                      /* get srl */
 890         sub     %o3, %o1, %o3                   /* decrease shift by tsb szc */
 891         st      %o3, [%o0]                      /* write srl */
 892         retl
 893         flush   %o0
 894         SET_SIZE(sfmmu_fix_ktlb_traptable)
 895 
 896         ENTRY_NP(sfmmu_fixup_ktsbbase)
 897         /*
 898          * %o0 = start of patch area
 899          * %o5 = kernel virtual or physical tsb base address
 900          * %o2, %o3 are used as scratch registers.
 901          */
 902         /* fixup sethi instruction */
 903         ld      [%o0], %o3
 904         srl     %o5, 10, %o2                    ! offset is bits 32:10
 905         or      %o3, %o2, %o3                   ! set imm22
 906         st      %o3, [%o0]
 907         /* fixup offset of lduw/ldx */
 908         add     %o0, I_SIZE, %o0                ! next instr
 909         ld      [%o0], %o3
 910         and     %o5, 0x3ff, %o2                 ! set imm13 to bits 9:0
 911         or      %o3, %o2, %o3
 912         st      %o3, [%o0]
 913         retl
 914         flush   %o0
 915         SET_SIZE(sfmmu_fixup_ktsbbase)
 916 
 917         ENTRY_NP(sfmmu_fixup_setx)
 918         /*
 919          * %o0 = start of patch area
 920          * %o4 = 64 bit value to patch
 921          * %o2, %o3 are used as scratch registers.
 922          *
 923          * Note: Assuming that all parts of the instructions which need to be
 924          *       patched correspond to RUNTIME_PATCH (aka 0)
 925          *
 926          * Note the implementation of setx which is being patched is as follows:
 927          *
 928          * sethi   %hh(RUNTIME_PATCH), tmp
 929          * sethi   %lm(RUNTIME_PATCH), dest
 930          * or      tmp, %hm(RUNTIME_PATCH), tmp
 931          * or      dest, %lo(RUNTIME_PATCH), dest
 932          * sllx    tmp, 32, tmp
 933          * nop
 934          * or      tmp, dest, dest
 935          *
 936          * which differs from the implementation in the
 937          * "SPARC Architecture Manual"
 938          */
 939         /* fixup sethi instruction */
 940         ld      [%o0], %o3
 941         srlx    %o4, 42, %o2                    ! bits [63:42]
 942         or      %o3, %o2, %o3                   ! set imm22
 943         st      %o3, [%o0]
 944         /* fixup sethi instruction */
 945         add     %o0, I_SIZE, %o0                ! next instr
 946         ld      [%o0], %o3
 947         sllx    %o4, 32, %o2                    ! clear upper bits
 948         srlx    %o2, 42, %o2                    ! bits [31:10]
 949         or      %o3, %o2, %o3                   ! set imm22
 950         st      %o3, [%o0]
 951         /* fixup or instruction */
 952         add     %o0, I_SIZE, %o0                ! next instr
 953         ld      [%o0], %o3
 954         srlx    %o4, 32, %o2                    ! bits [63:32]
 955         and     %o2, 0x3ff, %o2                 ! bits [41:32]
 956         or      %o3, %o2, %o3                   ! set imm
 957         st      %o3, [%o0]
 958         /* fixup or instruction */
 959         add     %o0, I_SIZE, %o0                ! next instr
 960         ld      [%o0], %o3
 961         and     %o4, 0x3ff, %o2                 ! bits [9:0]
 962         or      %o3, %o2, %o3                   ! set imm
 963         st      %o3, [%o0]
 964         retl
 965         flush   %o0
 966         SET_SIZE(sfmmu_fixup_setx)
 967 
 968         ENTRY_NP(sfmmu_fixup_or)
 969         /*
 970          * %o0 = start of patch area
 971          * %o4 = 32 bit value to patch
 972          * %o2, %o3 are used as scratch registers.
 973          * Note: Assuming that all parts of the instructions which need to be
 974          *       patched correspond to RUNTIME_PATCH (aka 0)
 975          */
 976         ld      [%o0], %o3
 977         and     %o4, 0x3ff, %o2                 ! bits [9:0]
 978         or      %o3, %o2, %o3                   ! set imm
 979         st      %o3, [%o0]
 980         retl
 981         flush   %o0
 982         SET_SIZE(sfmmu_fixup_or)
 983 
 984         ENTRY_NP(sfmmu_fixup_shiftx)
 985         /*
 986          * %o0 = start of patch area
 987          * %o4 = signed int immediate value to add to sllx/srlx imm field
 988          * %o2, %o3 are used as scratch registers.
 989          *
 990          * sllx/srlx store the 6 bit immediate value in the lowest order bits
 991          * so we do a simple add.  The caller must be careful to prevent
 992          * overflow, which could easily occur if the initial value is nonzero!
 993          */
 994         ld      [%o0], %o3                      ! %o3 = instruction to patch
 995         and     %o3, 0x3f, %o2                  ! %o2 = existing imm value
 996         add     %o2, %o4, %o2                   ! %o2 = new imm value
 997         andn    %o3, 0x3f, %o3                  ! clear old imm value
 998         and     %o2, 0x3f, %o2                  ! truncate new imm value
 999         or      %o3, %o2, %o3                   ! set new imm value
1000         st      %o3, [%o0]                      ! store updated instruction
1001         retl
1002         flush   %o0
1003         SET_SIZE(sfmmu_fixup_shiftx)
1004 
1005         ENTRY_NP(sfmmu_fixup_mmu_asi)
1006         /*
1007          * Patch imm_asi of all ldda instructions in the MMU
1008          * trap handlers.  We search MMU_PATCH_INSTR instructions
1009          * starting from the itlb miss handler (trap 0x64).
1010          * %o0 = address of tt[0,1]_itlbmiss
1011          * %o1 = imm_asi to setup, shifted by appropriate offset.
1012          * %o3 = number of instructions to search
1013          * %o4 = reserved by caller: called from leaf routine
1014          */
1015 1:      ldsw    [%o0], %o2                      ! load instruction to %o2
1016         brgez,pt %o2, 2f
1017           srl   %o2, 30, %o5
1018         btst    1, %o5                          ! test bit 30; skip if not set
1019         bz,pt   %icc, 2f
1020           sllx  %o2, 39, %o5                    ! bit 24 -> bit 63
1021         srlx    %o5, 58, %o5                    ! isolate op3 part of opcode
1022         xor     %o5, 0x13, %o5                  ! 01 0011 binary == ldda
1023         brnz,pt %o5, 2f                         ! skip if not a match
1024           or    %o2, %o1, %o2                   ! or in imm_asi
1025         st      %o2, [%o0]                      ! write patched instruction
1026 2:      dec     %o3
1027         brnz,a,pt %o3, 1b                       ! loop until we're done
1028           add   %o0, I_SIZE, %o0
1029         retl
1030         flush   %o0
1031         SET_SIZE(sfmmu_fixup_mmu_asi)
1032 
1033         /*
1034          * Patch immediate ASI used to access the TSB in the
1035          * trap table.
1036          * inputs: %o0 = value of ktsb_phys
1037          */
1038         ENTRY_NP(sfmmu_patch_mmu_asi)
1039         mov     %o7, %o4                        ! save return pc in %o4
1040         mov     ASI_QUAD_LDD_PHYS, %o3          ! set QUAD_LDD_PHYS by default
1041 
1042 #ifdef sun4v
1043 
1044         /*
1045          * Check ktsb_phys. It must be non-zero for sun4v, panic if not.
1046          */
1047 
1048         brnz,pt %o0, do_patch
1049         nop
1050 
1051         sethi   %hi(sfmmu_panic11), %o0
1052         call    panic
1053           or    %o0, %lo(sfmmu_panic11), %o0
1054 do_patch:
1055 
1056 #else /* sun4v */
1057         /*
1058          * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0).
1059          * Note that ASI_NQUAD_LD is not defined/used for sun4v
1060          */
1061         movrz   %o0, ASI_NQUAD_LD, %o3
1062 
1063 #endif /* sun4v */
1064 
1065         sll     %o3, 5, %o1                     ! imm_asi offset
1066         mov     6, %o3                          ! number of instructions
1067         sethi   %hi(dktsb), %o0                 ! to search
1068         call    sfmmu_fixup_mmu_asi             ! patch kdtlb miss
1069           or    %o0, %lo(dktsb), %o0
1070         mov     6, %o3                          ! number of instructions
1071         sethi   %hi(dktsb4m), %o0               ! to search
1072         call    sfmmu_fixup_mmu_asi             ! patch kdtlb4m miss
1073           or    %o0, %lo(dktsb4m), %o0
1074         mov     6, %o3                          ! number of instructions
1075         sethi   %hi(iktsb), %o0                 ! to search
1076         call    sfmmu_fixup_mmu_asi             ! patch kitlb miss
1077           or    %o0, %lo(iktsb), %o0
1078         mov     6, %o3                          ! number of instructions
1079         sethi   %hi(iktsb4m), %o0               ! to search
1080         call    sfmmu_fixup_mmu_asi             ! patch kitlb4m miss
1081           or    %o0, %lo(iktsb4m), %o0
1082         mov     %o4, %o7                        ! retore return pc -- leaf
1083         retl
1084         nop
1085         SET_SIZE(sfmmu_patch_mmu_asi)
1086 
1087 
1088         ENTRY_NP(sfmmu_patch_ktsb)
1089         /*
1090          * We need to fix iktsb, dktsb, et. al.
1091          */
1092         save    %sp, -SA(MINFRAME), %sp
1093         set     ktsb_phys, %o1
1094         ld      [%o1], %o4
1095         set     ktsb_base, %o5
1096         set     ktsb4m_base, %l1
1097         brz,pt  %o4, 1f
1098           nop
1099         set     ktsb_pbase, %o5
1100         set     ktsb4m_pbase, %l1
1101 1:
1102         sethi   %hi(ktsb_szcode), %o1
1103         ld      [%o1 + %lo(ktsb_szcode)], %o1   /* %o1 = ktsb size code */
1104 
1105         sethi   %hi(iktsb), %o0
1106         call    sfmmu_fix_ktlb_traptable
1107           or    %o0, %lo(iktsb), %o0
1108 
1109         sethi   %hi(dktsb), %o0
1110         call    sfmmu_fix_ktlb_traptable
1111           or    %o0, %lo(dktsb), %o0
1112 
1113         sethi   %hi(ktsb4m_szcode), %o1
1114         ld      [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */
1115 
1116         sethi   %hi(iktsb4m), %o0
1117         call    sfmmu_fix_ktlb_traptable
1118           or    %o0, %lo(iktsb4m), %o0
1119 
1120         sethi   %hi(dktsb4m), %o0
1121         call    sfmmu_fix_ktlb_traptable
1122           or    %o0, %lo(dktsb4m), %o0
1123 
1124 #ifndef sun4v
1125         mov     ASI_N, %o2
1126         movrnz  %o4, ASI_MEM, %o2       ! setup kernel 32bit ASI to patch
1127         mov     %o2, %o4                ! sfmmu_fixup_or needs this in %o4
1128         sethi   %hi(tsb_kernel_patch_asi), %o0
1129         call    sfmmu_fixup_or
1130           or    %o0, %lo(tsb_kernel_patch_asi), %o0
1131 #endif /* !sun4v */
1132 
1133         ldx     [%o5], %o4              ! load ktsb base addr (VA or PA)
1134 
1135         sethi   %hi(dktsbbase), %o0
1136         call    sfmmu_fixup_setx        ! patch value of ktsb base addr
1137           or    %o0, %lo(dktsbbase), %o0
1138 
1139         sethi   %hi(iktsbbase), %o0
1140         call    sfmmu_fixup_setx        ! patch value of ktsb base addr
1141           or    %o0, %lo(iktsbbase), %o0
1142 
1143         sethi   %hi(sfmmu_kprot_patch_ktsb_base), %o0
1144         call    sfmmu_fixup_setx        ! patch value of ktsb base addr
1145           or    %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1146 
1147 #ifdef sun4v
1148         sethi   %hi(sfmmu_dslow_patch_ktsb_base), %o0
1149         call    sfmmu_fixup_setx        ! patch value of ktsb base addr
1150           or    %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1151 #endif /* sun4v */
1152 
1153         ldx     [%l1], %o4              ! load ktsb4m base addr (VA or PA)
1154 
1155         sethi   %hi(dktsb4mbase), %o0
1156         call    sfmmu_fixup_setx        ! patch value of ktsb4m base addr
1157           or    %o0, %lo(dktsb4mbase), %o0
1158 
1159         sethi   %hi(iktsb4mbase), %o0
1160         call    sfmmu_fixup_setx        ! patch value of ktsb4m base addr
1161           or    %o0, %lo(iktsb4mbase), %o0
1162 
1163         sethi   %hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1164         call    sfmmu_fixup_setx        ! patch value of ktsb4m base addr
1165           or    %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1166 
1167 #ifdef sun4v
1168         sethi   %hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1169         call    sfmmu_fixup_setx        ! patch value of ktsb4m base addr
1170           or    %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1171 #endif /* sun4v */
1172 
1173         set     ktsb_szcode, %o4
1174         ld      [%o4], %o4
1175         sethi   %hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1176         call    sfmmu_fixup_or          ! patch value of ktsb_szcode
1177           or    %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1178 
1179 #ifdef sun4v
1180         sethi   %hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1181         call    sfmmu_fixup_or          ! patch value of ktsb_szcode
1182           or    %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1183 #endif /* sun4v */
1184 
1185         set     ktsb4m_szcode, %o4
1186         ld      [%o4], %o4
1187         sethi   %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1188         call    sfmmu_fixup_or          ! patch value of ktsb4m_szcode
1189           or    %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1190 
1191 #ifdef sun4v
1192         sethi   %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1193         call    sfmmu_fixup_or          ! patch value of ktsb4m_szcode
1194           or    %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1195 #endif /* sun4v */
1196 
1197         ret
1198         restore
1199         SET_SIZE(sfmmu_patch_ktsb)
1200 
1201         ENTRY_NP(sfmmu_kpm_patch_tlbm)
1202         /*
1203          * Fixup trap handlers in common segkpm case.  This is reserved
1204          * for future use should kpm TSB be changed to be other than the
1205          * kernel TSB.
1206          */
1207         retl
1208         nop
1209         SET_SIZE(sfmmu_kpm_patch_tlbm)
1210 
1211         ENTRY_NP(sfmmu_kpm_patch_tsbm)
1212         /*
1213          * nop the branch to sfmmu_kpm_dtsb_miss_small
1214          * in the case where we are using large pages for
1215          * seg_kpm (and hence must probe the second TSB for
1216          * seg_kpm VAs)
1217          */
1218         set     dktsb4m_kpmcheck_small, %o0
1219         MAKE_NOP_INSTR(%o1)
1220         st      %o1, [%o0]
1221         flush   %o0
1222         retl
1223         nop
1224         SET_SIZE(sfmmu_kpm_patch_tsbm)
1225 
1226         ENTRY_NP(sfmmu_patch_utsb)
1227 #ifdef UTSB_PHYS
1228         retl
1229         nop
1230 #else /* UTSB_PHYS */
1231         /*
1232          * We need to hot patch utsb_vabase and utsb4m_vabase
1233          */
1234         save    %sp, -SA(MINFRAME), %sp
1235 
1236         /* patch value of utsb_vabase */
1237         set     utsb_vabase, %o1
1238         ldx     [%o1], %o4
1239         sethi   %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1240         call    sfmmu_fixup_setx
1241           or    %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1242         sethi   %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1243         call    sfmmu_fixup_setx
1244           or    %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1245         sethi   %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1246         call    sfmmu_fixup_setx
1247           or    %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1248 
1249         /* patch value of utsb4m_vabase */
1250         set     utsb4m_vabase, %o1
1251         ldx     [%o1], %o4
1252         sethi   %hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1253         call    sfmmu_fixup_setx
1254           or    %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1255         sethi   %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1256         call    sfmmu_fixup_setx
1257           or    %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1258         sethi   %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1259         call    sfmmu_fixup_setx
1260           or    %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1261 
1262         /*
1263          * Patch TSB base register masks and shifts if needed.
1264          * By default the TSB base register contents are set up for 4M slab.
1265          * If we're using a smaller slab size and reserved VA range we need
1266          * to patch up those values here.
1267          */
1268         set     tsb_slab_shift, %o1
1269         set     MMU_PAGESHIFT4M, %o4
1270         lduw    [%o1], %o3
1271         subcc   %o4, %o3, %o4
1272         bz,pt   %icc, 1f
1273           /* delay slot safe */
1274 
1275         /* patch reserved VA range size if needed. */
1276         sethi   %hi(sfmmu_tsb_1st_resv_offset), %o0
1277         call    sfmmu_fixup_shiftx
1278           or    %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1279         call    sfmmu_fixup_shiftx
1280           add   %o0, I_SIZE, %o0
1281         sethi   %hi(sfmmu_tsb_2nd_resv_offset), %o0
1282         call    sfmmu_fixup_shiftx
1283           or    %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1284         call    sfmmu_fixup_shiftx
1285           add   %o0, I_SIZE, %o0
1286 1:
1287         /* patch TSBREG_VAMASK used to set up TSB base register */
1288         set     tsb_slab_mask, %o1
1289         ldx     [%o1], %o4
1290         sethi   %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1291         call    sfmmu_fixup_or
1292           or    %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1293         sethi   %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1294         call    sfmmu_fixup_or
1295           or    %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1296 
1297         ret
1298         restore
1299 #endif /* UTSB_PHYS */
1300         SET_SIZE(sfmmu_patch_utsb)
1301 
1302         ENTRY_NP(sfmmu_patch_shctx)
1303 #ifdef sun4u
1304         retl
1305           nop
1306 #else /* sun4u */
1307         set     sfmmu_shctx_cpu_mondo_patch, %o0
1308         MAKE_JMP_INSTR(5, %o1, %o2)     ! jmp       %g5
1309         st      %o1, [%o0]
1310         flush   %o0
1311         MAKE_NOP_INSTR(%o1)
1312         add     %o0, I_SIZE, %o0        ! next instr
1313         st      %o1, [%o0]
1314         flush   %o0
1315 
1316         set     sfmmu_shctx_user_rtt_patch, %o0
1317         st      %o1, [%o0]              ! nop 1st instruction
1318         flush   %o0
1319         add     %o0, I_SIZE, %o0
1320         st      %o1, [%o0]              ! nop 2nd instruction
1321         flush   %o0
1322         add     %o0, I_SIZE, %o0
1323         st      %o1, [%o0]              ! nop 3rd instruction
1324         flush   %o0
1325         add     %o0, I_SIZE, %o0
1326         st      %o1, [%o0]              ! nop 4th instruction
1327         flush   %o0
1328         add     %o0, I_SIZE, %o0
1329         st      %o1, [%o0]              ! nop 5th instruction
1330         flush   %o0
1331         add     %o0, I_SIZE, %o0
1332         st      %o1, [%o0]              ! nop 6th instruction
1333         retl
1334         flush   %o0
1335 #endif /* sun4u */
1336         SET_SIZE(sfmmu_patch_shctx)
1337 
1338         /*
1339          * Routine that loads an entry into a tsb using virtual addresses.
1340          * Locking is required since all cpus can use the same TSB.
1341          * Note that it is no longer required to have a valid context
1342          * when calling this function.
1343          */
1344         ENTRY_NP(sfmmu_load_tsbe)
1345         /*
1346          * %o0 = pointer to tsbe to load
1347          * %o1 = tsb tag
1348          * %o2 = virtual pointer to TTE
1349          * %o3 = 1 if physical address in %o0 else 0
1350          */
1351         rdpr    %pstate, %o5
1352 #ifdef DEBUG
1353         PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1354 #endif /* DEBUG */
1355 
1356         wrpr    %o5, PSTATE_IE, %pstate         /* disable interrupts */
1357 
1358         SETUP_TSB_ASI(%o3, %g3)
1359         TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8)
1360 
1361         wrpr    %g0, %o5, %pstate               /* enable interrupts */
1362 
1363         retl
1364         membar  #StoreStore|#StoreLoad
1365         SET_SIZE(sfmmu_load_tsbe)
1366 
1367         /*
1368          * Flush TSB of a given entry if the tag matches.
1369          */
1370         ENTRY(sfmmu_unload_tsbe)
1371         /*
1372          * %o0 = pointer to tsbe to be flushed
1373          * %o1 = tag to match
1374          * %o2 = 1 if physical address in %o0 else 0
1375          */
1376         SETUP_TSB_ASI(%o2, %g1)
1377         TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1378         retl
1379         membar  #StoreStore|#StoreLoad
1380         SET_SIZE(sfmmu_unload_tsbe)
1381 
1382         /*
1383          * Routine that loads a TTE into the kpm TSB from C code.
1384          * Locking is required since kpm TSB is shared among all CPUs.
1385          */
1386         ENTRY_NP(sfmmu_kpm_load_tsb)
1387         /*
1388          * %o0 = vaddr
1389          * %o1 = ttep
1390          * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1391          */
1392         rdpr    %pstate, %o5                    ! %o5 = saved pstate
1393 #ifdef DEBUG
1394         PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1395 #endif /* DEBUG */
1396         wrpr    %o5, PSTATE_IE, %pstate         ! disable interrupts
1397 
1398 #ifndef sun4v
1399         sethi   %hi(ktsb_phys), %o4
1400         mov     ASI_N, %o3
1401         ld      [%o4 + %lo(ktsb_phys)], %o4
1402         movrnz  %o4, ASI_MEM, %o3
1403         mov     %o3, %asi
1404 #endif /* !sun4v */
1405         mov     %o0, %g1                        ! %g1 = vaddr
1406 
1407         /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1408         GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1409         /* %g2 = tsbep, %g1 clobbered */
1410 
1411         srlx    %o0, TTARGET_VA_SHIFT, %g1;     ! %g1 = tag target
1412         /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1413         TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9)
1414 
1415         wrpr    %g0, %o5, %pstate               ! enable interrupts
1416         retl
1417           membar #StoreStore|#StoreLoad
1418         SET_SIZE(sfmmu_kpm_load_tsb)
1419 
1420         /*
1421          * Routine that shoots down a TTE in the kpm TSB or in the
1422          * kernel TSB depending on virtpg. Locking is required since
1423          * kpm/kernel TSB is shared among all CPUs.
1424          */
1425         ENTRY_NP(sfmmu_kpm_unload_tsb)
1426         /*
1427          * %o0 = vaddr
1428          * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1429          */
1430 #ifndef sun4v
1431         sethi   %hi(ktsb_phys), %o4
1432         mov     ASI_N, %o3
1433         ld      [%o4 + %lo(ktsb_phys)], %o4
1434         movrnz  %o4, ASI_MEM, %o3
1435         mov     %o3, %asi
1436 #endif /* !sun4v */
1437         mov     %o0, %g1                        ! %g1 = vaddr
1438 
1439         /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1440         GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1441         /* %g2 = tsbep, %g1 clobbered */
1442 
1443         srlx    %o0, TTARGET_VA_SHIFT, %g1;     ! %g1 = tag target
1444         /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1445         TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1446 
1447         retl
1448           membar        #StoreStore|#StoreLoad
1449         SET_SIZE(sfmmu_kpm_unload_tsb)
1450 
1451 
1452         ENTRY_NP(sfmmu_ttetopfn)
1453         ldx     [%o0], %g1                      /* read tte */
1454         TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1455         /*
1456          * g1 = pfn
1457          */
1458         retl
1459         mov     %g1, %o0
1460         SET_SIZE(sfmmu_ttetopfn)
1461 
1462 /*
1463  * These macros are used to update global sfmmu hme hash statistics
1464  * in perf critical paths. It is only enabled in debug kernels or
1465  * if SFMMU_STAT_GATHER is defined
1466  */
1467 #if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1468 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)                  \
1469         ldn     [tsbarea + TSBMISS_KHATID], tmp1                        ;\
1470         mov     HATSTAT_KHASH_SEARCH, tmp2                              ;\
1471         cmp     tmp1, hatid                                             ;\
1472         movne   %ncc, HATSTAT_UHASH_SEARCH, tmp2                        ;\
1473         set     sfmmu_global_stat, tmp1                                 ;\
1474         add     tmp1, tmp2, tmp1                                        ;\
1475         ld      [tmp1], tmp2                                            ;\
1476         inc     tmp2                                                    ;\
1477         st      tmp2, [tmp1]
1478 
1479 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)                    \
1480         ldn     [tsbarea + TSBMISS_KHATID], tmp1                        ;\
1481         mov     HATSTAT_KHASH_LINKS, tmp2                               ;\
1482         cmp     tmp1, hatid                                             ;\
1483         movne   %ncc, HATSTAT_UHASH_LINKS, tmp2                         ;\
1484         set     sfmmu_global_stat, tmp1                                 ;\
1485         add     tmp1, tmp2, tmp1                                        ;\
1486         ld      [tmp1], tmp2                                            ;\
1487         inc     tmp2                                                    ;\
1488         st      tmp2, [tmp1]
1489 
1490 
1491 #else /* DEBUG || SFMMU_STAT_GATHER */
1492 
1493 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1494 
1495 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1496 
1497 #endif  /* DEBUG || SFMMU_STAT_GATHER */
1498 
1499 /*
1500  * This macro is used to update global sfmmu kstas in non
1501  * perf critical areas so they are enabled all the time
1502  */
1503 #define HAT_GLOBAL_STAT(statname, tmp1, tmp2)                           \
1504         sethi   %hi(sfmmu_global_stat), tmp1                            ;\
1505         add     tmp1, statname, tmp1                                    ;\
1506         ld      [tmp1 + %lo(sfmmu_global_stat)], tmp2                   ;\
1507         inc     tmp2                                                    ;\
1508         st      tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1509 
1510 /*
1511  * These macros are used to update per cpu stats in non perf
1512  * critical areas so they are enabled all the time
1513  */
1514 #define HAT_PERCPU_STAT32(tsbarea, stat, tmp1)                          \
1515         ld      [tsbarea + stat], tmp1                                  ;\
1516         inc     tmp1                                                    ;\
1517         st      tmp1, [tsbarea + stat]
1518 
1519 /*
1520  * These macros are used to update per cpu stats in non perf
1521  * critical areas so they are enabled all the time
1522  */
1523 #define HAT_PERCPU_STAT16(tsbarea, stat, tmp1)                          \
1524         lduh    [tsbarea + stat], tmp1                                  ;\
1525         inc     tmp1                                                    ;\
1526         stuh    tmp1, [tsbarea + stat]
1527 
1528 #if defined(KPM_TLBMISS_STATS_GATHER)
1529         /*
1530          * Count kpm dtlb misses separately to allow a different
1531          * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1532          * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1533          */
1534 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)          \
1535         brgez   tagacc, label   /* KPM VA? */                           ;\
1536         nop                                                             ;\
1537         CPU_INDEX(tmp1, tsbma)                                          ;\
1538         sethi   %hi(kpmtsbm_area), tsbma                                ;\
1539         sllx    tmp1, KPMTSBM_SHIFT, tmp1                               ;\
1540         or      tsbma, %lo(kpmtsbm_area), tsbma                         ;\
1541         add     tsbma, tmp1, tsbma              /* kpmtsbm area */      ;\
1542         /* VA range check */                                            ;\
1543         ldx     [tsbma + KPMTSBM_VBASE], val                            ;\
1544         cmp     tagacc, val                                             ;\
1545         blu,pn  %xcc, label                                             ;\
1546           ldx   [tsbma + KPMTSBM_VEND], tmp1                            ;\
1547         cmp     tagacc, tmp1                                            ;\
1548         bgeu,pn %xcc, label                                             ;\
1549           lduw  [tsbma + KPMTSBM_DTLBMISS], val                         ;\
1550         inc     val                                                     ;\
1551         st      val, [tsbma + KPMTSBM_DTLBMISS]                         ;\
1552 label:
1553 #else
1554 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1555 #endif  /* KPM_TLBMISS_STATS_GATHER */
1556 
1557 #ifdef  PTL1_PANIC_DEBUG
1558         .seg    ".data"
1559         .global test_ptl1_panic
1560 test_ptl1_panic:
1561         .word   0
1562         .align  8
1563 
1564         .seg    ".text"
1565         .align  4
1566 #endif  /* PTL1_PANIC_DEBUG */
1567 
1568 
1569         ENTRY_NP(sfmmu_pagefault)
1570         SET_GL_REG(1)
1571         USE_ALTERNATE_GLOBALS(%g5)
1572         GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1573         rdpr    %tt, %g6
1574         cmp     %g6, FAST_IMMU_MISS_TT
1575         be,a,pn %icc, 1f
1576           mov   T_INSTR_MMU_MISS, %g3
1577         cmp     %g6, T_INSTR_MMU_MISS
1578         be,a,pn %icc, 1f
1579           mov   T_INSTR_MMU_MISS, %g3
1580         mov     %g5, %g2
1581         mov     T_DATA_PROT, %g3                /* arg2 = traptype */
1582         cmp     %g6, FAST_DMMU_MISS_TT
1583         move    %icc, T_DATA_MMU_MISS, %g3      /* arg2 = traptype */
1584         cmp     %g6, T_DATA_MMU_MISS
1585         move    %icc, T_DATA_MMU_MISS, %g3      /* arg2 = traptype */
1586 
1587 #ifdef  PTL1_PANIC_DEBUG
1588         /* check if we want to test the tl1 panic */
1589         sethi   %hi(test_ptl1_panic), %g4
1590         ld      [%g4 + %lo(test_ptl1_panic)], %g1
1591         st      %g0, [%g4 + %lo(test_ptl1_panic)]
1592         cmp     %g1, %g0
1593         bne,a,pn %icc, ptl1_panic
1594           or    %g0, PTL1_BAD_DEBUG, %g1
1595 #endif  /* PTL1_PANIC_DEBUG */
1596 1:
1597         HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1598         /*
1599          * g2 = tag access reg
1600          * g3.l = type
1601          * g3.h = 0
1602          */
1603         sethi   %hi(trap), %g1
1604         or      %g1, %lo(trap), %g1
1605 2:
1606         ba,pt   %xcc, sys_trap
1607           mov   -1, %g4
1608         SET_SIZE(sfmmu_pagefault)
1609 
1610         ENTRY_NP(sfmmu_mmu_trap)
1611         SET_GL_REG(1)
1612         USE_ALTERNATE_GLOBALS(%g5)
1613         GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1614         rdpr    %tt, %g6
1615         cmp     %g6, FAST_IMMU_MISS_TT
1616         be,a,pn %icc, 1f
1617           mov   T_INSTR_MMU_MISS, %g3
1618         cmp     %g6, T_INSTR_MMU_MISS
1619         be,a,pn %icc, 1f
1620           mov   T_INSTR_MMU_MISS, %g3
1621         mov     %g5, %g2
1622         mov     T_DATA_PROT, %g3                /* arg2 = traptype */
1623         cmp     %g6, FAST_DMMU_MISS_TT
1624         move    %icc, T_DATA_MMU_MISS, %g3      /* arg2 = traptype */
1625         cmp     %g6, T_DATA_MMU_MISS
1626         move    %icc, T_DATA_MMU_MISS, %g3      /* arg2 = traptype */
1627 1:
1628         /*
1629          * g2 = tag access reg
1630          * g3 = type
1631          */
1632         sethi   %hi(sfmmu_tsbmiss_exception), %g1
1633         or      %g1, %lo(sfmmu_tsbmiss_exception), %g1
1634         ba,pt   %xcc, sys_trap
1635           mov   -1, %g4
1636         /*NOTREACHED*/
1637         SET_SIZE(sfmmu_mmu_trap)
1638 
1639         ENTRY_NP(sfmmu_suspend_tl)
1640         SET_GL_REG(1)
1641         USE_ALTERNATE_GLOBALS(%g5)
1642         GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1643         rdpr    %tt, %g6
1644         cmp     %g6, FAST_IMMU_MISS_TT
1645         be,a,pn %icc, 1f
1646           mov   T_INSTR_MMU_MISS, %g3
1647         mov     %g5, %g2
1648         cmp     %g6, FAST_DMMU_MISS_TT
1649         move    %icc, T_DATA_MMU_MISS, %g3
1650         movne   %icc, T_DATA_PROT, %g3
1651 1:
1652         sethi   %hi(sfmmu_tsbmiss_suspended), %g1
1653         or      %g1, %lo(sfmmu_tsbmiss_suspended), %g1
1654         /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1655         ba,pt   %xcc, sys_trap
1656           mov   PIL_15, %g4
1657         /*NOTREACHED*/
1658         SET_SIZE(sfmmu_suspend_tl)
1659 
1660         /*
1661          * No %g registers in use at this point.
1662          */
1663         ENTRY_NP(sfmmu_window_trap)
1664         rdpr    %tpc, %g1
1665 #ifdef sun4v
1666 #ifdef DEBUG
1667         /* We assume previous %gl was 1 */
1668         rdpr    %tstate, %g4
1669         srlx    %g4, TSTATE_GL_SHIFT, %g4
1670         and     %g4, TSTATE_GL_MASK, %g4
1671         cmp     %g4, 1
1672         bne,a,pn %icc, ptl1_panic
1673           mov   PTL1_BAD_WTRAP, %g1
1674 #endif /* DEBUG */
1675         /* user miss at tl>1. better be the window handler or user_rtt */
1676         /* in user_rtt? */
1677         set     rtt_fill_start, %g4
1678         cmp     %g1, %g4
1679         blu,pn %xcc, 6f
1680          .empty
1681         set     rtt_fill_end, %g4
1682         cmp     %g1, %g4
1683         bgeu,pn %xcc, 6f
1684          nop
1685         set     fault_rtt_fn1, %g1
1686         wrpr    %g0, %g1, %tnpc
1687         ba,a    7f
1688 6:
1689         ! must save this trap level before descending trap stack
1690         ! no need to save %tnpc, either overwritten or discarded
1691         ! already got it: rdpr  %tpc, %g1
1692         rdpr    %tstate, %g6
1693         rdpr    %tt, %g7
1694         ! trap level saved, go get underlying trap type
1695         rdpr    %tl, %g5
1696         sub     %g5, 1, %g3
1697         wrpr    %g3, %tl
1698         rdpr    %tt, %g2
1699         wrpr    %g5, %tl
1700         ! restore saved trap level
1701         wrpr    %g1, %tpc
1702         wrpr    %g6, %tstate
1703         wrpr    %g7, %tt
1704 #else /* sun4v */
1705         /* user miss at tl>1. better be the window handler */
1706         rdpr    %tl, %g5
1707         sub     %g5, 1, %g3
1708         wrpr    %g3, %tl
1709         rdpr    %tt, %g2
1710         wrpr    %g5, %tl
1711 #endif /* sun4v */
1712         and     %g2, WTRAP_TTMASK, %g4
1713         cmp     %g4, WTRAP_TYPE
1714         bne,pn  %xcc, 1f
1715          nop
1716         /* tpc should be in the trap table */
1717         set     trap_table, %g4
1718         cmp     %g1, %g4
1719         blt,pn %xcc, 1f
1720          .empty
1721         set     etrap_table, %g4
1722         cmp     %g1, %g4
1723         bge,pn %xcc, 1f
1724          .empty
1725         andn    %g1, WTRAP_ALIGN, %g1   /* 128 byte aligned */
1726         add     %g1, WTRAP_FAULTOFF, %g1
1727         wrpr    %g0, %g1, %tnpc
1728 7:
1729         /*
1730          * some wbuf handlers will call systrap to resolve the fault
1731          * we pass the trap type so they figure out the correct parameters.
1732          * g5 = trap type, g6 = tag access reg
1733          */
1734 
1735         /*
1736          * only use g5, g6, g7 registers after we have switched to alternate
1737          * globals.
1738          */
1739         SET_GL_REG(1)
1740         USE_ALTERNATE_GLOBALS(%g5)
1741         GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1742         rdpr    %tt, %g7
1743         cmp     %g7, FAST_IMMU_MISS_TT
1744         be,a,pn %icc, ptl1_panic
1745           mov   PTL1_BAD_WTRAP, %g1
1746         cmp     %g7, T_INSTR_MMU_MISS
1747         be,a,pn %icc, ptl1_panic
1748           mov   PTL1_BAD_WTRAP, %g1
1749         mov     T_DATA_PROT, %g5
1750         cmp     %g7, FAST_DMMU_MISS_TT
1751         move    %icc, T_DATA_MMU_MISS, %g5
1752         cmp     %g7, T_DATA_MMU_MISS
1753         move    %icc, T_DATA_MMU_MISS, %g5
1754         ! XXXQ AGS re-check out this one
1755         done
1756 1:
1757         CPU_PADDR(%g1, %g4)
1758         add     %g1, CPU_TL1_HDLR, %g1
1759         lda     [%g1]ASI_MEM, %g4
1760         brnz,a,pt %g4, sfmmu_mmu_trap
1761           sta   %g0, [%g1]ASI_MEM
1762         ba,pt   %icc, ptl1_panic
1763           mov   PTL1_BAD_TRAP, %g1
1764         SET_SIZE(sfmmu_window_trap)
1765 
1766         ENTRY_NP(sfmmu_kpm_exception)
1767         /*
1768          * We have accessed an unmapped segkpm address or a legal segkpm
1769          * address which is involved in a VAC alias conflict prevention.
1770          * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1771          * set. If it is, we will instead note that a fault has occurred
1772          * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1773          * a "retry"). This will step over the faulting instruction.
1774          * Note that this means that a legal segkpm address involved in
1775          * a VAC alias conflict prevention (a rare case to begin with)
1776          * cannot be used in DTrace.
1777          */
1778         CPU_INDEX(%g1, %g2)
1779         set     cpu_core, %g2
1780         sllx    %g1, CPU_CORE_SHIFT, %g1
1781         add     %g1, %g2, %g1
1782         lduh    [%g1 + CPUC_DTRACE_FLAGS], %g2
1783         andcc   %g2, CPU_DTRACE_NOFAULT, %g0
1784         bz      0f
1785         or      %g2, CPU_DTRACE_BADADDR, %g2
1786         stuh    %g2, [%g1 + CPUC_DTRACE_FLAGS]
1787         GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1788         stx     %g3, [%g1 + CPUC_DTRACE_ILLVAL]
1789         done
1790 0:
1791         TSTAT_CHECK_TL1(1f, %g1, %g2)
1792 1:
1793         SET_GL_REG(1)
1794         USE_ALTERNATE_GLOBALS(%g5)
1795         GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1796         mov     T_DATA_MMU_MISS, %g3    /* arg2 = traptype */
1797         /*
1798          * g2=tagacc g3.l=type g3.h=0
1799          */
1800         sethi   %hi(trap), %g1
1801         or      %g1, %lo(trap), %g1
1802         ba,pt   %xcc, sys_trap
1803         mov     -1, %g4
1804         SET_SIZE(sfmmu_kpm_exception)
1805 
1806 #if (IMAP_SEG != 0)
1807 #error - ism_map->ism_seg offset is not zero
1808 #endif
1809 
1810 /*
1811  * Copies ism mapping for this ctx in param "ism" if this is a ISM
1812  * tlb miss and branches to label "ismhit". If this is not an ISM
1813  * process or an ISM tlb miss it falls thru.
1814  *
1815  * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1816  * this process.
1817  * If so, it will branch to label "ismhit".  If not, it will fall through.
1818  *
1819  * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1820  * so that any other threads of this process will not try and walk the ism
1821  * maps while they are being changed.
1822  *
1823  * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1824  *       will make sure of that. This means we can terminate our search on
1825  *       the first zero mapping we find.
1826  *
1827  * Parameters:
1828  * tagacc       = (pseudo-)tag access register (vaddr + ctx) (in)
1829  * tsbmiss      = address of tsb miss area (in)
1830  * ismseg       = contents of ism_seg for this ism map (out)
1831  * ismhat       = physical address of imap_ismhat for this ism map (out)
1832  * tmp1         = scratch reg (CLOBBERED)
1833  * tmp2         = scratch reg (CLOBBERED)
1834  * tmp3         = scratch reg (CLOBBERED)
1835  * label:    temporary labels
1836  * ismhit:   label where to jump to if an ism dtlb miss
1837  * exitlabel:label where to jump if hat is busy due to hat_unshare.
1838  */
1839 #define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1840         label, ismhit)                                                  \
1841         ldx     [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */     ;\
1842         brlz,pt  tmp1, label/**/3               /* exit if -1 */        ;\
1843           add   tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */       ;\
1844 label/**/1:                                                             ;\
1845         ldxa    [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */     ;\
1846         mov     tmp1, tmp3      /* update current ismblkpa head */      ;\
1847 label/**/2:                                                             ;\
1848         brz,pt  ismseg, label/**/3              /* no mapping */        ;\
1849           add   ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */  ;\
1850         lduba   [tmp1]ASI_MEM, tmp1             /* tmp1 = vb shift*/    ;\
1851         srlx    ismseg, tmp1, tmp2              /* tmp2 = vbase */      ;\
1852         srlx    tagacc, tmp1, tmp1              /* tmp1 =  va seg*/     ;\
1853         sub     tmp1, tmp2, tmp2                /* tmp2 = va - vbase */ ;\
1854         add     ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */    ;\
1855         lda     [tmp1]ASI_MEM, tmp1             /* tmp1 = sz_mask */    ;\
1856         and     ismseg, tmp1, tmp1              /* tmp1 = size */       ;\
1857         cmp     tmp2, tmp1                      /* check va <= offset*/      ;\
1858         blu,a,pt  %xcc, ismhit                  /* ism hit */           ;\
1859           add   ismhat, IMAP_ISMHAT, ismhat     /* ismhat = &ism_sfmmu*/ ;\
1860                                                                         ;\
1861         add     ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */  ;\
1862         add     tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1    ;\
1863         cmp     ismhat, tmp1                                            ;\
1864         bl,pt   %xcc, label/**/2                /* keep looking  */     ;\
1865           ldxa  [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */      ;\
1866                                                                         ;\
1867         add     tmp3, IBLK_NEXTPA, tmp1                                 ;\
1868         ldxa    [tmp1]ASI_MEM, tmp1             /* check blk->nextpa */      ;\
1869         brgez,pt tmp1, label/**/1               /* continue if not -1*/ ;\
1870           add   tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/        ;\
1871 label/**/3:
1872 
1873 /*
1874  * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1875  * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1876  * Parameters:
1877  * tagacc = reg containing virtual address
1878  * hatid = reg containing sfmmu pointer
1879  * hmeshift = constant/register to shift vaddr to obtain vapg
1880  * hmebp = register where bucket pointer will be stored
1881  * vapg = register where virtual page will be stored
1882  * tmp1, tmp2 = tmp registers
1883  */
1884 
1885 
1886 #define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,       \
1887         vapg, label, tmp1, tmp2)                                        \
1888         sllx    tagacc, TAGACC_CTX_LSHIFT, tmp1                         ;\
1889         brnz,a,pt tmp1, label/**/1                                      ;\
1890           ld    [tsbarea + TSBMISS_UHASHSZ], hmebp                      ;\
1891         ld      [tsbarea + TSBMISS_KHASHSZ], hmebp                      ;\
1892         ba,pt   %xcc, label/**/2                                        ;\
1893           ldx   [tsbarea + TSBMISS_KHASHSTART], tmp1                    ;\
1894 label/**/1:                                                             ;\
1895         ldx     [tsbarea + TSBMISS_UHASHSTART], tmp1                    ;\
1896 label/**/2:                                                             ;\
1897         srlx    tagacc, hmeshift, vapg                                  ;\
1898         xor     vapg, hatid, tmp2       /* hatid ^ (vaddr >> shift) */    ;\
1899         and     tmp2, hmebp, hmebp      /* index into hme_hash */       ;\
1900         mulx    hmebp, HMEBUCK_SIZE, hmebp                              ;\
1901         add     hmebp, tmp1, hmebp
1902 
1903 /*
1904  * hashtag includes bspage + hashno (64 bits).
1905  */
1906 
1907 #define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)            \
1908         sllx    vapg, hmeshift, vapg                                    ;\
1909         mov     hashno, hblktag                                         ;\
1910         sllx    hblktag, HTAG_REHASH_SHIFT, hblktag                     ;\
1911         or      vapg, hblktag, hblktag
1912 
1913 /*
1914  * Function to traverse hmeblk hash link list and find corresponding match.
1915  * The search is done using physical pointers. It returns the physical address
1916  * pointer to the hmeblk that matches with the tag provided.
1917  * Parameters:
1918  * hmebp        = register that points to hme hash bucket, also used as
1919  *                tmp reg (clobbered)
1920  * hmeblktag    = register with hmeblk tag match
1921  * hatid        = register with hatid
1922  * hmeblkpa     = register where physical ptr will be stored
1923  * tmp1         = tmp reg
1924  * label: temporary label
1925  */
1926 
1927 #define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea,      \
1928         tmp1, label)                                                    \
1929         add     hmebp, HMEBUCK_NEXTPA, hmeblkpa                         ;\
1930         ldxa    [hmeblkpa]ASI_MEM, hmeblkpa                             ;\
1931         HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)                 ;\
1932 label/**/1:                                                             ;\
1933         cmp     hmeblkpa, HMEBLK_ENDPA                                  ;\
1934         be,pn   %xcc, label/**/2                                        ;\
1935         HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)                   ;\
1936         add     hmeblkpa, HMEBLK_TAG, hmebp                             ;\
1937         ldxa    [hmebp]ASI_MEM, tmp1     /* read 1st part of tag */     ;\
1938         add     hmebp, CLONGSIZE, hmebp                                 ;\
1939         ldxa    [hmebp]ASI_MEM, hmebp   /* read 2nd part of tag */      ;\
1940         xor     tmp1, hmeblktag, tmp1                                   ;\
1941         xor     hmebp, hatid, hmebp                                     ;\
1942         or      hmebp, tmp1, hmebp                                      ;\
1943         brz,pn  hmebp, label/**/2       /* branch on hit */             ;\
1944           add   hmeblkpa, HMEBLK_NEXTPA, hmebp                          ;\
1945         ba,pt   %xcc, label/**/1                                        ;\
1946           ldxa  [hmebp]ASI_MEM, hmeblkpa        /* hmeblk ptr pa */     ;\
1947 label/**/2:
1948 
1949 /*
1950  * Function to traverse hmeblk hash link list and find corresponding match.
1951  * The search is done using physical pointers. It returns the physical address
1952  * pointer to the hmeblk that matches with the tag
1953  * provided.
1954  * Parameters:
1955  * hmeblktag    = register with hmeblk tag match (rid field is 0)
1956  * hatid        = register with hatid (pointer to SRD)
1957  * hmeblkpa     = register where physical ptr will be stored
1958  * tmp1         = tmp reg
1959  * tmp2         = tmp reg
1960  * label: temporary label
1961  */
1962 
1963 #define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea,        \
1964         tmp1, tmp2, label)                                              \
1965 label/**/1:                                                             ;\
1966         cmp     hmeblkpa, HMEBLK_ENDPA                                  ;\
1967         be,pn   %xcc, label/**/4                                        ;\
1968         HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)                    ;\
1969         add     hmeblkpa, HMEBLK_TAG, tmp2                              ;\
1970         ldxa    [tmp2]ASI_MEM, tmp1      /* read 1st part of tag */     ;\
1971         add     tmp2, CLONGSIZE, tmp2                                   ;\
1972         ldxa    [tmp2]ASI_MEM, tmp2     /* read 2nd part of tag */      ;\
1973         xor     tmp1, hmeblktag, tmp1                                   ;\
1974         xor     tmp2, hatid, tmp2                                       ;\
1975         brz,pn  tmp2, label/**/3        /* branch on hit */             ;\
1976           add   hmeblkpa, HMEBLK_NEXTPA, tmp2                           ;\
1977 label/**/2:                                                             ;\
1978         ba,pt   %xcc, label/**/1                                        ;\
1979           ldxa  [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */             ;\
1980 label/**/3:                                                             ;\
1981         cmp     tmp1, SFMMU_MAX_HME_REGIONS                             ;\
1982         bgeu,pt %xcc, label/**/2                                        ;\
1983           add   hmeblkpa, HMEBLK_NEXTPA, tmp2                           ;\
1984         and     tmp1, BT_ULMASK, tmp2                                   ;\
1985         srlx    tmp1, BT_ULSHIFT, tmp1                                  ;\
1986         sllx    tmp1, CLONGSHIFT, tmp1                                  ;\
1987         add     tsbarea, tmp1, tmp1                                     ;\
1988         ldx     [tmp1 + TSBMISS_SHMERMAP], tmp1                         ;\
1989         srlx    tmp1, tmp2, tmp1                                        ;\
1990         btst    0x1, tmp1                                               ;\
1991         bz,pn   %xcc, label/**/2                                        ;\
1992           add   hmeblkpa, HMEBLK_NEXTPA, tmp2                           ;\
1993 label/**/4:
1994 
1995 #if ((1 << SFHME_SHIFT) != SFHME_SIZE)
1996 #error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
1997 #endif
1998 
1999 /*
2000  * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2001  * the offset for the corresponding hment.
2002  * Parameters:
2003  * In:
2004  *      vaddr = register with virtual address
2005  *      hmeblkpa = physical pointer to hme_blk
2006  * Out:
2007  *      hmentoff = register where hment offset will be stored
2008  *      hmemisc = hblk_misc
2009  * Scratch:
2010  *      tmp1
2011  */
2012 #define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2013         add     hmeblkpa, HMEBLK_MISC, hmentoff                         ;\
2014         lda     [hmentoff]ASI_MEM, hmemisc                              ;\
2015         andcc   hmemisc, HBLK_SZMASK, %g0                               ;\
2016         bnz,a,pn  %icc, label1          /* if sz != TTE8K branch */     ;\
2017           or    %g0, HMEBLK_HME1, hmentoff                              ;\
2018         srl     vaddr, MMU_PAGESHIFT, tmp1                              ;\
2019         and     tmp1, NHMENTS - 1, tmp1         /* tmp1 = index */      ;\
2020         sllx    tmp1, SFHME_SHIFT, tmp1                                 ;\
2021         add     tmp1, HMEBLK_HME1, hmentoff                             ;\
2022 label1:
2023 
2024 /*
2025  * GET_TTE is a macro that returns a TTE given a tag and hatid.
2026  *
2027  * tagacc       = (pseudo-)tag access register (in)
2028  * hatid        = sfmmu pointer for TSB miss (in)
2029  * tte          = tte for TLB miss if found, otherwise clobbered (out)
2030  * hmeblkpa     = PA of hment if found, otherwise clobbered (out)
2031  * tsbarea      = pointer to the tsbmiss area for this cpu. (in)
2032  * hmemisc      = hblk_misc if TTE is found (out), otherwise clobbered
2033  * hmeshift     = constant/register to shift VA to obtain the virtual pfn
2034  *                for this page size.
2035  * hashno       = constant/register hash number
2036  * tmp          = temp value - clobbered
2037  * label        = temporary label for branching within macro.
2038  * foundlabel   = label to jump to when tte is found.
2039  * suspendlabel= label to jump to when tte is suspended.
2040  * exitlabel    = label to jump to when tte is not found.
2041  *
2042  */
2043 #define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \
2044                  hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \
2045                                                                         ;\
2046         stn     tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]      ;\
2047         stn     hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]    ;\
2048         HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,         \
2049                 hmeblkpa, label/**/5, hmemisc, tmp)                     ;\
2050                                                                         ;\
2051         /*                                                              ;\
2052          * tagacc = tagacc                                              ;\
2053          * hatid = hatid                                                ;\
2054          * tsbarea = tsbarea                                            ;\
2055          * tte   = hmebp (hme bucket pointer)                           ;\
2056          * hmeblkpa  = vapg  (virtual page)                             ;\
2057          * hmemisc, tmp = scratch                                       ;\
2058          */                                                             ;\
2059         MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)        ;\
2060         or      hmemisc, SFMMU_INVALID_SHMERID, hmemisc                 ;\
2061                                                                         ;\
2062         /*                                                              ;\
2063          * tagacc = tagacc                                              ;\
2064          * hatid = hatid                                                ;\
2065          * tte   = hmebp                                                ;\
2066          * hmeblkpa  = CLOBBERED                                        ;\
2067          * hmemisc  = htag_bspage+hashno+invalid_rid                    ;\
2068          * tmp  = scratch                                               ;\
2069          */                                                             ;\
2070         stn     tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]      ;\
2071         HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa,                   \
2072                 tsbarea, tagacc, label/**/1)                            ;\
2073         /*                                                              ;\
2074          * tagacc = CLOBBERED                                           ;\
2075          * tte = CLOBBERED                                              ;\
2076          * hmeblkpa = hmeblkpa                                          ;\
2077          * tmp = scratch                                                ;\
2078          */                                                             ;\
2079         cmp     hmeblkpa, HMEBLK_ENDPA                                  ;\
2080         bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
2081           ldn   [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc      ;\
2082         ba,pt   %xcc, exitlabel         /* exit if hblk not found */    ;\
2083           nop                                                           ;\
2084 label/**/4:                                                             ;\
2085         /*                                                              ;\
2086          * We have found the hmeblk containing the hment.               ;\
2087          * Now we calculate the corresponding tte.                      ;\
2088          *                                                              ;\
2089          * tagacc = tagacc                                              ;\
2090          * hatid = hatid                                                ;\
2091          * tte   = clobbered                                            ;\
2092          * hmeblkpa  = hmeblkpa                                         ;\
2093          * hmemisc  = hblktag                                           ;\
2094          * tmp = scratch                                                ;\
2095          */                                                             ;\
2096         HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,          \
2097                 label/**/2)                                             ;\
2098                                                                         ;\
2099         /*                                                              ;\
2100          * tagacc = tagacc                                              ;\
2101          * hatid = hmentoff                                             ;\
2102          * tte   = clobbered                                            ;\
2103          * hmeblkpa  = hmeblkpa                                         ;\
2104          * hmemisc  = hblk_misc                                         ;\
2105          * tmp = scratch                                                ;\
2106          */                                                             ;\
2107                                                                         ;\
2108         add     hatid, SFHME_TTE, hatid                                 ;\
2109         add     hmeblkpa, hatid, hmeblkpa                               ;\
2110         ldxa    [hmeblkpa]ASI_MEM, tte  /* MMU_READTTE through pa */    ;\
2111         ldn     [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid    ;\
2112         set     TTE_SUSPEND, hatid                                      ;\
2113         TTE_SUSPEND_INT_SHIFT(hatid)                                    ;\
2114         btst    tte, hatid                                              ;\
2115         bz,pt   %xcc, foundlabel                                        ;\
2116         ldn     [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid    ;\
2117                                                                         ;\
2118         /*                                                              ;\
2119          * Mapping is suspended, so goto suspend label.                 ;\
2120          */                                                             ;\
2121         ba,pt   %xcc, suspendlabel                                      ;\
2122           nop
2123 
2124 /*
2125  * GET_SHME_TTE is similar to GET_TTE() except it searches
2126  * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2127  * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2128  * either 0 (not part of scd) or 1 (part of scd).
2129  */
2130 #define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc,    \
2131                 hmeshift, hashno, tmp, label, foundlabel,               \
2132                 suspendlabel, exitlabel)                                \
2133                                                                         ;\
2134         stn     tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]      ;\
2135         stn     hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]    ;\
2136         HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,         \
2137                 hmeblkpa, label/**/5, hmemisc, tmp)                     ;\
2138                                                                         ;\
2139         /*                                                              ;\
2140          * tagacc = tagacc                                              ;\
2141          * hatid = hatid                                                ;\
2142          * tsbarea = tsbarea                                            ;\
2143          * tte   = hmebp (hme bucket pointer)                           ;\
2144          * hmeblkpa  = vapg  (virtual page)                             ;\
2145          * hmemisc, tmp = scratch                                       ;\
2146          */                                                             ;\
2147         MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)        ;\
2148                                                                         ;\
2149         /*                                                              ;\
2150          * tagacc = tagacc                                              ;\
2151          * hatid = hatid                                                ;\
2152          * tsbarea = tsbarea                                            ;\
2153          * tte   = hmebp                                                ;\
2154          * hmemisc  = htag_bspage + hashno + 0 (for rid)                ;\
2155          * hmeblkpa  = CLOBBERED                                        ;\
2156          * tmp = scratch                                                ;\
2157          */                                                             ;\
2158         stn     tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]      ;\
2159                                                                         ;\
2160         add     tte, HMEBUCK_NEXTPA, hmeblkpa                           ;\
2161         ldxa    [hmeblkpa]ASI_MEM, hmeblkpa                             ;\
2162         HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)                 ;\
2163                                                                         ;\
2164 label/**/8:                                                             ;\
2165         HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa,                   \
2166                 tsbarea, tagacc, tte, label/**/1)                       ;\
2167         /*                                                              ;\
2168          * tagacc = CLOBBERED                                           ;\
2169          * tte = CLOBBERED                                              ;\
2170          * hmeblkpa = hmeblkpa                                          ;\
2171          * tmp = scratch                                                ;\
2172          */                                                             ;\
2173         cmp     hmeblkpa, HMEBLK_ENDPA                                  ;\
2174         bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
2175           ldn   [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc      ;\
2176         ba,pt   %xcc, exitlabel         /* exit if hblk not found */    ;\
2177           nop                                                           ;\
2178 label/**/4:                                                             ;\
2179         /*                                                              ;\
2180          * We have found the hmeblk containing the hment.               ;\
2181          * Now we calculate the corresponding tte.                      ;\
2182          *                                                              ;\
2183          * tagacc = tagacc                                              ;\
2184          * hatid = hatid                                                ;\
2185          * tte   = clobbered                                            ;\
2186          * hmeblkpa  = hmeblkpa                                         ;\
2187          * hmemisc  = hblktag                                           ;\
2188          * tsbarea = tsbmiss area                                       ;\
2189          * tmp = scratch                                                ;\
2190          */                                                             ;\
2191         HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,          \
2192                 label/**/2)                                             ;\
2193                                                                         ;\
2194         /*                                                              ;\
2195          * tagacc = tagacc                                              ;\
2196          * hatid = hmentoff                                             ;\
2197          * tte = clobbered                                              ;\
2198          * hmeblkpa  = hmeblkpa                                         ;\
2199          * hmemisc  = hblk_misc                                         ;\
2200          * tsbarea = tsbmiss area                                       ;\
2201          * tmp = scratch                                                ;\
2202          */                                                             ;\
2203                                                                         ;\
2204         add     hatid, SFHME_TTE, hatid                                 ;\
2205         add     hmeblkpa, hatid, hmeblkpa                               ;\
2206         ldxa    [hmeblkpa]ASI_MEM, tte  /* MMU_READTTE through pa */    ;\
2207         brlz,pt tte, label/**/6                                         ;\
2208           nop                                                           ;\
2209         btst    HBLK_SZMASK, hmemisc                                    ;\
2210         bnz,a,pt %icc, label/**/7                                       ;\
2211           ldn   [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid    ;\
2212                                                                         ;\
2213         /*                                                              ;\
2214          * We found an invalid 8K tte in shme.                          ;\
2215          * it may not belong to shme's region since                     ;\
2216          * region size/alignment granularity is 8K but different        ;\
2217          * regions don't share hmeblks. Continue the search.            ;\
2218          */                                                             ;\
2219         sub     hmeblkpa, hatid, hmeblkpa                               ;\
2220         ldn     [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid    ;\
2221         srlx    tagacc, hmeshift, tte                                   ;\
2222         add     hmeblkpa, HMEBLK_NEXTPA, hmeblkpa                       ;\
2223         ldxa    [hmeblkpa]ASI_MEM, hmeblkpa                             ;\
2224         MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)             ;\
2225         ba,a,pt %xcc, label/**/8                                        ;\
2226 label/**/6:                                                             ;\
2227         GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)              ;\
2228         ldn     [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid    ;\
2229 label/**/7:                                                             ;\
2230         set     TTE_SUSPEND, hatid                                      ;\
2231         TTE_SUSPEND_INT_SHIFT(hatid)                                    ;\
2232         btst    tte, hatid                                              ;\
2233         bz,pt   %xcc, foundlabel                                        ;\
2234         ldn     [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid    ;\
2235                                                                         ;\
2236         /*                                                              ;\
2237          * Mapping is suspended, so goto suspend label.                 ;\
2238          */                                                             ;\
2239         ba,pt   %xcc, suspendlabel                                      ;\
2240           nop
2241 
2242         /*
2243          * KERNEL PROTECTION HANDLER
2244          *
2245          * g1 = tsb8k pointer register (clobbered)
2246          * g2 = tag access register (ro)
2247          * g3 - g7 = scratch registers
2248          *
2249          * Note: This function is patched at runtime for performance reasons.
2250          *       Any changes here require sfmmu_patch_ktsb fixed.
2251          */
2252         ENTRY_NP(sfmmu_kprot_trap)
2253         mov     %g2, %g7                ! TSB pointer macro clobbers tagacc
2254 sfmmu_kprot_patch_ktsb_base:
2255         RUNTIME_PATCH_SETX(%g1, %g6)
2256         /* %g1 = contents of ktsb_base or ktsb_pbase */
2257 sfmmu_kprot_patch_ktsb_szcode:
2258         or      %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched)
2259 
2260         GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2261         ! %g1 = First TSB entry pointer, as TSB miss handler expects
2262 
2263         mov     %g2, %g7                ! TSB pointer macro clobbers tagacc
2264 sfmmu_kprot_patch_ktsb4m_base:
2265         RUNTIME_PATCH_SETX(%g3, %g6)
2266         /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2267 sfmmu_kprot_patch_ktsb4m_szcode:
2268         or      %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched)
2269 
2270         GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2271         ! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2272 
2273         CPU_TSBMISS_AREA(%g6, %g7)
2274         HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2275         ba,pt   %xcc, sfmmu_tsb_miss_tt
2276           nop
2277 
2278         /*
2279          * USER PROTECTION HANDLER
2280          *
2281          * g1 = tsb8k pointer register (ro)
2282          * g2 = tag access register (ro)
2283          * g3 = faulting context (clobbered, currently not used)
2284          * g4 - g7 = scratch registers
2285          */
2286         ALTENTRY(sfmmu_uprot_trap)
2287 #ifdef sun4v
2288         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2289         /* %g1 = first TSB entry ptr now, %g2 preserved */
2290 
2291         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)   /* get 2nd utsbreg */
2292         brlz,pt %g3, 9f                         /* check for 2nd TSB */
2293           nop
2294 
2295         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2296         /* %g3 = second TSB entry ptr now, %g2 preserved */
2297 
2298 #else /* sun4v */
2299 #ifdef UTSB_PHYS
2300         /* g1 = first TSB entry ptr */
2301         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2302         brlz,pt %g3, 9f                 /* check for 2nd TSB */
2303           nop
2304 
2305         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2306         /* %g3 = second TSB entry ptr now, %g2 preserved */
2307 #else /* UTSB_PHYS */
2308         brgez,pt %g1, 9f                /* check for 2nd TSB */
2309           mov   -1, %g3                 /* set second tsbe ptr to -1 */
2310 
2311         mov     %g2, %g7
2312         GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2313         /* %g3 = second TSB entry ptr now, %g7 clobbered */
2314         mov     %g1, %g7
2315         GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2316 #endif /* UTSB_PHYS */
2317 #endif /* sun4v */
2318 9:
2319         CPU_TSBMISS_AREA(%g6, %g7)
2320         HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2321         ba,pt   %xcc, sfmmu_tsb_miss_tt         /* branch TSB miss handler */
2322           nop
2323 
2324         /*
2325          * Kernel 8K page iTLB miss.  We also get here if we took a
2326          * fast instruction access mmu miss trap while running in
2327          * invalid context.
2328          *
2329          * %g1 = 8K TSB pointer register (not used, clobbered)
2330          * %g2 = tag access register (used)
2331          * %g3 = faulting context id (used)
2332          * %g7 = TSB tag to match (used)
2333          */
2334         .align  64
2335         ALTENTRY(sfmmu_kitlb_miss)
2336         brnz,pn %g3, tsb_tl0_noctxt
2337           nop
2338 
2339         /* kernel miss */
2340         /* get kernel tsb pointer */
2341         /* we patch the next set of instructions at run time */
2342         /* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2343 iktsbbase:
2344         RUNTIME_PATCH_SETX(%g4, %g5)
2345         /* %g4 = contents of ktsb_base or ktsb_pbase */
2346 
2347 iktsb:  sllx    %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2348         srlx    %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2349         or      %g4, %g1, %g1                   ! form tsb ptr
2350         ldda    [%g1]RUNTIME_PATCH, %g4         ! %g4 = tag, %g5 = data
2351         cmp     %g4, %g7
2352         bne,pn  %xcc, iktsb4mbase               ! check 4m ktsb
2353           srlx    %g2, MMU_PAGESHIFT4M, %g3     ! use 4m virt-page as TSB index
2354 
2355         andcc %g5, TTE_EXECPRM_INT, %g0         ! check exec bit
2356         bz,pn   %icc, exec_fault
2357           nop
2358         TT_TRACE(trace_tsbhit)                  ! 2 instr traptrace
2359         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2360         retry
2361 
2362 iktsb4mbase:
2363         RUNTIME_PATCH_SETX(%g4, %g6)
2364         /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2365 iktsb4m:
2366         sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2367         srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2368         add     %g4, %g3, %g3                   ! %g3 = 4m tsbe ptr
2369         ldda    [%g3]RUNTIME_PATCH, %g4         ! %g4 = tag, %g5 = data
2370         cmp     %g4, %g7
2371         bne,pn  %xcc, sfmmu_tsb_miss_tt         ! branch on miss
2372           andcc %g5, TTE_EXECPRM_INT, %g0               ! check exec bit
2373         bz,pn   %icc, exec_fault
2374           nop
2375         TT_TRACE(trace_tsbhit)                  ! 2 instr traptrace
2376         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2377         retry
2378 
2379         /*
2380          * Kernel dTLB miss.  We also get here if we took a fast data
2381          * access mmu miss trap while running in invalid context.
2382          *
2383          * Note: for now we store kpm TTEs in the kernel TSB as usual.
2384          *      We select the TSB miss handler to branch to depending on
2385          *      the virtual address of the access.  In the future it may
2386          *      be desirable to separate kpm TTEs into their own TSB,
2387          *      in which case all that needs to be done is to set
2388          *      kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2389          *      early in the miss if we detect a kpm VA to a new handler.
2390          *
2391          * %g1 = 8K TSB pointer register (not used, clobbered)
2392          * %g2 = tag access register (used)
2393          * %g3 = faulting context id (used)
2394          */
2395         .align  64
2396         ALTENTRY(sfmmu_kdtlb_miss)
2397         brnz,pn %g3, tsb_tl0_noctxt             /* invalid context? */
2398           nop
2399 
2400         /* Gather some stats for kpm misses in the TLB. */
2401         /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2402         KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2403 
2404         /*
2405          * Get first TSB offset and look for 8K/64K/512K mapping
2406          * using the 8K virtual page as the index.
2407          *
2408          * We patch the next set of instructions at run time;
2409          * any changes here require sfmmu_patch_ktsb changes too.
2410          */
2411 dktsbbase:
2412         RUNTIME_PATCH_SETX(%g7, %g6)
2413         /* %g7 = contents of ktsb_base or ktsb_pbase */
2414 
2415 dktsb:  sllx    %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2416         srlx    %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2417 
2418         /*
2419          * At this point %g1 is our index into the TSB.
2420          * We just masked off enough bits of the VA depending
2421          * on our TSB size code.
2422          */
2423         ldda    [%g7 + %g1]RUNTIME_PATCH, %g4   ! %g4 = tag, %g5 = data
2424         srlx    %g2, TAG_VALO_SHIFT, %g6        ! make tag to compare
2425         cmp     %g6, %g4                        ! compare tag
2426         bne,pn  %xcc, dktsb4m_kpmcheck_small
2427           add   %g7, %g1, %g1                   /* form tsb ptr */
2428         TT_TRACE(trace_tsbhit)
2429         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2430         /* trapstat expects tte in %g5 */
2431         retry
2432 
2433         /*
2434          * If kpm is using large pages, the following instruction needs
2435          * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2436          * so that we will probe the 4M TSB regardless of the VA.  In
2437          * the case kpm is using small pages, we know no large kernel
2438          * mappings are located above 0x80000000.00000000 so we skip the
2439          * probe as an optimization.
2440          */
2441 dktsb4m_kpmcheck_small:
2442         brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2443           /* delay slot safe, below */
2444 
2445         /*
2446          * Get second TSB offset and look for 4M mapping
2447          * using 4M virtual page as the TSB index.
2448          *
2449          * Here:
2450          * %g1 = 8K TSB pointer.  Don't squash it.
2451          * %g2 = tag access register (we still need it)
2452          */
2453         srlx    %g2, MMU_PAGESHIFT4M, %g3
2454 
2455         /*
2456          * We patch the next set of instructions at run time;
2457          * any changes here require sfmmu_patch_ktsb changes too.
2458          */
2459 dktsb4mbase:
2460         RUNTIME_PATCH_SETX(%g7, %g6)
2461         /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2462 dktsb4m:
2463         sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2464         srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2465 
2466         /*
2467          * At this point %g3 is our index into the TSB.
2468          * We just masked off enough bits of the VA depending
2469          * on our TSB size code.
2470          */
2471         ldda    [%g7 + %g3]RUNTIME_PATCH, %g4   ! %g4 = tag, %g5 = data
2472         srlx    %g2, TAG_VALO_SHIFT, %g6        ! make tag to compare
2473         cmp     %g6, %g4                        ! compare tag
2474 
2475 dktsb4m_tsbmiss:
2476         bne,pn  %xcc, dktsb4m_kpmcheck
2477           add   %g7, %g3, %g3                   ! %g3 = kernel second TSB ptr
2478         TT_TRACE(trace_tsbhit)
2479         /* we don't check TTE size here since we assume 4M TSB is separate */
2480         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2481         /* trapstat expects tte in %g5 */
2482         retry
2483 
2484         /*
2485          * So, we failed to find a valid TTE to match the faulting
2486          * address in either TSB.  There are a few cases that could land
2487          * us here:
2488          *
2489          * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2490          *    to sfmmu_tsb_miss_tt to handle the miss.
2491          * 2) We missed on a kpm VA, and we didn't find the mapping in the
2492          *    4M TSB.  Let segkpm handle it.
2493          *
2494          * Note that we shouldn't land here in the case of a kpm VA when
2495          * kpm_smallpages is active -- we handled that case earlier at
2496          * dktsb4m_kpmcheck_small.
2497          *
2498          * At this point:
2499          *  g1 = 8K-indexed primary TSB pointer
2500          *  g2 = tag access register
2501          *  g3 = 4M-indexed secondary TSB pointer
2502          */
2503 dktsb4m_kpmcheck:
2504         cmp     %g2, %g0
2505         bl,pn   %xcc, sfmmu_kpm_dtsb_miss
2506           nop
2507         ba,a,pt %icc, sfmmu_tsb_miss_tt
2508           nop
2509 
2510 #ifdef sun4v
2511         /*
2512          * User instruction miss w/ single TSB.
2513          * The first probe covers 8K, 64K, and 512K page sizes,
2514          * because 64K and 512K mappings are replicated off 8K
2515          * pointer.
2516          *
2517          * g1 = tsb8k pointer register
2518          * g2 = tag access register
2519          * g3 - g6 = scratch registers
2520          * g7 = TSB tag to match
2521          */
2522         .align  64
2523         ALTENTRY(sfmmu_uitlb_fastpath)
2524 
2525         PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2526         /* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2527         ba,pn   %xcc, sfmmu_tsb_miss_tt
2528           mov   -1, %g3
2529 
2530         /*
2531          * User data miss w/ single TSB.
2532          * The first probe covers 8K, 64K, and 512K page sizes,
2533          * because 64K and 512K mappings are replicated off 8K
2534          * pointer.
2535          *
2536          * g1 = tsb8k pointer register
2537          * g2 = tag access register
2538          * g3 - g6 = scratch registers
2539          * g7 = TSB tag to match
2540          */
2541         .align 64
2542         ALTENTRY(sfmmu_udtlb_fastpath)
2543 
2544         PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2545         /* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2546         ba,pn   %xcc, sfmmu_tsb_miss_tt
2547           mov   -1, %g3
2548 
2549         /*
2550          * User instruction miss w/ multiple TSBs (sun4v).
2551          * The first probe covers 8K, 64K, and 512K page sizes,
2552          * because 64K and 512K mappings are replicated off 8K
2553          * pointer.  Second probe covers 4M page size only.
2554          *
2555          * Just like sfmmu_udtlb_slowpath, except:
2556          *   o Uses ASI_ITLB_IN
2557          *   o checks for execute permission
2558          *   o No ISM prediction.
2559          *
2560          * g1 = tsb8k pointer register
2561          * g2 = tag access register
2562          * g3 - g6 = scratch registers
2563          * g7 = TSB tag to match
2564          */
2565         .align  64
2566         ALTENTRY(sfmmu_uitlb_slowpath)
2567 
2568         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2569         PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2570         /* g4 - g5 = clobbered here */
2571 
2572         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2573         /* g1 = first TSB pointer, g3 = second TSB pointer */
2574         srlx    %g2, TAG_VALO_SHIFT, %g7
2575         PROBE_2ND_ITSB(%g3, %g7)
2576         /* NOT REACHED */
2577 
2578 #else /* sun4v */
2579 
2580         /*
2581          * User instruction miss w/ multiple TSBs (sun4u).
2582          * The first probe covers 8K, 64K, and 512K page sizes,
2583          * because 64K and 512K mappings are replicated off 8K
2584          * pointer.  Probe of 1st TSB has already been done prior to entry
2585          * into this routine. For the UTSB_PHYS case we probe up to 3
2586          * valid other TSBs in the following order:
2587          * 1) shared TSB for 4M-256M pages
2588          * 2) private TSB for 4M-256M pages
2589          * 3) shared TSB for 8K-512K pages
2590          *
2591          * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
2592          * 4M-256M pages.
2593          *
2594          * Just like sfmmu_udtlb_slowpath, except:
2595          *   o Uses ASI_ITLB_IN
2596          *   o checks for execute permission
2597          *   o No ISM prediction.
2598          *
2599          * g1 = tsb8k pointer register
2600          * g2 = tag access register
2601          * g4 - g6 = scratch registers
2602          * g7 = TSB tag to match
2603          */
2604         .align  64
2605         ALTENTRY(sfmmu_uitlb_slowpath)
2606 
2607 #ifdef UTSB_PHYS
2608 
2609        GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2610         brlz,pt %g6, 1f
2611           nop
2612         GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2613         PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
2614 1:
2615         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2616         brlz,pt %g3, 2f
2617           nop
2618         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2619         PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
2620 2:
2621         GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2622         brlz,pt %g6, sfmmu_tsb_miss_tt
2623           nop
2624         GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2625         PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
2626         ba,pn   %xcc, sfmmu_tsb_miss_tt
2627           nop
2628 
2629 #else /* UTSB_PHYS */
2630         mov     %g1, %g3        /* save tsb8k reg in %g3 */
2631         GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2632         PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2633         mov     %g2, %g6        /* GET_2ND_TSBE_PTR clobbers tagacc */
2634         mov     %g3, %g7        /* copy tsb8k reg in %g7 */
2635         GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2636        /* g1 = first TSB pointer, g3 = second TSB pointer */
2637         srlx    %g2, TAG_VALO_SHIFT, %g7
2638         PROBE_2ND_ITSB(%g3, %g7, isynth)
2639         ba,pn   %xcc, sfmmu_tsb_miss_tt
2640           nop
2641 
2642 #endif /* UTSB_PHYS */
2643 #endif /* sun4v */
2644 
2645 #if defined(sun4u) && defined(UTSB_PHYS)
2646 
2647         /*
2648          * We come here for ism predict DTLB_MISS case or if
2649          * if probe in first TSB failed.
2650          */
2651 
2652         .align 64
2653         ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
2654 
2655         /*
2656          * g1 = tsb8k pointer register
2657          * g2 = tag access register
2658          * g4 - %g6 = scratch registers
2659          * g7 = TSB tag to match
2660          */
2661 
2662         /*
2663          * ISM non-predict probe order
2664          * probe 1ST_TSB (8K index)
2665          * probe 2ND_TSB (4M index)
2666          * probe 4TH_TSB (4M index)
2667          * probe 3RD_TSB (8K index)
2668          *
2669          * We already probed first TSB in DTLB_MISS handler.
2670          */
2671 
2672         /*
2673          * Private 2ND TSB 4M-256 pages
2674          */
2675         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2676         brlz,pt %g3, 1f
2677           nop
2678         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2679         PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2680 
2681         /*
2682          * Shared Context 4TH TSB 4M-256 pages
2683          */
2684 1:
2685         GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2686         brlz,pt %g6, 2f
2687           nop
2688         GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2689         PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
2690 
2691         /*
2692          * Shared Context 3RD TSB 8K-512K pages
2693          */
2694 2:
2695         GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2696         brlz,pt %g6, sfmmu_tsb_miss_tt
2697           nop
2698         GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2699         PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
2700         ba,pn   %xcc, sfmmu_tsb_miss_tt
2701           nop
2702 
2703         .align 64
2704         ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
2705 
2706         /*
2707          * g1 = tsb8k pointer register
2708          * g2 = tag access register
2709          * g4 - g6 = scratch registers
2710          * g7 = TSB tag to match
2711          */
2712 
2713         /*
2714          * ISM predict probe order
2715          * probe 4TH_TSB (4M index)
2716          * probe 2ND_TSB (4M index)
2717          * probe 1ST_TSB (8K index)
2718          * probe 3RD_TSB (8K index)
2719 
2720         /*
2721          * Shared Context 4TH TSB 4M-256 pages
2722          */
2723         GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2724         brlz,pt %g6, 4f
2725           nop
2726         GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2727         PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
2728 
2729         /*
2730          * Private 2ND TSB 4M-256 pages
2731          */
2732 4:
2733         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2734         brlz,pt %g3, 5f
2735           nop
2736         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2737         PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
2738 
2739 5:
2740         PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
2741 
2742         /*
2743          * Shared Context 3RD TSB 8K-512K pages
2744          */
2745         GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2746         brlz,pt %g6, 6f
2747           nop
2748         GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2749         PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
2750 6:
2751         ba,pn   %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
2752           nop
2753 
2754 #else /* sun4u && UTSB_PHYS */
2755 
2756        .align 64
2757         ALTENTRY(sfmmu_udtlb_slowpath)
2758 
2759         srax    %g2, PREDISM_BASESHIFT, %g6     /* g6 > 0 : ISM predicted */
2760         brgz,pn %g6, udtlb_miss_probesecond     /* check for ISM */
2761           mov   %g1, %g3
2762 
2763 udtlb_miss_probefirst:
2764         /*
2765          * g1 = 8K TSB pointer register
2766          * g2 = tag access register
2767          * g3 = (potentially) second TSB entry ptr
2768          * g6 = ism pred.
2769          * g7 = vpg_4m
2770          */
2771 #ifdef sun4v
2772         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2773         PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2774 
2775         /*
2776          * Here:
2777          *   g1 = first TSB pointer
2778          *   g2 = tag access reg
2779          *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2780          */
2781         brgz,pn %g6, sfmmu_tsb_miss_tt
2782           nop
2783 #else /* sun4v */
2784         mov     %g1, %g4
2785         GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2786         PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2787 
2788         /*
2789          * Here:
2790          *   g1 = first TSB pointer
2791          *   g2 = tag access reg
2792          *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2793          */
2794         brgz,pn %g6, sfmmu_tsb_miss_tt
2795           nop
2796         ldxa    [%g0]ASI_DMMU_TSB_8K, %g3
2797         /* fall through in 8K->4M probe order */
2798 #endif /* sun4v */
2799 
2800 udtlb_miss_probesecond:
2801         /*
2802          * Look in the second TSB for the TTE
2803          * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2804          * g2 = tag access reg
2805          * g3 = 8K TSB pointer register
2806          * g6 = ism pred.
2807          * g7 = vpg_4m
2808          */
2809 #ifdef sun4v
2810         /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2811         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2812         /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2813 #else /* sun4v */
2814         mov     %g3, %g7
2815         GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2816         /* %g2 clobbered, %g3 =second tsbe ptr */
2817         mov     MMU_TAG_ACCESS, %g2
2818         ldxa    [%g2]ASI_DMMU, %g2
2819 #endif /* sun4v */
2820 
2821         srlx    %g2, TAG_VALO_SHIFT, %g7
2822         PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2823         /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2824         brgz,pn %g6, udtlb_miss_probefirst
2825           nop
2826 
2827         /* fall through to sfmmu_tsb_miss_tt */
2828 #endif /* sun4u && UTSB_PHYS */
2829 
2830 
2831         ALTENTRY(sfmmu_tsb_miss_tt)
2832         TT_TRACE(trace_tsbmiss)
2833         /*
2834          * We get here if there is a TSB miss OR a write protect trap.
2835          *
2836          * g1 = First TSB entry pointer
2837          * g2 = tag access register
2838          * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
2839          * g4 - g7 = scratch registers
2840          */
2841 
2842         ALTENTRY(sfmmu_tsb_miss)
2843 
2844         /*
2845          * If trapstat is running, we need to shift the %tpc and %tnpc to
2846          * point to trapstat's TSB miss return code (note that trapstat
2847          * itself will patch the correct offset to add).
2848          */
2849         rdpr    %tl, %g7
2850         cmp     %g7, 1
2851         ble,pt  %xcc, 0f
2852           sethi %hi(KERNELBASE), %g6
2853         rdpr    %tpc, %g7
2854         or      %g6, %lo(KERNELBASE), %g6
2855         cmp     %g7, %g6
2856         bgeu,pt %xcc, 0f
2857         /* delay slot safe */
2858 
2859         ALTENTRY(tsbmiss_trapstat_patch_point)
2860         add     %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
2861         wrpr    %g7, %tpc
2862         add     %g7, 4, %g7
2863         wrpr    %g7, %tnpc
2864 0:
2865         CPU_TSBMISS_AREA(%g6, %g7)
2866         stn     %g1, [%g6 + TSBMISS_TSBPTR]     /* save 1ST tsb pointer */
2867         stn     %g3, [%g6 + TSBMISS_TSBPTR4M]   /* save 2ND tsb pointer */
2868 
2869         sllx    %g2, TAGACC_CTX_LSHIFT, %g3
2870         brz,a,pn %g3, 1f                        /* skip ahead if kernel */
2871           ldn   [%g6 + TSBMISS_KHATID], %g7
2872         srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctxnum */
2873         ldn     [%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2874 
2875         HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2876 
2877         cmp     %g3, INVALID_CONTEXT
2878         be,pn   %icc, tsb_tl0_noctxt            /* no ctx miss exception */
2879           stn   %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2880 
2881 #if defined(sun4v) || defined(UTSB_PHYS)
2882         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */
2883         andn    %g7, HAT_CHKCTX1_FLAG, %g7      /* the previous tsb miss    */
2884         stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
2885 #endif /* sun4v || UTSB_PHYS */
2886 
2887         ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2888         /*
2889          * The miss wasn't in an ISM segment.
2890          *
2891          * %g1 %g3, %g4, %g5, %g7 all clobbered
2892          * %g2 = (pseudo) tag access
2893          */
2894 
2895         ba,pt   %icc, 2f
2896           ldn   [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2897 
2898 1:
2899         HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2900         /*
2901          * 8K and 64K hash.
2902          */
2903 2:
2904 
2905         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2906                 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte,
2907                 sfmmu_suspend_tl, tsb_512K)
2908         /* NOT REACHED */
2909 
2910 tsb_512K:
2911         sllx    %g2, TAGACC_CTX_LSHIFT, %g5
2912         brz,pn  %g5, 3f
2913           ldub  [%g6 + TSBMISS_UTTEFLAGS], %g4
2914         and     %g4, HAT_512K_FLAG, %g5
2915 
2916         /*
2917          * Note that there is a small window here where we may have
2918          * a 512k page in the hash list but have not set the HAT_512K_FLAG
2919          * flag yet, so we will skip searching the 512k hash list.
2920          * In this case we will end up in pagefault which will find
2921          * the mapping and return.  So, in this instance we will end up
2922          * spending a bit more time resolving this TSB miss, but it can
2923          * only happen once per process and even then, the chances of that
2924          * are very small, so it's not worth the extra overhead it would
2925          * take to close this window.
2926          */
2927         brz,pn  %g5, tsb_4M
2928           nop
2929 3:
2930         /*
2931          * 512K hash
2932          */
2933 
2934         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2935                 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte,
2936                 sfmmu_suspend_tl, tsb_4M)
2937         /* NOT REACHED */
2938 
2939 tsb_4M:
2940         sllx    %g2, TAGACC_CTX_LSHIFT, %g5
2941         brz,pn  %g5, 4f
2942           ldub  [%g6 + TSBMISS_UTTEFLAGS], %g4
2943         and     %g4, HAT_4M_FLAG, %g5
2944         brz,pn  %g5, tsb_32M
2945           nop
2946 4:
2947         /*
2948          * 4M hash
2949          */
2950 
2951         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2952                 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte,
2953                 sfmmu_suspend_tl, tsb_32M)
2954         /* NOT REACHED */
2955 
2956 tsb_32M:
2957         sllx    %g2, TAGACC_CTX_LSHIFT, %g5
2958 #ifdef  sun4v
2959         brz,pn  %g5, 6f
2960 #else
2961         brz,pn  %g5, tsb_pagefault
2962 #endif
2963           ldub  [%g6 + TSBMISS_UTTEFLAGS], %g4
2964         and     %g4, HAT_32M_FLAG, %g5
2965         brz,pn  %g5, tsb_256M
2966           nop
2967 5:
2968         /*
2969          * 32M hash
2970          */
2971 
2972         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2973                 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte,
2974                 sfmmu_suspend_tl, tsb_256M)
2975         /* NOT REACHED */
2976 
2977 #if defined(sun4u) && !defined(UTSB_PHYS)
2978 #define tsb_shme        tsb_pagefault
2979 #endif
2980 tsb_256M:
2981         ldub    [%g6 + TSBMISS_UTTEFLAGS], %g4
2982         and     %g4, HAT_256M_FLAG, %g5
2983         brz,pn  %g5, tsb_shme
2984           nop
2985 6:
2986         /*
2987          * 256M hash
2988          */
2989 
2990         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2991             MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte,
2992             sfmmu_suspend_tl, tsb_shme)
2993         /* NOT REACHED */
2994 
2995 tsb_checktte:
2996         /*
2997          * g1 = hblk_misc
2998          * g2 = tagacc
2999          * g3 = tte
3000          * g4 = tte pa
3001          * g6 = tsbmiss area
3002          * g7 = hatid
3003          */
3004         brlz,a,pt %g3, tsb_validtte
3005           rdpr  %tt, %g7
3006 
3007 #if defined(sun4u) && !defined(UTSB_PHYS)
3008 #undef tsb_shme
3009         ba      tsb_pagefault
3010           nop
3011 #else /* sun4u && !UTSB_PHYS */
3012 
3013 tsb_shme:
3014         /*
3015          * g2 = tagacc
3016          * g6 = tsbmiss area
3017          */
3018         sllx    %g2, TAGACC_CTX_LSHIFT, %g5
3019         brz,pn  %g5, tsb_pagefault
3020           nop
3021         ldx     [%g6 + TSBMISS_SHARED_UHATID], %g7      /* g7 = srdp */
3022         brz,pn  %g7, tsb_pagefault
3023           nop
3024 
3025         GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3026                 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte,
3027                 sfmmu_suspend_tl, tsb_shme_512K)
3028         /* NOT REACHED */
3029 
3030 tsb_shme_512K:
3031         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g4
3032         and     %g4, HAT_512K_FLAG, %g5
3033         brz,pn  %g5, tsb_shme_4M
3034           nop
3035 
3036         /*
3037          * 512K hash
3038          */
3039 
3040         GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3041                 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte,
3042                 sfmmu_suspend_tl, tsb_shme_4M)
3043         /* NOT REACHED */
3044 
3045 tsb_shme_4M:
3046         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g4
3047         and     %g4, HAT_4M_FLAG, %g5
3048         brz,pn  %g5, tsb_shme_32M
3049           nop
3050 4:
3051         /*
3052          * 4M hash
3053          */
3054         GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3055                 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte,
3056                 sfmmu_suspend_tl, tsb_shme_32M)
3057         /* NOT REACHED */
3058 
3059 tsb_shme_32M:
3060         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g4
3061         and     %g4, HAT_32M_FLAG, %g5
3062         brz,pn  %g5, tsb_shme_256M
3063           nop
3064 
3065         /*
3066          * 32M hash
3067          */
3068 
3069         GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3070                 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte,
3071                 sfmmu_suspend_tl, tsb_shme_256M)
3072         /* NOT REACHED */
3073 
3074 tsb_shme_256M:
3075         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g4
3076         and     %g4, HAT_256M_FLAG, %g5
3077         brz,pn  %g5, tsb_pagefault
3078           nop
3079 
3080         /*
3081          * 256M hash
3082          */
3083 
3084         GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3085             MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte,
3086             sfmmu_suspend_tl, tsb_pagefault)
3087         /* NOT REACHED */
3088 
3089 tsb_shme_checktte:
3090 
3091         brgez,pn %g3, tsb_pagefault
3092           rdpr  %tt, %g7
3093         /*
3094          * g1 = ctx1 flag
3095          * g3 = tte
3096          * g4 = tte pa
3097          * g6 = tsbmiss area
3098          * g7 = tt
3099          */
3100 
3101         brz,pt  %g1, tsb_validtte
3102           nop
3103         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3104           or    %g1, HAT_CHKCTX1_FLAG, %g1
3105         stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3106 
3107         SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3108 #endif /* sun4u && !UTSB_PHYS */
3109 
3110 tsb_validtte:
3111         /*
3112          * g3 = tte
3113          * g4 = tte pa
3114          * g6 = tsbmiss area
3115          * g7 = tt
3116          */
3117 
3118         /*
3119          * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3120          */
3121         cmp     %g7, FAST_PROT_TT
3122         bne,pt  %icc, 4f
3123           nop
3124 
3125         TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod,
3126             tsb_protfault)
3127 
3128         GET_MMU_D_TTARGET(%g2, %g7)             /* %g2 = ttarget */
3129 #ifdef sun4v
3130         MMU_FAULT_STATUS_AREA(%g7)
3131         ldx     [%g7 + MMFSA_D_ADDR], %g5       /* load fault addr for later */
3132 #else /* sun4v */
3133         mov     MMU_TAG_ACCESS, %g5
3134         ldxa    [%g5]ASI_DMMU, %g5
3135 #endif /* sun4v */
3136         ba,pt   %xcc, tsb_update_tl1
3137           nop
3138 4:
3139         /*
3140          * If ITLB miss check exec bit.
3141          * If not set treat as invalid TTE.
3142          */
3143         cmp     %g7, T_INSTR_MMU_MISS
3144         be,pn   %icc, 5f
3145           andcc   %g3, TTE_EXECPRM_INT, %g0     /* check execute bit is set */
3146         cmp     %g7, FAST_IMMU_MISS_TT
3147         bne,pt %icc, 3f
3148           andcc   %g3, TTE_EXECPRM_INT, %g0     /* check execute bit is set */
3149 5:
3150         bz,pn %icc, tsb_protfault
3151           nop
3152 
3153 3:
3154         /*
3155          * Set reference bit if not already set
3156          */
3157         TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref)
3158 
3159         /*
3160          * Now, load into TSB/TLB.  At this point:
3161          * g3 = tte
3162          * g4 = patte
3163          * g6 = tsbmiss area
3164          */
3165         rdpr    %tt, %g7
3166 #ifdef sun4v
3167         MMU_FAULT_STATUS_AREA(%g2)
3168         cmp     %g7, T_INSTR_MMU_MISS
3169         be,a,pt %icc, 9f
3170           nop
3171         cmp     %g7, FAST_IMMU_MISS_TT
3172         be,a,pt %icc, 9f
3173           nop
3174         add     %g2, MMFSA_D_, %g2
3175 9:
3176         ldx     [%g2 + MMFSA_CTX_], %g7
3177         sllx    %g7, TTARGET_CTX_SHIFT, %g7
3178         ldx     [%g2 + MMFSA_ADDR_], %g2
3179         mov     %g2, %g5                ! load the fault addr for later use
3180         srlx    %g2, TTARGET_VA_SHIFT, %g2
3181         or      %g2, %g7, %g2
3182 #else /* sun4v */
3183         mov     MMU_TAG_ACCESS, %g5
3184         cmp     %g7, FAST_IMMU_MISS_TT
3185         be,a,pt %icc, 9f
3186            ldxa  [%g0]ASI_IMMU, %g2
3187         ldxa    [%g0]ASI_DMMU, %g2
3188         ba,pt   %icc, tsb_update_tl1
3189            ldxa  [%g5]ASI_DMMU, %g5
3190 9:
3191         ldxa    [%g5]ASI_IMMU, %g5
3192 #endif /* sun4v */
3193 
3194 tsb_update_tl1:
3195         srlx    %g2, TTARGET_CTX_SHIFT, %g7
3196         brz,pn  %g7, tsb_kernel
3197 #ifdef sun4v
3198           and   %g3, TTE_SZ_BITS, %g7   ! assumes TTE_SZ_SHFT is 0
3199 #else  /* sun4v */
3200           srlx  %g3, TTE_SZ_SHFT, %g7
3201 #endif /* sun4v */
3202 
3203 tsb_user:
3204 #ifdef sun4v
3205         cmp     %g7, TTE4M
3206         bge,pn  %icc, tsb_user4m
3207           nop
3208 #else /* sun4v */
3209         cmp     %g7, TTESZ_VALID | TTE4M
3210         be,pn   %icc, tsb_user4m
3211           srlx  %g3, TTE_SZ2_SHFT, %g7
3212         andcc   %g7, TTE_SZ2_BITS, %g7          ! check 32/256MB
3213 #ifdef ITLB_32M_256M_SUPPORT
3214         bnz,pn  %icc, tsb_user4m
3215           nop
3216 #else /* ITLB_32M_256M_SUPPORT */
3217         bnz,a,pn %icc, tsb_user_pn_synth
3218          nop
3219 #endif /* ITLB_32M_256M_SUPPORT */
3220 #endif /* sun4v */
3221 
3222 tsb_user8k:
3223 #if defined(sun4v) || defined(UTSB_PHYS)
3224         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7
3225         and     %g7, HAT_CHKCTX1_FLAG, %g1
3226         brz,a,pn %g1, 1f
3227           ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3228         GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3229         brlz,a,pn %g1, ptl1_panic                       ! if no shared 3RD tsb
3230           mov PTL1_NO_SCDTSB8K, %g1                     ! panic
3231         GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
3232 1:
3233 #else /* defined(sun4v) || defined(UTSB_PHYS) */
3234         ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3235 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3236 
3237 #ifndef UTSB_PHYS
3238         mov     ASI_N, %g7      ! user TSBs accessed by VA
3239         mov     %g7, %asi
3240 #endif /* !UTSB_PHYS */
3241 
3242         TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3)
3243 
3244         rdpr    %tt, %g5
3245 #ifdef sun4v
3246         cmp     %g5, T_INSTR_MMU_MISS
3247         be,a,pn %xcc, 9f
3248           mov   %g3, %g5
3249 #endif /* sun4v */
3250         cmp     %g5, FAST_IMMU_MISS_TT
3251         be,pn   %xcc, 9f
3252           mov   %g3, %g5
3253 
3254         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3255         ! trapstat wants TTE in %g5
3256         retry
3257 9:
3258         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3259         ! trapstat wants TTE in %g5
3260         retry
3261 
3262 tsb_user4m:
3263 #if defined(sun4v) || defined(UTSB_PHYS)
3264         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7
3265         and     %g7, HAT_CHKCTX1_FLAG, %g1
3266         brz,a,pn %g1, 4f
3267           ldn   [%g6 + TSBMISS_TSBPTR4M], %g1           ! g1 = 2ND TSB ptr
3268         GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3269         brlz,a,pn %g1, 5f                               ! if no shared 4TH TSB
3270           nop
3271         GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3272 
3273 #else /* defined(sun4v) || defined(UTSB_PHYS) */
3274         ldn   [%g6 + TSBMISS_TSBPTR4M], %g1             ! g1 = 2ND TSB ptr
3275 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3276 4:
3277         brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */
3278           nop
3279 
3280 #ifndef UTSB_PHYS
3281         mov     ASI_N, %g7      ! user TSBs accessed by VA
3282         mov     %g7, %asi
3283 #endif /* UTSB_PHYS */
3284 
3285         TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4)
3286 
3287 5:
3288         rdpr    %tt, %g5
3289 #ifdef sun4v
3290         cmp     %g5, T_INSTR_MMU_MISS
3291         be,a,pn %xcc, 9f
3292           mov   %g3, %g5
3293 #endif /* sun4v */
3294         cmp     %g5, FAST_IMMU_MISS_TT
3295         be,pn   %xcc, 9f
3296         mov     %g3, %g5
3297 
3298         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3299         ! trapstat wants TTE in %g5
3300         retry
3301 9:
3302         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3303         ! trapstat wants TTE in %g5
3304         retry
3305 
3306 #if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3307         /*
3308          * Panther ITLB synthesis.
3309          * The Panther 32M and 256M ITLB code simulates these two large page
3310          * sizes with 4M pages, to provide support for programs, for example
3311          * Java, that may copy instructions into a 32M or 256M data page and
3312          * then execute them. The code below generates the 4M pfn bits and
3313          * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3314          * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3315          * are ignored by the hardware.
3316          *
3317          * Now, load into TSB/TLB.  At this point:
3318          * g2 = tagtarget
3319          * g3 = tte
3320          * g4 = patte
3321          * g5 = tt
3322          * g6 = tsbmiss area
3323          */
3324 tsb_user_pn_synth:
3325         rdpr %tt, %g5
3326         cmp    %g5, FAST_IMMU_MISS_TT
3327         be,pt   %xcc, tsb_user_itlb_synth       /* ITLB miss */
3328           andcc %g3, TTE_EXECPRM_INT, %g0       /* is execprm bit set */
3329         bz,pn %icc, 4b                          /* if not, been here before */
3330           ldn   [%g6 + TSBMISS_TSBPTR4M], %g1   /* g1 = tsbp */
3331         brlz,a,pn %g1, 5f                       /* no 2nd tsb */
3332           mov   %g3, %g5
3333 
3334         mov     MMU_TAG_ACCESS, %g7
3335         ldxa    [%g7]ASI_DMMU, %g6              /* get tag access va */
3336         GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)   /* make 4M pfn offset */
3337 
3338         mov     ASI_N, %g7      /* user TSBs always accessed by VA */
3339         mov     %g7, %asi
3340         TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */
3341 5:
3342         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3343         retry
3344 
3345 tsb_user_itlb_synth:
3346         ldn     [%g6 + TSBMISS_TSBPTR4M], %g1           /* g1 =  2ND TSB */
3347 
3348         mov     MMU_TAG_ACCESS, %g7
3349         ldxa    [%g7]ASI_IMMU, %g6              /* get tag access va */
3350         GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)   /* make 4M pfn offset */
3351         brlz,a,pn %g1, 7f       /* Check to see if we have 2nd TSB programmed */
3352           or    %g5, %g3, %g5                   /* add 4M bits to TTE */
3353 
3354         mov     ASI_N, %g7      /* user TSBs always accessed by VA */
3355         mov     %g7, %asi
3356         TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */
3357 7:
3358         SET_TTE4M_PN(%g5, %g7)                  /* add TTE4M pagesize to TTE */
3359         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3360         retry
3361 #endif /* sun4v && ITLB_32M_256M_SUPPORT */
3362 
3363 tsb_kernel:
3364         rdpr    %tt, %g5
3365 #ifdef sun4v
3366         cmp     %g7, TTE4M
3367         bge,pn  %icc, 5f
3368 #else
3369         cmp     %g7, TTESZ_VALID | TTE4M        ! no 32M or 256M support
3370         be,pn   %icc, 5f
3371 #endif /* sun4v */
3372           nop
3373         ldn     [%g6 + TSBMISS_TSBPTR], %g1     ! g1 = 8K TSB ptr
3374         ba,pt   %xcc, 6f
3375           nop
3376 5:
3377         ldn     [%g6 + TSBMISS_TSBPTR4M], %g1   ! g1 = 4M TSB ptr
3378         brlz,pn %g1, 3f         /* skip programming if 4M TSB ptr is -1 */
3379           nop
3380 6:
3381 #ifndef sun4v
3382 tsb_kernel_patch_asi:
3383         or      %g0, RUNTIME_PATCH, %g6
3384         mov     %g6, %asi       ! XXX avoid writing to %asi !!
3385 #endif
3386         TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7)
3387 3:
3388 #ifdef sun4v
3389         cmp     %g5, T_INSTR_MMU_MISS
3390         be,a,pn %icc, 1f
3391           mov   %g3, %g5                        ! trapstat wants TTE in %g5
3392 #endif /* sun4v */
3393         cmp     %g5, FAST_IMMU_MISS_TT
3394         be,pn   %icc, 1f
3395           mov   %g3, %g5                        ! trapstat wants TTE in %g5
3396         DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3397         ! trapstat wants TTE in %g5
3398         retry
3399 1:
3400         ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3401         ! trapstat wants TTE in %g5
3402         retry
3403 
3404 tsb_ism:
3405         /*
3406          * This is an ISM [i|d]tlb miss.  We optimize for largest
3407          * page size down to smallest.
3408          *
3409          * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3410          *      register
3411          * g3 = ismmap->ism_seg
3412          * g4 = physical address of ismmap->ism_sfmmu
3413          * g6 = tsbmiss area
3414          */
3415         ldna    [%g4]ASI_MEM, %g7               /* g7 = ism hatid */
3416         brz,a,pn %g7, ptl1_panic                /* if zero jmp ahead */
3417           mov   PTL1_BAD_ISM, %g1
3418                                                 /* g5 = pa of imap_vb_shift */
3419         sub     %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3420         lduba   [%g5]ASI_MEM, %g4               /* g4 = imap_vb_shift */
3421         srlx    %g3, %g4, %g3                   /* clr size field */
3422         set     TAGACC_CTX_MASK, %g1            /* mask off ctx number */
3423         sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3424         and     %g2, %g1, %g4                   /* g4 = ctx number */
3425         andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3426         sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3427         or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3428         sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3429         lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3430 #if defined(sun4v) || defined(UTSB_PHYS)
3431         and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3432         brz,pt %g5, tsb_chk4M_ism
3433           nop
3434         ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3435         or      %g5, HAT_CHKCTX1_FLAG, %g5
3436         stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3437         rdpr    %tt, %g5
3438         SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3439 #endif /* defined(sun4v) || defined(UTSB_PHYS) */
3440 
3441         /*
3442          * ISM pages are always locked down.
3443          * If we can't find the tte then pagefault
3444          * and let the spt segment driver resolve it.
3445          *
3446          * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3447          * g4 = imap_hatflags
3448          * g6 = tsb miss area
3449          * g7 = ISM hatid
3450          */
3451 
3452 tsb_chk4M_ism:
3453         and     %g4, HAT_4M_FLAG, %g5           /* g4 = imap_hatflags */
3454         brnz,pt %g5, tsb_ism_4M                 /* branch if 4M pages */
3455           nop
3456 
3457 tsb_ism_32M:
3458         and     %g4, HAT_32M_FLAG, %g5          /* check default 32M next */
3459         brz,pn  %g5, tsb_ism_256M
3460           nop
3461 
3462         /*
3463          * 32M hash.
3464          */
3465 
3466         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M,
3467             TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3468             tsb_ism_4M)
3469         /* NOT REACHED */
3470 
3471 tsb_ism_32M_found:
3472         brlz,a,pt %g3, tsb_validtte
3473           rdpr  %tt, %g7
3474         ba,pt   %xcc, tsb_ism_4M
3475           nop
3476 
3477 tsb_ism_256M:
3478         and     %g4, HAT_256M_FLAG, %g5         /* 256M is last resort */
3479         brz,a,pn %g5, ptl1_panic
3480           mov   PTL1_BAD_ISM, %g1
3481 
3482         /*
3483          * 256M hash.
3484          */
3485         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M,
3486             TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3487             tsb_ism_4M)
3488 
3489 tsb_ism_256M_found:
3490         brlz,a,pt %g3, tsb_validtte
3491           rdpr  %tt, %g7
3492 
3493 tsb_ism_4M:
3494         /*
3495          * 4M hash.
3496          */
3497         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M,
3498             TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3499             tsb_ism_8K)
3500         /* NOT REACHED */
3501 
3502 tsb_ism_4M_found:
3503         brlz,a,pt %g3, tsb_validtte
3504           rdpr  %tt, %g7
3505 
3506 tsb_ism_8K:
3507         /*
3508          * 8K and 64K hash.
3509          */
3510 
3511         GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K,
3512             TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3513             tsb_pagefault)
3514         /* NOT REACHED */
3515 
3516 tsb_ism_8K_found:
3517         brlz,a,pt %g3, tsb_validtte
3518           rdpr  %tt, %g7
3519 
3520 tsb_pagefault:
3521         rdpr    %tt, %g7
3522         cmp     %g7, FAST_PROT_TT
3523         be,a,pn %icc, tsb_protfault
3524           wrpr  %g0, FAST_DMMU_MISS_TT, %tt
3525 
3526 tsb_protfault:
3527         /*
3528          * we get here if we couldn't find a valid tte in the hash.
3529          *
3530          * If user and we are at tl>1 we go to window handling code.
3531          *
3532          * If kernel and the fault is on the same page as our stack
3533          * pointer, then we know the stack is bad and the trap handler
3534          * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3535          *
3536          * If this is a kernel trap and tl>1, panic.
3537          *
3538          * Otherwise we call pagefault.
3539          */
3540         cmp     %g7, FAST_IMMU_MISS_TT
3541 #ifdef sun4v
3542         MMU_FAULT_STATUS_AREA(%g4)
3543         ldx     [%g4 + MMFSA_I_CTX], %g5
3544         ldx     [%g4 + MMFSA_D_CTX], %g4
3545         move    %icc, %g5, %g4
3546         cmp     %g7, T_INSTR_MMU_MISS
3547         move    %icc, %g5, %g4
3548 #else
3549         mov     MMU_TAG_ACCESS, %g4
3550         ldxa    [%g4]ASI_DMMU, %g2
3551         ldxa    [%g4]ASI_IMMU, %g5
3552         move    %icc, %g5, %g2
3553         cmp     %g7, T_INSTR_MMU_MISS
3554         move    %icc, %g5, %g2
3555         sllx    %g2, TAGACC_CTX_LSHIFT, %g4
3556 #endif /* sun4v */
3557         brnz,pn %g4, 3f                         /* skip if not kernel */
3558           rdpr  %tl, %g5
3559 
3560         add     %sp, STACK_BIAS, %g3
3561         srlx    %g3, MMU_PAGESHIFT, %g3
3562         srlx    %g2, MMU_PAGESHIFT, %g4
3563         cmp     %g3, %g4
3564         be,a,pn %icc, ptl1_panic                /* panic if bad %sp */
3565           mov   PTL1_BAD_STACK, %g1
3566 
3567         cmp     %g5, 1
3568         ble,pt  %icc, 2f
3569           nop
3570         TSTAT_CHECK_TL1(2f, %g1, %g2)
3571         rdpr    %tt, %g2
3572         cmp     %g2, FAST_PROT_TT
3573         mov     PTL1_BAD_KPROT_FAULT, %g1
3574         movne   %icc, PTL1_BAD_KMISS, %g1
3575         ba,pt   %icc, ptl1_panic
3576           nop
3577 
3578 2:
3579         /*
3580          * We are taking a pagefault in the kernel on a kernel address.  If
3581          * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3582          * want to call sfmmu_pagefault -- we will instead note that a fault
3583          * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3584          * (instead of a "retry").  This will step over the faulting
3585          * instruction.
3586          */
3587         CPU_INDEX(%g1, %g2)
3588         set     cpu_core, %g2
3589         sllx    %g1, CPU_CORE_SHIFT, %g1
3590         add     %g1, %g2, %g1
3591         lduh    [%g1 + CPUC_DTRACE_FLAGS], %g2
3592         andcc   %g2, CPU_DTRACE_NOFAULT, %g0
3593         bz      sfmmu_pagefault
3594         or      %g2, CPU_DTRACE_BADADDR, %g2
3595         stuh    %g2, [%g1 + CPUC_DTRACE_FLAGS]
3596         GET_MMU_D_ADDR(%g3, %g4)
3597         stx     %g3, [%g1 + CPUC_DTRACE_ILLVAL]
3598         done
3599 
3600 3:
3601         cmp     %g5, 1
3602         ble,pt  %icc, 4f
3603           nop
3604         TSTAT_CHECK_TL1(4f, %g1, %g2)
3605         ba,pt   %icc, sfmmu_window_trap
3606           nop
3607 
3608 4:
3609         /*
3610          * We are taking a pagefault on a non-kernel address.  If we are in
3611          * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3612          * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3613          */
3614         CPU_INDEX(%g1, %g2)
3615         set     cpu_core, %g2
3616         sllx    %g1, CPU_CORE_SHIFT, %g1
3617         add     %g1, %g2, %g1
3618         lduh    [%g1 + CPUC_DTRACE_FLAGS], %g2
3619         andcc   %g2, CPU_DTRACE_NOFAULT, %g0
3620         bz      sfmmu_mmu_trap
3621         or      %g2, CPU_DTRACE_BADADDR, %g2
3622         stuh    %g2, [%g1 + CPUC_DTRACE_FLAGS]
3623         GET_MMU_D_ADDR(%g3, %g4)
3624         stx     %g3, [%g1 + CPUC_DTRACE_ILLVAL]
3625 
3626         /*
3627          * Be sure that we're actually taking this miss from the kernel --
3628          * otherwise we have managed to return to user-level with
3629          * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3630          */
3631         rdpr    %tstate, %g2
3632         btst    TSTATE_PRIV, %g2
3633         bz,a    ptl1_panic
3634           mov   PTL1_BAD_DTRACE_FLAGS, %g1
3635         done
3636 
3637         ALTENTRY(tsb_tl0_noctxt)
3638         /*
3639          * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3640          * if it is, indicated that we have faulted and issue a done.
3641          */
3642         CPU_INDEX(%g5, %g6)
3643         set     cpu_core, %g6
3644         sllx    %g5, CPU_CORE_SHIFT, %g5
3645         add     %g5, %g6, %g5
3646         lduh    [%g5 + CPUC_DTRACE_FLAGS], %g6
3647         andcc   %g6, CPU_DTRACE_NOFAULT, %g0
3648         bz      1f
3649         or      %g6, CPU_DTRACE_BADADDR, %g6
3650         stuh    %g6, [%g5 + CPUC_DTRACE_FLAGS]
3651         GET_MMU_D_ADDR(%g3, %g4)
3652         stx     %g3, [%g5 + CPUC_DTRACE_ILLVAL]
3653 
3654         /*
3655          * Be sure that we're actually taking this miss from the kernel --
3656          * otherwise we have managed to return to user-level with
3657          * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3658          */
3659         rdpr    %tstate, %g5
3660         btst    TSTATE_PRIV, %g5
3661         bz,a    ptl1_panic
3662           mov   PTL1_BAD_DTRACE_FLAGS, %g1
3663         TSTAT_CHECK_TL1(2f, %g1, %g2);
3664 2:
3665         done
3666 
3667 1:
3668         rdpr    %tt, %g5
3669         cmp     %g5, FAST_IMMU_MISS_TT
3670 #ifdef sun4v
3671         MMU_FAULT_STATUS_AREA(%g2)
3672         be,a,pt %icc, 2f
3673           ldx   [%g2 + MMFSA_I_CTX], %g3
3674         cmp     %g5, T_INSTR_MMU_MISS
3675         be,a,pt %icc, 2f
3676           ldx   [%g2 + MMFSA_I_CTX], %g3
3677         ldx     [%g2 + MMFSA_D_CTX], %g3
3678 2:
3679 #else
3680         mov     MMU_TAG_ACCESS, %g2
3681         be,a,pt %icc, 2f
3682           ldxa  [%g2]ASI_IMMU, %g3
3683         ldxa    [%g2]ASI_DMMU, %g3
3684 2:      sllx    %g3, TAGACC_CTX_LSHIFT, %g3
3685 #endif /* sun4v */
3686         brz,a,pn %g3, ptl1_panic                ! panic if called for kernel
3687           mov   PTL1_BAD_CTX_STEAL, %g1         ! since kernel ctx was stolen
3688         rdpr    %tl, %g5
3689         cmp     %g5, 1
3690         ble,pt  %icc, sfmmu_mmu_trap
3691           nop
3692         TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3693         ba,pt   %icc, sfmmu_window_trap
3694           nop
3695         SET_SIZE(sfmmu_tsb_miss)
3696 
3697         ENTRY_NP(sfmmu_vatopfn)
3698         /*
3699          * disable interrupts
3700          */
3701         rdpr    %pstate, %o3
3702 #ifdef DEBUG
3703         PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
3704 #endif
3705         /*
3706          * disable interrupts to protect the TSBMISS area
3707          */
3708         andn    %o3, PSTATE_IE, %o5
3709         wrpr    %o5, 0, %pstate
3710 
3711         /*
3712          * o0 = vaddr
3713          * o1 = sfmmup
3714          * o2 = ttep
3715          */
3716         CPU_TSBMISS_AREA(%g1, %o5)
3717         ldn     [%g1 + TSBMISS_KHATID], %o4
3718         cmp     %o4, %o1
3719         bne,pn  %ncc, vatopfn_nokernel
3720           mov   TTE64K, %g5                     /* g5 = rehash # */
3721         mov %g1,%o5                             /* o5 = tsbmiss_area */
3722         /*
3723          * o0 = vaddr
3724          * o1 & o4 = hatid
3725          * o2 = ttep
3726          * o5 = tsbmiss area
3727          */
3728         mov     HBLK_RANGE_SHIFT, %g6
3729 1:
3730 
3731         /*
3732          * o0 = vaddr
3733          * o1 = sfmmup
3734          * o2 = ttep
3735          * o3 = old %pstate
3736          * o4 = hatid
3737          * o5 = tsbmiss
3738          * g5 = rehash #
3739          * g6 = hmeshift
3740          *
3741          * The first arg to GET_TTE is actually tagaccess register
3742          * not just vaddr. Since this call is for kernel we need to clear
3743          * any lower vaddr bits that would be interpreted as ctx bits.
3744          */
3745         set     TAGACC_CTX_MASK, %g1
3746         andn    %o0, %g1, %o0
3747         GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3,
3748                 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3749 
3750 kvtop_hblk_found:
3751         /*
3752          * o0 = vaddr
3753          * o1 = sfmmup
3754          * o2 = ttep
3755          * g1 = tte
3756          * g2 = tte pa
3757          * g3 = scratch
3758          * o2 = tsbmiss area
3759          * o1 = hat id
3760          */
3761         brgez,a,pn %g1, 6f                      /* if tte invalid goto tl0 */
3762           mov   -1, %o0                         /* output = -1 (PFN_INVALID) */
3763         stx %g1,[%o2]                           /* put tte into *ttep */
3764         TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3765         /*
3766          * o0 = vaddr
3767          * o1 = sfmmup
3768          * o2 = ttep
3769          * g1 = pfn
3770          */
3771         ba,pt   %xcc, 6f
3772           mov   %g1, %o0
3773 
3774 kvtop_nohblk:
3775         /*
3776          * we get here if we couldn't find valid hblk in hash.  We rehash
3777          * if neccesary.
3778          */
3779         ldn     [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3780 #ifdef sun4v
3781         cmp     %g5, MAX_HASHCNT
3782 #else
3783         cmp     %g5, DEFAULT_MAX_HASHCNT        /* no 32/256M kernel pages */
3784 #endif /* sun4v */
3785         be,a,pn %icc, 6f
3786           mov   -1, %o0                         /* output = -1 (PFN_INVALID) */
3787         mov     %o1, %o4                        /* restore hatid */
3788 #ifdef sun4v
3789         add     %g5, 2, %g5
3790         cmp     %g5, 3
3791         move    %icc, MMU_PAGESHIFT4M, %g6
3792         ba,pt   %icc, 1b
3793         movne   %icc, MMU_PAGESHIFT256M, %g6
3794 #else
3795         inc     %g5
3796         cmp     %g5, 2
3797         move    %icc, MMU_PAGESHIFT512K, %g6
3798         ba,pt   %icc, 1b
3799         movne   %icc, MMU_PAGESHIFT4M, %g6
3800 #endif /* sun4v */
3801 6:
3802         retl
3803           wrpr  %g0, %o3, %pstate               /* re-enable interrupts */
3804 
3805 tsb_suspend:
3806         /*
3807          * o0 = vaddr
3808          * o1 = sfmmup
3809          * o2 = ttep
3810          * g1 = tte
3811          * g2 = tte pa
3812          * g3 = tte va
3813          * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3814          */
3815         stx %g1,[%o2]                           /* put tte into *ttep */
3816         brgez,a,pn %g1, 8f                      /* if tte invalid goto 8: */
3817           sub   %g0, 1, %o0                     /* output = PFN_INVALID */
3818         sub     %g0, 2, %o0                     /* output = PFN_SUSPENDED */
3819 8:
3820         retl
3821          wrpr   %g0, %o3, %pstate               /* enable interrupts */
3822 
3823 vatopfn_nokernel:
3824         /*
3825          * This routine does NOT support user addresses
3826          * There is a routine in C that supports this.
3827          * The only reason why we don't have the C routine
3828          * support kernel addresses as well is because
3829          * we do va_to_pa while holding the hashlock.
3830          */
3831         wrpr    %g0, %o3, %pstate               /* re-enable interrupts */
3832         save    %sp, -SA(MINFRAME), %sp
3833         sethi   %hi(sfmmu_panic3), %o0
3834         call    panic
3835          or     %o0, %lo(sfmmu_panic3), %o0
3836 
3837         SET_SIZE(sfmmu_vatopfn)
3838 
3839         /*
3840          * %o0 = vaddr
3841          * %o1 = hashno (aka szc)
3842          *
3843          *
3844          * This routine is similar to sfmmu_vatopfn() but will only look for
3845          * a kernel vaddr in the hash structure for the specified rehash value.
3846          * It's just an optimization for the case when pagesize for a given
3847          * va range is already known (e.g. large page heap) and we don't want
3848          * to start the search with rehash value 1 as sfmmu_vatopfn() does.
3849          *
3850          * Returns valid pfn or PFN_INVALID if
3851          * tte for specified rehash # is not found, invalid or suspended.
3852          */
3853         ENTRY_NP(sfmmu_kvaszc2pfn)
3854         /*
3855          * disable interrupts
3856          */
3857         rdpr    %pstate, %o3
3858 #ifdef DEBUG
3859         PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
3860 #endif
3861         /*
3862          * disable interrupts to protect the TSBMISS area
3863          */
3864         andn    %o3, PSTATE_IE, %o5
3865         wrpr    %o5, 0, %pstate
3866 
3867         CPU_TSBMISS_AREA(%g1, %o5)
3868         ldn     [%g1 + TSBMISS_KHATID], %o4
3869         sll     %o1, 1, %g6
3870         add     %g6, %o1, %g6
3871         add     %g6, MMU_PAGESHIFT, %g6
3872         /*
3873          * %o0 = vaddr
3874          * %o1 = hashno
3875          * %o3 = old %pstate
3876          * %o4 = ksfmmup
3877          * %g1 = tsbmiss area
3878          * %g6 = hmeshift
3879          */
3880 
3881         /*
3882          * The first arg to GET_TTE is actually tagaccess register
3883          * not just vaddr. Since this call is for kernel we need to clear
3884          * any lower vaddr bits that would be interpreted as ctx bits.
3885          */
3886         srlx    %o0, MMU_PAGESHIFT, %o0
3887         sllx    %o0, MMU_PAGESHIFT, %o0
3888         GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5,
3889                 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
3890                 kvaszc2pfn_nohblk)
3891 
3892 kvaszc2pfn_hblk_found:
3893         /*
3894          * %g3 = tte
3895          * %o0 = vaddr
3896          */
3897         brgez,a,pn %g3, 1f                      /* check if tte is invalid */
3898           mov   -1, %o0                         /* output = -1 (PFN_INVALID) */
3899         TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
3900         /*
3901          * g3 = pfn
3902          */
3903         ba,pt   %xcc, 1f
3904           mov   %g3, %o0
3905 
3906 kvaszc2pfn_nohblk:
3907         mov     -1, %o0
3908 
3909 1:
3910         retl
3911           wrpr  %g0, %o3, %pstate               /* re-enable interrupts */
3912 
3913         SET_SIZE(sfmmu_kvaszc2pfn)
3914 
3915 
3916 
3917 /*
3918  * kpm lock used between trap level tsbmiss handler and kpm C level.
3919  */
3920 #define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)                       \
3921         mov     0xff, tmp1                                              ;\
3922 label1:                                                                 ;\
3923         casa    [kpmlckp]asi, %g0, tmp1                                 ;\
3924         brnz,pn tmp1, label1                                            ;\
3925         mov     0xff, tmp1                                              ;\
3926         membar  #LoadLoad
3927 
3928 #define KPMLOCK_EXIT(kpmlckp, asi)                                      \
3929         membar  #LoadStore|#StoreStore                                  ;\
3930         sta     %g0, [kpmlckp]asi
3931 
3932 /*
3933  * Lookup a memseg for a given pfn and if found, return the physical
3934  * address of the corresponding struct memseg in mseg, otherwise
3935  * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3936  * tsbmp, %asi is assumed to be ASI_MEM.
3937  * This lookup is done by strictly traversing only the physical memseg
3938  * linkage. The more generic approach, to check the virtual linkage
3939  * before using the physical (used e.g. with hmehash buckets), cannot
3940  * be used here. Memory DR operations can run in parallel to this
3941  * lookup w/o any locks and updates of the physical and virtual linkage
3942  * cannot be done atomically wrt. to each other. Because physical
3943  * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3944  * as "physical NULL" pointer.
3945  */
3946 #define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3947         sethi   %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */  ;\
3948         ldx     [tmp3 + %lo(mhash_per_slot)], mseg                      ;\
3949         udivx   pfn, mseg, mseg                                         ;\
3950         ldx     [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1                     ;\
3951         and     mseg, SFMMU_N_MEM_SLOTS - 1, mseg                       ;\
3952         sllx    mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg                  ;\
3953         add     tmp1, mseg, tmp1                                        ;\
3954         ldxa    [tmp1]%asi, mseg                                        ;\
3955         cmp     mseg, MSEG_NULLPTR_PA                                   ;\
3956         be,pn   %xcc, label/**/1                /* if not found */      ;\
3957           nop                                                           ;\
3958         ldxa    [mseg + MEMSEG_PAGES_BASE]%asi, tmp1                    ;\
3959         cmp     pfn, tmp1                       /* pfn - pages_base */  ;\
3960         blu,pn  %xcc, label/**/1                                        ;\
3961           ldxa  [mseg + MEMSEG_PAGES_END]%asi, tmp2                     ;\
3962         cmp     pfn, tmp2                       /* pfn - pages_end */   ;\
3963         bgeu,pn %xcc, label/**/1                                        ;\
3964           sub   pfn, tmp1, tmp1                 /* pfn - pages_base */  ;\
3965         mulx    tmp1, PAGE_SIZE, tmp1                                   ;\
3966         ldxa    [mseg + MEMSEG_PAGESPA]%asi, tmp2       /* pages */     ;\
3967         add     tmp2, tmp1, tmp1                        /* pp */        ;\
3968         lduwa   [tmp1 + PAGE_PAGENUM]%asi, tmp2                         ;\
3969         cmp     tmp2, pfn                                               ;\
3970         be,pt   %xcc, label/**/_ok                      /* found */     ;\
3971 label/**/1:                                                             ;\
3972         /* brute force lookup */                                        ;\
3973         sethi   %hi(memsegspa), tmp3 /* no tsbmp use due to DR */       ;\
3974         ldx     [tmp3 + %lo(memsegspa)], mseg                           ;\
3975 label/**/2:                                                             ;\
3976         cmp     mseg, MSEG_NULLPTR_PA                                   ;\
3977         be,pn   %xcc, label/**/_ok              /* if not found */      ;\
3978           nop                                                           ;\
3979         ldxa    [mseg + MEMSEG_PAGES_BASE]%asi, tmp1                    ;\
3980         cmp     pfn, tmp1                       /* pfn - pages_base */  ;\
3981         blu,a,pt %xcc, label/**/2                                       ;\
3982           ldxa  [mseg + MEMSEG_NEXTPA]%asi, mseg                        ;\
3983         ldxa    [mseg + MEMSEG_PAGES_END]%asi, tmp2                     ;\
3984         cmp     pfn, tmp2                       /* pfn - pages_end */   ;\
3985         bgeu,a,pt %xcc, label/**/2                                      ;\
3986           ldxa  [mseg + MEMSEG_NEXTPA]%asi, mseg                        ;\
3987 label/**/_ok:
3988 
3989         /*
3990          * kpm tsb miss handler large pages
3991          * g1 = 8K kpm TSB entry pointer
3992          * g2 = tag access register
3993          * g3 = 4M kpm TSB entry pointer
3994          */
3995         ALTENTRY(sfmmu_kpm_dtsb_miss)
3996         TT_TRACE(trace_tsbmiss)
3997 
3998         CPU_INDEX(%g7, %g6)
3999         sethi   %hi(kpmtsbm_area), %g6
4000         sllx    %g7, KPMTSBM_SHIFT, %g7
4001         or      %g6, %lo(kpmtsbm_area), %g6
4002         add     %g6, %g7, %g6                   /* g6 = kpmtsbm ptr */
4003 
4004         /* check enable flag */
4005         ldub    [%g6 + KPMTSBM_FLAGS], %g4
4006         and     %g4, KPMTSBM_ENABLE_FLAG, %g5
4007         brz,pn  %g5, sfmmu_tsb_miss             /* if kpm not enabled */
4008           nop
4009 
4010         /* VA range check */
4011         ldx     [%g6 + KPMTSBM_VBASE], %g7
4012         cmp     %g2, %g7
4013         blu,pn  %xcc, sfmmu_tsb_miss
4014           ldx   [%g6 + KPMTSBM_VEND], %g5
4015         cmp     %g2, %g5
4016         bgeu,pn %xcc, sfmmu_tsb_miss
4017           stx   %g3, [%g6 + KPMTSBM_TSBPTR]
4018 
4019         /*
4020          * check TL tsbmiss handling flag
4021          * bump tsbmiss counter
4022          */
4023         lduw    [%g6 + KPMTSBM_TSBMISS], %g5
4024 #ifdef  DEBUG
4025         and     %g4, KPMTSBM_TLTSBM_FLAG, %g3
4026         inc     %g5
4027         brz,pn  %g3, sfmmu_kpm_exception
4028           st    %g5, [%g6 + KPMTSBM_TSBMISS]
4029 #else
4030         inc     %g5
4031         st      %g5, [%g6 + KPMTSBM_TSBMISS]
4032 #endif
4033         /*
4034          * At this point:
4035          *  g1 = 8K kpm TSB pointer (not used)
4036          *  g2 = tag access register
4037          *  g3 = clobbered
4038          *  g6 = per-CPU kpm tsbmiss area
4039          *  g7 = kpm_vbase
4040          */
4041 
4042         /* vaddr2pfn */
4043         ldub    [%g6 + KPMTSBM_SZSHIFT], %g3
4044         sub     %g2, %g7, %g4                   /* paddr = vaddr-kpm_vbase */
4045         srax    %g4, %g3, %g2                   /* which alias range (r) */
4046         brnz,pn %g2, sfmmu_kpm_exception        /* if (r != 0) goto C handler */
4047           srlx  %g4, MMU_PAGESHIFT, %g2         /* %g2 = pfn */
4048 
4049         /*
4050          * Setup %asi
4051          * mseg_pa = page_numtomemseg_nolock(pfn)
4052          * if (mseg_pa == NULL) sfmmu_kpm_exception
4053          * g2=pfn
4054          */
4055         mov     ASI_MEM, %asi
4056         PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4057         cmp     %g3, MSEG_NULLPTR_PA
4058         be,pn   %xcc, sfmmu_kpm_exception       /* if mseg not found */
4059           nop
4060 
4061         /*
4062          * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4063          * g2=pfn g3=mseg_pa
4064          */
4065         ldub    [%g6 + KPMTSBM_KPMP2PSHFT], %g5
4066         ldxa    [%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4067         srlx    %g2, %g5, %g4
4068         sllx    %g4, %g5, %g4
4069         sub     %g4, %g7, %g4
4070         srlx    %g4, %g5, %g4
4071 
4072         /*
4073          * Validate inx value
4074          * g2=pfn g3=mseg_pa g4=inx
4075          */
4076 #ifdef  DEBUG
4077         ldxa    [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4078         cmp     %g4, %g5                        /* inx - nkpmpgs */
4079         bgeu,pn %xcc, sfmmu_kpm_exception       /* if out of range */
4080           ld    [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4081 #else
4082         ld      [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4083 #endif
4084         /*
4085          * kp = &mseg_pa->kpm_pages[inx]
4086          */
4087         sllx    %g4, KPMPAGE_SHIFT, %g4         /* kpm_pages offset */
4088         ldxa    [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4089         add     %g5, %g4, %g5                   /* kp */
4090 
4091         /*
4092          * KPMP_HASH(kp)
4093          * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4094          */
4095         ldub    [%g6 + KPMTSBM_KPMPSHIFT], %g1  /* kpmp_shift */
4096         sub     %g7, 1, %g7                     /* mask */
4097         srlx    %g5, %g1, %g1                   /* x = ksp >> kpmp_shift */
4098         add     %g5, %g1, %g5                   /* y = ksp + x */
4099         and     %g5, %g7, %g5                   /* hashinx = y & mask */
4100 
4101         /*
4102          * Calculate physical kpm_page pointer
4103          * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4104          */
4105         ldxa    [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4106         add     %g1, %g4, %g1                   /* kp_pa */
4107 
4108         /*
4109          * Calculate physical hash lock address
4110          * g1=kp_refcntc_pa g2=pfn g5=hashinx
4111          */
4112         ldx     [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4113         sllx    %g5, KPMHLK_SHIFT, %g5
4114         add     %g4, %g5, %g3
4115         add     %g3, KPMHLK_LOCK, %g3           /* hlck_pa */
4116 
4117         /*
4118          * Assemble tte
4119          * g1=kp_pa g2=pfn g3=hlck_pa
4120          */
4121 #ifdef sun4v
4122         sethi   %hi(TTE_VALID_INT), %g5         /* upper part */
4123         sllx    %g5, 32, %g5
4124         mov     (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4125         or      %g4, TTE4M, %g4
4126         or      %g5, %g4, %g5
4127 #else
4128         sethi   %hi(TTE_VALID_INT), %g4
4129         mov     TTE4M, %g5
4130         sllx    %g5, TTE_SZ_SHFT_INT, %g5
4131         or      %g5, %g4, %g5                   /* upper part */
4132         sllx    %g5, 32, %g5
4133         mov     (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4134         or      %g5, %g4, %g5
4135 #endif
4136         sllx    %g2, MMU_PAGESHIFT, %g4
4137         or      %g5, %g4, %g5                   /* tte */
4138         ldx     [%g6 + KPMTSBM_TSBPTR], %g4
4139         GET_MMU_D_TTARGET(%g2, %g7)             /* %g2 = ttarget */
4140 
4141         /*
4142          * tsb dropin
4143          * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4144          */
4145 
4146         /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4147         KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4148 
4149         /* use C-handler if there's no go for dropin */
4150         ldsha   [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4151         cmp     %g7, -1
4152         bne,pn  %xcc, 5f        /* use C-handler if there's no go for dropin */
4153           nop
4154 
4155 #ifdef  DEBUG
4156         /* double check refcnt */
4157         ldsha   [%g1 + KPMPAGE_REFCNT]%asi, %g7
4158         brz,pn  %g7, 5f                 /* let C-handler deal with this */
4159           nop
4160 #endif
4161 
4162 #ifndef sun4v
4163         ldub    [%g6 + KPMTSBM_FLAGS], %g7
4164         mov     ASI_N, %g1
4165         andcc   %g7, KPMTSBM_TSBPHYS_FLAG, %g0
4166         movnz   %icc, ASI_MEM, %g1
4167         mov     %g1, %asi
4168 #endif
4169 
4170         /*
4171          * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4172          * If we fail to lock the TSB entry then just load the tte into the
4173          * TLB.
4174          */
4175         TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1)
4176 
4177         /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4178         TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4179 locked_tsb_l1:
4180         DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4181 
4182         /* KPMLOCK_EXIT(kpmlckp, asi) */
4183         KPMLOCK_EXIT(%g3, ASI_MEM)
4184 
4185         /*
4186          * If trapstat is running, we need to shift the %tpc and %tnpc to
4187          * point to trapstat's TSB miss return code (note that trapstat
4188          * itself will patch the correct offset to add).
4189          * Note: TTE is expected in %g5 (allows per pagesize reporting).
4190          */
4191         rdpr    %tl, %g7
4192         cmp     %g7, 1
4193         ble     %icc, 0f
4194         sethi   %hi(KERNELBASE), %g6
4195         rdpr    %tpc, %g7
4196         or      %g6, %lo(KERNELBASE), %g6
4197         cmp     %g7, %g6
4198         bgeu    %xcc, 0f
4199         ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4200         add     %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
4201         wrpr    %g7, %tpc
4202         add     %g7, 4, %g7
4203         wrpr    %g7, %tnpc
4204 0:
4205         retry
4206 5:
4207         /* g3=hlck_pa */
4208         KPMLOCK_EXIT(%g3, ASI_MEM)
4209         ba,pt   %icc, sfmmu_kpm_exception
4210           nop
4211         SET_SIZE(sfmmu_kpm_dtsb_miss)
4212 
4213         /*
4214          * kpm tsbmiss handler for smallpages
4215          * g1 = 8K kpm TSB pointer
4216          * g2 = tag access register
4217          * g3 = 4M kpm TSB pointer
4218          */
4219         ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4220         TT_TRACE(trace_tsbmiss)
4221         CPU_INDEX(%g7, %g6)
4222         sethi   %hi(kpmtsbm_area), %g6
4223         sllx    %g7, KPMTSBM_SHIFT, %g7
4224         or      %g6, %lo(kpmtsbm_area), %g6
4225         add     %g6, %g7, %g6                   /* g6 = kpmtsbm ptr */
4226 
4227         /* check enable flag */
4228         ldub    [%g6 + KPMTSBM_FLAGS], %g4
4229         and     %g4, KPMTSBM_ENABLE_FLAG, %g5
4230         brz,pn  %g5, sfmmu_tsb_miss             /* if kpm not enabled */
4231           nop
4232 
4233         /*
4234          * VA range check
4235          * On fail: goto sfmmu_tsb_miss
4236          */
4237         ldx     [%g6 + KPMTSBM_VBASE], %g7
4238         cmp     %g2, %g7
4239         blu,pn  %xcc, sfmmu_tsb_miss
4240           ldx   [%g6 + KPMTSBM_VEND], %g5
4241         cmp     %g2, %g5
4242         bgeu,pn %xcc, sfmmu_tsb_miss
4243           stx   %g1, [%g6 + KPMTSBM_TSBPTR]     /* save 8K kpm TSB pointer */
4244 
4245         /*
4246          * check TL tsbmiss handling flag
4247          * bump tsbmiss counter
4248          */
4249         lduw    [%g6 + KPMTSBM_TSBMISS], %g5
4250 #ifdef  DEBUG
4251         and     %g4, KPMTSBM_TLTSBM_FLAG, %g1
4252         inc     %g5
4253         brz,pn  %g1, sfmmu_kpm_exception
4254           st    %g5, [%g6 + KPMTSBM_TSBMISS]
4255 #else
4256         inc     %g5
4257         st      %g5, [%g6 + KPMTSBM_TSBMISS]
4258 #endif
4259         /*
4260          * At this point:
4261          *  g1 = clobbered
4262          *  g2 = tag access register
4263          *  g3 = 4M kpm TSB pointer (not used)
4264          *  g6 = per-CPU kpm tsbmiss area
4265          *  g7 = kpm_vbase
4266          */
4267 
4268         /*
4269          * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr)
4270          * which is defined in mach_kpm.h. Any changes in that macro
4271          * should also be ported back to this assembly code.
4272          */
4273         ldub    [%g6 + KPMTSBM_SZSHIFT], %g3    /* g3 = kpm_size_shift */
4274         sub     %g2, %g7, %g4                   /* paddr = vaddr-kpm_vbase */
4275         srax    %g4, %g3, %g7                   /* which alias range (r) */
4276         brz,pt  %g7, 2f
4277           sethi   %hi(vac_colors_mask), %g5
4278         ld      [%g5 + %lo(vac_colors_mask)], %g5
4279 
4280         srlx    %g2, MMU_PAGESHIFT, %g1         /* vaddr >> MMU_PAGESHIFT */
4281         and     %g1, %g5, %g1                   /* g1 = v */
4282         sllx    %g7, %g3, %g5                   /* g5 = r << kpm_size_shift */
4283         cmp     %g7, %g1                        /* if (r > v) */
4284         bleu,pn %xcc, 1f
4285           sub   %g4, %g5, %g4                   /* paddr -= r << kpm_size_shift */
4286         sub     %g7, %g1, %g5                   /* g5 = r - v */
4287         sllx    %g5, MMU_PAGESHIFT, %g7         /* (r-v) << MMU_PAGESHIFT */
4288         add     %g4, %g7, %g4                   /* paddr += (r-v)<<MMU_PAGESHIFT */
4289         ba      2f
4290           nop
4291 1:
4292         sllx    %g7, MMU_PAGESHIFT, %g5         /* else */
4293         sub     %g4, %g5, %g4                   /* paddr -= r << MMU_PAGESHIFT */
4294 
4295         /*
4296          * paddr2pfn
4297          *  g1 = vcolor (not used)
4298          *  g2 = tag access register
4299          *  g3 = clobbered
4300          *  g4 = paddr
4301          *  g5 = clobbered
4302          *  g6 = per-CPU kpm tsbmiss area
4303          *  g7 = clobbered
4304          */
4305 2:
4306         srlx    %g4, MMU_PAGESHIFT, %g2         /* g2 = pfn */
4307 
4308         /*
4309          * Setup %asi
4310          * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4311          * if (mseg not found) sfmmu_kpm_exception
4312          * g2=pfn g6=per-CPU kpm tsbmiss area
4313          * g4 g5 g7 for scratch use.
4314          */
4315         mov     ASI_MEM, %asi
4316         PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4317         cmp     %g3, MSEG_NULLPTR_PA
4318         be,pn   %xcc, sfmmu_kpm_exception       /* if mseg not found */
4319           nop
4320 
4321         /*
4322          * inx = pfn - mseg_pa->kpm_pbase
4323          * g2=pfn  g3=mseg_pa  g6=per-CPU kpm tsbmiss area
4324          */
4325         ldxa    [%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4326         sub     %g2, %g7, %g4
4327 
4328 #ifdef  DEBUG
4329         /*
4330          * Validate inx value
4331          * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area
4332          */
4333         ldxa    [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4334         cmp     %g4, %g5                        /* inx - nkpmpgs */
4335         bgeu,pn %xcc, sfmmu_kpm_exception       /* if out of range */
4336           ld    [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4337 #else
4338         ld      [%g6 + KPMTSBM_KPMPTABLESZ], %g7
4339 #endif
4340         /* ksp = &mseg_pa->kpm_spages[inx] */
4341         ldxa    [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4342         add     %g5, %g4, %g5                   /* ksp */
4343 
4344         /*
4345          * KPMP_SHASH(kp)
4346          * g2=pfn g3=mseg_pa g4=inx g5=ksp
4347          * g6=per-CPU kpm tsbmiss area  g7=kpmp_stable_sz
4348          */
4349         ldub    [%g6 + KPMTSBM_KPMPSHIFT], %g1  /* kpmp_shift */
4350         sub     %g7, 1, %g7                     /* mask */
4351         sllx    %g5, %g1, %g1                   /* x = ksp << kpmp_shift */
4352         add     %g5, %g1, %g5                   /* y = ksp + x */
4353         and     %g5, %g7, %g5                   /* hashinx = y & mask */
4354 
4355         /*
4356          * Calculate physical kpm_spage pointer
4357          * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4358          * g6=per-CPU kpm tsbmiss area
4359          */
4360         ldxa    [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4361         add     %g1, %g4, %g1                   /* ksp_pa */
4362 
4363         /*
4364          * Calculate physical hash lock address.
4365          * Note: Changes in kpm_shlk_t must be reflected here.
4366          * g1=ksp_pa g2=pfn g5=hashinx
4367          * g6=per-CPU kpm tsbmiss area
4368          */
4369         ldx     [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4370         sllx    %g5, KPMSHLK_SHIFT, %g5
4371         add     %g4, %g5, %g3                   /* hlck_pa */
4372 
4373         /*
4374          * Assemble non-cacheable tte initially
4375          * g1=ksp_pa g2=pfn g3=hlck_pa
4376          * g6=per-CPU kpm tsbmiss area
4377          */
4378         sethi   %hi(TTE_VALID_INT), %g5         /* upper part */
4379         sllx    %g5, 32, %g5
4380         mov     (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4381         or      %g5, %g4, %g5
4382         sllx    %g2, MMU_PAGESHIFT, %g4
4383         or      %g5, %g4, %g5                   /* tte */
4384         ldx     [%g6 + KPMTSBM_TSBPTR], %g4
4385         GET_MMU_D_TTARGET(%g2, %g7)             /* %g2 = ttarget */
4386 
4387         /*
4388          * tsb dropin
4389          * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable)
4390          * g6=per-CPU kpm tsbmiss area  g7=scratch register
4391          */
4392 
4393         /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4394         KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4395 
4396         /* use C-handler if there's no go for dropin */
4397         ldsba   [%g1 + KPMSPAGE_MAPPED]%asi, %g7        /* kp_mapped */
4398         andcc   %g7, KPM_MAPPED_GO, %g0                 /* go or no go ? */
4399         bz,pt   %icc, 5f                                /* no go */
4400           nop
4401         and     %g7, KPM_MAPPED_MASK, %g7               /* go */
4402         cmp     %g7, KPM_MAPPEDS                        /* cacheable ? */
4403         be,a,pn %xcc, 3f
4404           or    %g5, TTE_CV_INT, %g5                    /* cacheable */
4405 3:
4406 #ifndef sun4v
4407         ldub    [%g6 + KPMTSBM_FLAGS], %g7
4408         mov     ASI_N, %g1
4409         andcc   %g7, KPMTSBM_TSBPHYS_FLAG, %g0
4410         movnz   %icc, ASI_MEM, %g1
4411         mov     %g1, %asi
4412 #endif
4413 
4414         /*
4415          * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4416          * If we fail to lock the TSB entry then just load the tte into the
4417          * TLB.
4418          */
4419         TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2)
4420 
4421         /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4422         TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4423 locked_tsb_l2:
4424         DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4425 
4426         /* KPMLOCK_EXIT(kpmlckp, asi) */
4427         KPMLOCK_EXIT(%g3, ASI_MEM)
4428 
4429         /*
4430          * If trapstat is running, we need to shift the %tpc and %tnpc to
4431          * point to trapstat's TSB miss return code (note that trapstat
4432          * itself will patch the correct offset to add).
4433          * Note: TTE is expected in %g5 (allows per pagesize reporting).
4434          */
4435         rdpr    %tl, %g7
4436         cmp     %g7, 1
4437         ble     %icc, 0f
4438         sethi   %hi(KERNELBASE), %g6
4439         rdpr    %tpc, %g7
4440         or      %g6, %lo(KERNELBASE), %g6
4441         cmp     %g7, %g6
4442         bgeu    %xcc, 0f
4443         ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4444         add     %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */
4445         wrpr    %g7, %tpc
4446         add     %g7, 4, %g7
4447         wrpr    %g7, %tnpc
4448 0:
4449         retry
4450 5:
4451         /* g3=hlck_pa */
4452         KPMLOCK_EXIT(%g3, ASI_MEM)
4453         ba,pt   %icc, sfmmu_kpm_exception
4454           nop
4455         SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4456 
4457 #if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4458 #error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4459 #endif
4460 
4461         .seg    ".data"
4462 sfmmu_kpm_tsbmtl_panic:
4463         .ascii  "sfmmu_kpm_tsbmtl: interrupts disabled"
4464         .byte   0
4465 sfmmu_kpm_stsbmtl_panic:
4466         .ascii  "sfmmu_kpm_stsbmtl: interrupts disabled"
4467         .byte   0
4468         .align  4
4469         .seg    ".text"
4470 
4471         ENTRY_NP(sfmmu_kpm_tsbmtl)
4472         rdpr    %pstate, %o3
4473         /*
4474          * %o0 = &kp_refcntc
4475          * %o1 = &khl_lock
4476          * %o2 = 0/1 (off/on)
4477          * %o3 = pstate save
4478          */
4479 #ifdef DEBUG
4480         andcc   %o3, PSTATE_IE, %g0             /* if interrupts already */
4481         bnz,pt %icc, 1f                         /* disabled, panic       */
4482           nop
4483         save    %sp, -SA(MINFRAME), %sp
4484         sethi   %hi(sfmmu_kpm_tsbmtl_panic), %o0
4485         call    panic
4486          or     %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4487         ret
4488         restore
4489 1:
4490 #endif /* DEBUG */
4491         wrpr    %o3, PSTATE_IE, %pstate         /* disable interrupts */
4492 
4493         KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4494         mov     -1, %o5
4495         brz,a   %o2, 2f
4496           mov   0, %o5
4497 2:
4498         sth     %o5, [%o0]
4499         KPMLOCK_EXIT(%o1, ASI_N)
4500 
4501         retl
4502           wrpr  %g0, %o3, %pstate               /* enable interrupts */
4503         SET_SIZE(sfmmu_kpm_tsbmtl)
4504 
4505         ENTRY_NP(sfmmu_kpm_stsbmtl)
4506         rdpr    %pstate, %o3
4507         /*
4508          * %o0 = &mapped
4509          * %o1 = &kshl_lock
4510          * %o2 = val
4511          * %o3 = pstate save
4512          */
4513 #ifdef DEBUG
4514         andcc   %o3, PSTATE_IE, %g0             /* if interrupts already */
4515         bnz,pt %icc, 1f                         /* disabled, panic       */
4516           nop
4517         save    %sp, -SA(MINFRAME), %sp
4518         sethi   %hi(sfmmu_kpm_stsbmtl_panic), %o0
4519         call    panic
4520           or    %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4521         ret
4522         restore
4523 1:
4524 #endif /* DEBUG */
4525         wrpr    %o3, PSTATE_IE, %pstate         /* disable interrupts */
4526 
4527         KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4528         ldsb    [%o0], %o5
4529         stb     %o2, [%o0]
4530         KPMLOCK_EXIT(%o1, ASI_N)
4531 
4532         and     %o5, KPM_MAPPED_MASK, %o0       /* return old val */
4533         retl
4534           wrpr  %g0, %o3, %pstate               /* enable interrupts */
4535         SET_SIZE(sfmmu_kpm_stsbmtl)
4536 
4537 #ifdef sun4v
4538         /*
4539          * User/kernel data miss w// multiple TSBs
4540          * The first probe covers 8K, 64K, and 512K page sizes,
4541          * because 64K and 512K mappings are replicated off 8K
4542          * pointer.  Second probe covers 4M page size only.
4543          *
4544          * MMU fault area contains miss address and context.
4545          */
4546         ALTENTRY(sfmmu_slow_dmmu_miss)
4547         GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)     ! %g2 = ptagacc, %g3 = ctx type
4548 
4549 slow_miss_common:
4550         /*
4551          *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4552          *  %g3 = ctx (cannot be INVALID_CONTEXT)
4553          */
4554         brnz,pt %g3, 8f                 ! check for user context
4555           nop
4556 
4557         /*
4558          * Kernel miss
4559          * Get 8K and 4M TSB pointers in %g1 and %g3 and
4560          * branch to sfmmu_tsb_miss_tt to handle it.
4561          */
4562         mov     %g2, %g7                ! TSB pointer macro clobbers tagacc
4563 sfmmu_dslow_patch_ktsb_base:
4564         RUNTIME_PATCH_SETX(%g1, %g6)    ! %g1 = contents of ktsb_pbase
4565 sfmmu_dslow_patch_ktsb_szcode:
4566         or      %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched)
4567 
4568         GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4569         ! %g1 = First TSB entry pointer, as TSB miss handler expects
4570 
4571         mov     %g2, %g7                ! TSB pointer macro clobbers tagacc
4572 sfmmu_dslow_patch_ktsb4m_base:
4573         RUNTIME_PATCH_SETX(%g3, %g6)    ! %g3 = contents of ktsb4m_pbase
4574 sfmmu_dslow_patch_ktsb4m_szcode:
4575         or      %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched)
4576 
4577         GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4578         ! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4579         ba,a,pt %xcc, sfmmu_tsb_miss_tt
4580         .empty
4581 
4582 8:
4583         /*
4584          * User miss
4585          * Get first TSB pointer in %g1
4586          * Get second TSB pointer (or NULL if no second TSB) in %g3
4587          * Branch to sfmmu_tsb_miss_tt to handle it
4588          */
4589         GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4590         /* %g1 = first TSB entry ptr now, %g2 preserved */
4591 
4592         GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)   /* get 2nd utsbreg */
4593         brlz,pt %g3, sfmmu_tsb_miss_tt          /* done if no 2nd TSB */
4594           nop
4595 
4596         GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4597         /* %g3 = second TSB entry ptr now, %g2 preserved */
4598 9:
4599         ba,a,pt %xcc, sfmmu_tsb_miss_tt
4600         .empty
4601         SET_SIZE(sfmmu_slow_dmmu_miss)
4602 
4603 
4604         /*
4605          * User/kernel instruction miss w/ multiple TSBs
4606          * The first probe covers 8K, 64K, and 512K page sizes,
4607          * because 64K and 512K mappings are replicated off 8K
4608          * pointer.  Second probe covers 4M page size only.
4609          *
4610          * MMU fault area contains miss address and context.
4611          */
4612         ALTENTRY(sfmmu_slow_immu_miss)
4613         GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
4614         ba,a,pt %xcc, slow_miss_common
4615         SET_SIZE(sfmmu_slow_immu_miss)
4616 
4617 #endif /* sun4v */
4618 
4619 /*
4620  * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4621  */
4622         .seg    ".data"
4623         .align  64
4624         .global tsbmiss_area
4625 tsbmiss_area:
4626         .skip   (TSBMISS_SIZE * NCPU)
4627 
4628         .align  64
4629         .global kpmtsbm_area
4630 kpmtsbm_area:
4631         .skip   (KPMTSBM_SIZE * NCPU)