1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 */ 26 27 /* 28 * SFMMU primitives. These primitives should only be used by sfmmu 29 * routines. 30 */ 31 32 #include "assym.h" 33 34 #include <sys/asm_linkage.h> 35 #include <sys/machtrap.h> 36 #include <sys/machasi.h> 37 #include <sys/sun4asi.h> 38 #include <sys/pte.h> 39 #include <sys/mmu.h> 40 #include <vm/hat_sfmmu.h> 41 #include <vm/seg_spt.h> 42 #include <sys/machparam.h> 43 #include <sys/privregs.h> 44 #include <sys/scb.h> 45 #include <sys/intreg.h> 46 #include <sys/machthread.h> 47 #include <sys/intr.h> 48 #include <sys/clock.h> 49 #include <sys/trapstat.h> 50 51 #ifdef TRAPTRACE 52 #include <sys/traptrace.h> 53 54 /* 55 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 56 */ 57 #define TT_TRACE(label) \ 58 ba label ;\ 59 rd %pc, %g7 60 #else 61 62 #define TT_TRACE(label) 63 64 #endif /* TRAPTRACE */ 65 66 #if (TTE_SUSPEND_SHIFT > 0) 67 #define TTE_SUSPEND_INT_SHIFT(reg) \ 68 sllx reg, TTE_SUSPEND_SHIFT, reg 69 #else 70 #define TTE_SUSPEND_INT_SHIFT(reg) 71 #endif 72 73 /* 74 * Assumes TSBE_TAG is 0 75 * Assumes TSBE_INTHI is 0 76 * Assumes TSBREG.split is 0 77 */ 78 79 #if TSBE_TAG != 0 80 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0" 81 #endif 82 83 #if TSBTAG_INTHI != 0 84 #error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0" 85 #endif 86 87 /* 88 * The following code assumes the tsb is not split. 89 * 90 * With TSBs no longer shared between processes, it's no longer 91 * necessary to hash the context bits into the tsb index to get 92 * tsb coloring; the new implementation treats the TSB as a 93 * direct-mapped, virtually-addressed cache. 94 * 95 * In: 96 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 97 * tsbbase = base address of TSB (clobbered) 98 * tagacc = tag access register (clobbered) 99 * szc = size code of TSB (ro) 100 * tmp = scratch reg 101 * Out: 102 * tsbbase = pointer to entry in TSB 103 */ 104 #define GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp) \ 105 mov TSB_ENTRIES(0), tmp /* nentries in TSB size 0 */ ;\ 106 srlx tagacc, vpshift, tagacc ;\ 107 sllx tmp, szc, tmp /* tmp = nentries in TSB */ ;\ 108 sub tmp, 1, tmp /* mask = nentries - 1 */ ;\ 109 and tagacc, tmp, tmp /* tsbent = virtpage & mask */ ;\ 110 sllx tmp, TSB_ENTRY_SHIFT, tmp /* entry num --> ptr */ ;\ 111 add tsbbase, tmp, tsbbase /* add entry offset to TSB base */ 112 113 /* 114 * When the kpm TSB is used it is assumed that it is direct mapped 115 * using (vaddr>>vpshift)%tsbsz as the index. 116 * 117 * Note that, for now, the kpm TSB and kernel TSB are the same for 118 * each mapping size. However that need not always be the case. If 119 * the trap handlers are updated to search a different TSB for kpm 120 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz 121 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent. 122 * 123 * In: 124 * vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro) 125 * vaddr = virtual address (clobbered) 126 * tsbp, szc, tmp = scratch 127 * Out: 128 * tsbp = pointer to entry in TSB 129 */ 130 #define GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) \ 131 cmp vpshift, MMU_PAGESHIFT ;\ 132 bne,pn %icc, 1f /* branch if large case */ ;\ 133 sethi %hi(kpmsm_tsbsz), szc ;\ 134 sethi %hi(kpmsm_tsbbase), tsbp ;\ 135 ld [szc + %lo(kpmsm_tsbsz)], szc ;\ 136 ldx [tsbp + %lo(kpmsm_tsbbase)], tsbp ;\ 137 ba,pt %icc, 2f ;\ 138 nop ;\ 139 1: sethi %hi(kpm_tsbsz), szc ;\ 140 sethi %hi(kpm_tsbbase), tsbp ;\ 141 ld [szc + %lo(kpm_tsbsz)], szc ;\ 142 ldx [tsbp + %lo(kpm_tsbbase)], tsbp ;\ 143 2: GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp) 144 145 /* 146 * Lock the TSBE at virtual address tsbep. 147 * 148 * tsbep = TSBE va (ro) 149 * tmp1, tmp2 = scratch registers (clobbered) 150 * label = label to jump to if we fail to lock the tsb entry 151 * %asi = ASI to use for TSB access 152 * 153 * NOTE that we flush the TSB using fast VIS instructions that 154 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must 155 * not be treated as a locked entry or we'll get stuck spinning on 156 * an entry that isn't locked but really invalid. 157 */ 158 159 #if defined(UTSB_PHYS) 160 161 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 162 lda [tsbep]ASI_MEM, tmp1 ;\ 163 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 164 cmp tmp1, tmp2 ;\ 165 be,a,pn %icc, label /* if locked ignore */ ;\ 166 nop ;\ 167 casa [tsbep]ASI_MEM, tmp1, tmp2 ;\ 168 cmp tmp1, tmp2 ;\ 169 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 170 nop ;\ 171 /* tsbe lock acquired */ ;\ 172 membar #StoreStore 173 174 #else /* UTSB_PHYS */ 175 176 #define TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) \ 177 lda [tsbep]%asi, tmp1 ;\ 178 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 179 cmp tmp1, tmp2 ;\ 180 be,a,pn %icc, label /* if locked ignore */ ;\ 181 nop ;\ 182 casa [tsbep]%asi, tmp1, tmp2 ;\ 183 cmp tmp1, tmp2 ;\ 184 bne,a,pn %icc, label /* didn't lock so ignore */ ;\ 185 nop ;\ 186 /* tsbe lock acquired */ ;\ 187 membar #StoreStore 188 189 #endif /* UTSB_PHYS */ 190 191 /* 192 * Atomically write TSBE at virtual address tsbep. 193 * 194 * tsbep = TSBE va (ro) 195 * tte = TSBE TTE (ro) 196 * tagtarget = TSBE tag (ro) 197 * %asi = ASI to use for TSB access 198 */ 199 200 #if defined(UTSB_PHYS) 201 202 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) \ 203 add tsbep, TSBE_TTE, tmp1 ;\ 204 stxa tte, [tmp1]ASI_MEM /* write tte data */ ;\ 205 membar #StoreStore ;\ 206 add tsbep, TSBE_TAG, tmp1 ;\ 207 stxa tagtarget, [tmp1]ASI_MEM /* write tte tag & unlock */ 208 209 #else /* UTSB_PHYS */ 210 211 #define TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1) \ 212 stxa tte, [tsbep + TSBE_TTE]%asi /* write tte data */ ;\ 213 membar #StoreStore ;\ 214 stxa tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */ 215 216 #endif /* UTSB_PHYS */ 217 218 /* 219 * Load an entry into the TSB at TL > 0. 220 * 221 * tsbep = pointer to the TSBE to load as va (ro) 222 * tte = value of the TTE retrieved and loaded (wo) 223 * tagtarget = tag target register. To get TSBE tag to load, 224 * we need to mask off the context and leave only the va (clobbered) 225 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 226 * tmp1, tmp2 = scratch registers 227 * label = label to jump to if we fail to lock the tsb entry 228 * %asi = ASI to use for TSB access 229 */ 230 231 #if defined(UTSB_PHYS) 232 233 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 234 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 235 /* ;\ 236 * I don't need to update the TSB then check for the valid tte. ;\ 237 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 238 * we always invalidate the hash table before we unload the TSB.;\ 239 */ ;\ 240 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 241 ldxa [ttepa]ASI_MEM, tte ;\ 242 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 243 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 244 add tsbep, TSBE_TAG, tmp1 ;\ 245 brgez,a,pn tte, label ;\ 246 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 247 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 248 label: 249 250 #else /* UTSB_PHYS */ 251 252 #define TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 253 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 254 /* ;\ 255 * I don't need to update the TSB then check for the valid tte. ;\ 256 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 257 * we always invalidate the hash table before we unload the TSB.;\ 258 */ ;\ 259 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 260 ldxa [ttepa]ASI_MEM, tte ;\ 261 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 262 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 263 brgez,a,pn tte, label ;\ 264 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 265 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 266 label: 267 268 #endif /* UTSB_PHYS */ 269 270 /* 271 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0, 272 * for ITLB synthesis. 273 * 274 * tsbep = pointer to the TSBE to load as va (ro) 275 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out) 276 * with exec_perm turned off and exec_synth turned on 277 * tagtarget = tag target register. To get TSBE tag to load, 278 * we need to mask off the context and leave only the va (clobbered) 279 * ttepa = pointer to the TTE to retrieve/load as pa (ro) 280 * tmp1, tmp2 = scratch registers 281 * label = label to use for branch (text) 282 * %asi = ASI to use for TSB access 283 */ 284 285 #define TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \ 286 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 287 /* ;\ 288 * I don't need to update the TSB then check for the valid tte. ;\ 289 * TSB invalidate will spin till the entry is unlocked. Note, ;\ 290 * we always invalidate the hash table before we unload the TSB.;\ 291 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0 ;\ 292 * and exec_synth bit to 1. ;\ 293 */ ;\ 294 sllx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 295 mov tte, tmp1 ;\ 296 ldxa [ttepa]ASI_MEM, tte ;\ 297 srlx tagtarget, TTARGET_VA_SHIFT, tagtarget ;\ 298 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 299 brgez,a,pn tte, label ;\ 300 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 301 or tte, tmp1, tte ;\ 302 andn tte, TTE_EXECPRM_INT, tte ;\ 303 or tte, TTE_E_SYNTH_INT, tte ;\ 304 TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1) ;\ 305 label: 306 307 /* 308 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis. 309 * 310 * tte = value of the TTE, used to get tte_size bits (ro) 311 * tagaccess = tag access register, used to get 4M pfn bits (ro) 312 * pfn = 4M pfn bits shifted to offset for tte (out) 313 * tmp1 = scratch register 314 * label = label to use for branch (text) 315 */ 316 317 #define GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label) \ 318 /* ;\ 319 * Get 4M bits from tagaccess for 32M, 256M pagesizes. ;\ 320 * Return them, shifted, in pfn. ;\ 321 */ ;\ 322 srlx tagaccess, MMU_PAGESHIFT4M, tagaccess ;\ 323 srlx tte, TTE_SZ_SHFT, tmp /* isolate the */ ;\ 324 andcc tmp, TTE_SZ_BITS, %g0 /* tte_size bits */ ;\ 325 bz,a,pt %icc, label/**/f /* if 0, is */ ;\ 326 and tagaccess, 0x7, tagaccess /* 32M page size */ ;\ 327 and tagaccess, 0x3f, tagaccess /* else 256M page size */ ;\ 328 label: ;\ 329 sllx tagaccess, MMU_PAGESHIFT4M, pfn 330 331 /* 332 * Add 4M TTE size code to a tte for a Panther 32M/256M page, 333 * for ITLB synthesis. 334 * 335 * tte = value of the TTE, used to get tte_size bits (rw) 336 * tmp1 = scratch register 337 */ 338 339 #define SET_TTE4M_PN(tte, tmp) \ 340 /* ;\ 341 * Set 4M pagesize tte bits. ;\ 342 */ ;\ 343 set TTE4M, tmp ;\ 344 sllx tmp, TTE_SZ_SHFT, tmp ;\ 345 or tte, tmp, tte 346 347 /* 348 * Load an entry into the TSB at TL=0. 349 * 350 * tsbep = pointer to the TSBE to load as va (ro) 351 * tteva = pointer to the TTE to load as va (ro) 352 * tagtarget = TSBE tag to load (which contains no context), synthesized 353 * to match va of MMU tag target register only (ro) 354 * tmp1, tmp2 = scratch registers (clobbered) 355 * label = label to use for branches (text) 356 * %asi = ASI to use for TSB access 357 */ 358 359 #if defined(UTSB_PHYS) 360 361 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 362 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 363 ldx [tteva], tteva /* load tte */ ;\ 364 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 365 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 366 add tsbep, TSBE_TAG, tmp1 ;\ 367 brgez,a,pn tteva, label ;\ 368 sta tmp2, [tmp1]ASI_MEM /* unlock */ ;\ 369 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 370 label: 371 372 #else /* UTSB_PHYS */ 373 374 #define TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) \ 375 /* can't rd tteva after locking tsb because it can tlb miss */ ;\ 376 ldx [tteva], tteva /* load tte */ ;\ 377 TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label) ;\ 378 sethi %hi(TSBTAG_INVALID), tmp2 ;\ 379 brgez,a,pn tteva, label ;\ 380 sta tmp2, [tsbep + TSBE_TAG]%asi /* unlock */ ;\ 381 TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1) ;\ 382 label: 383 384 #endif /* UTSB_PHYS */ 385 386 /* 387 * Invalidate a TSB entry in the TSB. 388 * 389 * NOTE: TSBE_TAG is assumed to be zero. There is a compile time check 390 * about this earlier to ensure this is true. Thus when we are 391 * directly referencing tsbep below, we are referencing the tte_tag 392 * field of the TSBE. If this offset ever changes, the code below 393 * will need to be modified. 394 * 395 * tsbep = pointer to TSBE as va (ro) 396 * tag = invalidation is done if this matches the TSBE tag (ro) 397 * tmp1 - tmp3 = scratch registers (clobbered) 398 * label = label name to use for branches (text) 399 * %asi = ASI to use for TSB access 400 */ 401 402 #if defined(UTSB_PHYS) 403 404 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 405 lda [tsbep]ASI_MEM, tmp1 /* tmp1 = tsbe tag */ ;\ 406 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 407 label/**/1: ;\ 408 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 409 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 410 lda [tsbep]ASI_MEM, tmp1 /* reloading value each time */ ;\ 411 ldxa [tsbep]ASI_MEM, tmp3 /* tmp3 = tsbe tag */ ;\ 412 cmp tag, tmp3 /* compare tags */ ;\ 413 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 414 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 415 casa [tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */ ;\ 416 cmp tmp1, tmp3 /* if not successful */ ;\ 417 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 418 lda [tsbep]ASI_MEM, tmp1 /* reloading tsbe tag */ ;\ 419 label/**/2: 420 421 #else /* UTSB_PHYS */ 422 423 #define TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) \ 424 lda [tsbep]%asi, tmp1 /* tmp1 = tsbe tag */ ;\ 425 sethi %hi(TSBTAG_LOCKED), tmp2 ;\ 426 label/**/1: ;\ 427 cmp tmp1, tmp2 /* see if tsbe is locked, if */ ;\ 428 be,a,pn %icc, label/**/1 /* so, loop until unlocked */ ;\ 429 lda [tsbep]%asi, tmp1 /* reloading value each time */ ;\ 430 ldxa [tsbep]%asi, tmp3 /* tmp3 = tsbe tag */ ;\ 431 cmp tag, tmp3 /* compare tags */ ;\ 432 bne,pt %xcc, label/**/2 /* if different, do nothing */ ;\ 433 sethi %hi(TSBTAG_INVALID), tmp3 ;\ 434 casa [tsbep]%asi, tmp1, tmp3 /* try to set tag invalid */ ;\ 435 cmp tmp1, tmp3 /* if not successful */ ;\ 436 bne,a,pn %icc, label/**/1 /* start over from the top */ ;\ 437 lda [tsbep]%asi, tmp1 /* reloading tsbe tag */ ;\ 438 label/**/2: 439 440 #endif /* UTSB_PHYS */ 441 442 #if TSB_SOFTSZ_MASK < TSB_SZ_MASK 443 #error - TSB_SOFTSZ_MASK too small 444 #endif 445 446 447 /* 448 * An implementation of setx which will be hot patched at run time. 449 * since it is being hot patched, there is no value passed in. 450 * Thus, essentially we are implementing 451 * setx value, tmp, dest 452 * where value is RUNTIME_PATCH (aka 0) in this case. 453 */ 454 #define RUNTIME_PATCH_SETX(dest, tmp) \ 455 sethi %hh(RUNTIME_PATCH), tmp ;\ 456 sethi %lm(RUNTIME_PATCH), dest ;\ 457 or tmp, %hm(RUNTIME_PATCH), tmp ;\ 458 or dest, %lo(RUNTIME_PATCH), dest ;\ 459 sllx tmp, 32, tmp ;\ 460 nop /* for perf reasons */ ;\ 461 or tmp, dest, dest /* contents of patched value */ 462 463 464 .seg ".data" 465 .global sfmmu_panic1 466 sfmmu_panic1: 467 .asciz "sfmmu_asm: interrupts already disabled" 468 469 .global sfmmu_panic3 470 sfmmu_panic3: 471 .asciz "sfmmu_asm: sfmmu_vatopfn called for user" 472 473 .global sfmmu_panic4 474 sfmmu_panic4: 475 .asciz "sfmmu_asm: 4M tsb pointer mis-match" 476 477 .global sfmmu_panic5 478 sfmmu_panic5: 479 .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" 480 481 .global sfmmu_panic6 482 sfmmu_panic6: 483 .asciz "sfmmu_asm: interrupts not disabled" 484 485 .global sfmmu_panic7 486 sfmmu_panic7: 487 .asciz "sfmmu_asm: kernel as" 488 489 .global sfmmu_panic8 490 sfmmu_panic8: 491 .asciz "sfmmu_asm: gnum is zero" 492 493 .global sfmmu_panic9 494 sfmmu_panic9: 495 .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" 496 497 .global sfmmu_panic10 498 sfmmu_panic10: 499 .asciz "sfmmu_asm: valid SCD with no 3rd scd TSB" 500 501 .global sfmmu_panic11 502 sfmmu_panic11: 503 .asciz "sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform" 504 505 ENTRY(sfmmu_disable_intrs) 506 rdpr %pstate, %o0 507 #ifdef DEBUG 508 PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) 509 #endif /* DEBUG */ 510 retl 511 wrpr %o0, PSTATE_IE, %pstate 512 SET_SIZE(sfmmu_disable_intrs) 513 514 ENTRY(sfmmu_enable_intrs) 515 retl 516 wrpr %g0, %o0, %pstate 517 SET_SIZE(sfmmu_enable_intrs) 518 519 /* 520 * This routine is called both by resume() and sfmmu_get_ctx() to 521 * allocate a new context for the process on a MMU. 522 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . 523 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which 524 * is the case when sfmmu_alloc_ctx is called from resume(). 525 * 526 * The caller must disable interrupts before entering this routine. 527 * To reduce ctx switch overhead, the code contains both 'fast path' and 528 * 'slow path' code. The fast path code covers the common case where only 529 * a quick check is needed and the real ctx allocation is not required. 530 * It can be done without holding the per-process (PP) lock. 531 * The 'slow path' code must be protected by the PP Lock and performs ctx 532 * allocation. 533 * Hardware context register and HAT mmu cnum are updated accordingly. 534 * 535 * %o0 - sfmmup 536 * %o1 - allocflag 537 * %o2 - CPU 538 * %o3 - sfmmu private/shared flag 539 * 540 * ret - 0: no ctx is allocated 541 * 1: a ctx is allocated 542 */ 543 ENTRY_NP(sfmmu_alloc_ctx) 544 545 #ifdef DEBUG 546 sethi %hi(ksfmmup), %g1 547 ldx [%g1 + %lo(ksfmmup)], %g1 548 cmp %g1, %o0 549 bne,pt %xcc, 0f 550 nop 551 552 sethi %hi(panicstr), %g1 ! if kernel as, panic 553 ldx [%g1 + %lo(panicstr)], %g1 554 tst %g1 555 bnz,pn %icc, 7f 556 nop 557 558 sethi %hi(sfmmu_panic7), %o0 559 call panic 560 or %o0, %lo(sfmmu_panic7), %o0 561 562 7: 563 retl 564 mov %g0, %o0 ! %o0 = ret = 0 565 566 0: 567 PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) 568 #endif /* DEBUG */ 569 570 mov %o3, %g1 ! save sfmmu pri/sh flag in %g1 571 572 ! load global mmu_ctxp info 573 ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr 574 575 #ifdef sun4v 576 /* During suspend on sun4v, context domains can be temporary removed */ 577 brz,a,pn %o3, 0f 578 nop 579 #endif 580 581 lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index 582 583 ! load global mmu_ctxp gnum 584 ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum 585 586 #ifdef DEBUG 587 cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 588 bne,pt %xcc, 3f 589 nop 590 591 sethi %hi(panicstr), %g1 ! test if panicstr is already set 592 ldx [%g1 + %lo(panicstr)], %g1 593 tst %g1 594 bnz,pn %icc, 1f 595 nop 596 597 sethi %hi(sfmmu_panic8), %o0 598 call panic 599 or %o0, %lo(sfmmu_panic8), %o0 600 1: 601 retl 602 mov %g0, %o0 ! %o0 = ret = 0 603 3: 604 #endif 605 606 ! load HAT sfmmu_ctxs[mmuid] gnum, cnum 607 608 sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 609 add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 610 611 /* 612 * %g5 = sfmmu gnum returned 613 * %g6 = sfmmu cnum returned 614 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 615 * %g4 = scratch 616 * 617 * Fast path code, do a quick check. 618 */ 619 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 620 621 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 622 bne,pt %icc, 1f ! valid hat cnum, check gnum 623 nop 624 625 ! cnum == INVALID, check allocflag 626 mov %g0, %g4 ! %g4 = ret = 0 627 brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail 628 mov %g6, %o1 629 630 ! (invalid HAT cnum) && (allocflag == 1) 631 ba,pt %icc, 2f 632 nop 633 #ifdef sun4v 634 0: 635 set INVALID_CONTEXT, %o1 636 membar #LoadStore|#StoreStore 637 ba,pt %icc, 8f 638 mov %g0, %g4 ! %g4 = ret = 0 639 #endif 640 1: 641 ! valid HAT cnum, check gnum 642 cmp %g5, %o4 643 mov 1, %g4 !%g4 = ret = 1 644 be,a,pt %icc, 8f ! gnum unchanged, go to done 645 mov %g6, %o1 646 647 2: 648 /* 649 * Grab per process (PP) sfmmu_ctx_lock spinlock, 650 * followed by the 'slow path' code. 651 */ 652 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock 653 3: 654 brz %g3, 5f 655 nop 656 4: 657 brnz,a,pt %g3, 4b ! spin if lock is 1 658 ldub [%o0 + SFMMU_CTX_LOCK], %g3 659 ba %xcc, 3b ! retry the lock 660 ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock 661 662 5: 663 membar #LoadLoad 664 /* 665 * %g5 = sfmmu gnum returned 666 * %g6 = sfmmu cnum returned 667 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS 668 * %g4 = scratch 669 */ 670 SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) 671 672 cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? 673 bne,pt %icc, 1f ! valid hat cnum, check gnum 674 nop 675 676 ! cnum == INVALID, check allocflag 677 mov %g0, %g4 ! %g4 = ret = 0 678 brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw 679 mov %g6, %o1 680 681 ! (invalid HAT cnum) && (allocflag == 1) 682 ba,pt %icc, 6f 683 nop 684 1: 685 ! valid HAT cnum, check gnum 686 cmp %g5, %o4 687 mov 1, %g4 ! %g4 = ret = 1 688 be,a,pt %icc, 2f ! gnum unchanged, go to done 689 mov %g6, %o1 690 691 ba,pt %icc, 6f 692 nop 693 2: 694 membar #LoadStore|#StoreStore 695 ba,pt %icc, 8f 696 clrb [%o0 + SFMMU_CTX_LOCK] 697 6: 698 /* 699 * We get here if we do not have a valid context, or 700 * the HAT gnum does not match global gnum. We hold 701 * sfmmu_ctx_lock spinlock. Allocate that context. 702 * 703 * %o3 = mmu_ctxp 704 */ 705 add %o3, MMU_CTX_CNUM, %g3 706 ld [%o3 + MMU_CTX_NCTXS], %g4 707 708 /* 709 * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; 710 * %g3 = mmu cnum address 711 * %g4 = mmu nctxs 712 * 713 * %o0 = sfmmup 714 * %o1 = mmu current cnum value (used as new cnum) 715 * %o4 = mmu gnum 716 * 717 * %o5 = scratch 718 */ 719 ld [%g3], %o1 720 0: 721 cmp %o1, %g4 722 bl,a,pt %icc, 1f 723 add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 724 725 /* 726 * cnum reachs max, bail, so wrap around can be performed later. 727 */ 728 set INVALID_CONTEXT, %o1 729 mov %g0, %g4 ! %g4 = ret = 0 730 731 membar #LoadStore|#StoreStore 732 ba,pt %icc, 8f 733 clrb [%o0 + SFMMU_CTX_LOCK] 734 1: 735 ! %g3 = addr of mmu_ctxp->cnum 736 ! %o5 = mmu_ctxp->cnum + 1 737 cas [%g3], %o1, %o5 738 cmp %o1, %o5 739 bne,a,pn %xcc, 0b ! cas failed 740 ld [%g3], %o1 741 742 #ifdef DEBUG 743 set MAX_SFMMU_CTX_VAL, %o5 744 cmp %o1, %o5 745 ble,pt %icc, 2f 746 nop 747 748 sethi %hi(sfmmu_panic9), %o0 749 call panic 750 or %o0, %lo(sfmmu_panic9), %o0 751 2: 752 #endif 753 ! update hat gnum and cnum 754 sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 755 or %o4, %o1, %o4 756 stx %o4, [%g2 + SFMMU_CTXS] 757 758 membar #LoadStore|#StoreStore 759 clrb [%o0 + SFMMU_CTX_LOCK] 760 761 mov 1, %g4 ! %g4 = ret = 1 762 8: 763 /* 764 * program the secondary context register 765 * 766 * %o1 = cnum 767 * %g1 = sfmmu private/shared flag (0:private, 1:shared) 768 */ 769 770 /* 771 * When we come here and context is invalid, we want to set both 772 * private and shared ctx regs to INVALID. In order to 773 * do so, we set the sfmmu priv/shared flag to 'private' regardless 774 * so that private ctx reg will be set to invalid. 775 * Note that on sun4v values written to private context register are 776 * automatically written to corresponding shared context register as 777 * well. On sun4u SET_SECCTX() will invalidate shared context register 778 * when it sets a private secondary context register. 779 */ 780 781 cmp %o1, INVALID_CONTEXT 782 be,a,pn %icc, 9f 783 clr %g1 784 9: 785 786 #ifdef sun4u 787 ldub [%o0 + SFMMU_CEXT], %o2 788 sll %o2, CTXREG_EXT_SHIFT, %o2 789 or %o1, %o2, %o1 790 #endif /* sun4u */ 791 792 SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1) 793 794 retl 795 mov %g4, %o0 ! %o0 = ret 796 797 SET_SIZE(sfmmu_alloc_ctx) 798 799 ENTRY_NP(sfmmu_modifytte) 800 ldx [%o2], %g3 /* current */ 801 ldx [%o0], %g1 /* original */ 802 2: 803 ldx [%o1], %g2 /* modified */ 804 cmp %g2, %g3 /* is modified = current? */ 805 be,a,pt %xcc,1f /* yes, don't write */ 806 stx %g3, [%o0] /* update new original */ 807 casx [%o2], %g1, %g2 808 cmp %g1, %g2 809 be,pt %xcc, 1f /* cas succeeded - return */ 810 nop 811 ldx [%o2], %g3 /* new current */ 812 stx %g3, [%o0] /* save as new original */ 813 ba,pt %xcc, 2b 814 mov %g3, %g1 815 1: retl 816 membar #StoreLoad 817 SET_SIZE(sfmmu_modifytte) 818 819 ENTRY_NP(sfmmu_modifytte_try) 820 ldx [%o1], %g2 /* modified */ 821 ldx [%o2], %g3 /* current */ 822 ldx [%o0], %g1 /* original */ 823 cmp %g3, %g2 /* is modified = current? */ 824 be,a,pn %xcc,1f /* yes, don't write */ 825 mov 0, %o1 /* as if cas failed. */ 826 827 casx [%o2], %g1, %g2 828 membar #StoreLoad 829 cmp %g1, %g2 830 movne %xcc, -1, %o1 /* cas failed. */ 831 move %xcc, 1, %o1 /* cas succeeded. */ 832 1: 833 stx %g2, [%o0] /* report "current" value */ 834 retl 835 mov %o1, %o0 836 SET_SIZE(sfmmu_modifytte_try) 837 838 ENTRY_NP(sfmmu_copytte) 839 ldx [%o0], %g1 840 retl 841 stx %g1, [%o1] 842 SET_SIZE(sfmmu_copytte) 843 844 845 /* 846 * Calculate a TSB entry pointer for the given TSB, va, pagesize. 847 * %o0 = TSB base address (in), pointer to TSB entry (out) 848 * %o1 = vaddr (in) 849 * %o2 = vpshift (in) 850 * %o3 = tsb size code (in) 851 * %o4 = scratch register 852 */ 853 ENTRY_NP(sfmmu_get_tsbe) 854 GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4) 855 retl 856 nop 857 SET_SIZE(sfmmu_get_tsbe) 858 859 /* 860 * Return a TSB tag for the given va. 861 * %o0 = va (in/clobbered) 862 * %o0 = va shifted to be in tsb tag format (with no context) (out) 863 */ 864 ENTRY_NP(sfmmu_make_tsbtag) 865 retl 866 srln %o0, TTARGET_VA_SHIFT, %o0 867 SET_SIZE(sfmmu_make_tsbtag) 868 869 /* 870 * Other sfmmu primitives 871 */ 872 873 874 #define I_SIZE 4 875 876 ENTRY_NP(sfmmu_fix_ktlb_traptable) 877 /* 878 * %o0 = start of patch area 879 * %o1 = size code of TSB to patch 880 * %o3 = scratch 881 */ 882 /* fix sll */ 883 ld [%o0], %o3 /* get sll */ 884 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 885 st %o3, [%o0] /* write sll */ 886 flush %o0 887 /* fix srl */ 888 add %o0, I_SIZE, %o0 /* goto next instr. */ 889 ld [%o0], %o3 /* get srl */ 890 sub %o3, %o1, %o3 /* decrease shift by tsb szc */ 891 st %o3, [%o0] /* write srl */ 892 retl 893 flush %o0 894 SET_SIZE(sfmmu_fix_ktlb_traptable) 895 896 ENTRY_NP(sfmmu_fixup_ktsbbase) 897 /* 898 * %o0 = start of patch area 899 * %o5 = kernel virtual or physical tsb base address 900 * %o2, %o3 are used as scratch registers. 901 */ 902 /* fixup sethi instruction */ 903 ld [%o0], %o3 904 srl %o5, 10, %o2 ! offset is bits 32:10 905 or %o3, %o2, %o3 ! set imm22 906 st %o3, [%o0] 907 /* fixup offset of lduw/ldx */ 908 add %o0, I_SIZE, %o0 ! next instr 909 ld [%o0], %o3 910 and %o5, 0x3ff, %o2 ! set imm13 to bits 9:0 911 or %o3, %o2, %o3 912 st %o3, [%o0] 913 retl 914 flush %o0 915 SET_SIZE(sfmmu_fixup_ktsbbase) 916 917 ENTRY_NP(sfmmu_fixup_setx) 918 /* 919 * %o0 = start of patch area 920 * %o4 = 64 bit value to patch 921 * %o2, %o3 are used as scratch registers. 922 * 923 * Note: Assuming that all parts of the instructions which need to be 924 * patched correspond to RUNTIME_PATCH (aka 0) 925 * 926 * Note the implementation of setx which is being patched is as follows: 927 * 928 * sethi %hh(RUNTIME_PATCH), tmp 929 * sethi %lm(RUNTIME_PATCH), dest 930 * or tmp, %hm(RUNTIME_PATCH), tmp 931 * or dest, %lo(RUNTIME_PATCH), dest 932 * sllx tmp, 32, tmp 933 * nop 934 * or tmp, dest, dest 935 * 936 * which differs from the implementation in the 937 * "SPARC Architecture Manual" 938 */ 939 /* fixup sethi instruction */ 940 ld [%o0], %o3 941 srlx %o4, 42, %o2 ! bits [63:42] 942 or %o3, %o2, %o3 ! set imm22 943 st %o3, [%o0] 944 /* fixup sethi instruction */ 945 add %o0, I_SIZE, %o0 ! next instr 946 ld [%o0], %o3 947 sllx %o4, 32, %o2 ! clear upper bits 948 srlx %o2, 42, %o2 ! bits [31:10] 949 or %o3, %o2, %o3 ! set imm22 950 st %o3, [%o0] 951 /* fixup or instruction */ 952 add %o0, I_SIZE, %o0 ! next instr 953 ld [%o0], %o3 954 srlx %o4, 32, %o2 ! bits [63:32] 955 and %o2, 0x3ff, %o2 ! bits [41:32] 956 or %o3, %o2, %o3 ! set imm 957 st %o3, [%o0] 958 /* fixup or instruction */ 959 add %o0, I_SIZE, %o0 ! next instr 960 ld [%o0], %o3 961 and %o4, 0x3ff, %o2 ! bits [9:0] 962 or %o3, %o2, %o3 ! set imm 963 st %o3, [%o0] 964 retl 965 flush %o0 966 SET_SIZE(sfmmu_fixup_setx) 967 968 ENTRY_NP(sfmmu_fixup_or) 969 /* 970 * %o0 = start of patch area 971 * %o4 = 32 bit value to patch 972 * %o2, %o3 are used as scratch registers. 973 * Note: Assuming that all parts of the instructions which need to be 974 * patched correspond to RUNTIME_PATCH (aka 0) 975 */ 976 ld [%o0], %o3 977 and %o4, 0x3ff, %o2 ! bits [9:0] 978 or %o3, %o2, %o3 ! set imm 979 st %o3, [%o0] 980 retl 981 flush %o0 982 SET_SIZE(sfmmu_fixup_or) 983 984 ENTRY_NP(sfmmu_fixup_shiftx) 985 /* 986 * %o0 = start of patch area 987 * %o4 = signed int immediate value to add to sllx/srlx imm field 988 * %o2, %o3 are used as scratch registers. 989 * 990 * sllx/srlx store the 6 bit immediate value in the lowest order bits 991 * so we do a simple add. The caller must be careful to prevent 992 * overflow, which could easily occur if the initial value is nonzero! 993 */ 994 ld [%o0], %o3 ! %o3 = instruction to patch 995 and %o3, 0x3f, %o2 ! %o2 = existing imm value 996 add %o2, %o4, %o2 ! %o2 = new imm value 997 andn %o3, 0x3f, %o3 ! clear old imm value 998 and %o2, 0x3f, %o2 ! truncate new imm value 999 or %o3, %o2, %o3 ! set new imm value 1000 st %o3, [%o0] ! store updated instruction 1001 retl 1002 flush %o0 1003 SET_SIZE(sfmmu_fixup_shiftx) 1004 1005 ENTRY_NP(sfmmu_fixup_mmu_asi) 1006 /* 1007 * Patch imm_asi of all ldda instructions in the MMU 1008 * trap handlers. We search MMU_PATCH_INSTR instructions 1009 * starting from the itlb miss handler (trap 0x64). 1010 * %o0 = address of tt[0,1]_itlbmiss 1011 * %o1 = imm_asi to setup, shifted by appropriate offset. 1012 * %o3 = number of instructions to search 1013 * %o4 = reserved by caller: called from leaf routine 1014 */ 1015 1: ldsw [%o0], %o2 ! load instruction to %o2 1016 brgez,pt %o2, 2f 1017 srl %o2, 30, %o5 1018 btst 1, %o5 ! test bit 30; skip if not set 1019 bz,pt %icc, 2f 1020 sllx %o2, 39, %o5 ! bit 24 -> bit 63 1021 srlx %o5, 58, %o5 ! isolate op3 part of opcode 1022 xor %o5, 0x13, %o5 ! 01 0011 binary == ldda 1023 brnz,pt %o5, 2f ! skip if not a match 1024 or %o2, %o1, %o2 ! or in imm_asi 1025 st %o2, [%o0] ! write patched instruction 1026 2: dec %o3 1027 brnz,a,pt %o3, 1b ! loop until we're done 1028 add %o0, I_SIZE, %o0 1029 retl 1030 flush %o0 1031 SET_SIZE(sfmmu_fixup_mmu_asi) 1032 1033 /* 1034 * Patch immediate ASI used to access the TSB in the 1035 * trap table. 1036 * inputs: %o0 = value of ktsb_phys 1037 */ 1038 ENTRY_NP(sfmmu_patch_mmu_asi) 1039 mov %o7, %o4 ! save return pc in %o4 1040 mov ASI_QUAD_LDD_PHYS, %o3 ! set QUAD_LDD_PHYS by default 1041 1042 #ifdef sun4v 1043 1044 /* 1045 * Check ktsb_phys. It must be non-zero for sun4v, panic if not. 1046 */ 1047 1048 brnz,pt %o0, do_patch 1049 nop 1050 1051 sethi %hi(sfmmu_panic11), %o0 1052 call panic 1053 or %o0, %lo(sfmmu_panic11), %o0 1054 do_patch: 1055 1056 #else /* sun4v */ 1057 /* 1058 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0). 1059 * Note that ASI_NQUAD_LD is not defined/used for sun4v 1060 */ 1061 movrz %o0, ASI_NQUAD_LD, %o3 1062 1063 #endif /* sun4v */ 1064 1065 sll %o3, 5, %o1 ! imm_asi offset 1066 mov 6, %o3 ! number of instructions 1067 sethi %hi(dktsb), %o0 ! to search 1068 call sfmmu_fixup_mmu_asi ! patch kdtlb miss 1069 or %o0, %lo(dktsb), %o0 1070 mov 6, %o3 ! number of instructions 1071 sethi %hi(dktsb4m), %o0 ! to search 1072 call sfmmu_fixup_mmu_asi ! patch kdtlb4m miss 1073 or %o0, %lo(dktsb4m), %o0 1074 mov 6, %o3 ! number of instructions 1075 sethi %hi(iktsb), %o0 ! to search 1076 call sfmmu_fixup_mmu_asi ! patch kitlb miss 1077 or %o0, %lo(iktsb), %o0 1078 mov 6, %o3 ! number of instructions 1079 sethi %hi(iktsb4m), %o0 ! to search 1080 call sfmmu_fixup_mmu_asi ! patch kitlb4m miss 1081 or %o0, %lo(iktsb4m), %o0 1082 mov %o4, %o7 ! retore return pc -- leaf 1083 retl 1084 nop 1085 SET_SIZE(sfmmu_patch_mmu_asi) 1086 1087 1088 ENTRY_NP(sfmmu_patch_ktsb) 1089 /* 1090 * We need to fix iktsb, dktsb, et. al. 1091 */ 1092 save %sp, -SA(MINFRAME), %sp 1093 set ktsb_phys, %o1 1094 ld [%o1], %o4 1095 set ktsb_base, %o5 1096 set ktsb4m_base, %l1 1097 brz,pt %o4, 1f 1098 nop 1099 set ktsb_pbase, %o5 1100 set ktsb4m_pbase, %l1 1101 1: 1102 sethi %hi(ktsb_szcode), %o1 1103 ld [%o1 + %lo(ktsb_szcode)], %o1 /* %o1 = ktsb size code */ 1104 1105 sethi %hi(iktsb), %o0 1106 call sfmmu_fix_ktlb_traptable 1107 or %o0, %lo(iktsb), %o0 1108 1109 sethi %hi(dktsb), %o0 1110 call sfmmu_fix_ktlb_traptable 1111 or %o0, %lo(dktsb), %o0 1112 1113 sethi %hi(ktsb4m_szcode), %o1 1114 ld [%o1 + %lo(ktsb4m_szcode)], %o1 /* %o1 = ktsb4m size code */ 1115 1116 sethi %hi(iktsb4m), %o0 1117 call sfmmu_fix_ktlb_traptable 1118 or %o0, %lo(iktsb4m), %o0 1119 1120 sethi %hi(dktsb4m), %o0 1121 call sfmmu_fix_ktlb_traptable 1122 or %o0, %lo(dktsb4m), %o0 1123 1124 #ifndef sun4v 1125 mov ASI_N, %o2 1126 movrnz %o4, ASI_MEM, %o2 ! setup kernel 32bit ASI to patch 1127 mov %o2, %o4 ! sfmmu_fixup_or needs this in %o4 1128 sethi %hi(tsb_kernel_patch_asi), %o0 1129 call sfmmu_fixup_or 1130 or %o0, %lo(tsb_kernel_patch_asi), %o0 1131 #endif /* !sun4v */ 1132 1133 ldx [%o5], %o4 ! load ktsb base addr (VA or PA) 1134 1135 sethi %hi(dktsbbase), %o0 1136 call sfmmu_fixup_setx ! patch value of ktsb base addr 1137 or %o0, %lo(dktsbbase), %o0 1138 1139 sethi %hi(iktsbbase), %o0 1140 call sfmmu_fixup_setx ! patch value of ktsb base addr 1141 or %o0, %lo(iktsbbase), %o0 1142 1143 sethi %hi(sfmmu_kprot_patch_ktsb_base), %o0 1144 call sfmmu_fixup_setx ! patch value of ktsb base addr 1145 or %o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0 1146 1147 #ifdef sun4v 1148 sethi %hi(sfmmu_dslow_patch_ktsb_base), %o0 1149 call sfmmu_fixup_setx ! patch value of ktsb base addr 1150 or %o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0 1151 #endif /* sun4v */ 1152 1153 ldx [%l1], %o4 ! load ktsb4m base addr (VA or PA) 1154 1155 sethi %hi(dktsb4mbase), %o0 1156 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1157 or %o0, %lo(dktsb4mbase), %o0 1158 1159 sethi %hi(iktsb4mbase), %o0 1160 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1161 or %o0, %lo(iktsb4mbase), %o0 1162 1163 sethi %hi(sfmmu_kprot_patch_ktsb4m_base), %o0 1164 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1165 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0 1166 1167 #ifdef sun4v 1168 sethi %hi(sfmmu_dslow_patch_ktsb4m_base), %o0 1169 call sfmmu_fixup_setx ! patch value of ktsb4m base addr 1170 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0 1171 #endif /* sun4v */ 1172 1173 set ktsb_szcode, %o4 1174 ld [%o4], %o4 1175 sethi %hi(sfmmu_kprot_patch_ktsb_szcode), %o0 1176 call sfmmu_fixup_or ! patch value of ktsb_szcode 1177 or %o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0 1178 1179 #ifdef sun4v 1180 sethi %hi(sfmmu_dslow_patch_ktsb_szcode), %o0 1181 call sfmmu_fixup_or ! patch value of ktsb_szcode 1182 or %o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0 1183 #endif /* sun4v */ 1184 1185 set ktsb4m_szcode, %o4 1186 ld [%o4], %o4 1187 sethi %hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1188 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1189 or %o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0 1190 1191 #ifdef sun4v 1192 sethi %hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1193 call sfmmu_fixup_or ! patch value of ktsb4m_szcode 1194 or %o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0 1195 #endif /* sun4v */ 1196 1197 ret 1198 restore 1199 SET_SIZE(sfmmu_patch_ktsb) 1200 1201 ENTRY_NP(sfmmu_kpm_patch_tlbm) 1202 /* 1203 * Fixup trap handlers in common segkpm case. This is reserved 1204 * for future use should kpm TSB be changed to be other than the 1205 * kernel TSB. 1206 */ 1207 retl 1208 nop 1209 SET_SIZE(sfmmu_kpm_patch_tlbm) 1210 1211 ENTRY_NP(sfmmu_kpm_patch_tsbm) 1212 /* 1213 * nop the branch to sfmmu_kpm_dtsb_miss_small 1214 * in the case where we are using large pages for 1215 * seg_kpm (and hence must probe the second TSB for 1216 * seg_kpm VAs) 1217 */ 1218 set dktsb4m_kpmcheck_small, %o0 1219 MAKE_NOP_INSTR(%o1) 1220 st %o1, [%o0] 1221 flush %o0 1222 retl 1223 nop 1224 SET_SIZE(sfmmu_kpm_patch_tsbm) 1225 1226 ENTRY_NP(sfmmu_patch_utsb) 1227 #ifdef UTSB_PHYS 1228 retl 1229 nop 1230 #else /* UTSB_PHYS */ 1231 /* 1232 * We need to hot patch utsb_vabase and utsb4m_vabase 1233 */ 1234 save %sp, -SA(MINFRAME), %sp 1235 1236 /* patch value of utsb_vabase */ 1237 set utsb_vabase, %o1 1238 ldx [%o1], %o4 1239 sethi %hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1240 call sfmmu_fixup_setx 1241 or %o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0 1242 sethi %hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1243 call sfmmu_fixup_setx 1244 or %o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0 1245 sethi %hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1246 call sfmmu_fixup_setx 1247 or %o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0 1248 1249 /* patch value of utsb4m_vabase */ 1250 set utsb4m_vabase, %o1 1251 ldx [%o1], %o4 1252 sethi %hi(sfmmu_uprot_get_2nd_tsb_base), %o0 1253 call sfmmu_fixup_setx 1254 or %o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0 1255 sethi %hi(sfmmu_uitlb_get_2nd_tsb_base), %o0 1256 call sfmmu_fixup_setx 1257 or %o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0 1258 sethi %hi(sfmmu_udtlb_get_2nd_tsb_base), %o0 1259 call sfmmu_fixup_setx 1260 or %o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0 1261 1262 /* 1263 * Patch TSB base register masks and shifts if needed. 1264 * By default the TSB base register contents are set up for 4M slab. 1265 * If we're using a smaller slab size and reserved VA range we need 1266 * to patch up those values here. 1267 */ 1268 set tsb_slab_shift, %o1 1269 set MMU_PAGESHIFT4M, %o4 1270 lduw [%o1], %o3 1271 subcc %o4, %o3, %o4 1272 bz,pt %icc, 1f 1273 /* delay slot safe */ 1274 1275 /* patch reserved VA range size if needed. */ 1276 sethi %hi(sfmmu_tsb_1st_resv_offset), %o0 1277 call sfmmu_fixup_shiftx 1278 or %o0, %lo(sfmmu_tsb_1st_resv_offset), %o0 1279 call sfmmu_fixup_shiftx 1280 add %o0, I_SIZE, %o0 1281 sethi %hi(sfmmu_tsb_2nd_resv_offset), %o0 1282 call sfmmu_fixup_shiftx 1283 or %o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0 1284 call sfmmu_fixup_shiftx 1285 add %o0, I_SIZE, %o0 1286 1: 1287 /* patch TSBREG_VAMASK used to set up TSB base register */ 1288 set tsb_slab_mask, %o1 1289 ldx [%o1], %o4 1290 sethi %hi(sfmmu_tsb_1st_tsbreg_vamask), %o0 1291 call sfmmu_fixup_or 1292 or %o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0 1293 sethi %hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1294 call sfmmu_fixup_or 1295 or %o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0 1296 1297 ret 1298 restore 1299 #endif /* UTSB_PHYS */ 1300 SET_SIZE(sfmmu_patch_utsb) 1301 1302 ENTRY_NP(sfmmu_patch_shctx) 1303 #ifdef sun4u 1304 retl 1305 nop 1306 #else /* sun4u */ 1307 set sfmmu_shctx_cpu_mondo_patch, %o0 1308 MAKE_JMP_INSTR(5, %o1, %o2) ! jmp %g5 1309 st %o1, [%o0] 1310 flush %o0 1311 MAKE_NOP_INSTR(%o1) 1312 add %o0, I_SIZE, %o0 ! next instr 1313 st %o1, [%o0] 1314 flush %o0 1315 1316 set sfmmu_shctx_user_rtt_patch, %o0 1317 st %o1, [%o0] ! nop 1st instruction 1318 flush %o0 1319 add %o0, I_SIZE, %o0 1320 st %o1, [%o0] ! nop 2nd instruction 1321 flush %o0 1322 add %o0, I_SIZE, %o0 1323 st %o1, [%o0] ! nop 3rd instruction 1324 flush %o0 1325 add %o0, I_SIZE, %o0 1326 st %o1, [%o0] ! nop 4th instruction 1327 flush %o0 1328 add %o0, I_SIZE, %o0 1329 st %o1, [%o0] ! nop 5th instruction 1330 flush %o0 1331 add %o0, I_SIZE, %o0 1332 st %o1, [%o0] ! nop 6th instruction 1333 retl 1334 flush %o0 1335 #endif /* sun4u */ 1336 SET_SIZE(sfmmu_patch_shctx) 1337 1338 /* 1339 * Routine that loads an entry into a tsb using virtual addresses. 1340 * Locking is required since all cpus can use the same TSB. 1341 * Note that it is no longer required to have a valid context 1342 * when calling this function. 1343 */ 1344 ENTRY_NP(sfmmu_load_tsbe) 1345 /* 1346 * %o0 = pointer to tsbe to load 1347 * %o1 = tsb tag 1348 * %o2 = virtual pointer to TTE 1349 * %o3 = 1 if physical address in %o0 else 0 1350 */ 1351 rdpr %pstate, %o5 1352 #ifdef DEBUG 1353 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) 1354 #endif /* DEBUG */ 1355 1356 wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ 1357 1358 SETUP_TSB_ASI(%o3, %g3) 1359 TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8) 1360 1361 wrpr %g0, %o5, %pstate /* enable interrupts */ 1362 1363 retl 1364 membar #StoreStore|#StoreLoad 1365 SET_SIZE(sfmmu_load_tsbe) 1366 1367 /* 1368 * Flush TSB of a given entry if the tag matches. 1369 */ 1370 ENTRY(sfmmu_unload_tsbe) 1371 /* 1372 * %o0 = pointer to tsbe to be flushed 1373 * %o1 = tag to match 1374 * %o2 = 1 if physical address in %o0 else 0 1375 */ 1376 SETUP_TSB_ASI(%o2, %g1) 1377 TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe) 1378 retl 1379 membar #StoreStore|#StoreLoad 1380 SET_SIZE(sfmmu_unload_tsbe) 1381 1382 /* 1383 * Routine that loads a TTE into the kpm TSB from C code. 1384 * Locking is required since kpm TSB is shared among all CPUs. 1385 */ 1386 ENTRY_NP(sfmmu_kpm_load_tsb) 1387 /* 1388 * %o0 = vaddr 1389 * %o1 = ttep 1390 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift) 1391 */ 1392 rdpr %pstate, %o5 ! %o5 = saved pstate 1393 #ifdef DEBUG 1394 PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) 1395 #endif /* DEBUG */ 1396 wrpr %o5, PSTATE_IE, %pstate ! disable interrupts 1397 1398 #ifndef sun4v 1399 sethi %hi(ktsb_phys), %o4 1400 mov ASI_N, %o3 1401 ld [%o4 + %lo(ktsb_phys)], %o4 1402 movrnz %o4, ASI_MEM, %o3 1403 mov %o3, %asi 1404 #endif /* !sun4v */ 1405 mov %o0, %g1 ! %g1 = vaddr 1406 1407 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1408 GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4) 1409 /* %g2 = tsbep, %g1 clobbered */ 1410 1411 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1412 /* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */ 1413 TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9) 1414 1415 wrpr %g0, %o5, %pstate ! enable interrupts 1416 retl 1417 membar #StoreStore|#StoreLoad 1418 SET_SIZE(sfmmu_kpm_load_tsb) 1419 1420 /* 1421 * Routine that shoots down a TTE in the kpm TSB or in the 1422 * kernel TSB depending on virtpg. Locking is required since 1423 * kpm/kernel TSB is shared among all CPUs. 1424 */ 1425 ENTRY_NP(sfmmu_kpm_unload_tsb) 1426 /* 1427 * %o0 = vaddr 1428 * %o1 = virtpg to TSB index shift (e.g. TTE page shift) 1429 */ 1430 #ifndef sun4v 1431 sethi %hi(ktsb_phys), %o4 1432 mov ASI_N, %o3 1433 ld [%o4 + %lo(ktsb_phys)], %o4 1434 movrnz %o4, ASI_MEM, %o3 1435 mov %o3, %asi 1436 #endif /* !sun4v */ 1437 mov %o0, %g1 ! %g1 = vaddr 1438 1439 /* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */ 1440 GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4) 1441 /* %g2 = tsbep, %g1 clobbered */ 1442 1443 srlx %o0, TTARGET_VA_SHIFT, %g1; ! %g1 = tag target 1444 /* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */ 1445 TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval) 1446 1447 retl 1448 membar #StoreStore|#StoreLoad 1449 SET_SIZE(sfmmu_kpm_unload_tsb) 1450 1451 1452 ENTRY_NP(sfmmu_ttetopfn) 1453 ldx [%o0], %g1 /* read tte */ 1454 TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4) 1455 /* 1456 * g1 = pfn 1457 */ 1458 retl 1459 mov %g1, %o0 1460 SET_SIZE(sfmmu_ttetopfn) 1461 1462 /* 1463 * These macros are used to update global sfmmu hme hash statistics 1464 * in perf critical paths. It is only enabled in debug kernels or 1465 * if SFMMU_STAT_GATHER is defined 1466 */ 1467 #if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 1468 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1469 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1470 mov HATSTAT_KHASH_SEARCH, tmp2 ;\ 1471 cmp tmp1, hatid ;\ 1472 movne %ncc, HATSTAT_UHASH_SEARCH, tmp2 ;\ 1473 set sfmmu_global_stat, tmp1 ;\ 1474 add tmp1, tmp2, tmp1 ;\ 1475 ld [tmp1], tmp2 ;\ 1476 inc tmp2 ;\ 1477 st tmp2, [tmp1] 1478 1479 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) \ 1480 ldn [tsbarea + TSBMISS_KHATID], tmp1 ;\ 1481 mov HATSTAT_KHASH_LINKS, tmp2 ;\ 1482 cmp tmp1, hatid ;\ 1483 movne %ncc, HATSTAT_UHASH_LINKS, tmp2 ;\ 1484 set sfmmu_global_stat, tmp1 ;\ 1485 add tmp1, tmp2, tmp1 ;\ 1486 ld [tmp1], tmp2 ;\ 1487 inc tmp2 ;\ 1488 st tmp2, [tmp1] 1489 1490 1491 #else /* DEBUG || SFMMU_STAT_GATHER */ 1492 1493 #define HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1494 1495 #define HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) 1496 1497 #endif /* DEBUG || SFMMU_STAT_GATHER */ 1498 1499 /* 1500 * This macro is used to update global sfmmu kstas in non 1501 * perf critical areas so they are enabled all the time 1502 */ 1503 #define HAT_GLOBAL_STAT(statname, tmp1, tmp2) \ 1504 sethi %hi(sfmmu_global_stat), tmp1 ;\ 1505 add tmp1, statname, tmp1 ;\ 1506 ld [tmp1 + %lo(sfmmu_global_stat)], tmp2 ;\ 1507 inc tmp2 ;\ 1508 st tmp2, [tmp1 + %lo(sfmmu_global_stat)] 1509 1510 /* 1511 * These macros are used to update per cpu stats in non perf 1512 * critical areas so they are enabled all the time 1513 */ 1514 #define HAT_PERCPU_STAT32(tsbarea, stat, tmp1) \ 1515 ld [tsbarea + stat], tmp1 ;\ 1516 inc tmp1 ;\ 1517 st tmp1, [tsbarea + stat] 1518 1519 /* 1520 * These macros are used to update per cpu stats in non perf 1521 * critical areas so they are enabled all the time 1522 */ 1523 #define HAT_PERCPU_STAT16(tsbarea, stat, tmp1) \ 1524 lduh [tsbarea + stat], tmp1 ;\ 1525 inc tmp1 ;\ 1526 stuh tmp1, [tsbarea + stat] 1527 1528 #if defined(KPM_TLBMISS_STATS_GATHER) 1529 /* 1530 * Count kpm dtlb misses separately to allow a different 1531 * evaluation of hme and kpm tlbmisses. kpm tsb hits can 1532 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses). 1533 */ 1534 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) \ 1535 brgez tagacc, label /* KPM VA? */ ;\ 1536 nop ;\ 1537 CPU_INDEX(tmp1, tsbma) ;\ 1538 sethi %hi(kpmtsbm_area), tsbma ;\ 1539 sllx tmp1, KPMTSBM_SHIFT, tmp1 ;\ 1540 or tsbma, %lo(kpmtsbm_area), tsbma ;\ 1541 add tsbma, tmp1, tsbma /* kpmtsbm area */ ;\ 1542 /* VA range check */ ;\ 1543 ldx [tsbma + KPMTSBM_VBASE], val ;\ 1544 cmp tagacc, val ;\ 1545 blu,pn %xcc, label ;\ 1546 ldx [tsbma + KPMTSBM_VEND], tmp1 ;\ 1547 cmp tagacc, tmp1 ;\ 1548 bgeu,pn %xcc, label ;\ 1549 lduw [tsbma + KPMTSBM_DTLBMISS], val ;\ 1550 inc val ;\ 1551 st val, [tsbma + KPMTSBM_DTLBMISS] ;\ 1552 label: 1553 #else 1554 #define KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) 1555 #endif /* KPM_TLBMISS_STATS_GATHER */ 1556 1557 #ifdef PTL1_PANIC_DEBUG 1558 .seg ".data" 1559 .global test_ptl1_panic 1560 test_ptl1_panic: 1561 .word 0 1562 .align 8 1563 1564 .seg ".text" 1565 .align 4 1566 #endif /* PTL1_PANIC_DEBUG */ 1567 1568 1569 ENTRY_NP(sfmmu_pagefault) 1570 SET_GL_REG(1) 1571 USE_ALTERNATE_GLOBALS(%g5) 1572 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4) 1573 rdpr %tt, %g6 1574 cmp %g6, FAST_IMMU_MISS_TT 1575 be,a,pn %icc, 1f 1576 mov T_INSTR_MMU_MISS, %g3 1577 cmp %g6, T_INSTR_MMU_MISS 1578 be,a,pn %icc, 1f 1579 mov T_INSTR_MMU_MISS, %g3 1580 mov %g5, %g2 1581 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1582 cmp %g6, FAST_DMMU_MISS_TT 1583 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1584 cmp %g6, T_DATA_MMU_MISS 1585 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1586 1587 #ifdef PTL1_PANIC_DEBUG 1588 /* check if we want to test the tl1 panic */ 1589 sethi %hi(test_ptl1_panic), %g4 1590 ld [%g4 + %lo(test_ptl1_panic)], %g1 1591 st %g0, [%g4 + %lo(test_ptl1_panic)] 1592 cmp %g1, %g0 1593 bne,a,pn %icc, ptl1_panic 1594 or %g0, PTL1_BAD_DEBUG, %g1 1595 #endif /* PTL1_PANIC_DEBUG */ 1596 1: 1597 HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4) 1598 /* 1599 * g2 = tag access reg 1600 * g3.l = type 1601 * g3.h = 0 1602 */ 1603 sethi %hi(trap), %g1 1604 or %g1, %lo(trap), %g1 1605 2: 1606 ba,pt %xcc, sys_trap 1607 mov -1, %g4 1608 SET_SIZE(sfmmu_pagefault) 1609 1610 ENTRY_NP(sfmmu_mmu_trap) 1611 SET_GL_REG(1) 1612 USE_ALTERNATE_GLOBALS(%g5) 1613 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6) 1614 rdpr %tt, %g6 1615 cmp %g6, FAST_IMMU_MISS_TT 1616 be,a,pn %icc, 1f 1617 mov T_INSTR_MMU_MISS, %g3 1618 cmp %g6, T_INSTR_MMU_MISS 1619 be,a,pn %icc, 1f 1620 mov T_INSTR_MMU_MISS, %g3 1621 mov %g5, %g2 1622 mov T_DATA_PROT, %g3 /* arg2 = traptype */ 1623 cmp %g6, FAST_DMMU_MISS_TT 1624 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1625 cmp %g6, T_DATA_MMU_MISS 1626 move %icc, T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1627 1: 1628 /* 1629 * g2 = tag access reg 1630 * g3 = type 1631 */ 1632 sethi %hi(sfmmu_tsbmiss_exception), %g1 1633 or %g1, %lo(sfmmu_tsbmiss_exception), %g1 1634 ba,pt %xcc, sys_trap 1635 mov -1, %g4 1636 /*NOTREACHED*/ 1637 SET_SIZE(sfmmu_mmu_trap) 1638 1639 ENTRY_NP(sfmmu_suspend_tl) 1640 SET_GL_REG(1) 1641 USE_ALTERNATE_GLOBALS(%g5) 1642 GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3) 1643 rdpr %tt, %g6 1644 cmp %g6, FAST_IMMU_MISS_TT 1645 be,a,pn %icc, 1f 1646 mov T_INSTR_MMU_MISS, %g3 1647 mov %g5, %g2 1648 cmp %g6, FAST_DMMU_MISS_TT 1649 move %icc, T_DATA_MMU_MISS, %g3 1650 movne %icc, T_DATA_PROT, %g3 1651 1: 1652 sethi %hi(sfmmu_tsbmiss_suspended), %g1 1653 or %g1, %lo(sfmmu_tsbmiss_suspended), %g1 1654 /* g1 = TL0 handler, g2 = tagacc, g3 = trap type */ 1655 ba,pt %xcc, sys_trap 1656 mov PIL_15, %g4 1657 /*NOTREACHED*/ 1658 SET_SIZE(sfmmu_suspend_tl) 1659 1660 /* 1661 * No %g registers in use at this point. 1662 */ 1663 ENTRY_NP(sfmmu_window_trap) 1664 rdpr %tpc, %g1 1665 #ifdef sun4v 1666 #ifdef DEBUG 1667 /* We assume previous %gl was 1 */ 1668 rdpr %tstate, %g4 1669 srlx %g4, TSTATE_GL_SHIFT, %g4 1670 and %g4, TSTATE_GL_MASK, %g4 1671 cmp %g4, 1 1672 bne,a,pn %icc, ptl1_panic 1673 mov PTL1_BAD_WTRAP, %g1 1674 #endif /* DEBUG */ 1675 /* user miss at tl>1. better be the window handler or user_rtt */ 1676 /* in user_rtt? */ 1677 set rtt_fill_start, %g4 1678 cmp %g1, %g4 1679 blu,pn %xcc, 6f 1680 .empty 1681 set rtt_fill_end, %g4 1682 cmp %g1, %g4 1683 bgeu,pn %xcc, 6f 1684 nop 1685 set fault_rtt_fn1, %g1 1686 wrpr %g0, %g1, %tnpc 1687 ba,a 7f 1688 6: 1689 ! must save this trap level before descending trap stack 1690 ! no need to save %tnpc, either overwritten or discarded 1691 ! already got it: rdpr %tpc, %g1 1692 rdpr %tstate, %g6 1693 rdpr %tt, %g7 1694 ! trap level saved, go get underlying trap type 1695 rdpr %tl, %g5 1696 sub %g5, 1, %g3 1697 wrpr %g3, %tl 1698 rdpr %tt, %g2 1699 wrpr %g5, %tl 1700 ! restore saved trap level 1701 wrpr %g1, %tpc 1702 wrpr %g6, %tstate 1703 wrpr %g7, %tt 1704 #else /* sun4v */ 1705 /* user miss at tl>1. better be the window handler */ 1706 rdpr %tl, %g5 1707 sub %g5, 1, %g3 1708 wrpr %g3, %tl 1709 rdpr %tt, %g2 1710 wrpr %g5, %tl 1711 #endif /* sun4v */ 1712 and %g2, WTRAP_TTMASK, %g4 1713 cmp %g4, WTRAP_TYPE 1714 bne,pn %xcc, 1f 1715 nop 1716 /* tpc should be in the trap table */ 1717 set trap_table, %g4 1718 cmp %g1, %g4 1719 blt,pn %xcc, 1f 1720 .empty 1721 set etrap_table, %g4 1722 cmp %g1, %g4 1723 bge,pn %xcc, 1f 1724 .empty 1725 andn %g1, WTRAP_ALIGN, %g1 /* 128 byte aligned */ 1726 add %g1, WTRAP_FAULTOFF, %g1 1727 wrpr %g0, %g1, %tnpc 1728 7: 1729 /* 1730 * some wbuf handlers will call systrap to resolve the fault 1731 * we pass the trap type so they figure out the correct parameters. 1732 * g5 = trap type, g6 = tag access reg 1733 */ 1734 1735 /* 1736 * only use g5, g6, g7 registers after we have switched to alternate 1737 * globals. 1738 */ 1739 SET_GL_REG(1) 1740 USE_ALTERNATE_GLOBALS(%g5) 1741 GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/) 1742 rdpr %tt, %g7 1743 cmp %g7, FAST_IMMU_MISS_TT 1744 be,a,pn %icc, ptl1_panic 1745 mov PTL1_BAD_WTRAP, %g1 1746 cmp %g7, T_INSTR_MMU_MISS 1747 be,a,pn %icc, ptl1_panic 1748 mov PTL1_BAD_WTRAP, %g1 1749 mov T_DATA_PROT, %g5 1750 cmp %g7, FAST_DMMU_MISS_TT 1751 move %icc, T_DATA_MMU_MISS, %g5 1752 cmp %g7, T_DATA_MMU_MISS 1753 move %icc, T_DATA_MMU_MISS, %g5 1754 ! XXXQ AGS re-check out this one 1755 done 1756 1: 1757 CPU_PADDR(%g1, %g4) 1758 add %g1, CPU_TL1_HDLR, %g1 1759 lda [%g1]ASI_MEM, %g4 1760 brnz,a,pt %g4, sfmmu_mmu_trap 1761 sta %g0, [%g1]ASI_MEM 1762 ba,pt %icc, ptl1_panic 1763 mov PTL1_BAD_TRAP, %g1 1764 SET_SIZE(sfmmu_window_trap) 1765 1766 ENTRY_NP(sfmmu_kpm_exception) 1767 /* 1768 * We have accessed an unmapped segkpm address or a legal segkpm 1769 * address which is involved in a VAC alias conflict prevention. 1770 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is 1771 * set. If it is, we will instead note that a fault has occurred 1772 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of 1773 * a "retry"). This will step over the faulting instruction. 1774 * Note that this means that a legal segkpm address involved in 1775 * a VAC alias conflict prevention (a rare case to begin with) 1776 * cannot be used in DTrace. 1777 */ 1778 CPU_INDEX(%g1, %g2) 1779 set cpu_core, %g2 1780 sllx %g1, CPU_CORE_SHIFT, %g1 1781 add %g1, %g2, %g1 1782 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 1783 andcc %g2, CPU_DTRACE_NOFAULT, %g0 1784 bz 0f 1785 or %g2, CPU_DTRACE_BADADDR, %g2 1786 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 1787 GET_MMU_D_ADDR(%g3, /*scratch*/ %g4) 1788 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 1789 done 1790 0: 1791 TSTAT_CHECK_TL1(1f, %g1, %g2) 1792 1: 1793 SET_GL_REG(1) 1794 USE_ALTERNATE_GLOBALS(%g5) 1795 GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/) 1796 mov T_DATA_MMU_MISS, %g3 /* arg2 = traptype */ 1797 /* 1798 * g2=tagacc g3.l=type g3.h=0 1799 */ 1800 sethi %hi(trap), %g1 1801 or %g1, %lo(trap), %g1 1802 ba,pt %xcc, sys_trap 1803 mov -1, %g4 1804 SET_SIZE(sfmmu_kpm_exception) 1805 1806 #if (IMAP_SEG != 0) 1807 #error - ism_map->ism_seg offset is not zero 1808 #endif 1809 1810 /* 1811 * Copies ism mapping for this ctx in param "ism" if this is a ISM 1812 * tlb miss and branches to label "ismhit". If this is not an ISM 1813 * process or an ISM tlb miss it falls thru. 1814 * 1815 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for 1816 * this process. 1817 * If so, it will branch to label "ismhit". If not, it will fall through. 1818 * 1819 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT 1820 * so that any other threads of this process will not try and walk the ism 1821 * maps while they are being changed. 1822 * 1823 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare 1824 * will make sure of that. This means we can terminate our search on 1825 * the first zero mapping we find. 1826 * 1827 * Parameters: 1828 * tagacc = (pseudo-)tag access register (vaddr + ctx) (in) 1829 * tsbmiss = address of tsb miss area (in) 1830 * ismseg = contents of ism_seg for this ism map (out) 1831 * ismhat = physical address of imap_ismhat for this ism map (out) 1832 * tmp1 = scratch reg (CLOBBERED) 1833 * tmp2 = scratch reg (CLOBBERED) 1834 * tmp3 = scratch reg (CLOBBERED) 1835 * label: temporary labels 1836 * ismhit: label where to jump to if an ism dtlb miss 1837 * exitlabel:label where to jump if hat is busy due to hat_unshare. 1838 */ 1839 #define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \ 1840 label, ismhit) \ 1841 ldx [tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */ ;\ 1842 brlz,pt tmp1, label/**/3 /* exit if -1 */ ;\ 1843 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0] */ ;\ 1844 label/**/1: ;\ 1845 ldxa [ismhat]ASI_MEM, ismseg /* ismblk.map[0].ism_seg */ ;\ 1846 mov tmp1, tmp3 /* update current ismblkpa head */ ;\ 1847 label/**/2: ;\ 1848 brz,pt ismseg, label/**/3 /* no mapping */ ;\ 1849 add ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */ ;\ 1850 lduba [tmp1]ASI_MEM, tmp1 /* tmp1 = vb shift*/ ;\ 1851 srlx ismseg, tmp1, tmp2 /* tmp2 = vbase */ ;\ 1852 srlx tagacc, tmp1, tmp1 /* tmp1 = va seg*/ ;\ 1853 sub tmp1, tmp2, tmp2 /* tmp2 = va - vbase */ ;\ 1854 add ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */ ;\ 1855 lda [tmp1]ASI_MEM, tmp1 /* tmp1 = sz_mask */ ;\ 1856 and ismseg, tmp1, tmp1 /* tmp1 = size */ ;\ 1857 cmp tmp2, tmp1 /* check va <= offset*/ ;\ 1858 blu,a,pt %xcc, ismhit /* ism hit */ ;\ 1859 add ismhat, IMAP_ISMHAT, ismhat /* ismhat = &ism_sfmmu*/ ;\ 1860 ;\ 1861 add ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ ;\ 1862 add tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1 ;\ 1863 cmp ismhat, tmp1 ;\ 1864 bl,pt %xcc, label/**/2 /* keep looking */ ;\ 1865 ldxa [ismhat]ASI_MEM, ismseg /* ismseg = map[ismhat] */ ;\ 1866 ;\ 1867 add tmp3, IBLK_NEXTPA, tmp1 ;\ 1868 ldxa [tmp1]ASI_MEM, tmp1 /* check blk->nextpa */ ;\ 1869 brgez,pt tmp1, label/**/1 /* continue if not -1*/ ;\ 1870 add tmp1, IBLK_MAPS, ismhat /* ismhat = &ismblk.map[0]*/ ;\ 1871 label/**/3: 1872 1873 /* 1874 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid 1875 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift) 1876 * Parameters: 1877 * tagacc = reg containing virtual address 1878 * hatid = reg containing sfmmu pointer 1879 * hmeshift = constant/register to shift vaddr to obtain vapg 1880 * hmebp = register where bucket pointer will be stored 1881 * vapg = register where virtual page will be stored 1882 * tmp1, tmp2 = tmp registers 1883 */ 1884 1885 1886 #define HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp, \ 1887 vapg, label, tmp1, tmp2) \ 1888 sllx tagacc, TAGACC_CTX_LSHIFT, tmp1 ;\ 1889 brnz,a,pt tmp1, label/**/1 ;\ 1890 ld [tsbarea + TSBMISS_UHASHSZ], hmebp ;\ 1891 ld [tsbarea + TSBMISS_KHASHSZ], hmebp ;\ 1892 ba,pt %xcc, label/**/2 ;\ 1893 ldx [tsbarea + TSBMISS_KHASHSTART], tmp1 ;\ 1894 label/**/1: ;\ 1895 ldx [tsbarea + TSBMISS_UHASHSTART], tmp1 ;\ 1896 label/**/2: ;\ 1897 srlx tagacc, hmeshift, vapg ;\ 1898 xor vapg, hatid, tmp2 /* hatid ^ (vaddr >> shift) */ ;\ 1899 and tmp2, hmebp, hmebp /* index into hme_hash */ ;\ 1900 mulx hmebp, HMEBUCK_SIZE, hmebp ;\ 1901 add hmebp, tmp1, hmebp 1902 1903 /* 1904 * hashtag includes bspage + hashno (64 bits). 1905 */ 1906 1907 #define MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag) \ 1908 sllx vapg, hmeshift, vapg ;\ 1909 mov hashno, hblktag ;\ 1910 sllx hblktag, HTAG_REHASH_SHIFT, hblktag ;\ 1911 or vapg, hblktag, hblktag 1912 1913 /* 1914 * Function to traverse hmeblk hash link list and find corresponding match. 1915 * The search is done using physical pointers. It returns the physical address 1916 * pointer to the hmeblk that matches with the tag provided. 1917 * Parameters: 1918 * hmebp = register that points to hme hash bucket, also used as 1919 * tmp reg (clobbered) 1920 * hmeblktag = register with hmeblk tag match 1921 * hatid = register with hatid 1922 * hmeblkpa = register where physical ptr will be stored 1923 * tmp1 = tmp reg 1924 * label: temporary label 1925 */ 1926 1927 #define HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, \ 1928 tmp1, label) \ 1929 add hmebp, HMEBUCK_NEXTPA, hmeblkpa ;\ 1930 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 1931 HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1932 label/**/1: ;\ 1933 cmp hmeblkpa, HMEBLK_ENDPA ;\ 1934 be,pn %xcc, label/**/2 ;\ 1935 HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1) ;\ 1936 add hmeblkpa, HMEBLK_TAG, hmebp ;\ 1937 ldxa [hmebp]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 1938 add hmebp, CLONGSIZE, hmebp ;\ 1939 ldxa [hmebp]ASI_MEM, hmebp /* read 2nd part of tag */ ;\ 1940 xor tmp1, hmeblktag, tmp1 ;\ 1941 xor hmebp, hatid, hmebp ;\ 1942 or hmebp, tmp1, hmebp ;\ 1943 brz,pn hmebp, label/**/2 /* branch on hit */ ;\ 1944 add hmeblkpa, HMEBLK_NEXTPA, hmebp ;\ 1945 ba,pt %xcc, label/**/1 ;\ 1946 ldxa [hmebp]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 1947 label/**/2: 1948 1949 /* 1950 * Function to traverse hmeblk hash link list and find corresponding match. 1951 * The search is done using physical pointers. It returns the physical address 1952 * pointer to the hmeblk that matches with the tag 1953 * provided. 1954 * Parameters: 1955 * hmeblktag = register with hmeblk tag match (rid field is 0) 1956 * hatid = register with hatid (pointer to SRD) 1957 * hmeblkpa = register where physical ptr will be stored 1958 * tmp1 = tmp reg 1959 * tmp2 = tmp reg 1960 * label: temporary label 1961 */ 1962 1963 #define HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea, \ 1964 tmp1, tmp2, label) \ 1965 label/**/1: ;\ 1966 cmp hmeblkpa, HMEBLK_ENDPA ;\ 1967 be,pn %xcc, label/**/4 ;\ 1968 HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2) ;\ 1969 add hmeblkpa, HMEBLK_TAG, tmp2 ;\ 1970 ldxa [tmp2]ASI_MEM, tmp1 /* read 1st part of tag */ ;\ 1971 add tmp2, CLONGSIZE, tmp2 ;\ 1972 ldxa [tmp2]ASI_MEM, tmp2 /* read 2nd part of tag */ ;\ 1973 xor tmp1, hmeblktag, tmp1 ;\ 1974 xor tmp2, hatid, tmp2 ;\ 1975 brz,pn tmp2, label/**/3 /* branch on hit */ ;\ 1976 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 1977 label/**/2: ;\ 1978 ba,pt %xcc, label/**/1 ;\ 1979 ldxa [tmp2]ASI_MEM, hmeblkpa /* hmeblk ptr pa */ ;\ 1980 label/**/3: ;\ 1981 cmp tmp1, SFMMU_MAX_HME_REGIONS ;\ 1982 bgeu,pt %xcc, label/**/2 ;\ 1983 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 1984 and tmp1, BT_ULMASK, tmp2 ;\ 1985 srlx tmp1, BT_ULSHIFT, tmp1 ;\ 1986 sllx tmp1, CLONGSHIFT, tmp1 ;\ 1987 add tsbarea, tmp1, tmp1 ;\ 1988 ldx [tmp1 + TSBMISS_SHMERMAP], tmp1 ;\ 1989 srlx tmp1, tmp2, tmp1 ;\ 1990 btst 0x1, tmp1 ;\ 1991 bz,pn %xcc, label/**/2 ;\ 1992 add hmeblkpa, HMEBLK_NEXTPA, tmp2 ;\ 1993 label/**/4: 1994 1995 #if ((1 << SFHME_SHIFT) != SFHME_SIZE) 1996 #error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size 1997 #endif 1998 1999 /* 2000 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns 2001 * the offset for the corresponding hment. 2002 * Parameters: 2003 * In: 2004 * vaddr = register with virtual address 2005 * hmeblkpa = physical pointer to hme_blk 2006 * Out: 2007 * hmentoff = register where hment offset will be stored 2008 * hmemisc = hblk_misc 2009 * Scratch: 2010 * tmp1 2011 */ 2012 #define HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\ 2013 add hmeblkpa, HMEBLK_MISC, hmentoff ;\ 2014 lda [hmentoff]ASI_MEM, hmemisc ;\ 2015 andcc hmemisc, HBLK_SZMASK, %g0 ;\ 2016 bnz,a,pn %icc, label1 /* if sz != TTE8K branch */ ;\ 2017 or %g0, HMEBLK_HME1, hmentoff ;\ 2018 srl vaddr, MMU_PAGESHIFT, tmp1 ;\ 2019 and tmp1, NHMENTS - 1, tmp1 /* tmp1 = index */ ;\ 2020 sllx tmp1, SFHME_SHIFT, tmp1 ;\ 2021 add tmp1, HMEBLK_HME1, hmentoff ;\ 2022 label1: 2023 2024 /* 2025 * GET_TTE is a macro that returns a TTE given a tag and hatid. 2026 * 2027 * tagacc = (pseudo-)tag access register (in) 2028 * hatid = sfmmu pointer for TSB miss (in) 2029 * tte = tte for TLB miss if found, otherwise clobbered (out) 2030 * hmeblkpa = PA of hment if found, otherwise clobbered (out) 2031 * tsbarea = pointer to the tsbmiss area for this cpu. (in) 2032 * hmemisc = hblk_misc if TTE is found (out), otherwise clobbered 2033 * hmeshift = constant/register to shift VA to obtain the virtual pfn 2034 * for this page size. 2035 * hashno = constant/register hash number 2036 * tmp = temp value - clobbered 2037 * label = temporary label for branching within macro. 2038 * foundlabel = label to jump to when tte is found. 2039 * suspendlabel= label to jump to when tte is suspended. 2040 * exitlabel = label to jump to when tte is not found. 2041 * 2042 */ 2043 #define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \ 2044 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \ 2045 ;\ 2046 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2047 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2048 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2049 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2050 ;\ 2051 /* ;\ 2052 * tagacc = tagacc ;\ 2053 * hatid = hatid ;\ 2054 * tsbarea = tsbarea ;\ 2055 * tte = hmebp (hme bucket pointer) ;\ 2056 * hmeblkpa = vapg (virtual page) ;\ 2057 * hmemisc, tmp = scratch ;\ 2058 */ ;\ 2059 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2060 or hmemisc, SFMMU_INVALID_SHMERID, hmemisc ;\ 2061 ;\ 2062 /* ;\ 2063 * tagacc = tagacc ;\ 2064 * hatid = hatid ;\ 2065 * tte = hmebp ;\ 2066 * hmeblkpa = CLOBBERED ;\ 2067 * hmemisc = htag_bspage+hashno+invalid_rid ;\ 2068 * tmp = scratch ;\ 2069 */ ;\ 2070 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2071 HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, \ 2072 tsbarea, tagacc, label/**/1) ;\ 2073 /* ;\ 2074 * tagacc = CLOBBERED ;\ 2075 * tte = CLOBBERED ;\ 2076 * hmeblkpa = hmeblkpa ;\ 2077 * tmp = scratch ;\ 2078 */ ;\ 2079 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2080 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2081 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2082 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2083 nop ;\ 2084 label/**/4: ;\ 2085 /* ;\ 2086 * We have found the hmeblk containing the hment. ;\ 2087 * Now we calculate the corresponding tte. ;\ 2088 * ;\ 2089 * tagacc = tagacc ;\ 2090 * hatid = hatid ;\ 2091 * tte = clobbered ;\ 2092 * hmeblkpa = hmeblkpa ;\ 2093 * hmemisc = hblktag ;\ 2094 * tmp = scratch ;\ 2095 */ ;\ 2096 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2097 label/**/2) ;\ 2098 ;\ 2099 /* ;\ 2100 * tagacc = tagacc ;\ 2101 * hatid = hmentoff ;\ 2102 * tte = clobbered ;\ 2103 * hmeblkpa = hmeblkpa ;\ 2104 * hmemisc = hblk_misc ;\ 2105 * tmp = scratch ;\ 2106 */ ;\ 2107 ;\ 2108 add hatid, SFHME_TTE, hatid ;\ 2109 add hmeblkpa, hatid, hmeblkpa ;\ 2110 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2111 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2112 set TTE_SUSPEND, hatid ;\ 2113 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2114 btst tte, hatid ;\ 2115 bz,pt %xcc, foundlabel ;\ 2116 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2117 ;\ 2118 /* ;\ 2119 * Mapping is suspended, so goto suspend label. ;\ 2120 */ ;\ 2121 ba,pt %xcc, suspendlabel ;\ 2122 nop 2123 2124 /* 2125 * GET_SHME_TTE is similar to GET_TTE() except it searches 2126 * shared hmeblks via HMEHASH_SEARCH_SHME() macro. 2127 * If valid tte is found, hmemisc = shctx flag, i.e., shme is 2128 * either 0 (not part of scd) or 1 (part of scd). 2129 */ 2130 #define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, \ 2131 hmeshift, hashno, tmp, label, foundlabel, \ 2132 suspendlabel, exitlabel) \ 2133 ;\ 2134 stn tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)] ;\ 2135 stn hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)] ;\ 2136 HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte, \ 2137 hmeblkpa, label/**/5, hmemisc, tmp) ;\ 2138 ;\ 2139 /* ;\ 2140 * tagacc = tagacc ;\ 2141 * hatid = hatid ;\ 2142 * tsbarea = tsbarea ;\ 2143 * tte = hmebp (hme bucket pointer) ;\ 2144 * hmeblkpa = vapg (virtual page) ;\ 2145 * hmemisc, tmp = scratch ;\ 2146 */ ;\ 2147 MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc) ;\ 2148 ;\ 2149 /* ;\ 2150 * tagacc = tagacc ;\ 2151 * hatid = hatid ;\ 2152 * tsbarea = tsbarea ;\ 2153 * tte = hmebp ;\ 2154 * hmemisc = htag_bspage + hashno + 0 (for rid) ;\ 2155 * hmeblkpa = CLOBBERED ;\ 2156 * tmp = scratch ;\ 2157 */ ;\ 2158 stn tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)] ;\ 2159 ;\ 2160 add tte, HMEBUCK_NEXTPA, hmeblkpa ;\ 2161 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2162 HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte) ;\ 2163 ;\ 2164 label/**/8: ;\ 2165 HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, \ 2166 tsbarea, tagacc, tte, label/**/1) ;\ 2167 /* ;\ 2168 * tagacc = CLOBBERED ;\ 2169 * tte = CLOBBERED ;\ 2170 * hmeblkpa = hmeblkpa ;\ 2171 * tmp = scratch ;\ 2172 */ ;\ 2173 cmp hmeblkpa, HMEBLK_ENDPA ;\ 2174 bne,pn %xcc, label/**/4 /* branch if hmeblk found */ ;\ 2175 ldn [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc ;\ 2176 ba,pt %xcc, exitlabel /* exit if hblk not found */ ;\ 2177 nop ;\ 2178 label/**/4: ;\ 2179 /* ;\ 2180 * We have found the hmeblk containing the hment. ;\ 2181 * Now we calculate the corresponding tte. ;\ 2182 * ;\ 2183 * tagacc = tagacc ;\ 2184 * hatid = hatid ;\ 2185 * tte = clobbered ;\ 2186 * hmeblkpa = hmeblkpa ;\ 2187 * hmemisc = hblktag ;\ 2188 * tsbarea = tsbmiss area ;\ 2189 * tmp = scratch ;\ 2190 */ ;\ 2191 HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte, \ 2192 label/**/2) ;\ 2193 ;\ 2194 /* ;\ 2195 * tagacc = tagacc ;\ 2196 * hatid = hmentoff ;\ 2197 * tte = clobbered ;\ 2198 * hmeblkpa = hmeblkpa ;\ 2199 * hmemisc = hblk_misc ;\ 2200 * tsbarea = tsbmiss area ;\ 2201 * tmp = scratch ;\ 2202 */ ;\ 2203 ;\ 2204 add hatid, SFHME_TTE, hatid ;\ 2205 add hmeblkpa, hatid, hmeblkpa ;\ 2206 ldxa [hmeblkpa]ASI_MEM, tte /* MMU_READTTE through pa */ ;\ 2207 brlz,pt tte, label/**/6 ;\ 2208 nop ;\ 2209 btst HBLK_SZMASK, hmemisc ;\ 2210 bnz,a,pt %icc, label/**/7 ;\ 2211 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2212 ;\ 2213 /* ;\ 2214 * We found an invalid 8K tte in shme. ;\ 2215 * it may not belong to shme's region since ;\ 2216 * region size/alignment granularity is 8K but different ;\ 2217 * regions don't share hmeblks. Continue the search. ;\ 2218 */ ;\ 2219 sub hmeblkpa, hatid, hmeblkpa ;\ 2220 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2221 srlx tagacc, hmeshift, tte ;\ 2222 add hmeblkpa, HMEBLK_NEXTPA, hmeblkpa ;\ 2223 ldxa [hmeblkpa]ASI_MEM, hmeblkpa ;\ 2224 MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc) ;\ 2225 ba,a,pt %xcc, label/**/8 ;\ 2226 label/**/6: ;\ 2227 GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc) ;\ 2228 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid ;\ 2229 label/**/7: ;\ 2230 set TTE_SUSPEND, hatid ;\ 2231 TTE_SUSPEND_INT_SHIFT(hatid) ;\ 2232 btst tte, hatid ;\ 2233 bz,pt %xcc, foundlabel ;\ 2234 ldn [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid ;\ 2235 ;\ 2236 /* ;\ 2237 * Mapping is suspended, so goto suspend label. ;\ 2238 */ ;\ 2239 ba,pt %xcc, suspendlabel ;\ 2240 nop 2241 2242 /* 2243 * KERNEL PROTECTION HANDLER 2244 * 2245 * g1 = tsb8k pointer register (clobbered) 2246 * g2 = tag access register (ro) 2247 * g3 - g7 = scratch registers 2248 * 2249 * Note: This function is patched at runtime for performance reasons. 2250 * Any changes here require sfmmu_patch_ktsb fixed. 2251 */ 2252 ENTRY_NP(sfmmu_kprot_trap) 2253 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2254 sfmmu_kprot_patch_ktsb_base: 2255 RUNTIME_PATCH_SETX(%g1, %g6) 2256 /* %g1 = contents of ktsb_base or ktsb_pbase */ 2257 sfmmu_kprot_patch_ktsb_szcode: 2258 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 2259 2260 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 2261 ! %g1 = First TSB entry pointer, as TSB miss handler expects 2262 2263 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 2264 sfmmu_kprot_patch_ktsb4m_base: 2265 RUNTIME_PATCH_SETX(%g3, %g6) 2266 /* %g3 = contents of ktsb4m_base or ktsb4m_pbase */ 2267 sfmmu_kprot_patch_ktsb4m_szcode: 2268 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 2269 2270 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 2271 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 2272 2273 CPU_TSBMISS_AREA(%g6, %g7) 2274 HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7) 2275 ba,pt %xcc, sfmmu_tsb_miss_tt 2276 nop 2277 2278 /* 2279 * USER PROTECTION HANDLER 2280 * 2281 * g1 = tsb8k pointer register (ro) 2282 * g2 = tag access register (ro) 2283 * g3 = faulting context (clobbered, currently not used) 2284 * g4 - g7 = scratch registers 2285 */ 2286 ALTENTRY(sfmmu_uprot_trap) 2287 #ifdef sun4v 2288 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2289 /* %g1 = first TSB entry ptr now, %g2 preserved */ 2290 2291 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 2292 brlz,pt %g3, 9f /* check for 2nd TSB */ 2293 nop 2294 2295 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2296 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2297 2298 #else /* sun4v */ 2299 #ifdef UTSB_PHYS 2300 /* g1 = first TSB entry ptr */ 2301 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2302 brlz,pt %g3, 9f /* check for 2nd TSB */ 2303 nop 2304 2305 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2306 /* %g3 = second TSB entry ptr now, %g2 preserved */ 2307 #else /* UTSB_PHYS */ 2308 brgez,pt %g1, 9f /* check for 2nd TSB */ 2309 mov -1, %g3 /* set second tsbe ptr to -1 */ 2310 2311 mov %g2, %g7 2312 GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot) 2313 /* %g3 = second TSB entry ptr now, %g7 clobbered */ 2314 mov %g1, %g7 2315 GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot) 2316 #endif /* UTSB_PHYS */ 2317 #endif /* sun4v */ 2318 9: 2319 CPU_TSBMISS_AREA(%g6, %g7) 2320 HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7) 2321 ba,pt %xcc, sfmmu_tsb_miss_tt /* branch TSB miss handler */ 2322 nop 2323 2324 /* 2325 * Kernel 8K page iTLB miss. We also get here if we took a 2326 * fast instruction access mmu miss trap while running in 2327 * invalid context. 2328 * 2329 * %g1 = 8K TSB pointer register (not used, clobbered) 2330 * %g2 = tag access register (used) 2331 * %g3 = faulting context id (used) 2332 * %g7 = TSB tag to match (used) 2333 */ 2334 .align 64 2335 ALTENTRY(sfmmu_kitlb_miss) 2336 brnz,pn %g3, tsb_tl0_noctxt 2337 nop 2338 2339 /* kernel miss */ 2340 /* get kernel tsb pointer */ 2341 /* we patch the next set of instructions at run time */ 2342 /* NOTE: any changes here require sfmmu_patch_ktsb fixed */ 2343 iktsbbase: 2344 RUNTIME_PATCH_SETX(%g4, %g5) 2345 /* %g4 = contents of ktsb_base or ktsb_pbase */ 2346 2347 iktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2348 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2349 or %g4, %g1, %g1 ! form tsb ptr 2350 ldda [%g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2351 cmp %g4, %g7 2352 bne,pn %xcc, iktsb4mbase ! check 4m ktsb 2353 srlx %g2, MMU_PAGESHIFT4M, %g3 ! use 4m virt-page as TSB index 2354 2355 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2356 bz,pn %icc, exec_fault 2357 nop 2358 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2359 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2360 retry 2361 2362 iktsb4mbase: 2363 RUNTIME_PATCH_SETX(%g4, %g6) 2364 /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */ 2365 iktsb4m: 2366 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2367 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2368 add %g4, %g3, %g3 ! %g3 = 4m tsbe ptr 2369 ldda [%g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2370 cmp %g4, %g7 2371 bne,pn %xcc, sfmmu_tsb_miss_tt ! branch on miss 2372 andcc %g5, TTE_EXECPRM_INT, %g0 ! check exec bit 2373 bz,pn %icc, exec_fault 2374 nop 2375 TT_TRACE(trace_tsbhit) ! 2 instr traptrace 2376 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2377 retry 2378 2379 /* 2380 * Kernel dTLB miss. We also get here if we took a fast data 2381 * access mmu miss trap while running in invalid context. 2382 * 2383 * Note: for now we store kpm TTEs in the kernel TSB as usual. 2384 * We select the TSB miss handler to branch to depending on 2385 * the virtual address of the access. In the future it may 2386 * be desirable to separate kpm TTEs into their own TSB, 2387 * in which case all that needs to be done is to set 2388 * kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch 2389 * early in the miss if we detect a kpm VA to a new handler. 2390 * 2391 * %g1 = 8K TSB pointer register (not used, clobbered) 2392 * %g2 = tag access register (used) 2393 * %g3 = faulting context id (used) 2394 */ 2395 .align 64 2396 ALTENTRY(sfmmu_kdtlb_miss) 2397 brnz,pn %g3, tsb_tl0_noctxt /* invalid context? */ 2398 nop 2399 2400 /* Gather some stats for kpm misses in the TLB. */ 2401 /* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */ 2402 KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out) 2403 2404 /* 2405 * Get first TSB offset and look for 8K/64K/512K mapping 2406 * using the 8K virtual page as the index. 2407 * 2408 * We patch the next set of instructions at run time; 2409 * any changes here require sfmmu_patch_ktsb changes too. 2410 */ 2411 dktsbbase: 2412 RUNTIME_PATCH_SETX(%g7, %g6) 2413 /* %g7 = contents of ktsb_base or ktsb_pbase */ 2414 2415 dktsb: sllx %g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1 2416 srlx %g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1 2417 2418 /* 2419 * At this point %g1 is our index into the TSB. 2420 * We just masked off enough bits of the VA depending 2421 * on our TSB size code. 2422 */ 2423 ldda [%g7 + %g1]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2424 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2425 cmp %g6, %g4 ! compare tag 2426 bne,pn %xcc, dktsb4m_kpmcheck_small 2427 add %g7, %g1, %g1 /* form tsb ptr */ 2428 TT_TRACE(trace_tsbhit) 2429 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2430 /* trapstat expects tte in %g5 */ 2431 retry 2432 2433 /* 2434 * If kpm is using large pages, the following instruction needs 2435 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm) 2436 * so that we will probe the 4M TSB regardless of the VA. In 2437 * the case kpm is using small pages, we know no large kernel 2438 * mappings are located above 0x80000000.00000000 so we skip the 2439 * probe as an optimization. 2440 */ 2441 dktsb4m_kpmcheck_small: 2442 brlz,pn %g2, sfmmu_kpm_dtsb_miss_small 2443 /* delay slot safe, below */ 2444 2445 /* 2446 * Get second TSB offset and look for 4M mapping 2447 * using 4M virtual page as the TSB index. 2448 * 2449 * Here: 2450 * %g1 = 8K TSB pointer. Don't squash it. 2451 * %g2 = tag access register (we still need it) 2452 */ 2453 srlx %g2, MMU_PAGESHIFT4M, %g3 2454 2455 /* 2456 * We patch the next set of instructions at run time; 2457 * any changes here require sfmmu_patch_ktsb changes too. 2458 */ 2459 dktsb4mbase: 2460 RUNTIME_PATCH_SETX(%g7, %g6) 2461 /* %g7 = contents of ktsb4m_base or ktsb4m_pbase */ 2462 dktsb4m: 2463 sllx %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3 2464 srlx %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3 2465 2466 /* 2467 * At this point %g3 is our index into the TSB. 2468 * We just masked off enough bits of the VA depending 2469 * on our TSB size code. 2470 */ 2471 ldda [%g7 + %g3]RUNTIME_PATCH, %g4 ! %g4 = tag, %g5 = data 2472 srlx %g2, TAG_VALO_SHIFT, %g6 ! make tag to compare 2473 cmp %g6, %g4 ! compare tag 2474 2475 dktsb4m_tsbmiss: 2476 bne,pn %xcc, dktsb4m_kpmcheck 2477 add %g7, %g3, %g3 ! %g3 = kernel second TSB ptr 2478 TT_TRACE(trace_tsbhit) 2479 /* we don't check TTE size here since we assume 4M TSB is separate */ 2480 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 2481 /* trapstat expects tte in %g5 */ 2482 retry 2483 2484 /* 2485 * So, we failed to find a valid TTE to match the faulting 2486 * address in either TSB. There are a few cases that could land 2487 * us here: 2488 * 2489 * 1) This is a kernel VA below 0x80000000.00000000. We branch 2490 * to sfmmu_tsb_miss_tt to handle the miss. 2491 * 2) We missed on a kpm VA, and we didn't find the mapping in the 2492 * 4M TSB. Let segkpm handle it. 2493 * 2494 * Note that we shouldn't land here in the case of a kpm VA when 2495 * kpm_smallpages is active -- we handled that case earlier at 2496 * dktsb4m_kpmcheck_small. 2497 * 2498 * At this point: 2499 * g1 = 8K-indexed primary TSB pointer 2500 * g2 = tag access register 2501 * g3 = 4M-indexed secondary TSB pointer 2502 */ 2503 dktsb4m_kpmcheck: 2504 cmp %g2, %g0 2505 bl,pn %xcc, sfmmu_kpm_dtsb_miss 2506 nop 2507 ba,a,pt %icc, sfmmu_tsb_miss_tt 2508 nop 2509 2510 #ifdef sun4v 2511 /* 2512 * User instruction miss w/ single TSB. 2513 * The first probe covers 8K, 64K, and 512K page sizes, 2514 * because 64K and 512K mappings are replicated off 8K 2515 * pointer. 2516 * 2517 * g1 = tsb8k pointer register 2518 * g2 = tag access register 2519 * g3 - g6 = scratch registers 2520 * g7 = TSB tag to match 2521 */ 2522 .align 64 2523 ALTENTRY(sfmmu_uitlb_fastpath) 2524 2525 PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail) 2526 /* g4 - g5 = clobbered by PROBE_1ST_ITSB */ 2527 ba,pn %xcc, sfmmu_tsb_miss_tt 2528 mov -1, %g3 2529 2530 /* 2531 * User data miss w/ single TSB. 2532 * The first probe covers 8K, 64K, and 512K page sizes, 2533 * because 64K and 512K mappings are replicated off 8K 2534 * pointer. 2535 * 2536 * g1 = tsb8k pointer register 2537 * g2 = tag access register 2538 * g3 - g6 = scratch registers 2539 * g7 = TSB tag to match 2540 */ 2541 .align 64 2542 ALTENTRY(sfmmu_udtlb_fastpath) 2543 2544 PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail) 2545 /* g4 - g5 = clobbered by PROBE_1ST_DTSB */ 2546 ba,pn %xcc, sfmmu_tsb_miss_tt 2547 mov -1, %g3 2548 2549 /* 2550 * User instruction miss w/ multiple TSBs (sun4v). 2551 * The first probe covers 8K, 64K, and 512K page sizes, 2552 * because 64K and 512K mappings are replicated off 8K 2553 * pointer. Second probe covers 4M page size only. 2554 * 2555 * Just like sfmmu_udtlb_slowpath, except: 2556 * o Uses ASI_ITLB_IN 2557 * o checks for execute permission 2558 * o No ISM prediction. 2559 * 2560 * g1 = tsb8k pointer register 2561 * g2 = tag access register 2562 * g3 - g6 = scratch registers 2563 * g7 = TSB tag to match 2564 */ 2565 .align 64 2566 ALTENTRY(sfmmu_uitlb_slowpath) 2567 2568 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2569 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2570 /* g4 - g5 = clobbered here */ 2571 2572 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2573 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2574 srlx %g2, TAG_VALO_SHIFT, %g7 2575 PROBE_2ND_ITSB(%g3, %g7) 2576 /* NOT REACHED */ 2577 2578 #else /* sun4v */ 2579 2580 /* 2581 * User instruction miss w/ multiple TSBs (sun4u). 2582 * The first probe covers 8K, 64K, and 512K page sizes, 2583 * because 64K and 512K mappings are replicated off 8K 2584 * pointer. Probe of 1st TSB has already been done prior to entry 2585 * into this routine. For the UTSB_PHYS case we probe up to 3 2586 * valid other TSBs in the following order: 2587 * 1) shared TSB for 4M-256M pages 2588 * 2) private TSB for 4M-256M pages 2589 * 3) shared TSB for 8K-512K pages 2590 * 2591 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs 2592 * 4M-256M pages. 2593 * 2594 * Just like sfmmu_udtlb_slowpath, except: 2595 * o Uses ASI_ITLB_IN 2596 * o checks for execute permission 2597 * o No ISM prediction. 2598 * 2599 * g1 = tsb8k pointer register 2600 * g2 = tag access register 2601 * g4 - g6 = scratch registers 2602 * g7 = TSB tag to match 2603 */ 2604 .align 64 2605 ALTENTRY(sfmmu_uitlb_slowpath) 2606 2607 #ifdef UTSB_PHYS 2608 2609 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2610 brlz,pt %g6, 1f 2611 nop 2612 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2613 PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail) 2614 1: 2615 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2616 brlz,pt %g3, 2f 2617 nop 2618 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2619 PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail) 2620 2: 2621 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2622 brlz,pt %g6, sfmmu_tsb_miss_tt 2623 nop 2624 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2625 PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail) 2626 ba,pn %xcc, sfmmu_tsb_miss_tt 2627 nop 2628 2629 #else /* UTSB_PHYS */ 2630 mov %g1, %g3 /* save tsb8k reg in %g3 */ 2631 GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb) 2632 PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail) 2633 mov %g2, %g6 /* GET_2ND_TSBE_PTR clobbers tagacc */ 2634 mov %g3, %g7 /* copy tsb8k reg in %g7 */ 2635 GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb) 2636 /* g1 = first TSB pointer, g3 = second TSB pointer */ 2637 srlx %g2, TAG_VALO_SHIFT, %g7 2638 PROBE_2ND_ITSB(%g3, %g7, isynth) 2639 ba,pn %xcc, sfmmu_tsb_miss_tt 2640 nop 2641 2642 #endif /* UTSB_PHYS */ 2643 #endif /* sun4v */ 2644 2645 #if defined(sun4u) && defined(UTSB_PHYS) 2646 2647 /* 2648 * We come here for ism predict DTLB_MISS case or if 2649 * if probe in first TSB failed. 2650 */ 2651 2652 .align 64 2653 ALTENTRY(sfmmu_udtlb_slowpath_noismpred) 2654 2655 /* 2656 * g1 = tsb8k pointer register 2657 * g2 = tag access register 2658 * g4 - %g6 = scratch registers 2659 * g7 = TSB tag to match 2660 */ 2661 2662 /* 2663 * ISM non-predict probe order 2664 * probe 1ST_TSB (8K index) 2665 * probe 2ND_TSB (4M index) 2666 * probe 4TH_TSB (4M index) 2667 * probe 3RD_TSB (8K index) 2668 * 2669 * We already probed first TSB in DTLB_MISS handler. 2670 */ 2671 2672 /* 2673 * Private 2ND TSB 4M-256 pages 2674 */ 2675 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2676 brlz,pt %g3, 1f 2677 nop 2678 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2679 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2680 2681 /* 2682 * Shared Context 4TH TSB 4M-256 pages 2683 */ 2684 1: 2685 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2686 brlz,pt %g6, 2f 2687 nop 2688 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2689 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail) 2690 2691 /* 2692 * Shared Context 3RD TSB 8K-512K pages 2693 */ 2694 2: 2695 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2696 brlz,pt %g6, sfmmu_tsb_miss_tt 2697 nop 2698 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2699 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail) 2700 ba,pn %xcc, sfmmu_tsb_miss_tt 2701 nop 2702 2703 .align 64 2704 ALTENTRY(sfmmu_udtlb_slowpath_ismpred) 2705 2706 /* 2707 * g1 = tsb8k pointer register 2708 * g2 = tag access register 2709 * g4 - g6 = scratch registers 2710 * g7 = TSB tag to match 2711 */ 2712 2713 /* 2714 * ISM predict probe order 2715 * probe 4TH_TSB (4M index) 2716 * probe 2ND_TSB (4M index) 2717 * probe 1ST_TSB (8K index) 2718 * probe 3RD_TSB (8K index) 2719 2720 /* 2721 * Shared Context 4TH TSB 4M-256 pages 2722 */ 2723 GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6) 2724 brlz,pt %g6, 4f 2725 nop 2726 GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5) 2727 PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2) 2728 2729 /* 2730 * Private 2ND TSB 4M-256 pages 2731 */ 2732 4: 2733 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) 2734 brlz,pt %g3, 5f 2735 nop 2736 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2737 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2) 2738 2739 5: 2740 PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2) 2741 2742 /* 2743 * Shared Context 3RD TSB 8K-512K pages 2744 */ 2745 GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6) 2746 brlz,pt %g6, 6f 2747 nop 2748 GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5) 2749 PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2) 2750 6: 2751 ba,pn %xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */ 2752 nop 2753 2754 #else /* sun4u && UTSB_PHYS */ 2755 2756 .align 64 2757 ALTENTRY(sfmmu_udtlb_slowpath) 2758 2759 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 : ISM predicted */ 2760 brgz,pn %g6, udtlb_miss_probesecond /* check for ISM */ 2761 mov %g1, %g3 2762 2763 udtlb_miss_probefirst: 2764 /* 2765 * g1 = 8K TSB pointer register 2766 * g2 = tag access register 2767 * g3 = (potentially) second TSB entry ptr 2768 * g6 = ism pred. 2769 * g7 = vpg_4m 2770 */ 2771 #ifdef sun4v 2772 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 2773 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2774 2775 /* 2776 * Here: 2777 * g1 = first TSB pointer 2778 * g2 = tag access reg 2779 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2780 */ 2781 brgz,pn %g6, sfmmu_tsb_miss_tt 2782 nop 2783 #else /* sun4v */ 2784 mov %g1, %g4 2785 GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb) 2786 PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail) 2787 2788 /* 2789 * Here: 2790 * g1 = first TSB pointer 2791 * g2 = tag access reg 2792 * g3 = second TSB ptr IFF ISM pred. (else don't care) 2793 */ 2794 brgz,pn %g6, sfmmu_tsb_miss_tt 2795 nop 2796 ldxa [%g0]ASI_DMMU_TSB_8K, %g3 2797 /* fall through in 8K->4M probe order */ 2798 #endif /* sun4v */ 2799 2800 udtlb_miss_probesecond: 2801 /* 2802 * Look in the second TSB for the TTE 2803 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred. 2804 * g2 = tag access reg 2805 * g3 = 8K TSB pointer register 2806 * g6 = ism pred. 2807 * g7 = vpg_4m 2808 */ 2809 #ifdef sun4v 2810 /* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */ 2811 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 2812 /* %g2 is okay, no need to reload, %g3 = second tsbe ptr */ 2813 #else /* sun4v */ 2814 mov %g3, %g7 2815 GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb) 2816 /* %g2 clobbered, %g3 =second tsbe ptr */ 2817 mov MMU_TAG_ACCESS, %g2 2818 ldxa [%g2]ASI_DMMU, %g2 2819 #endif /* sun4v */ 2820 2821 srlx %g2, TAG_VALO_SHIFT, %g7 2822 PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail) 2823 /* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */ 2824 brgz,pn %g6, udtlb_miss_probefirst 2825 nop 2826 2827 /* fall through to sfmmu_tsb_miss_tt */ 2828 #endif /* sun4u && UTSB_PHYS */ 2829 2830 2831 ALTENTRY(sfmmu_tsb_miss_tt) 2832 TT_TRACE(trace_tsbmiss) 2833 /* 2834 * We get here if there is a TSB miss OR a write protect trap. 2835 * 2836 * g1 = First TSB entry pointer 2837 * g2 = tag access register 2838 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB 2839 * g4 - g7 = scratch registers 2840 */ 2841 2842 ALTENTRY(sfmmu_tsb_miss) 2843 2844 /* 2845 * If trapstat is running, we need to shift the %tpc and %tnpc to 2846 * point to trapstat's TSB miss return code (note that trapstat 2847 * itself will patch the correct offset to add). 2848 */ 2849 rdpr %tl, %g7 2850 cmp %g7, 1 2851 ble,pt %xcc, 0f 2852 sethi %hi(KERNELBASE), %g6 2853 rdpr %tpc, %g7 2854 or %g6, %lo(KERNELBASE), %g6 2855 cmp %g7, %g6 2856 bgeu,pt %xcc, 0f 2857 /* delay slot safe */ 2858 2859 ALTENTRY(tsbmiss_trapstat_patch_point) 2860 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 2861 wrpr %g7, %tpc 2862 add %g7, 4, %g7 2863 wrpr %g7, %tnpc 2864 0: 2865 CPU_TSBMISS_AREA(%g6, %g7) 2866 stn %g1, [%g6 + TSBMISS_TSBPTR] /* save 1ST tsb pointer */ 2867 stn %g3, [%g6 + TSBMISS_TSBPTR4M] /* save 2ND tsb pointer */ 2868 2869 sllx %g2, TAGACC_CTX_LSHIFT, %g3 2870 brz,a,pn %g3, 1f /* skip ahead if kernel */ 2871 ldn [%g6 + TSBMISS_KHATID], %g7 2872 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctxnum */ 2873 ldn [%g6 + TSBMISS_UHATID], %g7 /* g7 = hatid */ 2874 2875 HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5) 2876 2877 cmp %g3, INVALID_CONTEXT 2878 be,pn %icc, tsb_tl0_noctxt /* no ctx miss exception */ 2879 stn %g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)] 2880 2881 #if defined(sun4v) || defined(UTSB_PHYS) 2882 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 /* clear ctx1 flag set from */ 2883 andn %g7, HAT_CHKCTX1_FLAG, %g7 /* the previous tsb miss */ 2884 stub %g7, [%g6 + TSBMISS_URTTEFLAGS] 2885 #endif /* sun4v || UTSB_PHYS */ 2886 2887 ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism) 2888 /* 2889 * The miss wasn't in an ISM segment. 2890 * 2891 * %g1 %g3, %g4, %g5, %g7 all clobbered 2892 * %g2 = (pseudo) tag access 2893 */ 2894 2895 ba,pt %icc, 2f 2896 ldn [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7 2897 2898 1: 2899 HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5) 2900 /* 2901 * 8K and 64K hash. 2902 */ 2903 2: 2904 2905 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2906 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte, 2907 sfmmu_suspend_tl, tsb_512K) 2908 /* NOT REACHED */ 2909 2910 tsb_512K: 2911 sllx %g2, TAGACC_CTX_LSHIFT, %g5 2912 brz,pn %g5, 3f 2913 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2914 and %g4, HAT_512K_FLAG, %g5 2915 2916 /* 2917 * Note that there is a small window here where we may have 2918 * a 512k page in the hash list but have not set the HAT_512K_FLAG 2919 * flag yet, so we will skip searching the 512k hash list. 2920 * In this case we will end up in pagefault which will find 2921 * the mapping and return. So, in this instance we will end up 2922 * spending a bit more time resolving this TSB miss, but it can 2923 * only happen once per process and even then, the chances of that 2924 * are very small, so it's not worth the extra overhead it would 2925 * take to close this window. 2926 */ 2927 brz,pn %g5, tsb_4M 2928 nop 2929 3: 2930 /* 2931 * 512K hash 2932 */ 2933 2934 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2935 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte, 2936 sfmmu_suspend_tl, tsb_4M) 2937 /* NOT REACHED */ 2938 2939 tsb_4M: 2940 sllx %g2, TAGACC_CTX_LSHIFT, %g5 2941 brz,pn %g5, 4f 2942 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2943 and %g4, HAT_4M_FLAG, %g5 2944 brz,pn %g5, tsb_32M 2945 nop 2946 4: 2947 /* 2948 * 4M hash 2949 */ 2950 2951 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2952 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte, 2953 sfmmu_suspend_tl, tsb_32M) 2954 /* NOT REACHED */ 2955 2956 tsb_32M: 2957 sllx %g2, TAGACC_CTX_LSHIFT, %g5 2958 #ifdef sun4v 2959 brz,pn %g5, 6f 2960 #else 2961 brz,pn %g5, tsb_pagefault 2962 #endif 2963 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2964 and %g4, HAT_32M_FLAG, %g5 2965 brz,pn %g5, tsb_256M 2966 nop 2967 5: 2968 /* 2969 * 32M hash 2970 */ 2971 2972 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2973 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte, 2974 sfmmu_suspend_tl, tsb_256M) 2975 /* NOT REACHED */ 2976 2977 #if defined(sun4u) && !defined(UTSB_PHYS) 2978 #define tsb_shme tsb_pagefault 2979 #endif 2980 tsb_256M: 2981 ldub [%g6 + TSBMISS_UTTEFLAGS], %g4 2982 and %g4, HAT_256M_FLAG, %g5 2983 brz,pn %g5, tsb_shme 2984 nop 2985 6: 2986 /* 2987 * 256M hash 2988 */ 2989 2990 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 2991 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte, 2992 sfmmu_suspend_tl, tsb_shme) 2993 /* NOT REACHED */ 2994 2995 tsb_checktte: 2996 /* 2997 * g1 = hblk_misc 2998 * g2 = tagacc 2999 * g3 = tte 3000 * g4 = tte pa 3001 * g6 = tsbmiss area 3002 * g7 = hatid 3003 */ 3004 brlz,a,pt %g3, tsb_validtte 3005 rdpr %tt, %g7 3006 3007 #if defined(sun4u) && !defined(UTSB_PHYS) 3008 #undef tsb_shme 3009 ba tsb_pagefault 3010 nop 3011 #else /* sun4u && !UTSB_PHYS */ 3012 3013 tsb_shme: 3014 /* 3015 * g2 = tagacc 3016 * g6 = tsbmiss area 3017 */ 3018 sllx %g2, TAGACC_CTX_LSHIFT, %g5 3019 brz,pn %g5, tsb_pagefault 3020 nop 3021 ldx [%g6 + TSBMISS_SHARED_UHATID], %g7 /* g7 = srdp */ 3022 brz,pn %g7, tsb_pagefault 3023 nop 3024 3025 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3026 MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte, 3027 sfmmu_suspend_tl, tsb_shme_512K) 3028 /* NOT REACHED */ 3029 3030 tsb_shme_512K: 3031 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3032 and %g4, HAT_512K_FLAG, %g5 3033 brz,pn %g5, tsb_shme_4M 3034 nop 3035 3036 /* 3037 * 512K hash 3038 */ 3039 3040 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3041 MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte, 3042 sfmmu_suspend_tl, tsb_shme_4M) 3043 /* NOT REACHED */ 3044 3045 tsb_shme_4M: 3046 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3047 and %g4, HAT_4M_FLAG, %g5 3048 brz,pn %g5, tsb_shme_32M 3049 nop 3050 4: 3051 /* 3052 * 4M hash 3053 */ 3054 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3055 MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte, 3056 sfmmu_suspend_tl, tsb_shme_32M) 3057 /* NOT REACHED */ 3058 3059 tsb_shme_32M: 3060 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3061 and %g4, HAT_32M_FLAG, %g5 3062 brz,pn %g5, tsb_shme_256M 3063 nop 3064 3065 /* 3066 * 32M hash 3067 */ 3068 3069 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3070 MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte, 3071 sfmmu_suspend_tl, tsb_shme_256M) 3072 /* NOT REACHED */ 3073 3074 tsb_shme_256M: 3075 ldub [%g6 + TSBMISS_URTTEFLAGS], %g4 3076 and %g4, HAT_256M_FLAG, %g5 3077 brz,pn %g5, tsb_pagefault 3078 nop 3079 3080 /* 3081 * 256M hash 3082 */ 3083 3084 GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1, 3085 MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte, 3086 sfmmu_suspend_tl, tsb_pagefault) 3087 /* NOT REACHED */ 3088 3089 tsb_shme_checktte: 3090 3091 brgez,pn %g3, tsb_pagefault 3092 rdpr %tt, %g7 3093 /* 3094 * g1 = ctx1 flag 3095 * g3 = tte 3096 * g4 = tte pa 3097 * g6 = tsbmiss area 3098 * g7 = tt 3099 */ 3100 3101 brz,pt %g1, tsb_validtte 3102 nop 3103 ldub [%g6 + TSBMISS_URTTEFLAGS], %g1 3104 or %g1, HAT_CHKCTX1_FLAG, %g1 3105 stub %g1, [%g6 + TSBMISS_URTTEFLAGS] 3106 3107 SAVE_CTX1(%g7, %g2, %g1, tsb_shmel) 3108 #endif /* sun4u && !UTSB_PHYS */ 3109 3110 tsb_validtte: 3111 /* 3112 * g3 = tte 3113 * g4 = tte pa 3114 * g6 = tsbmiss area 3115 * g7 = tt 3116 */ 3117 3118 /* 3119 * Set ref/mod bits if this is a prot trap. Usually, it isn't. 3120 */ 3121 cmp %g7, FAST_PROT_TT 3122 bne,pt %icc, 4f 3123 nop 3124 3125 TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod, 3126 tsb_protfault) 3127 3128 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 3129 #ifdef sun4v 3130 MMU_FAULT_STATUS_AREA(%g7) 3131 ldx [%g7 + MMFSA_D_ADDR], %g5 /* load fault addr for later */ 3132 #else /* sun4v */ 3133 mov MMU_TAG_ACCESS, %g5 3134 ldxa [%g5]ASI_DMMU, %g5 3135 #endif /* sun4v */ 3136 ba,pt %xcc, tsb_update_tl1 3137 nop 3138 4: 3139 /* 3140 * If ITLB miss check exec bit. 3141 * If not set treat as invalid TTE. 3142 */ 3143 cmp %g7, T_INSTR_MMU_MISS 3144 be,pn %icc, 5f 3145 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3146 cmp %g7, FAST_IMMU_MISS_TT 3147 bne,pt %icc, 3f 3148 andcc %g3, TTE_EXECPRM_INT, %g0 /* check execute bit is set */ 3149 5: 3150 bz,pn %icc, tsb_protfault 3151 nop 3152 3153 3: 3154 /* 3155 * Set reference bit if not already set 3156 */ 3157 TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref) 3158 3159 /* 3160 * Now, load into TSB/TLB. At this point: 3161 * g3 = tte 3162 * g4 = patte 3163 * g6 = tsbmiss area 3164 */ 3165 rdpr %tt, %g7 3166 #ifdef sun4v 3167 MMU_FAULT_STATUS_AREA(%g2) 3168 cmp %g7, T_INSTR_MMU_MISS 3169 be,a,pt %icc, 9f 3170 nop 3171 cmp %g7, FAST_IMMU_MISS_TT 3172 be,a,pt %icc, 9f 3173 nop 3174 add %g2, MMFSA_D_, %g2 3175 9: 3176 ldx [%g2 + MMFSA_CTX_], %g7 3177 sllx %g7, TTARGET_CTX_SHIFT, %g7 3178 ldx [%g2 + MMFSA_ADDR_], %g2 3179 mov %g2, %g5 ! load the fault addr for later use 3180 srlx %g2, TTARGET_VA_SHIFT, %g2 3181 or %g2, %g7, %g2 3182 #else /* sun4v */ 3183 mov MMU_TAG_ACCESS, %g5 3184 cmp %g7, FAST_IMMU_MISS_TT 3185 be,a,pt %icc, 9f 3186 ldxa [%g0]ASI_IMMU, %g2 3187 ldxa [%g0]ASI_DMMU, %g2 3188 ba,pt %icc, tsb_update_tl1 3189 ldxa [%g5]ASI_DMMU, %g5 3190 9: 3191 ldxa [%g5]ASI_IMMU, %g5 3192 #endif /* sun4v */ 3193 3194 tsb_update_tl1: 3195 srlx %g2, TTARGET_CTX_SHIFT, %g7 3196 brz,pn %g7, tsb_kernel 3197 #ifdef sun4v 3198 and %g3, TTE_SZ_BITS, %g7 ! assumes TTE_SZ_SHFT is 0 3199 #else /* sun4v */ 3200 srlx %g3, TTE_SZ_SHFT, %g7 3201 #endif /* sun4v */ 3202 3203 tsb_user: 3204 #ifdef sun4v 3205 cmp %g7, TTE4M 3206 bge,pn %icc, tsb_user4m 3207 nop 3208 #else /* sun4v */ 3209 cmp %g7, TTESZ_VALID | TTE4M 3210 be,pn %icc, tsb_user4m 3211 srlx %g3, TTE_SZ2_SHFT, %g7 3212 andcc %g7, TTE_SZ2_BITS, %g7 ! check 32/256MB 3213 #ifdef ITLB_32M_256M_SUPPORT 3214 bnz,pn %icc, tsb_user4m 3215 nop 3216 #else /* ITLB_32M_256M_SUPPORT */ 3217 bnz,a,pn %icc, tsb_user_pn_synth 3218 nop 3219 #endif /* ITLB_32M_256M_SUPPORT */ 3220 #endif /* sun4v */ 3221 3222 tsb_user8k: 3223 #if defined(sun4v) || defined(UTSB_PHYS) 3224 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3225 and %g7, HAT_CHKCTX1_FLAG, %g1 3226 brz,a,pn %g1, 1f 3227 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3228 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1) 3229 brlz,a,pn %g1, ptl1_panic ! if no shared 3RD tsb 3230 mov PTL1_NO_SCDTSB8K, %g1 ! panic 3231 GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7) 3232 1: 3233 #else /* defined(sun4v) || defined(UTSB_PHYS) */ 3234 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 1ST TSB ptr 3235 #endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3236 3237 #ifndef UTSB_PHYS 3238 mov ASI_N, %g7 ! user TSBs accessed by VA 3239 mov %g7, %asi 3240 #endif /* !UTSB_PHYS */ 3241 3242 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3) 3243 3244 rdpr %tt, %g5 3245 #ifdef sun4v 3246 cmp %g5, T_INSTR_MMU_MISS 3247 be,a,pn %xcc, 9f 3248 mov %g3, %g5 3249 #endif /* sun4v */ 3250 cmp %g5, FAST_IMMU_MISS_TT 3251 be,pn %xcc, 9f 3252 mov %g3, %g5 3253 3254 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3255 ! trapstat wants TTE in %g5 3256 retry 3257 9: 3258 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3259 ! trapstat wants TTE in %g5 3260 retry 3261 3262 tsb_user4m: 3263 #if defined(sun4v) || defined(UTSB_PHYS) 3264 ldub [%g6 + TSBMISS_URTTEFLAGS], %g7 3265 and %g7, HAT_CHKCTX1_FLAG, %g1 3266 brz,a,pn %g1, 4f 3267 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3268 GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr 3269 brlz,a,pn %g1, 5f ! if no shared 4TH TSB 3270 nop 3271 GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7) 3272 3273 #else /* defined(sun4v) || defined(UTSB_PHYS) */ 3274 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 2ND TSB ptr 3275 #endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3276 4: 3277 brlz,pn %g1, 5f /* Check to see if we have 2nd TSB programmed */ 3278 nop 3279 3280 #ifndef UTSB_PHYS 3281 mov ASI_N, %g7 ! user TSBs accessed by VA 3282 mov %g7, %asi 3283 #endif /* UTSB_PHYS */ 3284 3285 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4) 3286 3287 5: 3288 rdpr %tt, %g5 3289 #ifdef sun4v 3290 cmp %g5, T_INSTR_MMU_MISS 3291 be,a,pn %xcc, 9f 3292 mov %g3, %g5 3293 #endif /* sun4v */ 3294 cmp %g5, FAST_IMMU_MISS_TT 3295 be,pn %xcc, 9f 3296 mov %g3, %g5 3297 3298 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3299 ! trapstat wants TTE in %g5 3300 retry 3301 9: 3302 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3303 ! trapstat wants TTE in %g5 3304 retry 3305 3306 #if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT) 3307 /* 3308 * Panther ITLB synthesis. 3309 * The Panther 32M and 256M ITLB code simulates these two large page 3310 * sizes with 4M pages, to provide support for programs, for example 3311 * Java, that may copy instructions into a 32M or 256M data page and 3312 * then execute them. The code below generates the 4M pfn bits and 3313 * saves them in the modified 32M/256M ttes in the TSB. If the tte is 3314 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits 3315 * are ignored by the hardware. 3316 * 3317 * Now, load into TSB/TLB. At this point: 3318 * g2 = tagtarget 3319 * g3 = tte 3320 * g4 = patte 3321 * g5 = tt 3322 * g6 = tsbmiss area 3323 */ 3324 tsb_user_pn_synth: 3325 rdpr %tt, %g5 3326 cmp %g5, FAST_IMMU_MISS_TT 3327 be,pt %xcc, tsb_user_itlb_synth /* ITLB miss */ 3328 andcc %g3, TTE_EXECPRM_INT, %g0 /* is execprm bit set */ 3329 bz,pn %icc, 4b /* if not, been here before */ 3330 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = tsbp */ 3331 brlz,a,pn %g1, 5f /* no 2nd tsb */ 3332 mov %g3, %g5 3333 3334 mov MMU_TAG_ACCESS, %g7 3335 ldxa [%g7]ASI_DMMU, %g6 /* get tag access va */ 3336 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1) /* make 4M pfn offset */ 3337 3338 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3339 mov %g7, %asi 3340 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */ 3341 5: 3342 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3343 retry 3344 3345 tsb_user_itlb_synth: 3346 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 /* g1 = 2ND TSB */ 3347 3348 mov MMU_TAG_ACCESS, %g7 3349 ldxa [%g7]ASI_IMMU, %g6 /* get tag access va */ 3350 GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2) /* make 4M pfn offset */ 3351 brlz,a,pn %g1, 7f /* Check to see if we have 2nd TSB programmed */ 3352 or %g5, %g3, %g5 /* add 4M bits to TTE */ 3353 3354 mov ASI_N, %g7 /* user TSBs always accessed by VA */ 3355 mov %g7, %asi 3356 TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */ 3357 7: 3358 SET_TTE4M_PN(%g5, %g7) /* add TTE4M pagesize to TTE */ 3359 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3360 retry 3361 #endif /* sun4v && ITLB_32M_256M_SUPPORT */ 3362 3363 tsb_kernel: 3364 rdpr %tt, %g5 3365 #ifdef sun4v 3366 cmp %g7, TTE4M 3367 bge,pn %icc, 5f 3368 #else 3369 cmp %g7, TTESZ_VALID | TTE4M ! no 32M or 256M support 3370 be,pn %icc, 5f 3371 #endif /* sun4v */ 3372 nop 3373 ldn [%g6 + TSBMISS_TSBPTR], %g1 ! g1 = 8K TSB ptr 3374 ba,pt %xcc, 6f 3375 nop 3376 5: 3377 ldn [%g6 + TSBMISS_TSBPTR4M], %g1 ! g1 = 4M TSB ptr 3378 brlz,pn %g1, 3f /* skip programming if 4M TSB ptr is -1 */ 3379 nop 3380 6: 3381 #ifndef sun4v 3382 tsb_kernel_patch_asi: 3383 or %g0, RUNTIME_PATCH, %g6 3384 mov %g6, %asi ! XXX avoid writing to %asi !! 3385 #endif 3386 TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7) 3387 3: 3388 #ifdef sun4v 3389 cmp %g5, T_INSTR_MMU_MISS 3390 be,a,pn %icc, 1f 3391 mov %g3, %g5 ! trapstat wants TTE in %g5 3392 #endif /* sun4v */ 3393 cmp %g5, FAST_IMMU_MISS_TT 3394 be,pn %icc, 1f 3395 mov %g3, %g5 ! trapstat wants TTE in %g5 3396 DTLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3397 ! trapstat wants TTE in %g5 3398 retry 3399 1: 3400 ITLB_STUFF(%g5, %g1, %g2, %g3, %g4) 3401 ! trapstat wants TTE in %g5 3402 retry 3403 3404 tsb_ism: 3405 /* 3406 * This is an ISM [i|d]tlb miss. We optimize for largest 3407 * page size down to smallest. 3408 * 3409 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access 3410 * register 3411 * g3 = ismmap->ism_seg 3412 * g4 = physical address of ismmap->ism_sfmmu 3413 * g6 = tsbmiss area 3414 */ 3415 ldna [%g4]ASI_MEM, %g7 /* g7 = ism hatid */ 3416 brz,a,pn %g7, ptl1_panic /* if zero jmp ahead */ 3417 mov PTL1_BAD_ISM, %g1 3418 /* g5 = pa of imap_vb_shift */ 3419 sub %g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5 3420 lduba [%g5]ASI_MEM, %g4 /* g4 = imap_vb_shift */ 3421 srlx %g3, %g4, %g3 /* clr size field */ 3422 set TAGACC_CTX_MASK, %g1 /* mask off ctx number */ 3423 sllx %g3, %g4, %g3 /* g3 = ism vbase */ 3424 and %g2, %g1, %g4 /* g4 = ctx number */ 3425 andn %g2, %g1, %g1 /* g1 = tlb miss vaddr */ 3426 sub %g1, %g3, %g2 /* g2 = offset in ISM seg */ 3427 or %g2, %g4, %g2 /* g2 = (pseudo-)tagacc */ 3428 sub %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5 3429 lduha [%g5]ASI_MEM, %g4 /* g5 = pa of imap_hatflags */ 3430 #if defined(sun4v) || defined(UTSB_PHYS) 3431 and %g4, HAT_CTX1_FLAG, %g5 /* g5 = imap_hatflags */ 3432 brz,pt %g5, tsb_chk4M_ism 3433 nop 3434 ldub [%g6 + TSBMISS_URTTEFLAGS], %g5 3435 or %g5, HAT_CHKCTX1_FLAG, %g5 3436 stub %g5, [%g6 + TSBMISS_URTTEFLAGS] 3437 rdpr %tt, %g5 3438 SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl) 3439 #endif /* defined(sun4v) || defined(UTSB_PHYS) */ 3440 3441 /* 3442 * ISM pages are always locked down. 3443 * If we can't find the tte then pagefault 3444 * and let the spt segment driver resolve it. 3445 * 3446 * g2 = tagacc w/ISM vaddr (offset in ISM seg) 3447 * g4 = imap_hatflags 3448 * g6 = tsb miss area 3449 * g7 = ISM hatid 3450 */ 3451 3452 tsb_chk4M_ism: 3453 and %g4, HAT_4M_FLAG, %g5 /* g4 = imap_hatflags */ 3454 brnz,pt %g5, tsb_ism_4M /* branch if 4M pages */ 3455 nop 3456 3457 tsb_ism_32M: 3458 and %g4, HAT_32M_FLAG, %g5 /* check default 32M next */ 3459 brz,pn %g5, tsb_ism_256M 3460 nop 3461 3462 /* 3463 * 32M hash. 3464 */ 3465 3466 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M, 3467 TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl, 3468 tsb_ism_4M) 3469 /* NOT REACHED */ 3470 3471 tsb_ism_32M_found: 3472 brlz,a,pt %g3, tsb_validtte 3473 rdpr %tt, %g7 3474 ba,pt %xcc, tsb_ism_4M 3475 nop 3476 3477 tsb_ism_256M: 3478 and %g4, HAT_256M_FLAG, %g5 /* 256M is last resort */ 3479 brz,a,pn %g5, ptl1_panic 3480 mov PTL1_BAD_ISM, %g1 3481 3482 /* 3483 * 256M hash. 3484 */ 3485 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M, 3486 TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl, 3487 tsb_ism_4M) 3488 3489 tsb_ism_256M_found: 3490 brlz,a,pt %g3, tsb_validtte 3491 rdpr %tt, %g7 3492 3493 tsb_ism_4M: 3494 /* 3495 * 4M hash. 3496 */ 3497 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M, 3498 TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl, 3499 tsb_ism_8K) 3500 /* NOT REACHED */ 3501 3502 tsb_ism_4M_found: 3503 brlz,a,pt %g3, tsb_validtte 3504 rdpr %tt, %g7 3505 3506 tsb_ism_8K: 3507 /* 3508 * 8K and 64K hash. 3509 */ 3510 3511 GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K, 3512 TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl, 3513 tsb_pagefault) 3514 /* NOT REACHED */ 3515 3516 tsb_ism_8K_found: 3517 brlz,a,pt %g3, tsb_validtte 3518 rdpr %tt, %g7 3519 3520 tsb_pagefault: 3521 rdpr %tt, %g7 3522 cmp %g7, FAST_PROT_TT 3523 be,a,pn %icc, tsb_protfault 3524 wrpr %g0, FAST_DMMU_MISS_TT, %tt 3525 3526 tsb_protfault: 3527 /* 3528 * we get here if we couldn't find a valid tte in the hash. 3529 * 3530 * If user and we are at tl>1 we go to window handling code. 3531 * 3532 * If kernel and the fault is on the same page as our stack 3533 * pointer, then we know the stack is bad and the trap handler 3534 * will fail, so we call ptl1_panic with PTL1_BAD_STACK. 3535 * 3536 * If this is a kernel trap and tl>1, panic. 3537 * 3538 * Otherwise we call pagefault. 3539 */ 3540 cmp %g7, FAST_IMMU_MISS_TT 3541 #ifdef sun4v 3542 MMU_FAULT_STATUS_AREA(%g4) 3543 ldx [%g4 + MMFSA_I_CTX], %g5 3544 ldx [%g4 + MMFSA_D_CTX], %g4 3545 move %icc, %g5, %g4 3546 cmp %g7, T_INSTR_MMU_MISS 3547 move %icc, %g5, %g4 3548 #else 3549 mov MMU_TAG_ACCESS, %g4 3550 ldxa [%g4]ASI_DMMU, %g2 3551 ldxa [%g4]ASI_IMMU, %g5 3552 move %icc, %g5, %g2 3553 cmp %g7, T_INSTR_MMU_MISS 3554 move %icc, %g5, %g2 3555 sllx %g2, TAGACC_CTX_LSHIFT, %g4 3556 #endif /* sun4v */ 3557 brnz,pn %g4, 3f /* skip if not kernel */ 3558 rdpr %tl, %g5 3559 3560 add %sp, STACK_BIAS, %g3 3561 srlx %g3, MMU_PAGESHIFT, %g3 3562 srlx %g2, MMU_PAGESHIFT, %g4 3563 cmp %g3, %g4 3564 be,a,pn %icc, ptl1_panic /* panic if bad %sp */ 3565 mov PTL1_BAD_STACK, %g1 3566 3567 cmp %g5, 1 3568 ble,pt %icc, 2f 3569 nop 3570 TSTAT_CHECK_TL1(2f, %g1, %g2) 3571 rdpr %tt, %g2 3572 cmp %g2, FAST_PROT_TT 3573 mov PTL1_BAD_KPROT_FAULT, %g1 3574 movne %icc, PTL1_BAD_KMISS, %g1 3575 ba,pt %icc, ptl1_panic 3576 nop 3577 3578 2: 3579 /* 3580 * We are taking a pagefault in the kernel on a kernel address. If 3581 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually 3582 * want to call sfmmu_pagefault -- we will instead note that a fault 3583 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done" 3584 * (instead of a "retry"). This will step over the faulting 3585 * instruction. 3586 */ 3587 CPU_INDEX(%g1, %g2) 3588 set cpu_core, %g2 3589 sllx %g1, CPU_CORE_SHIFT, %g1 3590 add %g1, %g2, %g1 3591 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3592 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3593 bz sfmmu_pagefault 3594 or %g2, CPU_DTRACE_BADADDR, %g2 3595 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3596 GET_MMU_D_ADDR(%g3, %g4) 3597 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3598 done 3599 3600 3: 3601 cmp %g5, 1 3602 ble,pt %icc, 4f 3603 nop 3604 TSTAT_CHECK_TL1(4f, %g1, %g2) 3605 ba,pt %icc, sfmmu_window_trap 3606 nop 3607 3608 4: 3609 /* 3610 * We are taking a pagefault on a non-kernel address. If we are in 3611 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags 3612 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above. 3613 */ 3614 CPU_INDEX(%g1, %g2) 3615 set cpu_core, %g2 3616 sllx %g1, CPU_CORE_SHIFT, %g1 3617 add %g1, %g2, %g1 3618 lduh [%g1 + CPUC_DTRACE_FLAGS], %g2 3619 andcc %g2, CPU_DTRACE_NOFAULT, %g0 3620 bz sfmmu_mmu_trap 3621 or %g2, CPU_DTRACE_BADADDR, %g2 3622 stuh %g2, [%g1 + CPUC_DTRACE_FLAGS] 3623 GET_MMU_D_ADDR(%g3, %g4) 3624 stx %g3, [%g1 + CPUC_DTRACE_ILLVAL] 3625 3626 /* 3627 * Be sure that we're actually taking this miss from the kernel -- 3628 * otherwise we have managed to return to user-level with 3629 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3630 */ 3631 rdpr %tstate, %g2 3632 btst TSTATE_PRIV, %g2 3633 bz,a ptl1_panic 3634 mov PTL1_BAD_DTRACE_FLAGS, %g1 3635 done 3636 3637 ALTENTRY(tsb_tl0_noctxt) 3638 /* 3639 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set; 3640 * if it is, indicated that we have faulted and issue a done. 3641 */ 3642 CPU_INDEX(%g5, %g6) 3643 set cpu_core, %g6 3644 sllx %g5, CPU_CORE_SHIFT, %g5 3645 add %g5, %g6, %g5 3646 lduh [%g5 + CPUC_DTRACE_FLAGS], %g6 3647 andcc %g6, CPU_DTRACE_NOFAULT, %g0 3648 bz 1f 3649 or %g6, CPU_DTRACE_BADADDR, %g6 3650 stuh %g6, [%g5 + CPUC_DTRACE_FLAGS] 3651 GET_MMU_D_ADDR(%g3, %g4) 3652 stx %g3, [%g5 + CPUC_DTRACE_ILLVAL] 3653 3654 /* 3655 * Be sure that we're actually taking this miss from the kernel -- 3656 * otherwise we have managed to return to user-level with 3657 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags. 3658 */ 3659 rdpr %tstate, %g5 3660 btst TSTATE_PRIV, %g5 3661 bz,a ptl1_panic 3662 mov PTL1_BAD_DTRACE_FLAGS, %g1 3663 TSTAT_CHECK_TL1(2f, %g1, %g2); 3664 2: 3665 done 3666 3667 1: 3668 rdpr %tt, %g5 3669 cmp %g5, FAST_IMMU_MISS_TT 3670 #ifdef sun4v 3671 MMU_FAULT_STATUS_AREA(%g2) 3672 be,a,pt %icc, 2f 3673 ldx [%g2 + MMFSA_I_CTX], %g3 3674 cmp %g5, T_INSTR_MMU_MISS 3675 be,a,pt %icc, 2f 3676 ldx [%g2 + MMFSA_I_CTX], %g3 3677 ldx [%g2 + MMFSA_D_CTX], %g3 3678 2: 3679 #else 3680 mov MMU_TAG_ACCESS, %g2 3681 be,a,pt %icc, 2f 3682 ldxa [%g2]ASI_IMMU, %g3 3683 ldxa [%g2]ASI_DMMU, %g3 3684 2: sllx %g3, TAGACC_CTX_LSHIFT, %g3 3685 #endif /* sun4v */ 3686 brz,a,pn %g3, ptl1_panic ! panic if called for kernel 3687 mov PTL1_BAD_CTX_STEAL, %g1 ! since kernel ctx was stolen 3688 rdpr %tl, %g5 3689 cmp %g5, 1 3690 ble,pt %icc, sfmmu_mmu_trap 3691 nop 3692 TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2) 3693 ba,pt %icc, sfmmu_window_trap 3694 nop 3695 SET_SIZE(sfmmu_tsb_miss) 3696 3697 ENTRY_NP(sfmmu_vatopfn) 3698 /* 3699 * disable interrupts 3700 */ 3701 rdpr %pstate, %o3 3702 #ifdef DEBUG 3703 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) 3704 #endif 3705 /* 3706 * disable interrupts to protect the TSBMISS area 3707 */ 3708 andn %o3, PSTATE_IE, %o5 3709 wrpr %o5, 0, %pstate 3710 3711 /* 3712 * o0 = vaddr 3713 * o1 = sfmmup 3714 * o2 = ttep 3715 */ 3716 CPU_TSBMISS_AREA(%g1, %o5) 3717 ldn [%g1 + TSBMISS_KHATID], %o4 3718 cmp %o4, %o1 3719 bne,pn %ncc, vatopfn_nokernel 3720 mov TTE64K, %g5 /* g5 = rehash # */ 3721 mov %g1,%o5 /* o5 = tsbmiss_area */ 3722 /* 3723 * o0 = vaddr 3724 * o1 & o4 = hatid 3725 * o2 = ttep 3726 * o5 = tsbmiss area 3727 */ 3728 mov HBLK_RANGE_SHIFT, %g6 3729 1: 3730 3731 /* 3732 * o0 = vaddr 3733 * o1 = sfmmup 3734 * o2 = ttep 3735 * o3 = old %pstate 3736 * o4 = hatid 3737 * o5 = tsbmiss 3738 * g5 = rehash # 3739 * g6 = hmeshift 3740 * 3741 * The first arg to GET_TTE is actually tagaccess register 3742 * not just vaddr. Since this call is for kernel we need to clear 3743 * any lower vaddr bits that would be interpreted as ctx bits. 3744 */ 3745 set TAGACC_CTX_MASK, %g1 3746 andn %o0, %g1, %o0 3747 GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3, 3748 vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk) 3749 3750 kvtop_hblk_found: 3751 /* 3752 * o0 = vaddr 3753 * o1 = sfmmup 3754 * o2 = ttep 3755 * g1 = tte 3756 * g2 = tte pa 3757 * g3 = scratch 3758 * o2 = tsbmiss area 3759 * o1 = hat id 3760 */ 3761 brgez,a,pn %g1, 6f /* if tte invalid goto tl0 */ 3762 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3763 stx %g1,[%o2] /* put tte into *ttep */ 3764 TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4) 3765 /* 3766 * o0 = vaddr 3767 * o1 = sfmmup 3768 * o2 = ttep 3769 * g1 = pfn 3770 */ 3771 ba,pt %xcc, 6f 3772 mov %g1, %o0 3773 3774 kvtop_nohblk: 3775 /* 3776 * we get here if we couldn't find valid hblk in hash. We rehash 3777 * if neccesary. 3778 */ 3779 ldn [%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0 3780 #ifdef sun4v 3781 cmp %g5, MAX_HASHCNT 3782 #else 3783 cmp %g5, DEFAULT_MAX_HASHCNT /* no 32/256M kernel pages */ 3784 #endif /* sun4v */ 3785 be,a,pn %icc, 6f 3786 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3787 mov %o1, %o4 /* restore hatid */ 3788 #ifdef sun4v 3789 add %g5, 2, %g5 3790 cmp %g5, 3 3791 move %icc, MMU_PAGESHIFT4M, %g6 3792 ba,pt %icc, 1b 3793 movne %icc, MMU_PAGESHIFT256M, %g6 3794 #else 3795 inc %g5 3796 cmp %g5, 2 3797 move %icc, MMU_PAGESHIFT512K, %g6 3798 ba,pt %icc, 1b 3799 movne %icc, MMU_PAGESHIFT4M, %g6 3800 #endif /* sun4v */ 3801 6: 3802 retl 3803 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3804 3805 tsb_suspend: 3806 /* 3807 * o0 = vaddr 3808 * o1 = sfmmup 3809 * o2 = ttep 3810 * g1 = tte 3811 * g2 = tte pa 3812 * g3 = tte va 3813 * o2 = tsbmiss area use o5 instead of o2 for tsbmiss 3814 */ 3815 stx %g1,[%o2] /* put tte into *ttep */ 3816 brgez,a,pn %g1, 8f /* if tte invalid goto 8: */ 3817 sub %g0, 1, %o0 /* output = PFN_INVALID */ 3818 sub %g0, 2, %o0 /* output = PFN_SUSPENDED */ 3819 8: 3820 retl 3821 wrpr %g0, %o3, %pstate /* enable interrupts */ 3822 3823 vatopfn_nokernel: 3824 /* 3825 * This routine does NOT support user addresses 3826 * There is a routine in C that supports this. 3827 * The only reason why we don't have the C routine 3828 * support kernel addresses as well is because 3829 * we do va_to_pa while holding the hashlock. 3830 */ 3831 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3832 save %sp, -SA(MINFRAME), %sp 3833 sethi %hi(sfmmu_panic3), %o0 3834 call panic 3835 or %o0, %lo(sfmmu_panic3), %o0 3836 3837 SET_SIZE(sfmmu_vatopfn) 3838 3839 /* 3840 * %o0 = vaddr 3841 * %o1 = hashno (aka szc) 3842 * 3843 * 3844 * This routine is similar to sfmmu_vatopfn() but will only look for 3845 * a kernel vaddr in the hash structure for the specified rehash value. 3846 * It's just an optimization for the case when pagesize for a given 3847 * va range is already known (e.g. large page heap) and we don't want 3848 * to start the search with rehash value 1 as sfmmu_vatopfn() does. 3849 * 3850 * Returns valid pfn or PFN_INVALID if 3851 * tte for specified rehash # is not found, invalid or suspended. 3852 */ 3853 ENTRY_NP(sfmmu_kvaszc2pfn) 3854 /* 3855 * disable interrupts 3856 */ 3857 rdpr %pstate, %o3 3858 #ifdef DEBUG 3859 PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1) 3860 #endif 3861 /* 3862 * disable interrupts to protect the TSBMISS area 3863 */ 3864 andn %o3, PSTATE_IE, %o5 3865 wrpr %o5, 0, %pstate 3866 3867 CPU_TSBMISS_AREA(%g1, %o5) 3868 ldn [%g1 + TSBMISS_KHATID], %o4 3869 sll %o1, 1, %g6 3870 add %g6, %o1, %g6 3871 add %g6, MMU_PAGESHIFT, %g6 3872 /* 3873 * %o0 = vaddr 3874 * %o1 = hashno 3875 * %o3 = old %pstate 3876 * %o4 = ksfmmup 3877 * %g1 = tsbmiss area 3878 * %g6 = hmeshift 3879 */ 3880 3881 /* 3882 * The first arg to GET_TTE is actually tagaccess register 3883 * not just vaddr. Since this call is for kernel we need to clear 3884 * any lower vaddr bits that would be interpreted as ctx bits. 3885 */ 3886 srlx %o0, MMU_PAGESHIFT, %o0 3887 sllx %o0, MMU_PAGESHIFT, %o0 3888 GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5, 3889 kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk, 3890 kvaszc2pfn_nohblk) 3891 3892 kvaszc2pfn_hblk_found: 3893 /* 3894 * %g3 = tte 3895 * %o0 = vaddr 3896 */ 3897 brgez,a,pn %g3, 1f /* check if tte is invalid */ 3898 mov -1, %o0 /* output = -1 (PFN_INVALID) */ 3899 TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5) 3900 /* 3901 * g3 = pfn 3902 */ 3903 ba,pt %xcc, 1f 3904 mov %g3, %o0 3905 3906 kvaszc2pfn_nohblk: 3907 mov -1, %o0 3908 3909 1: 3910 retl 3911 wrpr %g0, %o3, %pstate /* re-enable interrupts */ 3912 3913 SET_SIZE(sfmmu_kvaszc2pfn) 3914 3915 3916 3917 /* 3918 * kpm lock used between trap level tsbmiss handler and kpm C level. 3919 */ 3920 #define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) \ 3921 mov 0xff, tmp1 ;\ 3922 label1: ;\ 3923 casa [kpmlckp]asi, %g0, tmp1 ;\ 3924 brnz,pn tmp1, label1 ;\ 3925 mov 0xff, tmp1 ;\ 3926 membar #LoadLoad 3927 3928 #define KPMLOCK_EXIT(kpmlckp, asi) \ 3929 membar #LoadStore|#StoreStore ;\ 3930 sta %g0, [kpmlckp]asi 3931 3932 /* 3933 * Lookup a memseg for a given pfn and if found, return the physical 3934 * address of the corresponding struct memseg in mseg, otherwise 3935 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in 3936 * tsbmp, %asi is assumed to be ASI_MEM. 3937 * This lookup is done by strictly traversing only the physical memseg 3938 * linkage. The more generic approach, to check the virtual linkage 3939 * before using the physical (used e.g. with hmehash buckets), cannot 3940 * be used here. Memory DR operations can run in parallel to this 3941 * lookup w/o any locks and updates of the physical and virtual linkage 3942 * cannot be done atomically wrt. to each other. Because physical 3943 * address zero can be valid physical address, MSEG_NULLPTR_PA acts 3944 * as "physical NULL" pointer. 3945 */ 3946 #define PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \ 3947 sethi %hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */ ;\ 3948 ldx [tmp3 + %lo(mhash_per_slot)], mseg ;\ 3949 udivx pfn, mseg, mseg ;\ 3950 ldx [tsbmp + KPMTSBM_MSEGPHASHPA], tmp1 ;\ 3951 and mseg, SFMMU_N_MEM_SLOTS - 1, mseg ;\ 3952 sllx mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg ;\ 3953 add tmp1, mseg, tmp1 ;\ 3954 ldxa [tmp1]%asi, mseg ;\ 3955 cmp mseg, MSEG_NULLPTR_PA ;\ 3956 be,pn %xcc, label/**/1 /* if not found */ ;\ 3957 nop ;\ 3958 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3959 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3960 blu,pn %xcc, label/**/1 ;\ 3961 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3962 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3963 bgeu,pn %xcc, label/**/1 ;\ 3964 sub pfn, tmp1, tmp1 /* pfn - pages_base */ ;\ 3965 mulx tmp1, PAGE_SIZE, tmp1 ;\ 3966 ldxa [mseg + MEMSEG_PAGESPA]%asi, tmp2 /* pages */ ;\ 3967 add tmp2, tmp1, tmp1 /* pp */ ;\ 3968 lduwa [tmp1 + PAGE_PAGENUM]%asi, tmp2 ;\ 3969 cmp tmp2, pfn ;\ 3970 be,pt %xcc, label/**/_ok /* found */ ;\ 3971 label/**/1: ;\ 3972 /* brute force lookup */ ;\ 3973 sethi %hi(memsegspa), tmp3 /* no tsbmp use due to DR */ ;\ 3974 ldx [tmp3 + %lo(memsegspa)], mseg ;\ 3975 label/**/2: ;\ 3976 cmp mseg, MSEG_NULLPTR_PA ;\ 3977 be,pn %xcc, label/**/_ok /* if not found */ ;\ 3978 nop ;\ 3979 ldxa [mseg + MEMSEG_PAGES_BASE]%asi, tmp1 ;\ 3980 cmp pfn, tmp1 /* pfn - pages_base */ ;\ 3981 blu,a,pt %xcc, label/**/2 ;\ 3982 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3983 ldxa [mseg + MEMSEG_PAGES_END]%asi, tmp2 ;\ 3984 cmp pfn, tmp2 /* pfn - pages_end */ ;\ 3985 bgeu,a,pt %xcc, label/**/2 ;\ 3986 ldxa [mseg + MEMSEG_NEXTPA]%asi, mseg ;\ 3987 label/**/_ok: 3988 3989 /* 3990 * kpm tsb miss handler large pages 3991 * g1 = 8K kpm TSB entry pointer 3992 * g2 = tag access register 3993 * g3 = 4M kpm TSB entry pointer 3994 */ 3995 ALTENTRY(sfmmu_kpm_dtsb_miss) 3996 TT_TRACE(trace_tsbmiss) 3997 3998 CPU_INDEX(%g7, %g6) 3999 sethi %hi(kpmtsbm_area), %g6 4000 sllx %g7, KPMTSBM_SHIFT, %g7 4001 or %g6, %lo(kpmtsbm_area), %g6 4002 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4003 4004 /* check enable flag */ 4005 ldub [%g6 + KPMTSBM_FLAGS], %g4 4006 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4007 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4008 nop 4009 4010 /* VA range check */ 4011 ldx [%g6 + KPMTSBM_VBASE], %g7 4012 cmp %g2, %g7 4013 blu,pn %xcc, sfmmu_tsb_miss 4014 ldx [%g6 + KPMTSBM_VEND], %g5 4015 cmp %g2, %g5 4016 bgeu,pn %xcc, sfmmu_tsb_miss 4017 stx %g3, [%g6 + KPMTSBM_TSBPTR] 4018 4019 /* 4020 * check TL tsbmiss handling flag 4021 * bump tsbmiss counter 4022 */ 4023 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4024 #ifdef DEBUG 4025 and %g4, KPMTSBM_TLTSBM_FLAG, %g3 4026 inc %g5 4027 brz,pn %g3, sfmmu_kpm_exception 4028 st %g5, [%g6 + KPMTSBM_TSBMISS] 4029 #else 4030 inc %g5 4031 st %g5, [%g6 + KPMTSBM_TSBMISS] 4032 #endif 4033 /* 4034 * At this point: 4035 * g1 = 8K kpm TSB pointer (not used) 4036 * g2 = tag access register 4037 * g3 = clobbered 4038 * g6 = per-CPU kpm tsbmiss area 4039 * g7 = kpm_vbase 4040 */ 4041 4042 /* vaddr2pfn */ 4043 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 4044 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4045 srax %g4, %g3, %g2 /* which alias range (r) */ 4046 brnz,pn %g2, sfmmu_kpm_exception /* if (r != 0) goto C handler */ 4047 srlx %g4, MMU_PAGESHIFT, %g2 /* %g2 = pfn */ 4048 4049 /* 4050 * Setup %asi 4051 * mseg_pa = page_numtomemseg_nolock(pfn) 4052 * if (mseg_pa == NULL) sfmmu_kpm_exception 4053 * g2=pfn 4054 */ 4055 mov ASI_MEM, %asi 4056 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m) 4057 cmp %g3, MSEG_NULLPTR_PA 4058 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4059 nop 4060 4061 /* 4062 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase)); 4063 * g2=pfn g3=mseg_pa 4064 */ 4065 ldub [%g6 + KPMTSBM_KPMP2PSHFT], %g5 4066 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4067 srlx %g2, %g5, %g4 4068 sllx %g4, %g5, %g4 4069 sub %g4, %g7, %g4 4070 srlx %g4, %g5, %g4 4071 4072 /* 4073 * Validate inx value 4074 * g2=pfn g3=mseg_pa g4=inx 4075 */ 4076 #ifdef DEBUG 4077 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4078 cmp %g4, %g5 /* inx - nkpmpgs */ 4079 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4080 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4081 #else 4082 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4083 #endif 4084 /* 4085 * kp = &mseg_pa->kpm_pages[inx] 4086 */ 4087 sllx %g4, KPMPAGE_SHIFT, %g4 /* kpm_pages offset */ 4088 ldxa [%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */ 4089 add %g5, %g4, %g5 /* kp */ 4090 4091 /* 4092 * KPMP_HASH(kp) 4093 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz 4094 */ 4095 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4096 sub %g7, 1, %g7 /* mask */ 4097 srlx %g5, %g1, %g1 /* x = ksp >> kpmp_shift */ 4098 add %g5, %g1, %g5 /* y = ksp + x */ 4099 and %g5, %g7, %g5 /* hashinx = y & mask */ 4100 4101 /* 4102 * Calculate physical kpm_page pointer 4103 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4104 */ 4105 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */ 4106 add %g1, %g4, %g1 /* kp_pa */ 4107 4108 /* 4109 * Calculate physical hash lock address 4110 * g1=kp_refcntc_pa g2=pfn g5=hashinx 4111 */ 4112 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */ 4113 sllx %g5, KPMHLK_SHIFT, %g5 4114 add %g4, %g5, %g3 4115 add %g3, KPMHLK_LOCK, %g3 /* hlck_pa */ 4116 4117 /* 4118 * Assemble tte 4119 * g1=kp_pa g2=pfn g3=hlck_pa 4120 */ 4121 #ifdef sun4v 4122 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4123 sllx %g5, 32, %g5 4124 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4125 or %g4, TTE4M, %g4 4126 or %g5, %g4, %g5 4127 #else 4128 sethi %hi(TTE_VALID_INT), %g4 4129 mov TTE4M, %g5 4130 sllx %g5, TTE_SZ_SHFT_INT, %g5 4131 or %g5, %g4, %g5 /* upper part */ 4132 sllx %g5, 32, %g5 4133 mov (TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4134 or %g5, %g4, %g5 4135 #endif 4136 sllx %g2, MMU_PAGESHIFT, %g4 4137 or %g5, %g4, %g5 /* tte */ 4138 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4139 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4140 4141 /* 4142 * tsb dropin 4143 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area 4144 */ 4145 4146 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4147 KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM) 4148 4149 /* use C-handler if there's no go for dropin */ 4150 ldsha [%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */ 4151 cmp %g7, -1 4152 bne,pn %xcc, 5f /* use C-handler if there's no go for dropin */ 4153 nop 4154 4155 #ifdef DEBUG 4156 /* double check refcnt */ 4157 ldsha [%g1 + KPMPAGE_REFCNT]%asi, %g7 4158 brz,pn %g7, 5f /* let C-handler deal with this */ 4159 nop 4160 #endif 4161 4162 #ifndef sun4v 4163 ldub [%g6 + KPMTSBM_FLAGS], %g7 4164 mov ASI_N, %g1 4165 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4166 movnz %icc, ASI_MEM, %g1 4167 mov %g1, %asi 4168 #endif 4169 4170 /* 4171 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4172 * If we fail to lock the TSB entry then just load the tte into the 4173 * TLB. 4174 */ 4175 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1) 4176 4177 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4178 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4179 locked_tsb_l1: 4180 DTLB_STUFF(%g5, %g1, %g2, %g4, %g6) 4181 4182 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4183 KPMLOCK_EXIT(%g3, ASI_MEM) 4184 4185 /* 4186 * If trapstat is running, we need to shift the %tpc and %tnpc to 4187 * point to trapstat's TSB miss return code (note that trapstat 4188 * itself will patch the correct offset to add). 4189 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4190 */ 4191 rdpr %tl, %g7 4192 cmp %g7, 1 4193 ble %icc, 0f 4194 sethi %hi(KERNELBASE), %g6 4195 rdpr %tpc, %g7 4196 or %g6, %lo(KERNELBASE), %g6 4197 cmp %g7, %g6 4198 bgeu %xcc, 0f 4199 ALTENTRY(tsbmiss_trapstat_patch_point_kpm) 4200 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4201 wrpr %g7, %tpc 4202 add %g7, 4, %g7 4203 wrpr %g7, %tnpc 4204 0: 4205 retry 4206 5: 4207 /* g3=hlck_pa */ 4208 KPMLOCK_EXIT(%g3, ASI_MEM) 4209 ba,pt %icc, sfmmu_kpm_exception 4210 nop 4211 SET_SIZE(sfmmu_kpm_dtsb_miss) 4212 4213 /* 4214 * kpm tsbmiss handler for smallpages 4215 * g1 = 8K kpm TSB pointer 4216 * g2 = tag access register 4217 * g3 = 4M kpm TSB pointer 4218 */ 4219 ALTENTRY(sfmmu_kpm_dtsb_miss_small) 4220 TT_TRACE(trace_tsbmiss) 4221 CPU_INDEX(%g7, %g6) 4222 sethi %hi(kpmtsbm_area), %g6 4223 sllx %g7, KPMTSBM_SHIFT, %g7 4224 or %g6, %lo(kpmtsbm_area), %g6 4225 add %g6, %g7, %g6 /* g6 = kpmtsbm ptr */ 4226 4227 /* check enable flag */ 4228 ldub [%g6 + KPMTSBM_FLAGS], %g4 4229 and %g4, KPMTSBM_ENABLE_FLAG, %g5 4230 brz,pn %g5, sfmmu_tsb_miss /* if kpm not enabled */ 4231 nop 4232 4233 /* 4234 * VA range check 4235 * On fail: goto sfmmu_tsb_miss 4236 */ 4237 ldx [%g6 + KPMTSBM_VBASE], %g7 4238 cmp %g2, %g7 4239 blu,pn %xcc, sfmmu_tsb_miss 4240 ldx [%g6 + KPMTSBM_VEND], %g5 4241 cmp %g2, %g5 4242 bgeu,pn %xcc, sfmmu_tsb_miss 4243 stx %g1, [%g6 + KPMTSBM_TSBPTR] /* save 8K kpm TSB pointer */ 4244 4245 /* 4246 * check TL tsbmiss handling flag 4247 * bump tsbmiss counter 4248 */ 4249 lduw [%g6 + KPMTSBM_TSBMISS], %g5 4250 #ifdef DEBUG 4251 and %g4, KPMTSBM_TLTSBM_FLAG, %g1 4252 inc %g5 4253 brz,pn %g1, sfmmu_kpm_exception 4254 st %g5, [%g6 + KPMTSBM_TSBMISS] 4255 #else 4256 inc %g5 4257 st %g5, [%g6 + KPMTSBM_TSBMISS] 4258 #endif 4259 /* 4260 * At this point: 4261 * g1 = clobbered 4262 * g2 = tag access register 4263 * g3 = 4M kpm TSB pointer (not used) 4264 * g6 = per-CPU kpm tsbmiss area 4265 * g7 = kpm_vbase 4266 */ 4267 4268 /* 4269 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr) 4270 * which is defined in mach_kpm.h. Any changes in that macro 4271 * should also be ported back to this assembly code. 4272 */ 4273 ldub [%g6 + KPMTSBM_SZSHIFT], %g3 /* g3 = kpm_size_shift */ 4274 sub %g2, %g7, %g4 /* paddr = vaddr-kpm_vbase */ 4275 srax %g4, %g3, %g7 /* which alias range (r) */ 4276 brz,pt %g7, 2f 4277 sethi %hi(vac_colors_mask), %g5 4278 ld [%g5 + %lo(vac_colors_mask)], %g5 4279 4280 srlx %g2, MMU_PAGESHIFT, %g1 /* vaddr >> MMU_PAGESHIFT */ 4281 and %g1, %g5, %g1 /* g1 = v */ 4282 sllx %g7, %g3, %g5 /* g5 = r << kpm_size_shift */ 4283 cmp %g7, %g1 /* if (r > v) */ 4284 bleu,pn %xcc, 1f 4285 sub %g4, %g5, %g4 /* paddr -= r << kpm_size_shift */ 4286 sub %g7, %g1, %g5 /* g5 = r - v */ 4287 sllx %g5, MMU_PAGESHIFT, %g7 /* (r-v) << MMU_PAGESHIFT */ 4288 add %g4, %g7, %g4 /* paddr += (r-v)<<MMU_PAGESHIFT */ 4289 ba 2f 4290 nop 4291 1: 4292 sllx %g7, MMU_PAGESHIFT, %g5 /* else */ 4293 sub %g4, %g5, %g4 /* paddr -= r << MMU_PAGESHIFT */ 4294 4295 /* 4296 * paddr2pfn 4297 * g1 = vcolor (not used) 4298 * g2 = tag access register 4299 * g3 = clobbered 4300 * g4 = paddr 4301 * g5 = clobbered 4302 * g6 = per-CPU kpm tsbmiss area 4303 * g7 = clobbered 4304 */ 4305 2: 4306 srlx %g4, MMU_PAGESHIFT, %g2 /* g2 = pfn */ 4307 4308 /* 4309 * Setup %asi 4310 * mseg_pa = page_numtomemseg_nolock_pa(pfn) 4311 * if (mseg not found) sfmmu_kpm_exception 4312 * g2=pfn g6=per-CPU kpm tsbmiss area 4313 * g4 g5 g7 for scratch use. 4314 */ 4315 mov ASI_MEM, %asi 4316 PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m) 4317 cmp %g3, MSEG_NULLPTR_PA 4318 be,pn %xcc, sfmmu_kpm_exception /* if mseg not found */ 4319 nop 4320 4321 /* 4322 * inx = pfn - mseg_pa->kpm_pbase 4323 * g2=pfn g3=mseg_pa g6=per-CPU kpm tsbmiss area 4324 */ 4325 ldxa [%g3 + MEMSEG_KPM_PBASE]%asi, %g7 4326 sub %g2, %g7, %g4 4327 4328 #ifdef DEBUG 4329 /* 4330 * Validate inx value 4331 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area 4332 */ 4333 ldxa [%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5 4334 cmp %g4, %g5 /* inx - nkpmpgs */ 4335 bgeu,pn %xcc, sfmmu_kpm_exception /* if out of range */ 4336 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4337 #else 4338 ld [%g6 + KPMTSBM_KPMPTABLESZ], %g7 4339 #endif 4340 /* ksp = &mseg_pa->kpm_spages[inx] */ 4341 ldxa [%g3 + MEMSEG_KPM_SPAGES]%asi, %g5 4342 add %g5, %g4, %g5 /* ksp */ 4343 4344 /* 4345 * KPMP_SHASH(kp) 4346 * g2=pfn g3=mseg_pa g4=inx g5=ksp 4347 * g6=per-CPU kpm tsbmiss area g7=kpmp_stable_sz 4348 */ 4349 ldub [%g6 + KPMTSBM_KPMPSHIFT], %g1 /* kpmp_shift */ 4350 sub %g7, 1, %g7 /* mask */ 4351 sllx %g5, %g1, %g1 /* x = ksp << kpmp_shift */ 4352 add %g5, %g1, %g5 /* y = ksp + x */ 4353 and %g5, %g7, %g5 /* hashinx = y & mask */ 4354 4355 /* 4356 * Calculate physical kpm_spage pointer 4357 * g2=pfn g3=mseg_pa g4=offset g5=hashinx 4358 * g6=per-CPU kpm tsbmiss area 4359 */ 4360 ldxa [%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */ 4361 add %g1, %g4, %g1 /* ksp_pa */ 4362 4363 /* 4364 * Calculate physical hash lock address. 4365 * Note: Changes in kpm_shlk_t must be reflected here. 4366 * g1=ksp_pa g2=pfn g5=hashinx 4367 * g6=per-CPU kpm tsbmiss area 4368 */ 4369 ldx [%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */ 4370 sllx %g5, KPMSHLK_SHIFT, %g5 4371 add %g4, %g5, %g3 /* hlck_pa */ 4372 4373 /* 4374 * Assemble non-cacheable tte initially 4375 * g1=ksp_pa g2=pfn g3=hlck_pa 4376 * g6=per-CPU kpm tsbmiss area 4377 */ 4378 sethi %hi(TTE_VALID_INT), %g5 /* upper part */ 4379 sllx %g5, 32, %g5 4380 mov (TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4 4381 or %g5, %g4, %g5 4382 sllx %g2, MMU_PAGESHIFT, %g4 4383 or %g5, %g4, %g5 /* tte */ 4384 ldx [%g6 + KPMTSBM_TSBPTR], %g4 4385 GET_MMU_D_TTARGET(%g2, %g7) /* %g2 = ttarget */ 4386 4387 /* 4388 * tsb dropin 4389 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable) 4390 * g6=per-CPU kpm tsbmiss area g7=scratch register 4391 */ 4392 4393 /* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */ 4394 KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM) 4395 4396 /* use C-handler if there's no go for dropin */ 4397 ldsba [%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */ 4398 andcc %g7, KPM_MAPPED_GO, %g0 /* go or no go ? */ 4399 bz,pt %icc, 5f /* no go */ 4400 nop 4401 and %g7, KPM_MAPPED_MASK, %g7 /* go */ 4402 cmp %g7, KPM_MAPPEDS /* cacheable ? */ 4403 be,a,pn %xcc, 3f 4404 or %g5, TTE_CV_INT, %g5 /* cacheable */ 4405 3: 4406 #ifndef sun4v 4407 ldub [%g6 + KPMTSBM_FLAGS], %g7 4408 mov ASI_N, %g1 4409 andcc %g7, KPMTSBM_TSBPHYS_FLAG, %g0 4410 movnz %icc, ASI_MEM, %g1 4411 mov %g1, %asi 4412 #endif 4413 4414 /* 4415 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) 4416 * If we fail to lock the TSB entry then just load the tte into the 4417 * TLB. 4418 */ 4419 TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2) 4420 4421 /* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */ 4422 TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7) 4423 locked_tsb_l2: 4424 DTLB_STUFF(%g5, %g2, %g4, %g5, %g6) 4425 4426 /* KPMLOCK_EXIT(kpmlckp, asi) */ 4427 KPMLOCK_EXIT(%g3, ASI_MEM) 4428 4429 /* 4430 * If trapstat is running, we need to shift the %tpc and %tnpc to 4431 * point to trapstat's TSB miss return code (note that trapstat 4432 * itself will patch the correct offset to add). 4433 * Note: TTE is expected in %g5 (allows per pagesize reporting). 4434 */ 4435 rdpr %tl, %g7 4436 cmp %g7, 1 4437 ble %icc, 0f 4438 sethi %hi(KERNELBASE), %g6 4439 rdpr %tpc, %g7 4440 or %g6, %lo(KERNELBASE), %g6 4441 cmp %g7, %g6 4442 bgeu %xcc, 0f 4443 ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small) 4444 add %g7, RUNTIME_PATCH, %g7 /* must match TSTAT_TSBMISS_INSTR */ 4445 wrpr %g7, %tpc 4446 add %g7, 4, %g7 4447 wrpr %g7, %tnpc 4448 0: 4449 retry 4450 5: 4451 /* g3=hlck_pa */ 4452 KPMLOCK_EXIT(%g3, ASI_MEM) 4453 ba,pt %icc, sfmmu_kpm_exception 4454 nop 4455 SET_SIZE(sfmmu_kpm_dtsb_miss_small) 4456 4457 #if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE 4458 #error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct 4459 #endif 4460 4461 .seg ".data" 4462 sfmmu_kpm_tsbmtl_panic: 4463 .ascii "sfmmu_kpm_tsbmtl: interrupts disabled" 4464 .byte 0 4465 sfmmu_kpm_stsbmtl_panic: 4466 .ascii "sfmmu_kpm_stsbmtl: interrupts disabled" 4467 .byte 0 4468 .align 4 4469 .seg ".text" 4470 4471 ENTRY_NP(sfmmu_kpm_tsbmtl) 4472 rdpr %pstate, %o3 4473 /* 4474 * %o0 = &kp_refcntc 4475 * %o1 = &khl_lock 4476 * %o2 = 0/1 (off/on) 4477 * %o3 = pstate save 4478 */ 4479 #ifdef DEBUG 4480 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4481 bnz,pt %icc, 1f /* disabled, panic */ 4482 nop 4483 save %sp, -SA(MINFRAME), %sp 4484 sethi %hi(sfmmu_kpm_tsbmtl_panic), %o0 4485 call panic 4486 or %o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0 4487 ret 4488 restore 4489 1: 4490 #endif /* DEBUG */ 4491 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4492 4493 KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N) 4494 mov -1, %o5 4495 brz,a %o2, 2f 4496 mov 0, %o5 4497 2: 4498 sth %o5, [%o0] 4499 KPMLOCK_EXIT(%o1, ASI_N) 4500 4501 retl 4502 wrpr %g0, %o3, %pstate /* enable interrupts */ 4503 SET_SIZE(sfmmu_kpm_tsbmtl) 4504 4505 ENTRY_NP(sfmmu_kpm_stsbmtl) 4506 rdpr %pstate, %o3 4507 /* 4508 * %o0 = &mapped 4509 * %o1 = &kshl_lock 4510 * %o2 = val 4511 * %o3 = pstate save 4512 */ 4513 #ifdef DEBUG 4514 andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ 4515 bnz,pt %icc, 1f /* disabled, panic */ 4516 nop 4517 save %sp, -SA(MINFRAME), %sp 4518 sethi %hi(sfmmu_kpm_stsbmtl_panic), %o0 4519 call panic 4520 or %o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0 4521 ret 4522 restore 4523 1: 4524 #endif /* DEBUG */ 4525 wrpr %o3, PSTATE_IE, %pstate /* disable interrupts */ 4526 4527 KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N) 4528 ldsb [%o0], %o5 4529 stb %o2, [%o0] 4530 KPMLOCK_EXIT(%o1, ASI_N) 4531 4532 and %o5, KPM_MAPPED_MASK, %o0 /* return old val */ 4533 retl 4534 wrpr %g0, %o3, %pstate /* enable interrupts */ 4535 SET_SIZE(sfmmu_kpm_stsbmtl) 4536 4537 #ifdef sun4v 4538 /* 4539 * User/kernel data miss w// multiple TSBs 4540 * The first probe covers 8K, 64K, and 512K page sizes, 4541 * because 64K and 512K mappings are replicated off 8K 4542 * pointer. Second probe covers 4M page size only. 4543 * 4544 * MMU fault area contains miss address and context. 4545 */ 4546 ALTENTRY(sfmmu_slow_dmmu_miss) 4547 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) ! %g2 = ptagacc, %g3 = ctx type 4548 4549 slow_miss_common: 4550 /* 4551 * %g2 = tagacc register (needed for sfmmu_tsb_miss_tt) 4552 * %g3 = ctx (cannot be INVALID_CONTEXT) 4553 */ 4554 brnz,pt %g3, 8f ! check for user context 4555 nop 4556 4557 /* 4558 * Kernel miss 4559 * Get 8K and 4M TSB pointers in %g1 and %g3 and 4560 * branch to sfmmu_tsb_miss_tt to handle it. 4561 */ 4562 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4563 sfmmu_dslow_patch_ktsb_base: 4564 RUNTIME_PATCH_SETX(%g1, %g6) ! %g1 = contents of ktsb_pbase 4565 sfmmu_dslow_patch_ktsb_szcode: 4566 or %g0, RUNTIME_PATCH, %g3 ! ktsb_szcode (hot patched) 4567 4568 GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5) 4569 ! %g1 = First TSB entry pointer, as TSB miss handler expects 4570 4571 mov %g2, %g7 ! TSB pointer macro clobbers tagacc 4572 sfmmu_dslow_patch_ktsb4m_base: 4573 RUNTIME_PATCH_SETX(%g3, %g6) ! %g3 = contents of ktsb4m_pbase 4574 sfmmu_dslow_patch_ktsb4m_szcode: 4575 or %g0, RUNTIME_PATCH, %g6 ! ktsb4m_szcode (hot patched) 4576 4577 GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5) 4578 ! %g3 = 4M tsb entry pointer, as TSB miss handler expects 4579 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4580 .empty 4581 4582 8: 4583 /* 4584 * User miss 4585 * Get first TSB pointer in %g1 4586 * Get second TSB pointer (or NULL if no second TSB) in %g3 4587 * Branch to sfmmu_tsb_miss_tt to handle it 4588 */ 4589 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) 4590 /* %g1 = first TSB entry ptr now, %g2 preserved */ 4591 4592 GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3) /* get 2nd utsbreg */ 4593 brlz,pt %g3, sfmmu_tsb_miss_tt /* done if no 2nd TSB */ 4594 nop 4595 4596 GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5) 4597 /* %g3 = second TSB entry ptr now, %g2 preserved */ 4598 9: 4599 ba,a,pt %xcc, sfmmu_tsb_miss_tt 4600 .empty 4601 SET_SIZE(sfmmu_slow_dmmu_miss) 4602 4603 4604 /* 4605 * User/kernel instruction miss w/ multiple TSBs 4606 * The first probe covers 8K, 64K, and 512K page sizes, 4607 * because 64K and 512K mappings are replicated off 8K 4608 * pointer. Second probe covers 4M page size only. 4609 * 4610 * MMU fault area contains miss address and context. 4611 */ 4612 ALTENTRY(sfmmu_slow_immu_miss) 4613 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) 4614 ba,a,pt %xcc, slow_miss_common 4615 SET_SIZE(sfmmu_slow_immu_miss) 4616 4617 #endif /* sun4v */ 4618 4619 /* 4620 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers. 4621 */ 4622 .seg ".data" 4623 .align 64 4624 .global tsbmiss_area 4625 tsbmiss_area: 4626 .skip (TSBMISS_SIZE * NCPU) 4627 4628 .align 64 4629 .global kpmtsbm_area 4630 kpmtsbm_area: 4631 .skip (KPMTSBM_SIZE * NCPU)