1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * 25 * Assembly code support for the Cheetah+ module 26 */ 27 28 #include "assym.h" 29 30 #include <sys/asm_linkage.h> 31 #include <sys/mmu.h> 32 #include <vm/hat_sfmmu.h> 33 #include <sys/machparam.h> 34 #include <sys/machcpuvar.h> 35 #include <sys/machthread.h> 36 #include <sys/machtrap.h> 37 #include <sys/privregs.h> 38 #include <sys/asm_linkage.h> 39 #include <sys/trap.h> 40 #include <sys/cheetahregs.h> 41 #include <sys/us3_module.h> 42 #include <sys/xc_impl.h> 43 #include <sys/intreg.h> 44 #include <sys/async.h> 45 #include <sys/clock.h> 46 #include <sys/cheetahasm.h> 47 #include <sys/cmpregs.h> 48 49 #ifdef TRAPTRACE 50 #include <sys/traptrace.h> 51 #endif /* TRAPTRACE */ 52 53 54 /* BEGIN CSTYLED */ 55 56 /* 57 * Cheetah+ version to reflush an Ecache line by index. 58 * 59 * By default we assume the Ecache is 2-way so we flush both 60 * ways. Even if the cache is direct-mapped no harm will come 61 * from performing the flush twice, apart from perhaps a performance 62 * penalty. 63 * 64 * XXX - scr2 not used. 65 */ 66 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2) \ 67 ldxa [index]ASI_EC_DIAG, %g0; \ 68 ldxa [index + ec_set_size]ASI_EC_DIAG, %g0; 69 70 /* 71 * Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement 72 * Flush feature. 73 */ 74 #define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \ 75 sub ec_set_size, 1, scr1; \ 76 and physaddr, scr1, scr1; \ 77 set CHP_ECACHE_IDX_DISP_FLUSH, scr2; \ 78 or scr2, scr1, scr1; \ 79 ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2) 80 81 /* END CSTYLED */ 82 83 /* 84 * Panther version to reflush a line from both the L2 cache and L3 85 * cache by the respective indexes. Flushes all ways of the line from 86 * each cache. 87 * 88 * l2_index Index into the L2$ of the line to be flushed. This 89 * register will not be modified by this routine. 90 * l3_index Index into the L3$ of the line to be flushed. This 91 * register will not be modified by this routine. 92 * scr2 scratch register. 93 * scr3 scratch register. 94 * 95 */ 96 #define PN_ECACHE_REFLUSH_LINE(l2_index, l3_index, scr2, scr3) \ 97 set PN_L2_MAX_SET, scr2; \ 98 set PN_L2_SET_SIZE, scr3; \ 99 1: \ 100 ldxa [l2_index + scr2]ASI_L2_TAG, %g0; \ 101 cmp scr2, %g0; \ 102 bg,a 1b; \ 103 sub scr2, scr3, scr2; \ 104 mov 6, scr2; \ 105 7: \ 106 cmp scr2, %g0; \ 107 bg,a 7b; \ 108 sub scr2, 1, scr2; \ 109 set PN_L3_MAX_SET, scr2; \ 110 set PN_L3_SET_SIZE, scr3; \ 111 2: \ 112 ldxa [l3_index + scr2]ASI_EC_DIAG, %g0; \ 113 cmp scr2, %g0; \ 114 bg,a 2b; \ 115 sub scr2, scr3, scr2; 116 117 /* 118 * Panther version of ecache_flush_line. Flushes the line corresponding 119 * to physaddr from both the L2 cache and the L3 cache. 120 * 121 * physaddr Input: Physical address to flush. 122 * Output: Physical address to flush (preserved). 123 * l2_idx_out Input: scratch register. 124 * Output: Index into the L2$ of the line to be flushed. 125 * l3_idx_out Input: scratch register. 126 * Output: Index into the L3$ of the line to be flushed. 127 * scr3 scratch register. 128 * scr4 scratch register. 129 * 130 */ 131 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \ 132 set PN_L3_SET_SIZE, l2_idx_out; \ 133 sub l2_idx_out, 1, l2_idx_out; \ 134 and physaddr, l2_idx_out, l3_idx_out; \ 135 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \ 136 or l2_idx_out, l3_idx_out, l3_idx_out; \ 137 set PN_L2_SET_SIZE, l2_idx_out; \ 138 sub l2_idx_out, 1, l2_idx_out; \ 139 and physaddr, l2_idx_out, l2_idx_out; \ 140 set PN_L2_IDX_DISP_FLUSH, scr3; \ 141 or l2_idx_out, scr3, l2_idx_out; \ 142 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4) 143 144 /* 145 * Fast ECC error at TL>0 handler 146 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter 147 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 148 * For a complete description of the Fast ECC at TL>0 handling see the 149 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in 150 * us3_common_asm.s 151 */ 152 153 .section ".text" 154 .align 64 155 ENTRY_NP(fast_ecc_tl1_err) 156 157 /* 158 * This macro turns off the D$/I$ if they are on and saves their 159 * original state in ch_err_tl1_tmp, saves all the %g registers in the 160 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves 161 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will 162 * point to the ch_err_tl1_data structure and the original D$/I$ state 163 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 164 * will be available. 165 */ 166 CH_ERR_TL1_FECC_ENTER; 167 168 /* 169 * Get the diagnostic logout data. %g4 must be initialized to 170 * current CEEN state, %g5 must point to logout structure in 171 * ch_err_tl1_data_t. %g3 will contain the nesting count upon 172 * return. 173 */ 174 ldxa [%g0]ASI_ESTATE_ERR, %g4 175 and %g4, EN_REG_CEEN, %g4 176 add %g1, CH_ERR_TL1_LOGOUT, %g5 177 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) 178 179 /* 180 * If the logout nesting count is exceeded, we're probably 181 * not making any progress, try to panic instead. 182 */ 183 cmp %g3, CLO_NESTING_MAX 184 bge fecc_tl1_err 185 nop 186 187 /* 188 * Save the current CEEN and NCEEN state in %g7 and turn them off 189 * before flushing the Ecache. 190 */ 191 ldxa [%g0]ASI_ESTATE_ERR, %g7 192 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 193 stxa %g5, [%g0]ASI_ESTATE_ERR 194 membar #Sync 195 196 /* 197 * Flush the Ecache, using the largest possible cache size with the 198 * smallest possible line size since we can't get the actual sizes 199 * from the cpu_node due to DTLB misses. 200 */ 201 PN_L2_FLUSHALL(%g3, %g4, %g5) 202 203 set CH_ECACHE_MAX_SIZE, %g4 204 set CH_ECACHE_MIN_LSIZE, %g5 205 206 GET_CPU_IMPL(%g6) 207 cmp %g6, PANTHER_IMPL 208 bne %xcc, 2f 209 nop 210 set PN_L3_SIZE, %g4 211 2: 212 mov %g6, %g3 213 CHP_ECACHE_FLUSHALL(%g4, %g5, %g3) 214 215 /* 216 * Restore CEEN and NCEEN to the previous state. 217 */ 218 stxa %g7, [%g0]ASI_ESTATE_ERR 219 membar #Sync 220 221 /* 222 * If we turned off the D$, then flush it and turn it back on. 223 */ 224 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 225 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 226 bz %xcc, 3f 227 nop 228 229 /* 230 * Flush the D$. 231 */ 232 ASM_LD(%g4, dcache_size) 233 ASM_LD(%g5, dcache_linesize) 234 CH_DCACHE_FLUSHALL(%g4, %g5, %g6) 235 236 /* 237 * Turn the D$ back on. 238 */ 239 ldxa [%g0]ASI_DCU, %g3 240 or %g3, DCU_DC, %g3 241 stxa %g3, [%g0]ASI_DCU 242 membar #Sync 243 3: 244 /* 245 * If we turned off the I$, then flush it and turn it back on. 246 */ 247 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 248 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 249 bz %xcc, 4f 250 nop 251 252 /* 253 * Flush the I$. Panther has different I$ parameters, and we 254 * can't access the logout I$ params without possibly generating 255 * a MMU miss. 256 */ 257 GET_CPU_IMPL(%g6) 258 set PN_ICACHE_SIZE, %g3 259 set CH_ICACHE_SIZE, %g4 260 mov CH_ICACHE_LSIZE, %g5 261 cmp %g6, PANTHER_IMPL 262 movz %xcc, %g3, %g4 263 movz %xcc, PN_ICACHE_LSIZE, %g5 264 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) 265 266 /* 267 * Turn the I$ back on. Changing DCU_IC requires flush. 268 */ 269 ldxa [%g0]ASI_DCU, %g3 270 or %g3, DCU_IC, %g3 271 stxa %g3, [%g0]ASI_DCU 272 flush %g0 273 4: 274 275 #ifdef TRAPTRACE 276 /* 277 * Get current trap trace entry physical pointer. 278 */ 279 CPU_INDEX(%g6, %g5) 280 sll %g6, TRAPTR_SIZE_SHIFT, %g6 281 set trap_trace_ctl, %g5 282 add %g6, %g5, %g6 283 ld [%g6 + TRAPTR_LIMIT], %g5 284 tst %g5 285 be %icc, skip_traptrace 286 nop 287 ldx [%g6 + TRAPTR_PBASE], %g5 288 ld [%g6 + TRAPTR_OFFSET], %g4 289 add %g5, %g4, %g5 290 291 /* 292 * Create trap trace entry. 293 */ 294 rd %asi, %g7 295 wr %g0, TRAPTR_ASI, %asi 296 rd STICK, %g4 297 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 298 rdpr %tl, %g4 299 stha %g4, [%g5 + TRAP_ENT_TL]%asi 300 rdpr %tt, %g4 301 stha %g4, [%g5 + TRAP_ENT_TT]%asi 302 rdpr %tpc, %g4 303 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 304 rdpr %tstate, %g4 305 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 306 stna %sp, [%g5 + TRAP_ENT_SP]%asi 307 stna %g0, [%g5 + TRAP_ENT_TR]%asi 308 wr %g0, %g7, %asi 309 ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 310 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 311 wr %g0, TRAPTR_ASI, %asi 312 stna %g3, [%g5 + TRAP_ENT_F1]%asi 313 stna %g4, [%g5 + TRAP_ENT_F2]%asi 314 wr %g0, %g7, %asi 315 ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 316 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 317 wr %g0, TRAPTR_ASI, %asi 318 stna %g3, [%g5 + TRAP_ENT_F3]%asi 319 stna %g4, [%g5 + TRAP_ENT_F4]%asi 320 wr %g0, %g7, %asi 321 322 /* 323 * Advance trap trace pointer. 324 */ 325 ld [%g6 + TRAPTR_OFFSET], %g5 326 ld [%g6 + TRAPTR_LIMIT], %g4 327 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 328 add %g5, TRAP_ENT_SIZE, %g5 329 sub %g4, TRAP_ENT_SIZE, %g4 330 cmp %g5, %g4 331 movge %icc, 0, %g5 332 st %g5, [%g6 + TRAPTR_OFFSET] 333 skip_traptrace: 334 #endif /* TRAPTRACE */ 335 336 /* 337 * If nesting count is not zero, skip all the AFSR/AFAR 338 * handling and just do the necessary cache-flushing. 339 */ 340 ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 341 brnz %g2, 6f 342 nop 343 344 /* 345 * If a UCU or L3_UCU followed by a WDU has occurred go ahead 346 * and panic since a UE will occur (on the retry) before the 347 * UCU and WDU messages are enqueued. On a Panther processor, 348 * we need to also see an L3_WDU before panicking. Note that 349 * we avoid accessing the _EXT ASIs if not on a Panther. 350 */ 351 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g3 352 set 1, %g4 353 sllx %g4, C_AFSR_UCU_SHIFT, %g4 354 btst %g4, %g3 ! UCU in original shadow AFSR? 355 bnz %xcc, 5f 356 nop 357 GET_CPU_IMPL(%g6) 358 cmp %g6, PANTHER_IMPL 359 bne %xcc, 6f ! not Panther, no UCU, skip the rest 360 nop 361 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR_EXT]%asi, %g3 362 btst C_AFSR_L3_UCU, %g3 ! L3_UCU in original shadow AFSR_EXT? 363 bz %xcc, 6f ! neither UCU nor L3_UCU was seen 364 nop 365 5: 366 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 ! original AFSR 367 ldxa [%g0]ASI_AFSR, %g3 ! current AFSR 368 or %g3, %g4, %g3 ! %g3 = original + current AFSR 369 set 1, %g4 370 sllx %g4, C_AFSR_WDU_SHIFT, %g4 371 btst %g4, %g3 ! WDU in original or current AFSR? 372 bz %xcc, 6f ! no WDU, skip remaining tests 373 nop 374 GET_CPU_IMPL(%g6) 375 cmp %g6, PANTHER_IMPL 376 bne %xcc, fecc_tl1_err ! if not Panther, panic (saw UCU, WDU) 377 nop 378 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR_EXT]%asi, %g4 ! original AFSR_EXT 379 set ASI_AFSR_EXT_VA, %g6 ! ASI of current AFSR_EXT 380 ldxa [%g6]ASI_AFSR, %g3 ! value of current AFSR_EXT 381 or %g3, %g4, %g3 ! %g3 = original + current AFSR_EXT 382 btst C_AFSR_L3_WDU, %g3 ! L3_WDU in original or current AFSR? 383 bnz %xcc, fecc_tl1_err ! panic (saw L3_WDU and UCU or L3_UCU) 384 nop 385 6: 386 /* 387 * We fall into this macro if we've successfully logged the error in 388 * the ch_err_tl1_data structure and want the PIL15 softint to pick 389 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 390 * Restores the %g registers and issues retry. 391 */ 392 CH_ERR_TL1_EXIT; 393 394 /* 395 * Establish panic exit label. 396 */ 397 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); 398 399 SET_SIZE(fast_ecc_tl1_err) 400 401 402 /* 403 * scrubphys - Pass in the aligned physical memory address 404 * that you want to scrub, along with the ecache set size. 405 * 406 * 1) Displacement flush the E$ line corresponding to %addr. 407 * The first ldxa guarantees that the %addr is no longer in 408 * M, O, or E (goes to I or S (if instruction fetch also happens). 409 * 2) "Write" the data using a CAS %addr,%g0,%g0. 410 * The casxa guarantees a transition from I to M or S to M. 411 * 3) Displacement flush the E$ line corresponding to %addr. 412 * The second ldxa pushes the M line out of the ecache, into the 413 * writeback buffers, on the way to memory. 414 * 4) The "membar #Sync" pushes the cache line out of the writeback 415 * buffers onto the bus, on the way to dram finally. 416 * 417 * This is a modified version of the algorithm suggested by Gary Lauterbach. 418 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line 419 * as modified, but then we found out that for spitfire, if it misses in the 420 * E$ it will probably install as an M, but if it hits in the E$, then it 421 * will stay E, if the store doesn't happen. So the first displacement flush 422 * should ensure that the CAS will miss in the E$. Arrgh. 423 */ 424 ENTRY(scrubphys) 425 rdpr %pstate, %o4 426 andn %o4, PSTATE_IE | PSTATE_AM, %o5 427 wrpr %o5, %g0, %pstate ! clear IE, AM bits 428 429 GET_CPU_IMPL(%o5) ! Panther Ecache is flushed differently 430 cmp %o5, PANTHER_IMPL 431 bne scrubphys_1 432 nop 433 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5) 434 casxa [%o0]ASI_MEM, %g0, %g0 435 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) 436 b scrubphys_2 437 nop 438 scrubphys_1: 439 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 440 casxa [%o0]ASI_MEM, %g0, %g0 441 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 442 scrubphys_2: 443 wrpr %g0, %o4, %pstate ! restore earlier pstate register value 444 445 retl 446 membar #Sync ! move the data out of the load buffer 447 SET_SIZE(scrubphys) 448 449 450 /* 451 * clearphys - Pass in the physical memory address of the checkblock 452 * that you want to push out, cleared with a recognizable pattern, 453 * from the ecache. 454 * 455 * To ensure that the ecc gets recalculated after the bad data is cleared, 456 * we must write out enough data to fill the w$ line (64 bytes). So we read 457 * in an entire ecache subblock's worth of data, and write it back out. 458 * Then we overwrite the 16 bytes of bad data with the pattern. 459 */ 460 ENTRY(clearphys) 461 /* turn off IE, AM bits */ 462 rdpr %pstate, %o4 463 andn %o4, PSTATE_IE | PSTATE_AM, %o5 464 wrpr %o5, %g0, %pstate 465 466 /* turn off NCEEN */ 467 ldxa [%g0]ASI_ESTATE_ERR, %o5 468 andn %o5, EN_REG_NCEEN, %o3 469 stxa %o3, [%g0]ASI_ESTATE_ERR 470 membar #Sync 471 472 /* align address passed with 64 bytes subblock size */ 473 mov CH_ECACHE_SUBBLK_SIZE, %o2 474 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 475 476 /* move the good data into the W$ */ 477 clearphys_1: 478 subcc %o2, 8, %o2 479 ldxa [%g1 + %o2]ASI_MEM, %g2 480 bge clearphys_1 481 stxa %g2, [%g1 + %o2]ASI_MEM 482 483 /* now overwrite the bad data */ 484 setx 0xbadecc00badecc01, %g1, %g2 485 stxa %g2, [%o0]ASI_MEM 486 mov 8, %g1 487 stxa %g2, [%o0 + %g1]ASI_MEM 488 489 GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently 490 cmp %o3, PANTHER_IMPL 491 bne clearphys_2 492 nop 493 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %g1) 494 casxa [%o0]ASI_MEM, %g0, %g0 495 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) 496 b clearphys_3 497 nop 498 clearphys_2: 499 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 500 casxa [%o0]ASI_MEM, %g0, %g0 501 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 502 clearphys_3: 503 /* clear the AFSR */ 504 ldxa [%g0]ASI_AFSR, %o1 505 stxa %o1, [%g0]ASI_AFSR 506 membar #Sync 507 508 /* turn NCEEN back on */ 509 stxa %o5, [%g0]ASI_ESTATE_ERR 510 membar #Sync 511 512 /* return and re-enable IE and AM */ 513 retl 514 wrpr %g0, %o4, %pstate 515 SET_SIZE(clearphys) 516 517 518 /* 519 * Cheetah+ Ecache displacement flush the specified line from the E$ 520 * 521 * For Panther, this means flushing the specified line from both the 522 * L2 cache and L3 cache. 523 * 524 * Register usage: 525 * %o0 - 64 bit physical address for flushing 526 * %o1 - Ecache set size 527 */ 528 ENTRY(ecache_flush_line) 529 530 GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently 531 cmp %o3, PANTHER_IMPL 532 bne ecache_flush_line_1 533 nop 534 535 PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4) 536 b ecache_flush_line_2 537 nop 538 ecache_flush_line_1: 539 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 540 ecache_flush_line_2: 541 retl 542 nop 543 SET_SIZE(ecache_flush_line) 544 545 ENTRY(set_afsr_ext) 546 set ASI_AFSR_EXT_VA, %o1 547 stxa %o0, [%o1]ASI_AFSR ! afsr_ext reg 548 membar #Sync 549 retl 550 nop 551 SET_SIZE(set_afsr_ext) 552 553 554 /* 555 * The CPU jumps here from the MMU exception handler if an ITLB parity 556 * error is detected and we are running on Panther. 557 * 558 * In this routine we collect diagnostic information and write it to our 559 * logout structure (if possible) and clear all ITLB entries that may have 560 * caused our parity trap. 561 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 562 * and log any error messages. As for parameters to cpu_tlb_parity_error, we 563 * send two: 564 * 565 * %g2 - Contains the VA whose lookup in the ITLB caused the parity error 566 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, 567 * regardless of whether or not we actually used the logout struct. 568 * 569 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two 570 * parameters to the data contained in the logout structure in order to 571 * determine whether the logout information is valid for this particular 572 * error or not. 573 */ 574 ENTRY_NP(itlb_parity_trap) 575 /* 576 * Collect important information about the trap which will be 577 * used as a parameter to the TL0 handler. 578 */ 579 wr %g0, ASI_IMMU, %asi 580 rdpr %tpc, %g2 ! VA that caused the IMMU trap 581 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page size 582 set PN_ITLB_PGSZ_MASK, %g4 583 and %g3, %g4, %g3 584 ldxa [MMU_TAG_ACCESS]%asi, %g4 585 set TAGREAD_CTX_MASK, %g5 586 and %g4, %g5, %g4 587 or %g4, %g3, %g3 ! 'or' in the trap context and 588 mov 1, %g4 ! add the IMMU flag to complete 589 sllx %g4, PN_TLO_INFO_IMMU_SHIFT, %g4 590 or %g4, %g3, %g3 ! the tlo_info field for logout 591 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR 592 membar #Sync 593 594 /* 595 * at this point: 596 * %g2 - contains the VA whose lookup caused the trap 597 * %g3 - contains the tlo_info field 598 * 599 * Next, we calculate the TLB index value for the failing VA. 600 */ 601 mov %g2, %g4 ! We need the ITLB index 602 set PN_ITLB_PGSZ_MASK, %g5 603 and %g3, %g5, %g5 604 srlx %g5, PN_ITLB_PGSZ_SHIFT, %g5 605 PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the index 606 sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place 607 set PN_ITLB_T512, %g5 608 or %g4, %g5, %g4 ! and add in the TLB ID 609 610 /* 611 * at this point: 612 * %g2 - contains the VA whose lookup caused the trap 613 * %g3 - contains the tlo_info field 614 * %g4 - contains the TLB access index value for the 615 * VA/PgSz in question 616 * 617 * Check to see if the logout structure is available. 618 */ 619 set CHPR_TLB_LOGOUT, %g6 620 GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, itlb_parity_trap_1) 621 set LOGOUT_INVALID_U32, %g6 622 sllx %g6, 32, %g6 ! if our logout structure is 623 set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is 624 or %g5, %g6, %g5 ! already being used, then we 625 ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic 626 cmp %g6, %g5 ! information before clearing 627 bne itlb_parity_trap_1 ! and logging the error. 628 nop 629 630 /* 631 * Record the logout information. %g4 contains our index + TLB ID 632 * for use in ASI_ITLB_ACCESS and ASI_ITLB_TAGREAD. %g1 contains 633 * the pointer to our logout struct. 634 */ 635 stx %g3, [%g1 + PN_TLO_INFO] 636 stx %g2, [%g1 + PN_TLO_ADDR] 637 stx %g2, [%g1 + PN_TLO_PC] ! %tpc == fault addr for IMMU 638 639 add %g1, PN_TLO_ITLB_TTE, %g1 ! move up the pointer 640 641 ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data 642 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away 643 ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag 644 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away 645 646 set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 647 or %g4, %g6, %g4 648 add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer 649 650 ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data 651 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away 652 ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag 653 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away 654 655 andn %g4, %g6, %g4 ! back to way 0 656 657 itlb_parity_trap_1: 658 /* 659 * at this point: 660 * %g2 - contains the VA whose lookup caused the trap 661 * %g3 - contains the tlo_info field 662 * %g4 - contains the TLB access index value for the 663 * VA/PgSz in question 664 * 665 * Here we will clear the errors from the TLB. 666 */ 667 set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of 668 stxa %g0, [%g5]ASI_IMMU ! 0 as it will be invalid. 669 stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write the data and tag 670 membar #Sync 671 672 set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 673 or %g4, %g6, %g4 674 675 stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write same data and tag 676 membar #Sync 677 678 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a 679 flush %g6 ! flush after writing MMU regs 680 681 /* 682 * at this point: 683 * %g2 - contains the VA whose lookup caused the trap 684 * %g3 - contains the tlo_info field 685 * 686 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're 687 * already at PIL 15. */ 688 set cpu_tlb_parity_error, %g1 689 rdpr %pil, %g4 690 cmp %g4, PIL_14 691 movl %icc, PIL_14, %g4 692 ba sys_trap 693 nop 694 SET_SIZE(itlb_parity_trap) 695 696 /* 697 * The CPU jumps here from the MMU exception handler if a DTLB parity 698 * error is detected and we are running on Panther. 699 * 700 * In this routine we collect diagnostic information and write it to our 701 * logout structure (if possible) and clear all DTLB entries that may have 702 * caused our parity trap. 703 * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 704 * and log any error messages. As for parameters to cpu_tlb_parity_error, we 705 * send two: 706 * 707 * %g2 - Contains the VA whose lookup in the DTLB caused the parity error 708 * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, 709 * regardless of whether or not we actually used the logout struct. 710 * 711 * In the TL0 handler (cpu_tlb_parity_error) we will compare those two 712 * parameters to the data contained in the logout structure in order to 713 * determine whether the logout information is valid for this particular 714 * error or not. 715 */ 716 ENTRY_NP(dtlb_parity_trap) 717 /* 718 * Collect important information about the trap which will be 719 * used as a parameter to the TL0 handler. 720 */ 721 wr %g0, ASI_DMMU, %asi 722 ldxa [MMU_SFAR]%asi, %g2 ! VA that caused the IMMU trap 723 ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page sizes 724 set PN_DTLB_PGSZ_MASK, %g4 725 and %g3, %g4, %g3 726 ldxa [MMU_TAG_ACCESS]%asi, %g4 727 set TAGREAD_CTX_MASK, %g5 ! 'or' in the trap context 728 and %g4, %g5, %g4 ! to complete the tlo_info 729 or %g4, %g3, %g3 ! field for logout 730 stxa %g0,[MMU_SFSR]%asi ! clear the SFSR 731 membar #Sync 732 733 /* 734 * at this point: 735 * %g2 - contains the VA whose lookup caused the trap 736 * %g3 - contains the tlo_info field 737 * 738 * Calculate the TLB index values for the failing VA. Since the T512 739 * TLBs can be configured for different page sizes, we need to find 740 * the index into each one separately. 741 */ 742 mov %g2, %g4 ! First we get the DTLB_0 index 743 set PN_DTLB_PGSZ0_MASK, %g5 744 and %g3, %g5, %g5 745 srlx %g5, PN_DTLB_PGSZ0_SHIFT, %g5 746 PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the DTLB_0 index 747 sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place 748 set PN_DTLB_T512_0, %g5 749 or %g4, %g5, %g4 ! and add in the TLB ID 750 751 mov %g2, %g7 ! Next we get the DTLB_1 index 752 set PN_DTLB_PGSZ1_MASK, %g5 753 and %g3, %g5, %g5 754 srlx %g5, PN_DTLB_PGSZ1_SHIFT, %g5 755 PN_GET_TLB_INDEX(%g7, %g5) ! %g7 has the DTLB_1 index 756 sllx %g7, PN_TLB_ACC_IDX_SHIFT, %g7 ! shift the index into place 757 set PN_DTLB_T512_1, %g5 758 or %g7, %g5, %g7 ! and add in the TLB ID 759 760 /* 761 * at this point: 762 * %g2 - contains the VA whose lookup caused the trap 763 * %g3 - contains the tlo_info field 764 * %g4 - contains the T512_0 access index value for the 765 * VA/PgSz in question 766 * %g7 - contains the T512_1 access index value for the 767 * VA/PgSz in question 768 * 769 * If this trap happened at TL>0, then we don't want to mess 770 * with the normal logout struct since that could caused a TLB 771 * miss. 772 */ 773 rdpr %tl, %g6 ! read current trap level 774 cmp %g6, 1 ! skip over the tl>1 code 775 ble dtlb_parity_trap_1 ! if TL <= 1. 776 nop 777 778 /* 779 * If we are here, then the trap happened at TL>1. Simply 780 * update our tlo_info field and then skip to the TLB flush 781 * code. 782 */ 783 mov 1, %g6 784 sllx %g6, PN_TLO_INFO_TL1_SHIFT, %g6 785 or %g6, %g3, %g3 786 ba dtlb_parity_trap_2 787 nop 788 789 dtlb_parity_trap_1: 790 /* 791 * at this point: 792 * %g2 - contains the VA whose lookup caused the trap 793 * %g3 - contains the tlo_info field 794 * %g4 - contains the T512_0 access index value for the 795 * VA/PgSz in question 796 * %g7 - contains the T512_1 access index value for the 797 * VA/PgSz in question 798 * 799 * Check to see if the logout structure is available. 800 */ 801 set CHPR_TLB_LOGOUT, %g6 802 GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, dtlb_parity_trap_2) 803 set LOGOUT_INVALID_U32, %g6 804 sllx %g6, 32, %g6 ! if our logout structure is 805 set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is 806 or %g5, %g6, %g5 ! already being used, then we 807 ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic 808 cmp %g6, %g5 ! information before clearing 809 bne dtlb_parity_trap_2 ! and logging the error. 810 nop 811 812 /* 813 * Record the logout information. %g4 contains our DTLB_0 814 * index + TLB ID and %g7 contains our DTLB_1 index + TLB ID 815 * both of which will be used for ASI_DTLB_ACCESS and 816 * ASI_DTLB_TAGREAD. %g1 contains the pointer to our logout 817 * struct. 818 */ 819 stx %g3, [%g1 + PN_TLO_INFO] 820 stx %g2, [%g1 + PN_TLO_ADDR] 821 rdpr %tpc, %g5 822 stx %g5, [%g1 + PN_TLO_PC] 823 824 add %g1, PN_TLO_DTLB_TTE, %g1 ! move up the pointer 825 826 ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 827 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 0 and store it away 828 ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 829 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 0 and store it away 830 831 ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 0 832 stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] 833 ldxa [%g7]ASI_DTLB_TAGREAD, %g5 834 stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] 835 836 set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 837 or %g4, %g6, %g4 ! of each TLB. 838 or %g7, %g6, %g7 839 add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer 840 841 ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 842 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 1 and store it away 843 ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 844 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 1 and store it away 845 846 ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 1 847 stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] 848 ldxa [%g7]ASI_DTLB_TAGREAD, %g5 849 stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] 850 851 andn %g4, %g6, %g4 ! back to way 0 852 andn %g7, %g6, %g7 ! back to way 0 853 854 dtlb_parity_trap_2: 855 /* 856 * at this point: 857 * %g2 - contains the VA whose lookup caused the trap 858 * %g3 - contains the tlo_info field 859 * %g4 - contains the T512_0 access index value for the 860 * VA/PgSz in question 861 * %g7 - contains the T512_1 access index value for the 862 * VA/PgSz in question 863 * 864 * Here we will clear the errors from the DTLB. 865 */ 866 set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of 867 stxa %g0, [%g5]ASI_DMMU ! 0 as it will be invalid. 868 stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write the data and tag. 869 stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 870 membar #Sync 871 872 set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 873 or %g4, %g6, %g4 874 or %g7, %g6, %g7 875 876 stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write same data and tag. 877 stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 878 membar #Sync 879 880 sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a 881 flush %g6 ! flush after writing MMU regs 882 883 /* 884 * at this point: 885 * %g2 - contains the VA whose lookup caused the trap 886 * %g3 - contains the tlo_info field 887 * 888 * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're 889 * already at PIL 15. We do this even for TL>1 traps since 890 * those will lead to a system panic. 891 */ 892 set cpu_tlb_parity_error, %g1 893 rdpr %pil, %g4 894 cmp %g4, PIL_14 895 movl %icc, PIL_14, %g4 896 ba sys_trap 897 nop 898 SET_SIZE(dtlb_parity_trap) 899 900 901 /* 902 * Calculates the Panther TLB index based on a virtual address and page size 903 * 904 * Register usage: 905 * %o0 - virtual address whose index we want 906 * %o1 - Page Size of the TLB in question as encoded in the 907 * ASI_[D|I]MMU_TAG_ACCESS_EXT register. 908 */ 909 ENTRY(pn_get_tlb_index) 910 911 PN_GET_TLB_INDEX(%o0, %o1) 912 913 retl 914 nop 915 SET_SIZE(pn_get_tlb_index) 916 917 918 /* 919 * For Panther CPUs we need to flush the IPB after any I$ or D$ 920 * parity errors are detected. 921 */ 922 ENTRY(flush_ipb) 923 clr %o0 924 925 flush_ipb_1: 926 stxa %g0, [%o0]ASI_IPB_TAG 927 membar #Sync 928 cmp %o0, PN_IPB_TAG_ADDR_MAX 929 blt flush_ipb_1 930 add %o0, PN_IPB_TAG_ADDR_LINESIZE, %o0 931 932 sethi %hi(FLUSH_ADDR), %o0 933 flush %o0 934 retl 935 nop 936 SET_SIZE(flush_ipb) 937 938