1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Assembly code support for the jalapeno module 27 */ 28 29 #include "assym.h" 30 31 #include <sys/asm_linkage.h> 32 #include <sys/mmu.h> 33 #include <vm/hat_sfmmu.h> 34 #include <sys/machparam.h> 35 #include <sys/machcpuvar.h> 36 #include <sys/machthread.h> 37 #include <sys/machtrap.h> 38 #include <sys/privregs.h> 39 #include <sys/asm_linkage.h> 40 #include <sys/trap.h> 41 #include <sys/cheetahregs.h> 42 #include <sys/us3_module.h> 43 #include <sys/xc_impl.h> 44 #include <sys/intreg.h> 45 #include <sys/async.h> 46 #include <sys/clock.h> 47 #include <sys/cheetahasm.h> 48 49 #ifdef TRAPTRACE 50 #include <sys/traptrace.h> 51 #endif /* TRAPTRACE */ 52 53 /* BEGIN CSTYLED */ 54 55 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 56 57 #define CHK_JP_ERRATA85_ENABLED(scr, label) \ 58 ASM_LD(scr, jp_errata_85_active); \ 59 cmp scr, 1; \ 60 bne %icc, label; \ 61 nop 62 63 #define SET_64BIT_PA(dest, scr, hi32, lo32) \ 64 set hi32, scr; \ 65 sllx scr, 32, scr; \ 66 sethi %hi(lo32), dest; \ 67 or dest, %lo(lo32), dest; \ 68 or scr, dest, dest 69 70 /* 71 * Macro to trigger Jalapeno/Tomatillo speed change 72 * j_chng_pa - scratch register 73 * scr - scratch register 74 */ 75 #define JP_ESTAR_TRIGGER(j_chng_pa, scr) \ 76 SET_64BIT_PA(j_chng_pa, scr, TOM_HIGH_PA, M_T_J_CHNG_INIT_PA); \ 77 ldxa [j_chng_pa]ASI_IO, scr; \ 78 5: \ 79 and scr, TOM_TRIGGER_MASK, scr; \ 80 cmp scr, TOM_TRIGGER; \ 81 be,pt %icc, 5b; /* wait while 10 */ \ 82 ldxa [j_chng_pa]ASI_IO, scr; \ 83 andn scr, TOM_TRIGGER_MASK, scr; \ 84 stxa scr, [j_chng_pa]ASI_IO; /* clear j_chng[1:0] */ \ 85 or scr, TOM_TRIGGER, scr; \ 86 stxa scr, [j_chng_pa]ASI_IO; /* trigger j_chng */ \ 87 ldxa [j_chng_pa]ASI_IO, scr; \ 88 6: \ 89 and scr, TOM_TRIGGER_MASK, scr; \ 90 cmp scr, TOM_TRIGGER; \ 91 be,pt %icc, 6b; /* wait while 10 */ \ 92 ldxa [j_chng_pa]ASI_IO, scr; \ 93 andn scr, TOM_TRIGGER_MASK, scr; \ 94 stxa scr, [j_chng_pa]ASI_IO; /* deassert j_chng */ 95 96 /* 97 * Macro to set Jalapeno CPU speed 98 * speed - new speed constant 99 * scr1 - scratch register 100 * scr2 - scratch register 101 */ 102 #define SET_JP_SPEED(speed, scr1, scr2) \ 103 ldxa [%g0]ASI_JBUS_CONFIG, scr1; \ 104 set JBUS_CONFIG_ECLK_MASK, scr2; \ 105 andn scr1, scr2, scr1; \ 106 set speed, scr2; \ 107 or scr1, scr2, scr1; \ 108 stxa scr1, [%g0]ASI_JBUS_CONFIG; 109 110 /* 111 * macro to set Master Tomatillo speed 112 * speed - tomatillo speed constant 113 * tpa - tomatillo estar control register PA 114 * scr - scratch register 115 */ 116 #define SET_TOM_SPEED(speed, tpa, scr) \ 117 ldxa [tpa]ASI_IO, scr; \ 118 andn scr, TOM_ESTAR_ELCK_MASK, scr; \ 119 or scr, speed, scr; \ 120 stxa scr, [tpa]ASI_IO; 121 122 /* 123 * macro to check and set Slave Tomatillo speed 124 * speed - tomatillo speed constant 125 * scr1 - scratch register 126 * scr2 - scratch register 127 */ 128 129 #define SET_SLAVE_T_SPEED(speed, scr1, scr2) \ 130 ldxa [%g0]ASI_JBUS_CONFIG, scr2; \ 131 srlx scr2, JBUS_SLAVE_T_PORT_BIT, scr2; \ 132 btst 1, scr2; \ 133 bz,pt %icc, 4f; \ 134 nop; \ 135 SET_64BIT_PA(scr1, scr2, TOM_HIGH_PA, S_T_ESTAR_CTRL_PA); \ 136 SET_TOM_SPEED(speed, scr1, scr2); \ 137 4: 138 139 140 /* 141 * macro to adjust ASI_MCU_CTL_REG1[26:25] fsm bits according to 142 * new cpu speed: fsm[1:0]=11b for full speed, fsm[1:0]=0 for estar speed 143 * value - fsm bit value constant 144 * scr1 - scratch register 145 * scr2 - scratch register 146 */ 147 #define JP_ADJUST_FSM(value, scr1, scr2) \ 148 ldxa [%g0]ASI_MCU_CTRL, scr1; \ 149 set JP_MCU_FSM_MASK, scr2; \ 150 andn scr1, scr2, scr1; \ 151 set value, scr2; \ 152 or scr1, scr2, scr1; \ 153 stxa scr1, [%g0]ASI_MCU_CTRL; \ 154 membar #Sync; 155 156 /* 157 * JP_FORCE_FULL_SPEED and its fellow macros are for Jalapeno 158 * workstation to work around Errata 85. The front portion of 159 * it packs JP speed(14..13) and Tomatillo speed(5..0) into one 160 * register. 161 * 162 * Current code assumes that these two fields are non-overlapping. 163 * If that assumption changes, then this code won't work. If so, we 164 * force a compile time error by not defining the JP_FORCE_FULL_SPEED 165 * and JP_RESTORE_SPEED macros below. 166 */ 167 168 #if !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) 169 170 /* 171 * Macro to force Jalapeno/Tomatillo to full speed 172 * old_lvl - register used to save original cpu, tomatillo speed 173 * scr2 - scratch register 174 * scr3 - scratch register 175 * scr4 - scratch register 176 */ 177 #define JP_FORCE_FULL_SPEED(old_lvl, scr2, scr3, scr4) \ 178 ldxa [%g0]ASI_JBUS_CONFIG, old_lvl; \ 179 set JBUS_CONFIG_ECLK_MASK, scr4; \ 180 and old_lvl, scr4, old_lvl; \ 181 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \ 182 ldxa [scr2]ASI_IO, scr3; \ 183 set TOM_ESTAR_ELCK_MASK, scr4; \ 184 and scr3, scr4, scr3; \ 185 or old_lvl, scr3, old_lvl; \ 186 /* original jp and tomatillo speed saved in old_lvl */ \ 187 \ 188 /* either intended or currently at full speed */ \ 189 set JBUS_CONFIG_ECLK_MASK, scr4; \ 190 andcc old_lvl, scr4, %g0; \ 191 bz,pt %icc, 8f; \ 192 nop; \ 193 /* go through 1/2 speed. */ \ 194 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \ 195 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \ 196 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \ 197 JP_ADJUST_FSM(0, scr3, scr4); \ 198 set jp_estar_tl0_data, scr3; \ 199 ldx [scr3], %g0; \ 200 membar #Sync; /* or busy wait 1us */ \ 201 JP_ESTAR_TRIGGER(scr3, scr4); \ 202 8: \ 203 /* bring to 1:1 speed */ \ 204 SET_JP_SPEED(JBUS_CONFIG_ECLK_1, scr3, scr4); \ 205 SET_TOM_SPEED(TOM_FULL_SPEED, scr2, scr3); \ 206 SET_SLAVE_T_SPEED(TOM_FULL_SPEED, scr3, scr4); \ 207 JP_ADJUST_FSM(JP_MCU_FSM_MASK, scr3, scr4); \ 208 JP_ESTAR_TRIGGER(scr3, scr4) 209 210 211 /* 212 * Macro to restore Jalapeno/Tomatillo to original speed 213 * old_lvl - register contains saved original cpu, tomatillo speed 214 * scr2 - scratch register 215 * scr3 - scratch register 216 * scr4 - scratch register 217 * 218 * If trap had occured in the middle of ppm cpu speed transtion, then 219 * old_lvl[31:10] contains the intended new speed written into jbus_config. 220 * if old_lvl[9:0] is inconsistent with old_lvl[31:10], then the trap surely 221 * interrupted the ppm cpu speed transition, otherwise nothing for sure. 222 * We'll restore the intended/then-current speed, that should cause no 223 * trouble to subsequent ppm cpu speed change code. 224 */ 225 #define JP_RESTORE_SPEED(old_lvl, scr2, scr3, scr4) \ 226 srlx old_lvl, JBUS_CONFIG_ECLK_SHIFT, scr2; \ 227 and scr2, 3, scr2; \ 228 add scr2, 1, scr2; \ 229 cmp scr2, 3; \ 230 bne,pt %icc, 7f; \ 231 nop; \ 232 set TOM_SLOW_SPEED, scr2; \ 233 /* scr2 contains tom speed according to intended jp speed */ \ 234 7: \ 235 andn old_lvl, TOM_ESTAR_ELCK_MASK, old_lvl; \ 236 or scr2, old_lvl, old_lvl; \ 237 /* updated old_lvl to contain intended jp and tom speed */ \ 238 andcc old_lvl, TOM_FULL_SPEED, %g0; \ 239 bnz,pt %icc, 9f; /* intended full, already at full */ \ 240 nop; \ 241 \ 242 /* go to half speed */ \ 243 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \ 244 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \ 245 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \ 246 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \ 247 JP_ADJUST_FSM(0, scr3, scr4); \ 248 set jp_estar_tl0_data, scr3; \ 249 ldx [scr3], %g0; \ 250 membar #Sync; \ 251 JP_ESTAR_TRIGGER(scr3, scr4); \ 252 andcc old_lvl, TOM_SLOW_SPEED, %g0; \ 253 bz,pt %icc, 9f; /* intended 1:2, already at 1:2 */ \ 254 nop; \ 255 \ 256 /* go to 1:32 speed */ \ 257 SET_JP_SPEED(JBUS_CONFIG_ECLK_32, scr3, scr4); \ 258 SET_TOM_SPEED(TOM_SLOW_SPEED, scr2, scr3); \ 259 SET_SLAVE_T_SPEED(TOM_SLOW_SPEED, scr3, scr4); \ 260 JP_ESTAR_TRIGGER(scr3, scr4); \ 261 9: 262 263 #endif /* !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) */ 264 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 265 266 /* 267 * Jalapeno version to reflush an Ecache line by index. 268 * Will flush all 4 ways (with only one scratch register). 269 * Note that the code will be faster if we use 2 scratch registers. 270 */ 271 #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr1) \ 272 JP_EC_DIAG_ACCESS_MEMBAR; \ 273 ldxa [index]ASI_EC_DIAG, %g0; \ 274 JP_EC_DIAG_ACCESS_MEMBAR; \ 275 mov 1, scr1; \ 276 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 277 add scr1, index, scr1; \ 278 JP_EC_DIAG_ACCESS_MEMBAR; \ 279 ldxa [scr1]ASI_EC_DIAG, %g0; \ 280 JP_EC_DIAG_ACCESS_MEMBAR; \ 281 mov 2, scr1; \ 282 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 283 add scr1, index, scr1; \ 284 JP_EC_DIAG_ACCESS_MEMBAR; \ 285 ldxa [scr1]ASI_EC_DIAG, %g0; \ 286 JP_EC_DIAG_ACCESS_MEMBAR; \ 287 mov 3, scr1; \ 288 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 289 add scr1, index, scr1; \ 290 JP_EC_DIAG_ACCESS_MEMBAR; \ 291 ldxa [scr1]ASI_EC_DIAG, %g0; \ 292 JP_EC_DIAG_ACCESS_MEMBAR 293 294 /* 295 * Jalapeno version of ecache_flush_line. Uses Jalapeno Ecache Displacement 296 * Flush feature to flush all 4 sets/ways. 297 */ 298 #define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \ 299 CPU_INDEX(scr1, scr2); \ 300 sllx scr1, JP_ECFLUSH_PORTID_SHIFT, scr1; \ 301 set JP_ECACHE_IDX_DISP_FLUSH, scr2; \ 302 or scr2, scr1, scr2; \ 303 sub ec_set_size, 1, scr1; \ 304 and physaddr, scr1, scr1; \ 305 or scr2, scr1, scr1; \ 306 ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2) 307 308 /* 309 * Macro for getting ecache size from cpunodes structure 310 * scr1: Scratch, ecache size returned in this 311 * scr2: Scratch 312 */ 313 #define GET_ECACHE_SIZE(scr1, scr2) \ 314 CPU_INDEX(scr1, scr2); \ 315 mulx scr1, CPU_NODE_SIZE, scr1; \ 316 set cpunodes + ECACHE_SIZE, scr2; \ 317 ld [scr1 + scr2], scr1 318 319 /* END CSTYLED */ 320 321 /* 322 * Ship mondo to aid using implicit busy/nack pair (bn ignored) 323 */ 324 ENTRY_NP(shipit) 325 sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = agent id 326 or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70 327 stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch 328 membar #Sync 329 retl 330 nop 331 SET_SIZE(shipit) 332 333 334 /* 335 * flush_ecache: 336 * %o0 - 64 bit physical address 337 * %o1 - ecache size 338 * %o2 - ecache linesize 339 */ 340 341 ENTRY(flush_ecache) 342 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 343 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_1); 344 JP_FORCE_FULL_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */ 345 flush_ecache_1: 346 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 347 348 /* 349 * Flush the entire Ecache using displacement flush. 350 */ 351 ECACHE_FLUSHALL(%o1, %o2, %o0, %o4) 352 353 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 354 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_2); 355 JP_RESTORE_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */ 356 flush_ecache_2: 357 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 358 359 retl 360 nop 361 SET_SIZE(flush_ecache) 362 363 364 .section ".text" 365 .align 64 366 ENTRY_NP(fast_ecc_err) 367 368 /* 369 * Turn off CEEN and NCEEN. 370 */ 371 ldxa [%g0]ASI_ESTATE_ERR, %g3 372 andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 373 stxa %g4, [%g0]ASI_ESTATE_ERR 374 membar #Sync ! membar sync required 375 376 /* 377 * Do the CPU log out capture. 378 * %g3 = "failed?" return value. 379 * %g2 = Input = AFAR. Output the clo_flags info which is passed 380 * into this macro via %g4. Output only valid if cpu_private 381 * struct has not been initialized. 382 * CHPR_FECCTL0_LOGOUT = cpu logout structure offset input 383 * %g4 = Trap information stored in the cpu logout flags field 384 * %g5 = scr1 385 * %g6 = scr2 386 * %g3 = scr3 387 * %g4 = scr4 388 */ 389 and %g3, EN_REG_CEEN, %g4 ! store the CEEN value, TL=0 390 set CHPR_FECCTL0_LOGOUT, %g6 391 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 392 393 /* 394 * Flush the Ecache to get the error out of the Ecache. If the UCC 395 * or UCU is on a dirty line, then the following flush will turn 396 * that into a WDC or WDU, respectively. 397 */ 398 CPU_INDEX(%g4, %g5) 399 mulx %g4, CPU_NODE_SIZE, %g4 400 set cpunodes, %g5 401 add %g4, %g5, %g4 402 ld [%g4 + ECACHE_LINESIZE], %g5 403 ld [%g4 + ECACHE_SIZE], %g4 404 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 405 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_1); 406 set jp_estar_tl0_data, %g6 407 stx %g2, [%g6 + 0] 408 stx %g3, [%g6 + 8] 409 JP_FORCE_FULL_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */ 410 fast_ecc_err_1: 411 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 412 ECACHE_FLUSHALL(%g4, %g5, %g6, %g7) 413 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 414 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_2); 415 JP_RESTORE_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */ 416 set jp_estar_tl0_data, %g6 417 ldx [%g6 + 0], %g2 418 ldx [%g6 + 8], %g3 419 fast_ecc_err_2: 420 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 421 422 /* 423 * Flush the Dcache. Since bad data could have been installed in 424 * the Dcache we must flush it before re-enabling it. 425 */ 426 ASM_LD(%g5, dcache_size) 427 ASM_LD(%g6, dcache_linesize) 428 CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 429 430 /* 431 * Flush the Icache. Since we turned off the Icache to capture the 432 * Icache line it is now stale or corrupted and we must flush it 433 * before re-enabling it. 434 */ 435 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_4); 436 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 437 ba,pt %icc, 5f 438 ld [%g5 + CHPR_ICACHE_SIZE], %g5 439 fast_ecc_err_4: 440 ASM_LD(%g5, icache_size) 441 ASM_LD(%g6, icache_linesize) 442 5: 443 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 444 445 /* 446 * Restore the Dcache and Icache to the previous state. 447 */ 448 stxa %g1, [%g0]ASI_DCU 449 flush %g0 /* flush required after changing the IC bit */ 450 451 /* 452 * Make sure our CPU logout operation was successful. 453 */ 454 cmp %g3, %g0 455 be 8f 456 nop 457 458 /* 459 * If the logout structure had been busy, how many times have 460 * we tried to use it and failed (nesting count)? If we have 461 * already recursed a substantial number of times, then we can 462 * assume things are not going to get better by themselves and 463 * so it would be best to panic. 464 */ 465 cmp %g3, CLO_NESTING_MAX 466 blt 7f 467 nop 468 469 call ptl1_panic 470 mov PTL1_BAD_ECC, %g1 471 472 7: 473 /* 474 * Otherwise, if the logout structure was busy but we have not 475 * nested more times than our maximum value, then we simply 476 * issue a retry. Our TL=0 trap handler code will check and 477 * clear the AFSR after it is done logging what is currently 478 * in the logout struct and handle this event at that time. 479 */ 480 retry 481 8: 482 /* 483 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're 484 * already at PIL 15. 485 */ 486 set cpu_fast_ecc_error, %g1 487 rdpr %pil, %g4 488 cmp %g4, PIL_14 489 ba sys_trap 490 movl %icc, PIL_14, %g4 491 492 SET_SIZE(fast_ecc_err) 493 494 495 /* 496 * Fast ECC error at TL>0 handler 497 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter 498 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 499 * For a complete description of the Fast ECC at TL>0 handling see the 500 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in 501 * us3_common_asm.s 502 */ 503 504 .section ".text" 505 .align 64 506 ENTRY_NP(fast_ecc_tl1_err) 507 508 /* 509 * This macro turns off the D$/I$ if they are on and saves their 510 * original state in ch_err_tl1_tmp, saves all the %g registers in the 511 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves 512 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will 513 * point to the ch_err_tl1_data structure and the original D$/I$ state 514 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 515 * will be available. 516 */ 517 CH_ERR_TL1_FECC_ENTER; 518 519 /* 520 * Get the diagnostic logout data. %g4 must be initialized to 521 * current CEEN state, %g5 must point to logout structure in 522 * ch_err_tl1_data_t. %g3 will contain the nesting count upon 523 * return. 524 */ 525 ldxa [%g0]ASI_ESTATE_ERR, %g4 526 and %g4, EN_REG_CEEN, %g4 527 add %g1, CH_ERR_TL1_LOGOUT, %g5 528 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) 529 530 /* 531 * If the logout nesting count is exceeded, we're probably 532 * not making any progress, try to panic instead. 533 */ 534 cmp %g3, CLO_NESTING_MAX 535 bge fecc_tl1_err 536 nop 537 538 /* 539 * Save the current CEEN and NCEEN state in %g7 and turn them off 540 * before flushing the Ecache. 541 */ 542 ldxa [%g0]ASI_ESTATE_ERR, %g7 543 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 544 stxa %g5, [%g0]ASI_ESTATE_ERR 545 membar #Sync 546 547 /* 548 * Flush the Ecache, using the largest possible cache size with the 549 * smallest possible line size since we can't get the actual sizes 550 * from the cpu_node due to DTLB misses. 551 */ 552 set JP_ECACHE_MAX_SIZE, %g4 553 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 554 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_1); 555 set jp_estar_tl1_data, %g6 556 stx %g2, [%g6 + 0] 557 stx %g3, [%g6 + 8] 558 JP_FORCE_FULL_SPEED(%g2, %g3, %g5, %g6) 559 fast_ecc_tl1_err_1: 560 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 561 ECACHE_FLUSHALL(%g4, JP_ECACHE_MAX_LSIZE, %g5, %g6) 562 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 563 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_2); 564 JP_RESTORE_SPEED(%g2, %g3, %g5, %g6) 565 set jp_estar_tl1_data, %g6 566 ldx [%g6 + 0], %g2 567 ldx [%g6 + 8], %g3 568 fast_ecc_tl1_err_2: 569 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 570 571 /* 572 * Restore CEEN and NCEEN to the previous state. 573 */ 574 stxa %g7, [%g0]ASI_ESTATE_ERR 575 membar #Sync 576 577 /* 578 * If we turned off the D$, then flush it and turn it back on. 579 */ 580 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 581 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 582 bz %xcc, 3f 583 nop 584 585 /* 586 * Flush the D$. 587 */ 588 ASM_LD(%g4, dcache_size) 589 ASM_LD(%g5, dcache_linesize) 590 CH_DCACHE_FLUSHALL(%g4, %g5, %g6) 591 592 /* 593 * Turn the D$ back on. 594 */ 595 ldxa [%g0]ASI_DCU, %g3 596 or %g3, DCU_DC, %g3 597 stxa %g3, [%g0]ASI_DCU 598 membar #Sync 599 3: 600 /* 601 * If we turned off the I$, then flush it and turn it back on. 602 */ 603 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 604 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 605 bz %xcc, 4f 606 nop 607 608 /* 609 * Flush the I$. 610 */ 611 ASM_LD(%g4, icache_size) 612 ASM_LD(%g5, icache_linesize) 613 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) 614 615 /* 616 * Turn the I$ back on. Changing DCU_IC requires flush. 617 */ 618 ldxa [%g0]ASI_DCU, %g3 619 or %g3, DCU_IC, %g3 620 stxa %g3, [%g0]ASI_DCU 621 flush %g0 622 4: 623 624 #ifdef TRAPTRACE 625 /* 626 * Get current trap trace entry physical pointer. 627 */ 628 CPU_INDEX(%g6, %g5) 629 sll %g6, TRAPTR_SIZE_SHIFT, %g6 630 set trap_trace_ctl, %g5 631 add %g6, %g5, %g6 632 ld [%g6 + TRAPTR_LIMIT], %g5 633 tst %g5 634 be %icc, skip_traptrace 635 nop 636 ldx [%g6 + TRAPTR_PBASE], %g5 637 ld [%g6 + TRAPTR_OFFSET], %g4 638 add %g5, %g4, %g5 639 640 /* 641 * Create trap trace entry. 642 */ 643 rd %asi, %g7 644 wr %g0, TRAPTR_ASI, %asi 645 rd STICK, %g4 646 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 647 rdpr %tl, %g4 648 stha %g4, [%g5 + TRAP_ENT_TL]%asi 649 rdpr %tt, %g4 650 stha %g4, [%g5 + TRAP_ENT_TT]%asi 651 rdpr %tpc, %g4 652 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 653 rdpr %tstate, %g4 654 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 655 stna %sp, [%g5 + TRAP_ENT_SP]%asi 656 stna %g0, [%g5 + TRAP_ENT_TR]%asi 657 wr %g0, %g7, %asi 658 ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 659 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 660 wr %g0, TRAPTR_ASI, %asi 661 stna %g3, [%g5 + TRAP_ENT_F1]%asi 662 stna %g4, [%g5 + TRAP_ENT_F2]%asi 663 wr %g0, %g7, %asi 664 ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 665 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 666 wr %g0, TRAPTR_ASI, %asi 667 stna %g3, [%g5 + TRAP_ENT_F3]%asi 668 stna %g4, [%g5 + TRAP_ENT_F4]%asi 669 wr %g0, %g7, %asi 670 671 /* 672 * Advance trap trace pointer. 673 */ 674 ld [%g6 + TRAPTR_OFFSET], %g5 675 ld [%g6 + TRAPTR_LIMIT], %g4 676 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 677 add %g5, TRAP_ENT_SIZE, %g5 678 sub %g4, TRAP_ENT_SIZE, %g4 679 cmp %g5, %g4 680 movge %icc, 0, %g5 681 st %g5, [%g6 + TRAPTR_OFFSET] 682 skip_traptrace: 683 #endif /* TRAPTRACE */ 684 685 /* 686 * If nesting count is not zero, skip all the AFSR/AFAR 687 * handling and just do the necessary cache-flushing. 688 */ 689 ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 690 brnz %g2, 6f 691 nop 692 693 /* 694 * If a UCU followed by a WDU has occurred go ahead and panic 695 * since a UE will occur (on the retry) before the UCU and WDU 696 * messages are enqueued. 697 */ 698 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g3 699 set 1, %g4 700 sllx %g4, C_AFSR_UCU_SHIFT, %g4 701 btst %g4, %g3 ! UCU in original AFSR? 702 bz %xcc, 6f 703 nop 704 ldxa [%g0]ASI_AFSR, %g4 ! current AFSR 705 or %g3, %g4, %g3 ! %g3 = original + current AFSR 706 set 1, %g4 707 sllx %g4, C_AFSR_WDU_SHIFT, %g4 708 btst %g4, %g3 ! WDU in original or current AFSR? 709 bnz %xcc, fecc_tl1_err 710 nop 711 712 6: 713 /* 714 * We fall into this macro if we've successfully logged the error in 715 * the ch_err_tl1_data structure and want the PIL15 softint to pick 716 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 717 * Restores the %g registers and issues retry. 718 */ 719 CH_ERR_TL1_EXIT; 720 /* 721 * Establish panic exit label. 722 */ 723 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); 724 725 SET_SIZE(fast_ecc_tl1_err) 726 727 728 ENTRY(get_jbus_config) 729 ldxa [%g0]ASI_JBUS_CONFIG, %o0 730 retl 731 nop 732 SET_SIZE(get_jbus_config) 733 734 ENTRY(set_jbus_config) 735 stxa %o0, [%g0]ASI_JBUS_CONFIG 736 membar #Sync 737 retl 738 nop 739 SET_SIZE(set_jbus_config) 740 741 742 ENTRY(get_mcu_ctl_reg1) 743 ldxa [%g0]ASI_MCU_CTRL, %o0 ! MCU control reg1 is at offset 0 744 retl 745 nop 746 SET_SIZE(get_mcu_ctl_reg1) 747 748 749 ENTRY(set_mcu_ctl_reg1) 750 stxa %o0, [%g0]ASI_MCU_CTRL ! MCU control reg1 is at offset 0 751 membar #Sync 752 retl 753 nop 754 SET_SIZE(set_mcu_ctl_reg1) 755 756 757 /* 758 * scrubphys - Pass in the aligned physical memory address 759 * that you want to scrub, along with the ecache set size. 760 * 761 * 1) Displacement flush the E$ line corresponding to %addr. 762 * The first ldxa guarantees that the %addr is no longer in 763 * M, O, or E (goes to I or S (if instruction fetch also happens). 764 * 2) "Write" the data using a CAS %addr,%g0,%g0. 765 * The casxa guarantees a transition from I to M or S to M. 766 * 3) Displacement flush the E$ line corresponding to %addr. 767 * The second ldxa pushes the M line out of the ecache, into the 768 * writeback buffers, on the way to memory. 769 * 4) The "membar #Sync" pushes the cache line out of the writeback 770 * buffers onto the bus, on the way to dram finally. 771 * 772 * This is a modified version of the algorithm suggested by Gary Lauterbach. 773 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line 774 * as modified, but then we found out that for spitfire, if it misses in the 775 * E$ it will probably install as an M, but if it hits in the E$, then it 776 * will stay E, if the store doesn't happen. So the first displacement flush 777 * should ensure that the CAS will miss in the E$. Arrgh. 778 */ 779 ENTRY(scrubphys) 780 rdpr %pstate, %o4 781 andn %o4, PSTATE_IE | PSTATE_AM, %o5 782 wrpr %o5, %g0, %pstate ! clear IE, AM bits 783 784 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 785 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_1); 786 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 787 scrubphys_1: 788 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 789 790 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 791 casxa [%o0]ASI_MEM, %g0, %g0 792 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 793 794 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 795 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_2); 796 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 797 scrubphys_2: 798 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 799 800 wrpr %g0, %o4, %pstate ! restore earlier pstate register value 801 802 retl 803 membar #Sync ! move the data out of the load buffer 804 SET_SIZE(scrubphys) 805 806 /* 807 * clearphys - Pass in the physical memory address of the checkblock 808 * that you want to push out, cleared with a recognizable pattern, 809 * from the ecache. 810 * 811 * To ensure that the ecc gets recalculated after the bad data is cleared, 812 * we must write out enough data to fill the w$ line (64 bytes). So we read 813 * in an entire ecache subblock's worth of data, and write it back out. 814 * Then we overwrite the 16 bytes of bad data with the pattern. 815 */ 816 ENTRY(clearphys) 817 /* turn off IE, AM bits */ 818 rdpr %pstate, %o4 819 andn %o4, PSTATE_IE | PSTATE_AM, %o5 820 wrpr %o5, %g0, %pstate 821 822 /* turn off NCEEN */ 823 ldxa [%g0]ASI_ESTATE_ERR, %o5 824 andn %o5, EN_REG_NCEEN, %o3 825 stxa %o3, [%g0]ASI_ESTATE_ERR 826 membar #Sync 827 828 /* align address passed with 64 bytes subblock size */ 829 mov CH_ECACHE_SUBBLK_SIZE, %o2 830 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 831 832 /* move the good data into the W$ */ 833 1: 834 subcc %o2, 8, %o2 835 ldxa [%g1 + %o2]ASI_MEM, %g2 836 bge 1b 837 stxa %g2, [%g1 + %o2]ASI_MEM 838 839 /* now overwrite the bad data */ 840 setx 0xbadecc00badecc01, %g1, %g2 841 stxa %g2, [%o0]ASI_MEM 842 mov 8, %g1 843 stxa %g2, [%o0 + %g1]ASI_MEM 844 845 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 846 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_1); 847 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 848 clearphys_1: 849 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 850 851 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 852 casxa [%o0]ASI_MEM, %g0, %g0 853 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 854 855 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 856 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_2); 857 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 858 clearphys_2: 859 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 860 861 /* clear the AFSR */ 862 ldxa [%g0]ASI_AFSR, %o1 863 stxa %o1, [%g0]ASI_AFSR 864 membar #Sync 865 866 /* turn NCEEN back on */ 867 stxa %o5, [%g0]ASI_ESTATE_ERR 868 membar #Sync 869 870 /* return and re-enable IE and AM */ 871 retl 872 wrpr %g0, %o4, %pstate 873 SET_SIZE(clearphys) 874 875 876 /* 877 * Jalapeno Ecache displacement flush the specified line from the E$ 878 * 879 * Register usage: 880 * %o0 - 64 bit physical address for flushing 881 * %o1 - Ecache set size 882 */ 883 ENTRY(ecache_flush_line) 884 885 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 886 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_1); 887 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 888 ecache_flush_line_1: 889 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 890 891 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 892 893 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 894 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_2); 895 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 896 ecache_flush_line_2: 897 #endif /* JALAPENO && JALAPENO_ERRATA_85 */ 898 899 retl 900 nop 901 SET_SIZE(ecache_flush_line) 902 903 904 /* 905 * Perform necessary cpu workaround to ensure jbus ordering. 906 * Called only from Fire systems. 907 * CPU's internal "invalidate FIFOs" are flushed. 908 */ 909 910 #define VIS_BLOCKSIZE 64 911 912 .seg ".data" 913 .align VIS_BLOCKSIZE 914 .type sync_buf, #object 915 sync_buf: 916 .skip VIS_BLOCKSIZE 917 .size sync_buf, VIS_BLOCKSIZE 918 919 ENTRY(jbus_stst_order) 920 set sync_buf, %o1 921 922 rd %fprs, %o2 ! %o2 = saved fprs 923 or %o2, FPRS_FEF, %o3 924 wr %g0, %o3, %fprs ! make sure fp is enabled 925 stda %d0, [%o1]ASI_BLK_COMMIT_P 926 wr %o2, 0, %fprs ! restore fprs 927 928 retl 929 membar #Sync 930 SET_SIZE(jbus_stst_order) 931 932 /* 933 * This routine will not be called in Jalapeno systems. 934 */ 935 ENTRY(flush_ipb) 936 retl 937 nop 938 SET_SIZE(flush_ipb) 939