1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * 25 * Assembly code support for the Cheetah+ module 26 */ 27 28 #include "assym.h" 29 30 #include <sys/asm_linkage.h> 31 #include <sys/mmu.h> 32 #include <vm/hat_sfmmu.h> 33 #include <sys/machparam.h> 34 #include <sys/machcpuvar.h> 35 #include <sys/machthread.h> 36 #include <sys/machtrap.h> 37 #include <sys/privregs.h> 38 #include <sys/asm_linkage.h> 39 #include <sys/trap.h> 40 #include <sys/cheetahregs.h> 41 #include <sys/xc_impl.h> 42 #include <sys/intreg.h> 43 #include <sys/async.h> 44 #include <sys/clock.h> 45 #include <sys/cheetahasm.h> 46 #include <sys/cmpregs.h> 47 48 #ifdef TRAPTRACE 49 #include <sys/traptrace.h> 50 #endif /* TRAPTRACE */ 51 52 53 .global retire_l2_start 54 .global retire_l2_end 55 .global unretire_l2_start 56 .global unretire_l2_end 57 .global retire_l3_start 58 .global retire_l3_end 59 .global unretire_l3_start 60 .global unretire_l3_end 61 62 /* 63 * Panther version to reflush a line from both the L2 cache and L3 64 * cache by the respective indexes. Flushes all ways of the line from 65 * each cache. 66 * 67 * l2_index Index into the L2$ of the line to be flushed. This 68 * register will not be modified by this routine. 69 * l3_index Index into the L3$ of the line to be flushed. This 70 * register will not be modified by this routine. 71 * scr2 scratch register. 72 * scr3 scratch register. 73 * 74 */ 75 #define PN_ECACHE_REFLUSH_LINE(l2_index, l3_index, scr2, scr3) \ 76 set PN_L2_MAX_SET, scr2; \ 77 set PN_L2_SET_SIZE, scr3; \ 78 1: \ 79 ldxa [l2_index + scr2]ASI_L2_TAG, %g0; \ 80 cmp scr2, %g0; \ 81 bg,a 1b; \ 82 sub scr2, scr3, scr2; \ 83 mov 6, scr2; \ 84 6: \ 85 cmp scr2, %g0; \ 86 bg,a 6b; \ 87 sub scr2, 1, scr2; \ 88 set PN_L3_MAX_SET, scr2; \ 89 set PN_L3_SET_SIZE, scr3; \ 90 2: \ 91 ldxa [l3_index + scr2]ASI_EC_DIAG, %g0; \ 92 cmp scr2, %g0; \ 93 bg,a 2b; \ 94 sub scr2, scr3, scr2; 95 96 /* 97 * Panther version of ecache_flush_line. Flushes the line corresponding 98 * to physaddr from both the L2 cache and the L3 cache. 99 * 100 * physaddr Input: Physical address to flush. 101 * Output: Physical address to flush (preserved). 102 * l2_idx_out Input: scratch register. 103 * Output: Index into the L2$ of the line to be flushed. 104 * l3_idx_out Input: scratch register. 105 * Output: Index into the L3$ of the line to be flushed. 106 * scr3 scratch register. 107 * scr4 scratch register. 108 * 109 */ 110 #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \ 111 set PN_L3_SET_SIZE, l2_idx_out; \ 112 sub l2_idx_out, 1, l2_idx_out; \ 113 and physaddr, l2_idx_out, l3_idx_out; \ 114 set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \ 115 or l2_idx_out, l3_idx_out, l3_idx_out; \ 116 set PN_L2_SET_SIZE, l2_idx_out; \ 117 sub l2_idx_out, 1, l2_idx_out; \ 118 and physaddr, l2_idx_out, l2_idx_out; \ 119 set PN_L2_IDX_DISP_FLUSH, scr3; \ 120 or l2_idx_out, scr3, l2_idx_out; \ 121 PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4) 122 123 124 .align 4096 125 ENTRY(retire_l2) 126 retire_l2_start: 127 128 ! since we disable interrupts, we don't need to do kpreempt_disable() 129 rdpr %pstate, %o2 130 andn %o2, PSTATE_IE, %g1 131 wrpr %g0, %g1, %pstate ! disable interrupts 132 /* 133 * Save current DCU state. Turn off IPS 134 */ 135 setx DCU_IPS_MASK, %g2, %o3 136 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 137 andn %g1, %o3, %g4 138 stxa %g4, [%g0]ASI_DCU 139 flush %g0 140 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 141 clr %o5 ! assume success 142 8: 143 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3) 144 1: 145 ! Check if line is invalid; if so, NA it. 146 ldxa [%o0]ASI_L2_TAG, %o3 147 btst 0x7, %o3 148 bnz %xcc, 2f 149 nop 150 stxa %o1, [%o0]ASI_L2_TAG 151 membar #Sync ! still on same cache line 152 ! now delay 15 cycles so we don't have hazard when we return 153 mov 16, %o1 154 1: 155 brnz,pt %o1, 1b 156 dec %o1 157 9: 158 ! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary 159 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 160 /* 161 * Restore the DCU 162 */ 163 stxa %g1, [%g0]ASI_DCU 164 flush %g0 165 wrpr %g0, %o2, %pstate !restore pstate 166 retl 167 mov %o5, %o0 168 2: 169 ! It is OK to have STATE as NA (if so, nothing to do!) 170 and %o3, 0x7, %o3 171 cmp %o3, 0x5 172 be,a,pt %xcc, 9b 173 mov 1, %o5 ! indicate was already NA 174 ! Hmm. Not INV, not NA. 175 cmp %o5, 0 176 be,a,pt %xcc, 8b ! Flush the cacheline again 177 mov 2, %o5 ! indicate retry was done 178 ! We already Flushed cacheline second time. Return -1 179 clr %o5 180 ba 9b 181 dec %o5 182 retire_l2_end: 183 SET_SIZE(retire_l2) 184 185 ENTRY(unretire_l2) 186 unretire_l2_start: 187 188 ! since we disable interrupts, we don't need to do kpreempt_disable() 189 rdpr %pstate, %o2 190 andn %o2, PSTATE_IE, %g1 191 wrpr %g0, %g1, %pstate ! disable interrupts 192 /* 193 * Save current DCU state. Turn off IPS 194 */ 195 setx DCU_IPS_MASK, %g2, %o3 196 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 197 andn %g1, %o3, %g4 198 stxa %g4, [%g0]ASI_DCU 199 flush %g0 /* flush required after changing the IC bit */ 200 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 201 202 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 203 1: 204 clr %o5 ! assume success 205 ! Check that line is in NA state; if so, INV it. 206 ldxa [%o0]ASI_L2_TAG, %o3 207 and %o3, 0x7, %o3 208 cmp %o3, 0x5 209 bne,a,pt %xcc, 9f ! Wasn't NA, so something is wrong 210 dec %o5 ! indicate not NA 211 stxa %g0, [%o0]ASI_L2_TAG 212 membar #Sync 213 ! now delay 15 cycles so we don't have hazard when we return 214 mov 16, %o1 215 1: 216 brnz,pt %o1, 1b 217 dec %o1 218 9: 219 ! UNPARK-SIBLING_CORE is 7 instructions 220 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 221 /* 222 * Restore the DCU 223 */ 224 stxa %g1, [%g0]ASI_DCU 225 flush %g0 226 wrpr %g0, %o2, %pstate !restore pstate 227 retl 228 mov %o5, %o0 229 unretire_l2_end: 230 SET_SIZE(unretire_l2) 231 232 ENTRY(retire_l3) 233 retire_l3_start: 234 235 ! since we disable interrupts, we don't need to do kpreempt_disable() 236 rdpr %pstate, %o2 237 andn %o2, PSTATE_IE, %g1 238 wrpr %g0, %g1, %pstate ! disable interrupts 239 /* 240 * Save current DCU state. Turn off IPS 241 */ 242 setx DCU_IPS_MASK, %g2, %o3 243 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 244 andn %g1, %o3, %g4 245 stxa %g4, [%g0]ASI_DCU 246 flush %g0 /* flush required after changing the IC bit */ 247 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 248 249 ! PN-ECACHE-FLUSH_LINE is 30 instructions 250 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 251 1: 252 clr %o5 ! assume success 253 ! Check if line is invalid; if so, NA it. 254 ldxa [%o0]ASI_EC_DIAG, %o3 255 btst 0x7, %o3 256 bnz %xcc, 2f 257 nop 258 stxa %o1, [%o0]ASI_EC_DIAG 259 membar #Sync ! still on same cache line 260 ! now delay 15 cycles so we don't have hazard when we return 261 mov 16, %o1 262 1: 263 brnz,pt %o1, 1b 264 dec %o1 265 9: 266 ! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary 267 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 268 /* 269 * Restore the DCU 270 */ 271 stxa %g1, [%g0]ASI_DCU 272 flush %g0 273 wrpr %g0, %o2, %pstate !restore pstate 274 retl 275 mov %o5, %o0 276 2: 277 ! It is OK to have STATE as NA (if so, nothing to do!) 278 and %o3, 0x7, %o3 279 cmp %o3, 0x5 280 be,a,pt %xcc, 9b 281 inc %o5 ! indicate was already NA 282 ! Hmm. Not INV, not NA 283 ba 9b 284 dec %o5 285 retire_l3_end: 286 SET_SIZE(retire_l3) 287 288 ENTRY(unretire_l3) 289 unretire_l3_start: 290 291 ! since we disable interrupts, we don't need to do kpreempt_disable() 292 rdpr %pstate, %o2 293 andn %o2, PSTATE_IE, %g1 294 wrpr %g0, %g1, %pstate ! disable interrupts 295 /* 296 * Save current DCU state. Turn off IPS 297 */ 298 setx DCU_IPS_MASK, %g2, %o3 299 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 300 andn %g1, %o3, %g4 301 stxa %g4, [%g0]ASI_DCU 302 flush %g0 /* flush required after changing the IC bit */ 303 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 304 305 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 306 1: 307 clr %o5 ! assume success 308 ! Check that line is in NA state; if so, INV it. 309 ldxa [%o0]ASI_EC_DIAG, %o3 310 and %o3, 0x7, %o3 311 cmp %o3, 0x5 312 bne,a,pt %xcc, 9f ! Wasn't NA, so something is wrong 313 dec %o5 ! indicate not NA 314 stxa %g0, [%o0]ASI_EC_DIAG 315 membar #Sync 316 ! now delay 15 cycles so we don't have hazard when we return 317 mov 16, %o1 318 1: 319 brnz,pt %o1, 1b 320 dec %o1 321 9: 322 ! UNPARK-SIBLING_CORE is 7 instructions 323 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 324 /* 325 * Restore the DCU 326 */ 327 stxa %g1, [%g0]ASI_DCU 328 flush %g0 329 wrpr %g0, %o2, %pstate !restore pstate 330 retl 331 mov %o5, %o0 332 unretire_l3_end: 333 SET_SIZE(unretire_l3) 334 335 .align 2048 336 337 ENTRY(retire_l2_alternate) 338 339 ! since we disable interrupts, we don't need to do kpreempt_disable() 340 rdpr %pstate, %o2 341 andn %o2, PSTATE_IE, %g1 342 wrpr %g0, %g1, %pstate ! disable interrupts 343 /* 344 * Save current DCU state. Turn off IPS 345 */ 346 setx DCU_IPS_MASK, %g2, %o3 347 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 348 andn %g1, %o3, %g4 349 stxa %g4, [%g0]ASI_DCU 350 flush %g0 351 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 352 clr %o5 ! assume success 353 8: 354 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3) 355 1: 356 ! Check if line is invalid; if so, NA it. 357 ldxa [%o0]ASI_L2_TAG, %o3 358 btst 0x7, %o3 359 bnz %xcc, 2f 360 nop 361 stxa %o1, [%o0]ASI_L2_TAG 362 membar #Sync ! still on same cache line 363 ! now delay 15 cycles so we don't have hazard when we return 364 mov 16, %o1 365 1: 366 brnz,pt %o1, 1b 367 dec %o1 368 9: 369 ! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary 370 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 371 /* 372 * Restore the DCU 373 */ 374 stxa %g1, [%g0]ASI_DCU 375 flush %g0 376 wrpr %g0, %o2, %pstate !restore pstate 377 retl 378 mov %o5, %o0 379 2: 380 ! It is OK to have STATE as NA (if so, nothing to do!) 381 and %o3, 0x7, %o3 382 cmp %o3, 0x5 383 be,a,pt %xcc, 9b 384 mov 1, %o5 ! indicate was already NA 385 ! Hmm. Not INV, not NA. 386 cmp %o5, 0 387 be,a,pt %xcc, 8b ! Flush the cacheline again 388 mov 2, %o5 ! indicate retry was done 389 ! We already Flushed cacheline second time. Return -1 390 clr %o5 391 ba 9b 392 dec %o5 393 SET_SIZE(retire_l2_alternate) 394 395 ENTRY(unretire_l2_alternate) 396 397 ! since we disable interrupts, we don't need to do kpreempt_disable() 398 rdpr %pstate, %o2 399 andn %o2, PSTATE_IE, %g1 400 wrpr %g0, %g1, %pstate ! disable interrupts 401 /* 402 * Save current DCU state. Turn off IPS 403 */ 404 setx DCU_IPS_MASK, %g2, %o3 405 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 406 andn %g1, %o3, %g4 407 stxa %g4, [%g0]ASI_DCU 408 flush %g0 /* flush required after changing the IC bit */ 409 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 410 411 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 412 1: 413 clr %o5 ! assume success 414 ! Check that line is in NA state; if so, INV it. 415 ldxa [%o0]ASI_L2_TAG, %o3 416 and %o3, 0x7, %o3 417 cmp %o3, 0x5 418 bne,a,pt %xcc, 9f ! Wasn't NA, so something is wrong 419 dec %o5 ! indicate not NA 420 stxa %g0, [%o0]ASI_L2_TAG 421 membar #Sync 422 ! now delay 15 cycles so we don't have hazard when we return 423 mov 16, %o1 424 1: 425 brnz,pt %o1, 1b 426 dec %o1 427 9: 428 ! UNPARK-SIBLING_CORE is 7 instructions 429 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 430 /* 431 * Restore the DCU 432 */ 433 stxa %g1, [%g0]ASI_DCU 434 flush %g0 435 wrpr %g0, %o2, %pstate !restore pstate 436 retl 437 mov %o5, %o0 438 SET_SIZE(unretire_l2_alternate) 439 440 ENTRY(retire_l3_alternate) 441 442 ! since we disable interrupts, we don't need to do kpreempt_disable() 443 rdpr %pstate, %o2 444 andn %o2, PSTATE_IE, %g1 445 wrpr %g0, %g1, %pstate ! disable interrupts 446 /* 447 * Save current DCU state. Turn off IPS 448 */ 449 setx DCU_IPS_MASK, %g2, %o3 450 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 451 andn %g1, %o3, %g4 452 stxa %g4, [%g0]ASI_DCU 453 flush %g0 /* flush required after changing the IC bit */ 454 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 455 456 ! PN-ECACHE-FLUSH_LINE is 30 instructions 457 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 458 1: 459 clr %o5 ! assume success 460 ! Check if line is invalid; if so, NA it. 461 ldxa [%o0]ASI_EC_DIAG, %o3 462 btst 0x7, %o3 463 bnz %xcc, 2f 464 nop 465 stxa %o1, [%o0]ASI_EC_DIAG 466 membar #Sync ! still on same cache line 467 ! now delay 15 cycles so we don't have hazard when we return 468 mov 16, %o1 469 1: 470 brnz,pt %o1, 1b 471 dec %o1 472 9: 473 ! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary 474 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 475 /* 476 * Restore the DCU 477 */ 478 stxa %g1, [%g0]ASI_DCU 479 flush %g0 480 wrpr %g0, %o2, %pstate !restore pstate 481 retl 482 mov %o5, %o0 483 2: 484 ! It is OK to have STATE as NA (if so, nothing to do!) 485 and %o3, 0x7, %o3 486 cmp %o3, 0x5 487 be,a,pt %xcc, 9b 488 inc %o5 ! indicate was already NA 489 ! Hmm. Not INV, not NA 490 ba 9b 491 dec %o5 492 SET_SIZE(retire_l3_alternate) 493 494 ENTRY(unretire_l3_alternate) 495 496 ! since we disable interrupts, we don't need to do kpreempt_disable() 497 rdpr %pstate, %o2 498 andn %o2, PSTATE_IE, %g1 499 wrpr %g0, %g1, %pstate ! disable interrupts 500 /* 501 * Save current DCU state. Turn off IPS 502 */ 503 setx DCU_IPS_MASK, %g2, %o3 504 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 505 andn %g1, %o3, %g4 506 stxa %g4, [%g0]ASI_DCU 507 flush %g0 /* flush required after changing the IC bit */ 508 PARK_SIBLING_CORE(%g1, %o3, %o4) ! %g1 has DCU value 509 510 PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2) 511 1: 512 clr %o5 ! assume success 513 ! Check that line is in NA state; if so, INV it. 514 ldxa [%o0]ASI_EC_DIAG, %o3 515 and %o3, 0x7, %o3 516 cmp %o3, 0x5 517 bne,a,pt %xcc, 9f ! Wasn't NA, so something is wrong 518 dec %o5 ! indicate not NA 519 stxa %g0, [%o0]ASI_EC_DIAG 520 membar #Sync 521 ! now delay 15 cycles so we don't have hazard when we return 522 mov 16, %o1 523 1: 524 brnz,pt %o1, 1b 525 dec %o1 526 9: 527 ! UNPARK-SIBLING_CORE is 7 instructions 528 UNPARK_SIBLING_CORE(%g1, %o3, %o4) ! 7 instructions 529 /* 530 * Restore the DCU 531 */ 532 stxa %g1, [%g0]ASI_DCU 533 flush %g0 534 wrpr %g0, %o2, %pstate !restore pstate 535 retl 536 mov %o5, %o0 537 SET_SIZE(unretire_l3_alternate) 538 539 ENTRY(get_ecache_dtags_tl1) 540 541 542 PARK_SIBLING_CORE(%g3, %g4, %g5) 543 add %g2, CH_CLO_DATA + CH_CHD_EC_DATA, %g2 544 rd %asi, %g4 545 wr %g0, ASI_N, %asi 546 GET_ECACHE_DTAGS(%g1, %g2, %g5, %g6, %g7) 547 wr %g4, %asi 548 UNPARK_SIBLING_CORE(%g3, %g4, %g5) ! can use %g3 again 549 550 retry 551 SET_SIZE(get_ecache_dtags_tl1) 552 553 ENTRY(get_l2_tag_tl1) 554 555 /* 556 * Now read the tag data 557 */ 558 ldxa [%g1]ASI_L2_TAG, %g4 ! save tag_data 559 stx %g4, [%g2] 560 561 retry 562 SET_SIZE(get_l2_tag_tl1) 563 564 ENTRY(get_l3_tag_tl1) 565 566 /* 567 * Now read the tag data 568 */ 569 ldxa [%g1]ASI_EC_DIAG, %g4 ! save tag_data 570 stx %g4, [%g2] 571 572 retry 573 SET_SIZE(get_l3_tag_tl1) 574