1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * General assembly language routines. 28 * It is the intent of this file to contain routines that are 29 * independent of the specific kernel architecture, and those that are 30 * common across kernel architectures. 31 * As architectures diverge, and implementations of specific 32 * architecture-dependent routines change, the routines should be moved 33 * from this file into the respective ../`arch -k`/subr.s file. 34 * Or, if you want to be really nice, move them to a file whose 35 * name has something to do with the routine you are moving. 36 */ 37 38 #include <sys/asm_linkage.h> 39 #include <sys/privregs.h> 40 #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */ 41 #include <sys/machthread.h> 42 #include <sys/clock.h> 43 #include <sys/psr_compat.h> 44 #include <sys/isa_defs.h> 45 #include <sys/dditypes.h> 46 #include <sys/panic.h> 47 #include <sys/machlock.h> 48 #include <sys/ontrap.h> 49 50 #include "assym.h" 51 52 .seg ".text" 53 .align 4 54 55 /* 56 * Macro to raise processor priority level. 57 * Avoid dropping processor priority if already at high level. 58 * Also avoid going below CPU->cpu_base_spl, which could've just been set by 59 * a higher-level interrupt thread that just blocked. 60 * 61 * level can be %o0 (not other regs used here) or a constant. 62 */ 63 #define RAISE(level) \ 64 rdpr %pil, %o1; /* get current PIL */ \ 65 cmp %o1, level; /* is PIL high enough? */ \ 66 bge 1f; /* yes, return */ \ 67 nop; \ 68 wrpr %g0, PIL_MAX, %pil; /* freeze CPU_BASE_SPL */ \ 69 ldn [THREAD_REG + T_CPU], %o2; \ 70 ld [%o2 + CPU_BASE_SPL], %o2; \ 71 cmp %o2, level; /* compare new to base */ \ 72 movl %xcc, level, %o2; /* use new if base lower */ \ 73 wrpr %g0, %o2, %pil; \ 74 1: \ 75 retl; \ 76 mov %o1, %o0 /* return old PIL */ 77 78 /* 79 * Macro to raise processor priority level to level >= DISP_LEVEL. 80 * Doesn't require comparison to CPU->cpu_base_spl. 81 * 82 * newpil can be %o0 (not other regs used here) or a constant. 83 */ 84 #define RAISE_HIGH(level) \ 85 rdpr %pil, %o1; /* get current PIL */ \ 86 cmp %o1, level; /* is PIL high enough? */ \ 87 bge 1f; /* yes, return */ \ 88 nop; \ 89 wrpr %g0, level, %pil; /* use chose value */ \ 90 1: \ 91 retl; \ 92 mov %o1, %o0 /* return old PIL */ 93 94 /* 95 * Macro to set the priority to a specified level. 96 * Avoid dropping the priority below CPU->cpu_base_spl. 97 * 98 * newpil can be %o0 (not other regs used here) or a constant with 99 * the new PIL in the PSR_PIL field of the level arg. 100 */ 101 #define SETPRI(level) \ 102 rdpr %pil, %o1; /* get current PIL */ \ 103 wrpr %g0, PIL_MAX, %pil; /* freeze CPU_BASE_SPL */ \ 104 ldn [THREAD_REG + T_CPU], %o2; \ 105 ld [%o2 + CPU_BASE_SPL], %o2; \ 106 cmp %o2, level; /* compare new to base */ \ 107 movl %xcc, level, %o2; /* use new if base lower */ \ 108 wrpr %g0, %o2, %pil; \ 109 retl; \ 110 mov %o1, %o0 /* return old PIL */ 111 112 /* 113 * Macro to set the priority to a specified level at or above LOCK_LEVEL. 114 * Doesn't require comparison to CPU->cpu_base_spl. 115 * 116 * newpil can be %o0 (not other regs used here) or a constant with 117 * the new PIL in the PSR_PIL field of the level arg. 118 */ 119 #define SETPRI_HIGH(level) \ 120 rdpr %pil, %o1; /* get current PIL */ \ 121 wrpr %g0, level, %pil; \ 122 retl; \ 123 mov %o1, %o0 /* return old PIL */ 124 125 /* 126 * Berkley 4.3 introduced symbolically named interrupt levels 127 * as a way deal with priority in a machine independent fashion. 128 * Numbered priorities are machine specific, and should be 129 * discouraged where possible. 130 * 131 * Note, for the machine specific priorities there are 132 * examples listed for devices that use a particular priority. 133 * It should not be construed that all devices of that 134 * type should be at that priority. It is currently were 135 * the current devices fit into the priority scheme based 136 * upon time criticalness. 137 * 138 * The underlying assumption of these assignments is that 139 * SPARC9 IPL 10 is the highest level from which a device 140 * routine can call wakeup. Devices that interrupt from higher 141 * levels are restricted in what they can do. If they need 142 * kernels services they should schedule a routine at a lower 143 * level (via software interrupt) to do the required 144 * processing. 145 * 146 * Examples of this higher usage: 147 * Level Usage 148 * 15 Asynchronous memory exceptions 149 * 14 Profiling clock (and PROM uart polling clock) 150 * 13 Audio device 151 * 12 Serial ports 152 * 11 Floppy controller 153 * 154 * The serial ports request lower level processing on level 6. 155 * Audio and floppy request lower level processing on level 4. 156 * 157 * Also, almost all splN routines (where N is a number or a 158 * mnemonic) will do a RAISE(), on the assumption that they are 159 * never used to lower our priority. 160 * The exceptions are: 161 * spl8() Because you can't be above 15 to begin with! 162 * splzs() Because this is used at boot time to lower our 163 * priority, to allow the PROM to poll the uart. 164 * spl0() Used to lower priority to 0. 165 */ 166 167 /* locks out all interrupts, including memory errors */ 168 ENTRY(spl8) 169 SETPRI_HIGH(15) 170 SET_SIZE(spl8) 171 172 /* just below the level that profiling runs */ 173 ENTRY(spl7) 174 RAISE_HIGH(13) 175 SET_SIZE(spl7) 176 177 /* sun specific - highest priority onboard serial i/o zs ports */ 178 ENTRY(splzs) 179 SETPRI_HIGH(12) /* Can't be a RAISE, as it's used to lower us */ 180 SET_SIZE(splzs) 181 182 /* 183 * should lock out clocks and all interrupts, 184 * as you can see, there are exceptions 185 */ 186 ENTRY(splhi) 187 ALTENTRY(splhigh) 188 ALTENTRY(spl6) 189 ALTENTRY(i_ddi_splhigh) 190 RAISE_HIGH(DISP_LEVEL) 191 SET_SIZE(i_ddi_splhigh) 192 SET_SIZE(spl6) 193 SET_SIZE(splhigh) 194 SET_SIZE(splhi) 195 196 /* allow all interrupts */ 197 ENTRY(spl0) 198 SETPRI(0) 199 SET_SIZE(spl0) 200 201 /* 202 * splx - set PIL back to that indicated by the old %pil passed as an argument, 203 * or to the CPU's base priority, whichever is higher. 204 */ 205 206 ENTRY(splx) 207 ALTENTRY(i_ddi_splx) 208 SETPRI(%o0) /* set PIL */ 209 SET_SIZE(i_ddi_splx) 210 SET_SIZE(splx) 211 212 /* 213 * splr() 214 * 215 * splr is like splx but will only raise the priority and never drop it 216 * Be careful not to set priority lower than CPU->cpu_base_pri, 217 * even though it seems we're raising the priority, it could be set higher 218 * at any time by an interrupt routine, so we must block interrupts and 219 * look at CPU->cpu_base_pri. 220 */ 221 222 ENTRY(splr) 223 RAISE(%o0) 224 SET_SIZE(splr) 225 226 /* 227 * on_fault() 228 * Catch lofault faults. Like setjmp except it returns one 229 * if code following causes uncorrectable fault. Turned off 230 * by calling no_fault(). 231 */ 232 233 ENTRY(on_fault) 234 membar #Sync ! sync error barrier (see copy.s) 235 stn %o0, [THREAD_REG + T_ONFAULT] 236 set catch_fault, %o1 237 b setjmp ! let setjmp do the rest 238 stn %o1, [THREAD_REG + T_LOFAULT] ! put catch_fault in t_lofault 239 240 catch_fault: 241 save %sp, -SA(WINDOWSIZE), %sp ! goto next window so that we can rtn 242 ldn [THREAD_REG + T_ONFAULT], %o0 243 membar #Sync ! sync error barrier 244 stn %g0, [THREAD_REG + T_ONFAULT] ! turn off onfault 245 b longjmp ! let longjmp do the rest 246 stn %g0, [THREAD_REG + T_LOFAULT] ! turn off lofault 247 SET_SIZE(on_fault) 248 249 /* 250 * no_fault() 251 * turn off fault catching. 252 */ 253 254 ENTRY(no_fault) 255 membar #Sync ! sync error barrier 256 stn %g0, [THREAD_REG + T_ONFAULT] 257 retl 258 stn %g0, [THREAD_REG + T_LOFAULT] ! turn off lofault 259 SET_SIZE(no_fault) 260 261 /* 262 * Default trampoline code for on_trap() (see <sys/ontrap.h>). On sparcv9, 263 * the trap code will complete trap processing but reset the return %pc to 264 * ot_trampoline, which will by default be set to the address of this code. 265 * We longjmp(&curthread->t_ontrap->ot_jmpbuf) to return back to on_trap(). 266 */ 267 268 ENTRY(on_trap_trampoline) 269 ldn [THREAD_REG + T_ONTRAP], %o0 270 b longjmp 271 add %o0, OT_JMPBUF, %o0 272 SET_SIZE(on_trap_trampoline) 273 274 /* 275 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 276 * more information about the on_trap() mechanism. If the on_trap_data is the 277 * same as the topmost stack element, we just modify that element. 278 * On UltraSPARC, we need to issue a membar #Sync before modifying t_ontrap. 279 * The issue barrier is defined to force all deferred errors to complete before 280 * we go any further. We want these errors to be processed before we modify 281 * our current error protection. 282 */ 283 284 ENTRY(on_trap) 285 membar #Sync ! force error barrier 286 sth %o1, [%o0 + OT_PROT] ! ot_prot = prot 287 sth %g0, [%o0 + OT_TRAP] ! ot_trap = 0 288 set on_trap_trampoline, %o2 ! %o2 = &on_trap_trampoline 289 stn %o2, [%o0 + OT_TRAMPOLINE] ! ot_trampoline = %o2 290 stn %g0, [%o0 + OT_HANDLE] ! ot_handle = NULL 291 ldn [THREAD_REG + T_ONTRAP], %o2 ! %o2 = curthread->t_ontrap 292 cmp %o0, %o2 ! if (otp == %o2) 293 be 0f ! don't modify t_ontrap 294 stn %g0, [%o0 + OT_PAD1] ! delay - ot_pad1 = NULL 295 296 stn %o2, [%o0 + OT_PREV] ! ot_prev = t_ontrap 297 membar #Sync ! force error barrier 298 stn %o0, [THREAD_REG + T_ONTRAP] ! t_ontrap = otp 299 300 0: b setjmp ! let setjmp do the rest 301 add %o0, OT_JMPBUF, %o0 ! %o0 = &ot_jmpbuf 302 SET_SIZE(on_trap) 303 304 /* 305 * Setjmp and longjmp implement non-local gotos using state vectors 306 * type label_t. 307 */ 308 309 ENTRY(setjmp) 310 stn %o7, [%o0 + L_PC] ! save return address 311 stn %sp, [%o0 + L_SP] ! save stack ptr 312 retl 313 clr %o0 ! return 0 314 SET_SIZE(setjmp) 315 316 317 ENTRY(longjmp) 318 ! 319 ! The following save is required so that an extra register 320 ! window is flushed. Flushw flushes nwindows-2 321 ! register windows. If setjmp and longjmp are called from 322 ! within the same window, that window will not get pushed 323 ! out onto the stack without the extra save below. Tail call 324 ! optimization can lead to callers of longjmp executing 325 ! from a window that could be the same as the setjmp, 326 ! thus the need for the following save. 327 ! 328 save %sp, -SA(MINFRAME), %sp 329 flushw ! flush all but this window 330 ldn [%i0 + L_PC], %i7 ! restore return addr 331 ldn [%i0 + L_SP], %fp ! restore sp for dest on foreign stack 332 ret ! return 1 333 restore %g0, 1, %o0 ! takes underflow, switches stacks 334 SET_SIZE(longjmp) 335 336 /* 337 * movtuc(length, from, to, table) 338 * 339 * VAX movtuc instruction (sort of). 340 */ 341 342 ENTRY(movtuc) 343 tst %o0 344 ble,pn %ncc, 2f ! check length 345 clr %o4 346 347 ldub [%o1 + %o4], %g1 ! get next byte in string 348 0: 349 ldub [%o3 + %g1], %g1 ! get corresponding table entry 350 tst %g1 ! escape char? 351 bnz 1f 352 stb %g1, [%o2 + %o4] ! delay slot, store it 353 354 retl ! return (bytes moved) 355 mov %o4, %o0 356 1: 357 inc %o4 ! increment index 358 cmp %o4, %o0 ! index < length ? 359 bl,a,pt %ncc, 0b 360 ldub [%o1 + %o4], %g1 ! delay slot, get next byte in string 361 2: 362 retl ! return (bytes moved) 363 mov %o4, %o0 364 SET_SIZE(movtuc) 365 366 /* 367 * scanc(length, string, table, mask) 368 * 369 * VAX scanc instruction. 370 */ 371 372 ENTRY(scanc) 373 tst %o0 374 ble,pn %ncc, 1f ! check length 375 clr %o4 376 0: 377 ldub [%o1 + %o4], %g1 ! get next byte in string 378 cmp %o4, %o0 ! interlock slot, index < length ? 379 ldub [%o2 + %g1], %g1 ! get corresponding table entry 380 bge,pn %ncc, 1f ! interlock slot 381 btst %o3, %g1 ! apply the mask 382 bz,a 0b 383 inc %o4 ! delay slot, increment index 384 1: 385 retl ! return(length - index) 386 sub %o0, %o4, %o0 387 SET_SIZE(scanc) 388 389 /* 390 * if a() calls b() calls caller(), 391 * caller() returns return address in a(). 392 */ 393 394 ENTRY(caller) 395 retl 396 mov %i7, %o0 397 SET_SIZE(caller) 398 399 /* 400 * if a() calls callee(), callee() returns the 401 * return address in a(); 402 */ 403 404 ENTRY(callee) 405 retl 406 mov %o7, %o0 407 SET_SIZE(callee) 408 409 /* 410 * return the current frame pointer 411 */ 412 413 ENTRY(getfp) 414 retl 415 mov %fp, %o0 416 SET_SIZE(getfp) 417 418 /* 419 * Get vector base register 420 */ 421 422 ENTRY(gettbr) 423 retl 424 mov %tbr, %o0 425 SET_SIZE(gettbr) 426 427 /* 428 * Get processor state register, V9 faked to look like V8. 429 * Note: does not provide ccr.xcc and provides FPRS.FEF instead of 430 * PSTATE.PEF, because PSTATE.PEF is always on in order to allow the 431 * libc_psr memcpy routines to run without hitting the fp_disabled trap. 432 */ 433 434 ENTRY(getpsr) 435 rd %ccr, %o1 ! get ccr 436 sll %o1, PSR_ICC_SHIFT, %o0 ! move icc to V8 psr.icc 437 rd %fprs, %o1 ! get fprs 438 and %o1, FPRS_FEF, %o1 ! mask out dirty upper/lower 439 sllx %o1, PSR_FPRS_FEF_SHIFT, %o1 ! shift fef to V8 psr.ef 440 or %o0, %o1, %o0 ! or into psr.ef 441 set V9_PSR_IMPLVER, %o1 ! SI assigned impl/ver: 0xef 442 retl 443 or %o0, %o1, %o0 ! or into psr.impl/ver 444 SET_SIZE(getpsr) 445 446 /* 447 * Get current processor interrupt level 448 */ 449 450 ENTRY(getpil) 451 retl 452 rdpr %pil, %o0 453 SET_SIZE(getpil) 454 455 ENTRY(setpil) 456 retl 457 wrpr %g0, %o0, %pil 458 SET_SIZE(setpil) 459 460 461 /* 462 * _insque(entryp, predp) 463 * 464 * Insert entryp after predp in a doubly linked list. 465 */ 466 467 ENTRY(_insque) 468 ldn [%o1], %g1 ! predp->forw 469 stn %o1, [%o0 + CPTRSIZE] ! entryp->back = predp 470 stn %g1, [%o0] ! entryp->forw = predp->forw 471 stn %o0, [%o1] ! predp->forw = entryp 472 retl 473 stn %o0, [%g1 + CPTRSIZE] ! predp->forw->back = entryp 474 SET_SIZE(_insque) 475 476 /* 477 * _remque(entryp) 478 * 479 * Remove entryp from a doubly linked list 480 */ 481 482 ENTRY(_remque) 483 ldn [%o0], %g1 ! entryp->forw 484 ldn [%o0 + CPTRSIZE], %g2 ! entryp->back 485 stn %g1, [%g2] ! entryp->back->forw = entryp->forw 486 retl 487 stn %g2, [%g1 + CPTRSIZE] ! entryp->forw->back = entryp->back 488 SET_SIZE(_remque) 489 490 491 /* 492 * strlen(str) 493 * 494 * Returns the number of non-NULL bytes in string argument. 495 * 496 * XXX - why is this here, rather than the traditional file? 497 * why does it have local labels which don't start with a `.'? 498 */ 499 500 ENTRY(strlen) 501 mov %o0, %o1 502 andcc %o1, 3, %o3 ! is src word aligned 503 bz $nowalgnd 504 clr %o0 ! length of non-zero bytes 505 cmp %o3, 2 ! is src half-word aligned 506 be $s2algn 507 cmp %o3, 3 ! src is byte aligned 508 ldub [%o1], %o3 ! move 1 or 3 bytes to align it 509 inc 1, %o1 ! in either case, safe to do a byte 510 be $s3algn 511 tst %o3 512 $s1algn: 513 bnz,a $s2algn ! now go align dest 514 inc 1, %o0 515 b,a $done 516 517 $s2algn: 518 lduh [%o1], %o3 ! know src is half-byte aligned 519 inc 2, %o1 520 srl %o3, 8, %o4 521 tst %o4 ! is the first byte zero 522 bnz,a 1f 523 inc %o0 524 b,a $done 525 1: andcc %o3, 0xff, %o3 ! is the second byte zero 526 bnz,a $nowalgnd 527 inc %o0 528 b,a $done 529 $s3algn: 530 bnz,a $nowalgnd 531 inc 1, %o0 532 b,a $done 533 534 $nowalgnd: 535 ! use trick to check if any read bytes of a word are zero 536 ! the following two constants will generate "byte carries" 537 ! and check if any bit in a byte is set, if all characters 538 ! are 7bits (unsigned) this allways works, otherwise 539 ! there is a specil case that rarely happens, see below 540 541 set 0x7efefeff, %o3 542 set 0x81010100, %o4 543 544 3: ld [%o1], %o2 ! main loop 545 inc 4, %o1 546 add %o2, %o3, %o5 ! generate byte-carries 547 xor %o5, %o2, %o5 ! see if orignal bits set 548 and %o5, %o4, %o5 549 cmp %o5, %o4 ! if ==, no zero bytes 550 be,a 3b 551 inc 4, %o0 552 553 ! check for the zero byte and increment the count appropriately 554 ! some information (the carry bit) is lost if bit 31 555 ! was set (very rare), if this is the rare condition, 556 ! return to the main loop again 557 558 sethi %hi(0xff000000), %o5 ! mask used to test for terminator 559 andcc %o2, %o5, %g0 ! check if first byte was zero 560 bnz 1f 561 srl %o5, 8, %o5 562 $done: 563 retl 564 nop 565 1: andcc %o2, %o5, %g0 ! check if second byte was zero 566 bnz 1f 567 srl %o5, 8, %o5 568 $done1: 569 retl 570 inc %o0 571 1: andcc %o2, %o5, %g0 ! check if third byte was zero 572 bnz 1f 573 andcc %o2, 0xff, %g0 ! check if last byte is zero 574 $done2: 575 retl 576 inc 2, %o0 577 1: bnz,a 3b 578 inc 4, %o0 ! count of bytes 579 $done3: 580 retl 581 inc 3, %o0 582 SET_SIZE(strlen) 583 584 /* 585 * Provide a C callable interface to the membar instruction. 586 */ 587 588 ENTRY(membar_ldld) 589 retl 590 membar #LoadLoad 591 SET_SIZE(membar_ldld) 592 593 ENTRY(membar_stld) 594 retl 595 membar #StoreLoad 596 SET_SIZE(membar_stld) 597 598 ENTRY(membar_ldst) 599 retl 600 membar #LoadStore 601 SET_SIZE(membar_ldst) 602 603 ENTRY(membar_stst) 604 retl 605 membar #StoreStore 606 SET_SIZE(membar_stst) 607 608 ENTRY(membar_ldld_stld) 609 ALTENTRY(membar_stld_ldld) 610 retl 611 membar #LoadLoad|#StoreLoad 612 SET_SIZE(membar_stld_ldld) 613 SET_SIZE(membar_ldld_stld) 614 615 ENTRY(membar_ldld_ldst) 616 ALTENTRY(membar_ldst_ldld) 617 retl 618 membar #LoadLoad|#LoadStore 619 SET_SIZE(membar_ldst_ldld) 620 SET_SIZE(membar_ldld_ldst) 621 622 ENTRY(membar_ldld_stst) 623 ALTENTRY(membar_stst_ldld) 624 retl 625 membar #LoadLoad|#StoreStore 626 SET_SIZE(membar_stst_ldld) 627 SET_SIZE(membar_ldld_stst) 628 629 ENTRY(membar_stld_ldst) 630 ALTENTRY(membar_ldst_stld) 631 retl 632 membar #StoreLoad|#LoadStore 633 SET_SIZE(membar_ldst_stld) 634 SET_SIZE(membar_stld_ldst) 635 636 ENTRY(membar_stld_stst) 637 ALTENTRY(membar_stst_stld) 638 retl 639 membar #StoreLoad|#StoreStore 640 SET_SIZE(membar_stst_stld) 641 SET_SIZE(membar_stld_stst) 642 643 ENTRY(membar_ldst_stst) 644 ALTENTRY(membar_stst_ldst) 645 retl 646 membar #LoadStore|#StoreStore 647 SET_SIZE(membar_stst_ldst) 648 SET_SIZE(membar_ldst_stst) 649 650 ENTRY(membar_lookaside) 651 retl 652 membar #Lookaside 653 SET_SIZE(membar_lookaside) 654 655 ENTRY(membar_memissue) 656 retl 657 membar #MemIssue 658 SET_SIZE(membar_memissue) 659 660 ENTRY(membar_sync) 661 retl 662 membar #Sync 663 SET_SIZE(membar_sync) 664 665 666 /* 667 * Since all of the fuword() variants are so similar, we have a macro to spit 668 * them out. 669 */ 670 671 #define FUWORD(NAME, LOAD, STORE, COPYOP) \ 672 ENTRY(NAME); \ 673 sethi %hi(1f), %o5; \ 674 ldn [THREAD_REG + T_LOFAULT], %o3; \ 675 or %o5, %lo(1f), %o5; \ 676 membar #Sync; \ 677 stn %o5, [THREAD_REG + T_LOFAULT]; \ 678 LOAD [%o0]ASI_USER, %o2; \ 679 membar #Sync; \ 680 stn %o3, [THREAD_REG + T_LOFAULT]; \ 681 mov 0, %o0; \ 682 retl; \ 683 STORE %o2, [%o1]; \ 684 1: \ 685 membar #Sync; \ 686 stn %o3, [THREAD_REG + T_LOFAULT]; \ 687 ldn [THREAD_REG + T_COPYOPS], %o2; \ 688 brz %o2, 2f; \ 689 nop; \ 690 ldn [%o2 + COPYOP], %g1; \ 691 jmp %g1; \ 692 nop; \ 693 2: \ 694 retl; \ 695 mov -1, %o0; \ 696 SET_SIZE(NAME) 697 698 FUWORD(fuword64, ldxa, stx, CP_FUWORD64) 699 FUWORD(fuword32, lda, st, CP_FUWORD32) 700 FUWORD(fuword16, lduha, sth, CP_FUWORD16) 701 FUWORD(fuword8, lduba, stb, CP_FUWORD8) 702 703 704 /* 705 * Since all of the suword() variants are so similar, we have a macro to spit 706 * them out. 707 */ 708 709 #define SUWORD(NAME, STORE, COPYOP) \ 710 ENTRY(NAME) \ 711 sethi %hi(1f), %o5; \ 712 ldn [THREAD_REG + T_LOFAULT], %o3; \ 713 or %o5, %lo(1f), %o5; \ 714 membar #Sync; \ 715 stn %o5, [THREAD_REG + T_LOFAULT]; \ 716 STORE %o1, [%o0]ASI_USER; \ 717 membar #Sync; \ 718 stn %o3, [THREAD_REG + T_LOFAULT]; \ 719 retl; \ 720 clr %o0; \ 721 1: \ 722 membar #Sync; \ 723 stn %o3, [THREAD_REG + T_LOFAULT]; \ 724 ldn [THREAD_REG + T_COPYOPS], %o2; \ 725 brz %o2, 2f; \ 726 nop; \ 727 ldn [%o2 + COPYOP], %g1; \ 728 jmp %g1; \ 729 nop; \ 730 2: \ 731 retl; \ 732 mov -1, %o0; \ 733 SET_SIZE(NAME) 734 735 SUWORD(suword64, stxa, CP_SUWORD64) 736 SUWORD(suword32, sta, CP_SUWORD32) 737 SUWORD(suword16, stha, CP_SUWORD16) 738 SUWORD(suword8, stba, CP_SUWORD8) 739 740 ENTRY(fuword8_noerr) 741 lduba [%o0]ASI_USER, %o0 742 retl 743 stb %o0, [%o1] 744 SET_SIZE(fuword8_noerr) 745 746 ENTRY(fuword16_noerr) 747 lduha [%o0]ASI_USER, %o0 748 retl 749 sth %o0, [%o1] 750 SET_SIZE(fuword16_noerr) 751 752 ENTRY(fuword32_noerr) 753 lda [%o0]ASI_USER, %o0 754 retl 755 st %o0, [%o1] 756 SET_SIZE(fuword32_noerr) 757 758 ENTRY(fuword64_noerr) 759 ldxa [%o0]ASI_USER, %o0 760 retl 761 stx %o0, [%o1] 762 SET_SIZE(fuword64_noerr) 763 764 ENTRY(suword8_noerr) 765 retl 766 stba %o1, [%o0]ASI_USER 767 SET_SIZE(suword8_noerr) 768 769 ENTRY(suword16_noerr) 770 retl 771 stha %o1, [%o0]ASI_USER 772 SET_SIZE(suword16_noerr) 773 774 ENTRY(suword32_noerr) 775 retl 776 sta %o1, [%o0]ASI_USER 777 SET_SIZE(suword32_noerr) 778 779 ENTRY(suword64_noerr) 780 retl 781 stxa %o1, [%o0]ASI_USER 782 SET_SIZE(suword64_noerr) 783 784 .weak subyte 785 subyte=suword8 786 .weak subyte_noerr 787 subyte_noerr=suword8_noerr 788 #ifdef _LP64 789 .weak fulword 790 fulword=fuword64 791 .weak fulword_noerr 792 fulword_noerr=fuword64_noerr 793 .weak sulword 794 sulword=suword64 795 .weak sulword_noerr 796 sulword_noerr=suword64_noerr 797 #else 798 .weak fulword 799 fulword=fuword32 800 .weak fulword_noerr 801 fulword_noerr=fuword32_noerr 802 .weak sulword 803 sulword=suword32 804 .weak sulword_noerr 805 sulword_noerr=suword32_noerr 806 #endif /* LP64 */ 807 808 /* 809 * We define rdtick here, but not for sun4v. On sun4v systems, the %tick 810 * and %stick should not be read directly without considering the tick 811 * and stick offset kernel variables introduced to support sun4v OS 812 * suspension. 813 */ 814 #if !defined (sun4v) 815 816 ENTRY(rdtick) 817 retl 818 rd %tick, %o0 819 SET_SIZE(rdtick) 820 821 #endif /* !sun4v */ 822 823 /* 824 * Set tba to given address, no side effects. 825 */ 826 827 ENTRY(set_tba) 828 mov %o0, %o1 829 rdpr %tba, %o0 830 wrpr %o1, %tba 831 retl 832 nop 833 SET_SIZE(set_tba) 834 835 ENTRY(get_tba) 836 retl 837 rdpr %tba, %o0 838 SET_SIZE(get_tba) 839 840 ENTRY_NP(setpstate) 841 retl 842 wrpr %g0, %o0, %pstate 843 SET_SIZE(setpstate) 844 845 ENTRY_NP(getpstate) 846 retl 847 rdpr %pstate, %o0 848 SET_SIZE(getpstate) 849 850 ENTRY_NP(dtrace_interrupt_disable) 851 rdpr %pstate, %o0 852 andn %o0, PSTATE_IE, %o1 853 retl 854 wrpr %g0, %o1, %pstate 855 SET_SIZE(dtrace_interrupt_disable) 856 857 ENTRY_NP(dtrace_interrupt_enable) 858 retl 859 wrpr %g0, %o0, %pstate 860 SET_SIZE(dtrace_interrupt_enable) 861 862 #ifdef SF_ERRATA_51 863 .align 32 864 ENTRY(dtrace_membar_return) 865 retl 866 nop 867 SET_SIZE(dtrace_membar_return) 868 #define DTRACE_MEMBAR_RETURN ba,pt %icc, dtrace_membar_return 869 #else 870 #define DTRACE_MEMBAR_RETURN retl 871 #endif 872 873 ENTRY(dtrace_membar_producer) 874 DTRACE_MEMBAR_RETURN 875 membar #StoreStore 876 SET_SIZE(dtrace_membar_producer) 877 878 ENTRY(dtrace_membar_consumer) 879 DTRACE_MEMBAR_RETURN 880 membar #LoadLoad 881 SET_SIZE(dtrace_membar_consumer) 882 883 ENTRY_NP(dtrace_flush_windows) 884 retl 885 flushw 886 SET_SIZE(dtrace_flush_windows) 887 888 /* 889 * %g1 pcstack 890 * %g2 iteration count 891 * %g3 final %fp 892 * %g4 final %i7 893 * %g5 saved %cwp (so we can get back to the original window) 894 * 895 * %o0 pcstack / return value (iteration count) 896 * %o1 limit / saved %cansave 897 * %o2 lastfp 898 * %o3 lastpc 899 * %o4 saved %canrestore 900 * %o5 saved %pstate (to restore interrupts) 901 * 902 * Note: The frame pointer returned via lastfp is safe to use as 903 * long as getpcstack_top() returns either (0) or a value less 904 * than (limit). 905 */ 906 ENTRY_NP(getpcstack_top) 907 908 rdpr %pstate, %o5 909 andn %o5, PSTATE_IE, %g1 910 wrpr %g0, %g1, %pstate ! disable interrupts 911 912 mov %o0, %g1 ! we need the pcstack pointer while 913 ! we're visiting other windows 914 915 rdpr %canrestore, %g2 ! number of available windows 916 sub %g2, 1, %g2 ! account for skipped frame 917 cmp %g2, %o1 ! compare with limit 918 movg %icc, %o1, %g2 ! %g2 = min(%canrestore-1, limit) 919 920 brlez,a,pn %g2, 3f ! Use slow path if count <= 0 -- 921 clr %o0 ! return zero. 922 923 mov %g2, %o0 ! set up return value 924 925 rdpr %cwp, %g5 ! remember the register window state 926 rdpr %cansave, %o1 ! 'restore' changes, so we can undo 927 rdpr %canrestore, %o4 ! its effects when we finish. 928 929 restore ! skip caller's frame 930 1: 931 st %i7, [%g1] ! stash return address in pcstack 932 restore ! go to the next frame 933 subcc %g2, 1, %g2 ! decrement the count 934 bnz,pt %icc, 1b ! loop until count reaches 0 935 add %g1, 4, %g1 ! increment pcstack 936 937 mov %i6, %g3 ! copy the final %fp and return PC 938 mov %i7, %g4 ! aside so we can return them to our 939 ! caller 940 941 wrpr %g0, %g5, %cwp ! jump back to the original window 942 wrpr %g0, %o1, %cansave ! and restore the original register 943 wrpr %g0, %o4, %canrestore ! window state. 944 2: 945 stn %g3, [%o2] ! store the frame pointer and pc 946 st %g4, [%o3] ! so our caller can continue the trace 947 948 retl ! return to caller 949 wrpr %g0, %o5, %pstate ! restore interrupts 950 951 3: 952 flushw ! flush register windows, then 953 ldn [%fp + STACK_BIAS + 14*CLONGSIZE], %g3 ! load initial fp 954 ba 2b 955 ldn [%fp + STACK_BIAS + 15*CLONGSIZE], %g4 ! and pc 956 SET_SIZE(getpcstack_top) 957 958 ENTRY_NP(setwstate) 959 retl 960 wrpr %g0, %o0, %wstate 961 SET_SIZE(setwstate) 962 963 964 ENTRY_NP(getwstate) 965 retl 966 rdpr %wstate, %o0 967 SET_SIZE(getwstate) 968 969 970 /* 971 * int panic_trigger(int *tp) 972 * 973 * A panic trigger is a word which is updated atomically and can only be set 974 * once. We atomically store 0xFF into the high byte and load the old value. 975 * If the byte was 0xFF, the trigger has already been activated and we fail. 976 * If the previous value was 0 or not 0xFF, we succeed. This allows a 977 * partially corrupt trigger to still trigger correctly. DTrace has its own 978 * version of this function to allow it to panic correctly from probe context. 979 */ 980 981 ENTRY_NP(panic_trigger) 982 ldstub [%o0], %o0 ! store 0xFF, load byte into %o0 983 cmp %o0, 0xFF ! compare %o0 to 0xFF 984 set 1, %o1 ! %o1 = 1 985 be,a 0f ! if (%o0 == 0xFF) goto 0f (else annul) 986 set 0, %o1 ! delay - %o1 = 0 987 0: retl 988 mov %o1, %o0 ! return (%o1); 989 SET_SIZE(panic_trigger) 990 991 ENTRY_NP(dtrace_panic_trigger) 992 ldstub [%o0], %o0 ! store 0xFF, load byte into %o0 993 cmp %o0, 0xFF ! compare %o0 to 0xFF 994 set 1, %o1 ! %o1 = 1 995 be,a 0f ! if (%o0 == 0xFF) goto 0f (else annul) 996 set 0, %o1 ! delay - %o1 = 0 997 0: retl 998 mov %o1, %o0 ! return (%o1); 999 SET_SIZE(dtrace_panic_trigger) 1000 1001 /* 1002 * void vpanic(const char *format, va_list alist) 1003 * 1004 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 1005 * into the panic code implemented in panicsys(). vpanic() is responsible 1006 * for passing through the format string and arguments, and constructing a 1007 * regs structure on the stack into which it saves the current register 1008 * values. If we are not dying due to a fatal trap, these registers will 1009 * then be preserved in panicbuf as the current processor state. Before 1010 * invoking panicsys(), vpanic() activates the first panic trigger (see 1011 * common/os/panic.c) and switches to the panic_stack if successful. Note that 1012 * DTrace takes a slightly different panic path if it must panic from probe 1013 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 1014 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 1015 * branches back into vpanic(). 1016 */ 1017 1018 ENTRY_NP(vpanic) 1019 1020 save %sp, -SA(MINFRAME + REGSIZE), %sp ! save and allocate regs 1021 1022 ! 1023 ! The v9 struct regs has a 64-bit r_tstate field, which we use here 1024 ! to store the %ccr, %asi, %pstate, and %cwp as they would appear 1025 ! in %tstate if a trap occurred. We leave it up to the debugger to 1026 ! realize what happened and extract the register values. 1027 ! 1028 rd %ccr, %l0 ! %l0 = %ccr 1029 sllx %l0, TSTATE_CCR_SHIFT, %l0 ! %l0 <<= CCR_SHIFT 1030 rd %asi, %l1 ! %l1 = %asi 1031 sllx %l1, TSTATE_ASI_SHIFT, %l1 ! %l1 <<= ASI_SHIFT 1032 or %l0, %l1, %l0 ! %l0 |= %l1 1033 rdpr %pstate, %l1 ! %l1 = %pstate 1034 sllx %l1, TSTATE_PSTATE_SHIFT, %l1 ! %l1 <<= PSTATE_SHIFT 1035 or %l0, %l1, %l0 ! %l0 |= %l1 1036 rdpr %cwp, %l1 ! %l1 = %cwp 1037 sllx %l1, TSTATE_CWP_SHIFT, %l1 ! %l1 <<= CWP_SHIFT 1038 or %l0, %l1, %l0 ! %l0 |= %l1 1039 1040 set vpanic, %l1 ! %l1 = %pc (vpanic) 1041 add %l1, 4, %l2 ! %l2 = %npc (vpanic+4) 1042 rd %y, %l3 ! %l3 = %y 1043 ! 1044 ! Flush register windows before panic_trigger() in order to avoid a 1045 ! problem that a dump hangs if flush_windows() causes another panic. 1046 ! 1047 call flush_windows 1048 nop 1049 1050 sethi %hi(panic_quiesce), %o0 1051 call panic_trigger 1052 or %o0, %lo(panic_quiesce), %o0 ! if (!panic_trigger( 1053 1054 vpanic_common: 1055 tst %o0 ! &panic_quiesce)) 1056 be 0f ! goto 0f; 1057 mov %o0, %l4 ! delay - %l4 = %o0 1058 1059 ! 1060 ! If panic_trigger() was successful, we are the first to initiate a 1061 ! panic: switch to the panic_stack. 1062 ! 1063 set panic_stack, %o0 ! %o0 = panic_stack 1064 set PANICSTKSIZE, %o1 ! %o1 = size of stack 1065 add %o0, %o1, %o0 ! %o0 = top of stack 1066 1067 sub %o0, SA(MINFRAME + REGSIZE) + STACK_BIAS, %sp 1068 1069 ! 1070 ! Now that we've got everything set up, store each register to its 1071 ! designated location in the regs structure allocated on the stack. 1072 ! The register set we store is the equivalent of the registers at 1073 ! the time the %pc was pointing to vpanic, thus the %i's now contain 1074 ! what the %o's contained prior to the save instruction. 1075 ! 1076 0: stx %l0, [%sp + STACK_BIAS + SA(MINFRAME) + TSTATE_OFF] 1077 stx %g1, [%sp + STACK_BIAS + SA(MINFRAME) + G1_OFF] 1078 stx %g2, [%sp + STACK_BIAS + SA(MINFRAME) + G2_OFF] 1079 stx %g3, [%sp + STACK_BIAS + SA(MINFRAME) + G3_OFF] 1080 stx %g4, [%sp + STACK_BIAS + SA(MINFRAME) + G4_OFF] 1081 stx %g5, [%sp + STACK_BIAS + SA(MINFRAME) + G5_OFF] 1082 stx %g6, [%sp + STACK_BIAS + SA(MINFRAME) + G6_OFF] 1083 stx %g7, [%sp + STACK_BIAS + SA(MINFRAME) + G7_OFF] 1084 stx %i0, [%sp + STACK_BIAS + SA(MINFRAME) + O0_OFF] 1085 stx %i1, [%sp + STACK_BIAS + SA(MINFRAME) + O1_OFF] 1086 stx %i2, [%sp + STACK_BIAS + SA(MINFRAME) + O2_OFF] 1087 stx %i3, [%sp + STACK_BIAS + SA(MINFRAME) + O3_OFF] 1088 stx %i4, [%sp + STACK_BIAS + SA(MINFRAME) + O4_OFF] 1089 stx %i5, [%sp + STACK_BIAS + SA(MINFRAME) + O5_OFF] 1090 stx %i6, [%sp + STACK_BIAS + SA(MINFRAME) + O6_OFF] 1091 stx %i7, [%sp + STACK_BIAS + SA(MINFRAME) + O7_OFF] 1092 stn %l1, [%sp + STACK_BIAS + SA(MINFRAME) + PC_OFF] 1093 stn %l2, [%sp + STACK_BIAS + SA(MINFRAME) + NPC_OFF] 1094 st %l3, [%sp + STACK_BIAS + SA(MINFRAME) + Y_OFF] 1095 1096 mov %l4, %o3 ! %o3 = on_panic_stack 1097 add %sp, STACK_BIAS + SA(MINFRAME), %o2 ! %o2 = ®s 1098 mov %i1, %o1 ! %o1 = alist 1099 call panicsys ! panicsys(); 1100 mov %i0, %o0 ! %o0 = format 1101 ret 1102 restore 1103 1104 SET_SIZE(vpanic) 1105 1106 ENTRY_NP(dtrace_vpanic) 1107 1108 save %sp, -SA(MINFRAME + REGSIZE), %sp ! save and allocate regs 1109 1110 ! 1111 ! The v9 struct regs has a 64-bit r_tstate field, which we use here 1112 ! to store the %ccr, %asi, %pstate, and %cwp as they would appear 1113 ! in %tstate if a trap occurred. We leave it up to the debugger to 1114 ! realize what happened and extract the register values. 1115 ! 1116 rd %ccr, %l0 ! %l0 = %ccr 1117 sllx %l0, TSTATE_CCR_SHIFT, %l0 ! %l0 <<= CCR_SHIFT 1118 rd %asi, %l1 ! %l1 = %asi 1119 sllx %l1, TSTATE_ASI_SHIFT, %l1 ! %l1 <<= ASI_SHIFT 1120 or %l0, %l1, %l0 ! %l0 |= %l1 1121 rdpr %pstate, %l1 ! %l1 = %pstate 1122 sllx %l1, TSTATE_PSTATE_SHIFT, %l1 ! %l1 <<= PSTATE_SHIFT 1123 or %l0, %l1, %l0 ! %l0 |= %l1 1124 rdpr %cwp, %l1 ! %l1 = %cwp 1125 sllx %l1, TSTATE_CWP_SHIFT, %l1 ! %l1 <<= CWP_SHIFT 1126 or %l0, %l1, %l0 ! %l0 |= %l1 1127 1128 set dtrace_vpanic, %l1 ! %l1 = %pc (vpanic) 1129 add %l1, 4, %l2 ! %l2 = %npc (vpanic+4) 1130 rd %y, %l3 ! %l3 = %y 1131 ! 1132 ! Flush register windows before panic_trigger() in order to avoid a 1133 ! problem that a dump hangs if flush_windows() causes another panic. 1134 ! 1135 call dtrace_flush_windows 1136 nop 1137 1138 sethi %hi(panic_quiesce), %o0 1139 call dtrace_panic_trigger 1140 or %o0, %lo(panic_quiesce), %o0 ! if (!panic_trigger( 1141 1142 ba,a vpanic_common 1143 SET_SIZE(dtrace_vpanic) 1144 1145 ENTRY(get_subcc_ccr) 1146 wr %g0, %ccr ! clear condition codes 1147 subcc %o0, %o1, %g0 1148 retl 1149 rd %ccr, %o0 ! return condition codes 1150 SET_SIZE(get_subcc_ccr) 1151 1152 ENTRY_NP(ftrace_interrupt_disable) 1153 rdpr %pstate, %o0 1154 andn %o0, PSTATE_IE, %o1 1155 retl 1156 wrpr %g0, %o1, %pstate 1157 SET_SIZE(ftrace_interrupt_disable) 1158 1159 ENTRY_NP(ftrace_interrupt_enable) 1160 retl 1161 wrpr %g0, %o0, %pstate 1162 SET_SIZE(ftrace_interrupt_enable) 1163