OS-7753 THREAD_KPRI_RELEASE does nothing of the sort Reviewed by: Bryan Cantrill <bryan@joyent.com> Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #if defined(lint) 29 #include <sys/types.h> 30 #include <sys/thread.h> 31 #include <sys/cpuvar.h> 32 #else /* lint */ 33 #include "assym.h" 34 #endif /* lint */ 35 36 #include <sys/t_lock.h> 37 #include <sys/mutex.h> 38 #include <sys/mutex_impl.h> 39 #include <sys/rwlock_impl.h> 40 #include <sys/asm_linkage.h> 41 #include <sys/machlock.h> 42 #include <sys/machthread.h> 43 #include <sys/lockstat.h> 44 45 /* #define DEBUG */ 46 47 #ifdef DEBUG 48 #include <sys/machparam.h> 49 #endif /* DEBUG */ 50 51 /************************************************************************ 52 * ATOMIC OPERATIONS 53 */ 54 55 /* 56 * uint8_t ldstub(uint8_t *cp) 57 * 58 * Store 0xFF at the specified location, and return its previous content. 59 */ 60 61 #if defined(lint) 62 uint8_t 63 ldstub(uint8_t *cp) 64 { 65 uint8_t rv; 66 rv = *cp; 67 *cp = 0xFF; 68 return rv; 69 } 70 #else /* lint */ 71 72 ENTRY(ldstub) 73 retl 74 ldstub [%o0], %o0 75 SET_SIZE(ldstub) 76 77 #endif /* lint */ 78 79 /************************************************************************ 80 * MEMORY BARRIERS -- see atomic.h for full descriptions. 81 */ 82 83 #if defined(lint) 84 85 void 86 membar_enter(void) 87 {} 88 89 void 90 membar_exit(void) 91 {} 92 93 void 94 membar_producer(void) 95 {} 96 97 void 98 membar_consumer(void) 99 {} 100 101 #else /* lint */ 102 103 #ifdef SF_ERRATA_51 104 .align 32 105 ENTRY(membar_return) 106 retl 107 nop 108 SET_SIZE(membar_return) 109 #define MEMBAR_RETURN ba,pt %icc, membar_return 110 #else 111 #define MEMBAR_RETURN retl 112 #endif 113 114 ENTRY(membar_enter) 115 MEMBAR_RETURN 116 membar #StoreLoad|#StoreStore 117 SET_SIZE(membar_enter) 118 119 ENTRY(membar_exit) 120 MEMBAR_RETURN 121 membar #LoadStore|#StoreStore 122 SET_SIZE(membar_exit) 123 124 ENTRY(membar_producer) 125 MEMBAR_RETURN 126 membar #StoreStore 127 SET_SIZE(membar_producer) 128 129 ENTRY(membar_consumer) 130 MEMBAR_RETURN 131 membar #LoadLoad 132 SET_SIZE(membar_consumer) 133 134 #endif /* lint */ 135 136 /************************************************************************ 137 * MINIMUM LOCKS 138 */ 139 140 #if defined(lint) 141 142 /* 143 * lock_try(lp), ulock_try(lp) 144 * - returns non-zero on success. 145 * - doesn't block interrupts so don't use this to spin on a lock. 146 * - uses "0xFF is busy, anything else is free" model. 147 * 148 * ulock_try() is for a lock in the user address space. 149 * For all V7/V8 sparc systems they are same since the kernel and 150 * user are mapped in a user' context. 151 * For V9 platforms the lock_try and ulock_try are different impl. 152 */ 153 154 int 155 lock_try(lock_t *lp) 156 { 157 return (0xFF ^ ldstub(lp)); 158 } 159 160 int 161 lock_spin_try(lock_t *lp) 162 { 163 return (0xFF ^ ldstub(lp)); 164 } 165 166 void 167 lock_set(lock_t *lp) 168 { 169 extern void lock_set_spin(lock_t *); 170 171 if (!lock_try(lp)) 172 lock_set_spin(lp); 173 membar_enter(); 174 } 175 176 void 177 lock_clear(lock_t *lp) 178 { 179 membar_exit(); 180 *lp = 0; 181 } 182 183 int 184 ulock_try(lock_t *lp) 185 { 186 return (0xFF ^ ldstub(lp)); 187 } 188 189 void 190 ulock_clear(lock_t *lp) 191 { 192 membar_exit(); 193 *lp = 0; 194 } 195 196 #else /* lint */ 197 198 .align 32 199 ENTRY(lock_try) 200 ldstub [%o0], %o1 ! try to set lock, get value in %o1 201 brnz,pn %o1, 1f 202 membar #LoadLoad 203 .lock_try_lockstat_patch_point: 204 retl 205 or %o0, 1, %o0 ! ensure lo32 != 0 206 1: 207 retl 208 clr %o0 209 SET_SIZE(lock_try) 210 211 .align 32 212 ENTRY(lock_spin_try) 213 ldstub [%o0], %o1 ! try to set lock, get value in %o1 214 brnz,pn %o1, 1f 215 membar #LoadLoad 216 retl 217 or %o0, 1, %o0 ! ensure lo32 != 0 218 1: 219 retl 220 clr %o0 221 SET_SIZE(lock_spin_try) 222 223 .align 32 224 ENTRY(lock_set) 225 ldstub [%o0], %o1 226 brnz,pn %o1, 1f ! go to C for the hard case 227 membar #LoadLoad 228 .lock_set_lockstat_patch_point: 229 retl 230 nop 231 1: 232 sethi %hi(lock_set_spin), %o2 ! load up for jump to C 233 jmp %o2 + %lo(lock_set_spin) 234 nop ! delay: do nothing 235 SET_SIZE(lock_set) 236 237 ENTRY(lock_clear) 238 membar #LoadStore|#StoreStore 239 .lock_clear_lockstat_patch_point: 240 retl 241 clrb [%o0] 242 SET_SIZE(lock_clear) 243 244 .align 32 245 ENTRY(ulock_try) 246 ldstuba [%o0]ASI_USER, %o1 ! try to set lock, get value in %o1 247 xor %o1, 0xff, %o0 ! delay - return non-zero if success 248 retl 249 membar #LoadLoad 250 SET_SIZE(ulock_try) 251 252 ENTRY(ulock_clear) 253 membar #LoadStore|#StoreStore 254 retl 255 stba %g0, [%o0]ASI_USER ! clear lock 256 SET_SIZE(ulock_clear) 257 258 #endif /* lint */ 259 260 261 /* 262 * lock_set_spl(lp, new_pil, *old_pil_addr) 263 * Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr. 264 */ 265 266 #if defined(lint) 267 268 /* ARGSUSED */ 269 void 270 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil_addr) 271 { 272 extern int splr(int); 273 extern void lock_set_spl_spin(lock_t *, int, u_short *, int); 274 int old_pil; 275 276 old_pil = splr(new_pil); 277 if (!lock_try(lp)) { 278 lock_set_spl_spin(lp, new_pil, old_pil_addr, old_pil); 279 } else { 280 *old_pil_addr = (u_short)old_pil; 281 membar_enter(); 282 } 283 } 284 285 #else /* lint */ 286 287 ENTRY(lock_set_spl) 288 rdpr %pil, %o3 ! %o3 = current pil 289 cmp %o3, %o1 ! is current pil high enough? 290 bl,a,pt %icc, 1f ! if not, write %pil in delay 291 wrpr %g0, %o1, %pil 292 1: 293 ldstub [%o0], %o4 ! try the lock 294 brnz,pn %o4, 2f ! go to C for the miss case 295 membar #LoadLoad 296 .lock_set_spl_lockstat_patch_point: 297 retl 298 sth %o3, [%o2] ! delay - save original pil 299 2: 300 sethi %hi(lock_set_spl_spin), %o5 ! load up jmp to C 301 jmp %o5 + %lo(lock_set_spl_spin) ! jmp to lock_set_spl_spin 302 nop ! delay: do nothing 303 SET_SIZE(lock_set_spl) 304 305 #endif /* lint */ 306 307 /* 308 * lock_clear_splx(lp, s) 309 */ 310 311 #if defined(lint) 312 313 void 314 lock_clear_splx(lock_t *lp, int s) 315 { 316 extern void splx(int); 317 318 lock_clear(lp); 319 splx(s); 320 } 321 322 #else /* lint */ 323 324 ENTRY(lock_clear_splx) 325 ldn [THREAD_REG + T_CPU], %o2 ! get CPU pointer 326 membar #LoadStore|#StoreStore 327 ld [%o2 + CPU_BASE_SPL], %o2 328 clrb [%o0] ! clear lock 329 cmp %o2, %o1 ! compare new to base 330 movl %xcc, %o1, %o2 ! use new pri if base is less 331 .lock_clear_splx_lockstat_patch_point: 332 retl 333 wrpr %g0, %o2, %pil 334 SET_SIZE(lock_clear_splx) 335 336 #endif /* lint */ 337 338 /* 339 * mutex_enter() and mutex_exit(). 340 * 341 * These routines handle the simple cases of mutex_enter() (adaptive 342 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 343 * If anything complicated is going on we punt to mutex_vector_enter(). 344 * 345 * mutex_tryenter() is similar to mutex_enter() but returns zero if 346 * the lock cannot be acquired, nonzero on success. 347 * 348 * If mutex_exit() gets preempted in the window between checking waiters 349 * and clearing the lock, we can miss wakeups. Disabling preemption 350 * in the mutex code is prohibitively expensive, so instead we detect 351 * mutex preemption by examining the trapped PC in the interrupt path. 352 * If we interrupt a thread in mutex_exit() that has not yet cleared 353 * the lock, pil_interrupt() resets its PC back to the beginning of 354 * mutex_exit() so it will check again for waiters when it resumes. 355 * 356 * The lockstat code below is activated when the lockstat driver 357 * calls lockstat_hot_patch() to hot-patch the kernel mutex code. 358 * Note that we don't need to test lockstat_event_mask here -- we won't 359 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. 360 */ 361 362 #if defined (lint) 363 364 /* ARGSUSED */ 365 void 366 mutex_enter(kmutex_t *lp) 367 {} 368 369 /* ARGSUSED */ 370 int 371 mutex_tryenter(kmutex_t *lp) 372 { return (0); } 373 374 /* ARGSUSED */ 375 void 376 mutex_exit(kmutex_t *lp) 377 {} 378 379 /* ARGSUSED */ 380 void * 381 mutex_owner_running(mutex_impl_t *lp) 382 { return (NULL); } 383 384 #else 385 .align 32 386 ENTRY(mutex_enter) 387 mov THREAD_REG, %o1 388 casx [%o0], %g0, %o1 ! try to acquire as adaptive 389 brnz,pn %o1, 1f ! locked or wrong type 390 membar #LoadLoad 391 .mutex_enter_lockstat_patch_point: 392 retl 393 nop 394 1: 395 sethi %hi(mutex_vector_enter), %o2 ! load up for jump to C 396 jmp %o2 + %lo(mutex_vector_enter) 397 nop 398 SET_SIZE(mutex_enter) 399 400 ENTRY(mutex_tryenter) 401 mov THREAD_REG, %o1 402 casx [%o0], %g0, %o1 ! try to acquire as adaptive 403 brnz,pn %o1, 1f ! locked or wrong type continue 404 membar #LoadLoad 405 .mutex_tryenter_lockstat_patch_point: 406 retl 407 or %o0, 1, %o0 ! ensure lo32 != 0 408 1: 409 sethi %hi(mutex_vector_tryenter), %o2 ! hi bits 410 jmp %o2 + %lo(mutex_vector_tryenter) ! go to C 411 nop 412 SET_SIZE(mutex_tryenter) 413 414 ENTRY(mutex_adaptive_tryenter) 415 mov THREAD_REG, %o1 416 casx [%o0], %g0, %o1 ! try to acquire as adaptive 417 brnz,pn %o1, 0f ! locked or wrong type 418 membar #LoadLoad 419 retl 420 or %o0, 1, %o0 ! ensure lo32 != 0 421 0: 422 retl 423 mov %g0, %o0 424 SET_SIZE(mutex_adaptive_tryenter) 425 426 ! these need to be together and cache aligned for performance. 427 .align 64 428 .global mutex_exit_critical_size 429 .global mutex_exit_critical_start 430 .global mutex_owner_running_critical_size 431 .global mutex_owner_running_critical_start 432 433 mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start 434 435 .align 32 436 437 ENTRY(mutex_exit) 438 mutex_exit_critical_start: ! If we are interrupted, restart here 439 ldn [%o0], %o1 ! get the owner field 440 membar #LoadStore|#StoreStore 441 cmp THREAD_REG, %o1 ! do we own lock with no waiters? 442 be,a,pt %ncc, 1f ! if so, drive on ... 443 stn %g0, [%o0] ! delay: clear lock if we owned it 444 .mutex_exit_critical_end: ! for pil_interrupt() hook 445 ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases 446 1: 447 .mutex_exit_lockstat_patch_point: 448 retl 449 nop 450 SET_SIZE(mutex_exit) 451 452 mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start 453 454 .align 32 455 456 ENTRY(mutex_owner_running) 457 mutex_owner_running_critical_start: ! If interrupted restart here 458 ldn [%o0], %o1 ! get the owner field 459 and %o1, MUTEX_THREAD, %o1 ! remove the waiters bit if any 460 brz,pn %o1, 1f ! if so, drive on ... 461 nop 462 ldn [%o1+T_CPU], %o2 ! get owner->t_cpu 463 ldn [%o2+CPU_THREAD], %o3 ! get owner->t_cpu->cpu_thread 464 .mutex_owner_running_critical_end: ! for pil_interrupt() hook 465 cmp %o1, %o3 ! owner == running thread? 466 be,a,pt %xcc, 2f ! yes, go return cpu 467 nop 468 1: 469 retl 470 mov %g0, %o0 ! return 0 (owner not running) 471 2: 472 retl 473 mov %o2, %o0 ! owner running, return cpu 474 SET_SIZE(mutex_owner_running) 475 476 #endif /* lint */ 477 478 /* 479 * rw_enter() and rw_exit(). 480 * 481 * These routines handle the simple cases of rw_enter (write-locking an unheld 482 * lock or read-locking a lock that's neither write-locked nor write-wanted) 483 * and rw_exit (no waiters or not the last reader). If anything complicated 484 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 485 */ 486 #if defined(lint) 487 488 /* ARGSUSED */ 489 void 490 rw_enter(krwlock_t *lp, krw_t rw) 491 {} 492 493 /* ARGSUSED */ 494 void 495 rw_exit(krwlock_t *lp) 496 {} 497 498 #else 499 500 .align 16 501 ENTRY(rw_enter) 502 cmp %o1, RW_WRITER ! entering as writer? 503 be,a,pn %icc, 2f ! if so, go do it ... 504 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner 505 ld [THREAD_REG + T_KPRI_REQ], %o3 ! begin THREAD_KPRI_REQUEST() 506 ldn [%o0], %o4 ! %o4 = old lock value 507 inc %o3 ! bump kpri 508 st %o3, [THREAD_REG + T_KPRI_REQ] ! store new kpri 509 1: 510 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted? 511 bz,pt %xcc, 3f ! if so, prepare to block 512 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count 513 sethi %hi(rw_enter_sleep), %o2 ! load up jump 514 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 515 nop ! delay: do nothing 516 3: 517 casx [%o0], %o4, %o5 ! try to grab read lock 518 cmp %o4, %o5 ! did we get it? 519 #ifdef sun4v 520 be,a,pt %xcc, 0f 521 membar #LoadLoad 522 sethi %hi(rw_enter_sleep), %o2 ! load up jump 523 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep 524 nop ! delay: do nothing 525 0: 526 #else /* sun4v */ 527 bne,pn %xcc, 1b ! if not, try again 528 mov %o5, %o4 ! delay: %o4 = old lock value 529 membar #LoadLoad 530 #endif /* sun4v */ 531 .rw_read_enter_lockstat_patch_point: 532 retl 533 nop 534 2: 535 casx [%o0], %g0, %o5 ! try to grab write lock 536 brz,pt %o5, 4f ! branch around if we got it 537 membar #LoadLoad ! done regardless of where we go 538 sethi %hi(rw_enter_sleep), %o2 539 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not 540 nop ! delay: do nothing 541 4: 542 .rw_write_enter_lockstat_patch_point: 543 retl 544 nop 545 SET_SIZE(rw_enter) 546 547 .align 16 548 ENTRY(rw_exit) 549 ldn [%o0], %o4 ! %o4 = old lock value 550 membar #LoadStore|#StoreStore ! membar_exit() 551 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader 552 bnz,pn %xcc, 2f ! single reader, no waiters? 553 clr %o1 554 1: 555 ld [THREAD_REG + T_KPRI_REQ], %g1 ! begin THREAD_KPRI_RELEASE() 556 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat) 557 casx [%o0], %o4, %o5 ! try to drop lock 558 cmp %o4, %o5 ! did we succeed? 559 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 560 dec %g1 ! delay: drop kpri 561 .rw_read_exit_lockstat_patch_point: 562 retl 563 st %g1, [THREAD_REG + T_KPRI_REQ] ! delay: store new kpri 564 2: 565 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer? 566 bnz,a,pt %xcc, 3f 567 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner 568 cmp %o5, RW_READ_LOCK ! would lock still be held? 569 bge,pt %xcc, 1b ! if so, go ahead and drop it 570 nop 571 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters 572 nop 573 3: 574 casx [%o0], %o4, %o1 ! try to drop write lock 575 cmp %o4, %o1 ! did we succeed? 576 bne,pn %xcc, rw_exit_wakeup ! if not, go to C 577 nop 578 .rw_write_exit_lockstat_patch_point: 579 retl 580 nop 581 SET_SIZE(rw_exit) 582 583 #endif 584 585 #if defined(lint) 586 587 void 588 lockstat_hot_patch(void) 589 {} 590 591 #else 592 593 #define RETL 0x81c3e008 594 #define NOP 0x01000000 595 #define BA 0x10800000 596 597 #define DISP22 ((1 << 22) - 1) 598 #define ANNUL 0x20000000 599 600 #define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs) \ 601 ba 1f; \ 602 rd %pc, %o0; \ 603 save %sp, -SA(MINFRAME), %sp; \ 604 set lockstat_probemap, %l1; \ 605 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 606 brz,pn %o0, 0f; \ 607 ldub [THREAD_REG + T_LOCKSTAT], %l0; \ 608 add %l0, 1, %l2; \ 609 stub %l2, [THREAD_REG + T_LOCKSTAT]; \ 610 set lockstat_probe, %g1; \ 611 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ 612 brz,a,pn %o0, 0f; \ 613 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 614 ldn [%g1], %g2; \ 615 mov rs, %o2; \ 616 jmpl %g2, %o7; \ 617 mov %i0, %o1; \ 618 stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 619 0: ret; \ 620 restore %g0, 1, %o0; /* for mutex_tryenter / lock_try */ \ 621 1: set addr, %o1; \ 622 sub %o0, %o1, %o0; \ 623 srl %o0, 2, %o0; \ 624 inc %o0; \ 625 set DISP22, %o1; \ 626 and %o1, %o0, %o0; \ 627 set BA, %o1; \ 628 or %o1, %o0, %o0; \ 629 sethi %hi(annul), %o2; \ 630 add %o0, %o2, %o2; \ 631 set addr, %o0; \ 632 set normal_instr, %o1; \ 633 ld [%i0 + (event * DTRACE_IDSIZE)], %o3; \ 634 tst %o3; \ 635 movnz %icc, %o2, %o1; \ 636 call hot_patch_kernel_text; \ 637 mov 4, %o2; \ 638 membar #Sync 639 640 #define HOT_PATCH(addr, event, normal_instr) \ 641 HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1) 642 643 #define HOT_PATCH_ARG(addr, event, normal_instr, arg) \ 644 HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg) 645 646 #define HOT_PATCH_ANNULLED(addr, event, normal_instr) \ 647 HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1) 648 649 ENTRY(lockstat_hot_patch) 650 save %sp, -SA(MINFRAME), %sp 651 set lockstat_probemap, %i0 652 HOT_PATCH(.mutex_enter_lockstat_patch_point, 653 LS_MUTEX_ENTER_ACQUIRE, RETL) 654 HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point, 655 LS_MUTEX_TRYENTER_ACQUIRE, RETL) 656 HOT_PATCH(.mutex_exit_lockstat_patch_point, 657 LS_MUTEX_EXIT_RELEASE, RETL) 658 HOT_PATCH(.rw_write_enter_lockstat_patch_point, 659 LS_RW_ENTER_ACQUIRE, RETL) 660 HOT_PATCH(.rw_read_enter_lockstat_patch_point, 661 LS_RW_ENTER_ACQUIRE, RETL) 662 HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point, 663 LS_RW_EXIT_RELEASE, RETL, RW_WRITER) 664 HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point, 665 LS_RW_EXIT_RELEASE, RETL, RW_READER) 666 HOT_PATCH(.lock_set_lockstat_patch_point, 667 LS_LOCK_SET_ACQUIRE, RETL) 668 HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point, 669 LS_LOCK_TRY_ACQUIRE, RETL) 670 HOT_PATCH(.lock_clear_lockstat_patch_point, 671 LS_LOCK_CLEAR_RELEASE, RETL) 672 HOT_PATCH(.lock_set_spl_lockstat_patch_point, 673 LS_LOCK_SET_SPL_ACQUIRE, RETL) 674 HOT_PATCH(.lock_clear_splx_lockstat_patch_point, 675 LS_LOCK_CLEAR_SPLX_RELEASE, RETL) 676 ret 677 restore 678 SET_SIZE(lockstat_hot_patch) 679 680 #endif /* lint */ 681 682 /* 683 * asm_mutex_spin_enter(mutex_t *) 684 * 685 * For use by assembly interrupt handler only. 686 * Does not change spl, since the interrupt handler is assumed to be 687 * running at high level already. 688 * Traps may be off, so cannot panic. 689 * Does not keep statistics on the lock. 690 * 691 * Entry: %l6 - points to mutex 692 * %l7 - address of call (returns to %l7+8) 693 * Uses: %l6, %l5 694 */ 695 #ifndef lint 696 .align 16 697 ENTRY_NP(asm_mutex_spin_enter) 698 ldstub [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5 699 1: 700 tst %l5 701 bnz 3f ! lock already held - go spin 702 nop 703 2: 704 jmp %l7 + 8 ! return 705 membar #LoadLoad 706 ! 707 ! Spin on lock without using an atomic operation to prevent the caches 708 ! from unnecessarily moving ownership of the line around. 709 ! 710 3: 711 ldub [%l6 + M_SPINLOCK], %l5 712 4: 713 tst %l5 714 bz,a 1b ! lock appears to be free, try again 715 ldstub [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock 716 717 sethi %hi(panicstr) , %l5 718 ldn [%l5 + %lo(panicstr)], %l5 719 tst %l5 720 bnz 2b ! after panic, feign success 721 nop 722 b 4b 723 ldub [%l6 + M_SPINLOCK], %l5 ! delay - reload lock 724 SET_SIZE(asm_mutex_spin_enter) 725 #endif /* lint */ 726 727 /* 728 * asm_mutex_spin_exit(mutex_t *) 729 * 730 * For use by assembly interrupt handler only. 731 * Does not change spl, since the interrupt handler is assumed to be 732 * running at high level already. 733 * 734 * Entry: %l6 - points to mutex 735 * %l7 - address of call (returns to %l7+8) 736 * Uses: none 737 */ 738 #ifndef lint 739 ENTRY_NP(asm_mutex_spin_exit) 740 membar #LoadStore|#StoreStore 741 jmp %l7 + 8 ! return 742 clrb [%l6 + M_SPINLOCK] ! delay - clear lock 743 SET_SIZE(asm_mutex_spin_exit) 744 #endif /* lint */ 745 746 /* 747 * thread_onproc() 748 * Set thread in onproc state for the specified CPU. 749 * Also set the thread lock pointer to the CPU's onproc lock. 750 * Since the new lock isn't held, the store ordering is important. 751 * If not done in assembler, the compiler could reorder the stores. 752 */ 753 #if defined(lint) 754 755 void 756 thread_onproc(kthread_id_t t, cpu_t *cp) 757 { 758 t->t_state = TS_ONPROC; 759 t->t_lockp = &cp->cpu_thread_lock; 760 } 761 762 #else /* lint */ 763 764 ENTRY(thread_onproc) 765 set TS_ONPROC, %o2 ! TS_ONPROC state 766 st %o2, [%o0 + T_STATE] ! store state 767 add %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running 768 retl ! return 769 stn %o3, [%o0 + T_LOCKP] ! delay - store new lock pointer 770 SET_SIZE(thread_onproc) 771 772 #endif /* lint */ 773 774 /* delay function used in some mutex code - just do 3 nop cas ops */ 775 #if defined(lint) 776 777 /* ARGSUSED */ 778 void 779 cas_delay(void *addr) 780 {} 781 #else /* lint */ 782 ENTRY(cas_delay) 783 casx [%o0], %g0, %g0 784 casx [%o0], %g0, %g0 785 retl 786 casx [%o0], %g0, %g0 787 SET_SIZE(cas_delay) 788 #endif /* lint */ 789 790 #if defined(lint) 791 792 /* 793 * alternative delay function for some niagara processors. The rd 794 * instruction uses less resources than casx on those cpus. 795 */ 796 /* ARGSUSED */ 797 void 798 rdccr_delay(void) 799 {} 800 #else /* lint */ 801 ENTRY(rdccr_delay) 802 rd %ccr, %g0 803 rd %ccr, %g0 804 retl 805 rd %ccr, %g0 806 SET_SIZE(rdccr_delay) 807 #endif /* lint */ 808 809 /* 810 * mutex_delay_default(void) 811 * Spins for approx a few hundred processor cycles and returns to caller. 812 */ 813 #if defined(lint) 814 815 void 816 mutex_delay_default(void) 817 {} 818 819 #else /* lint */ 820 821 ENTRY(mutex_delay_default) 822 mov 72,%o0 823 1: brgz %o0, 1b 824 dec %o0 825 retl 826 nop 827 SET_SIZE(mutex_delay_default) 828 829 #endif /* lint */ --- EOF ---