1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27 
  28 #if defined(lint)
  29 #include <sys/types.h>
  30 #include <sys/thread.h>
  31 #include <sys/cpuvar.h>
  32 #else   /* lint */
  33 #include "assym.h"
  34 #endif  /* lint */
  35 
  36 #include <sys/t_lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/mutex_impl.h>
  39 #include <sys/rwlock_impl.h>
  40 #include <sys/asm_linkage.h>
  41 #include <sys/machlock.h>
  42 #include <sys/machthread.h>
  43 #include <sys/lockstat.h>
  44 
  45 /* #define DEBUG */
  46 
  47 #ifdef DEBUG
  48 #include <sys/machparam.h>
  49 #endif /* DEBUG */
  50 
  51 /************************************************************************
  52  *              ATOMIC OPERATIONS
  53  */
  54 
  55 /*
  56  * uint8_t      ldstub(uint8_t *cp)
  57  *
  58  * Store 0xFF at the specified location, and return its previous content.
  59  */
  60 
  61 #if defined(lint)
  62 uint8_t
  63 ldstub(uint8_t *cp)
  64 {
  65         uint8_t rv;
  66         rv = *cp;
  67         *cp = 0xFF;
  68         return rv;
  69 }
  70 #else   /* lint */
  71 
  72         ENTRY(ldstub)
  73         retl
  74         ldstub  [%o0], %o0
  75         SET_SIZE(ldstub)
  76 
  77 #endif  /* lint */
  78 
  79 /************************************************************************
  80  *              MEMORY BARRIERS -- see atomic.h for full descriptions.
  81  */
  82 
  83 #if defined(lint)
  84 
  85 void
  86 membar_enter(void)
  87 {}
  88 
  89 void
  90 membar_exit(void)
  91 {}
  92 
  93 void
  94 membar_producer(void)
  95 {}
  96 
  97 void
  98 membar_consumer(void)
  99 {}
 100 
 101 #else   /* lint */
 102 
 103 #ifdef SF_ERRATA_51
 104         .align 32
 105         ENTRY(membar_return)
 106         retl
 107         nop
 108         SET_SIZE(membar_return)
 109 #define MEMBAR_RETURN   ba,pt %icc, membar_return
 110 #else
 111 #define MEMBAR_RETURN   retl
 112 #endif
 113 
 114         ENTRY(membar_enter)
 115         MEMBAR_RETURN
 116         membar  #StoreLoad|#StoreStore
 117         SET_SIZE(membar_enter)
 118 
 119         ENTRY(membar_exit)
 120         MEMBAR_RETURN
 121         membar  #LoadStore|#StoreStore
 122         SET_SIZE(membar_exit)
 123 
 124         ENTRY(membar_producer)
 125         MEMBAR_RETURN
 126         membar  #StoreStore
 127         SET_SIZE(membar_producer)
 128 
 129         ENTRY(membar_consumer)
 130         MEMBAR_RETURN
 131         membar  #LoadLoad
 132         SET_SIZE(membar_consumer)
 133 
 134 #endif  /* lint */
 135 
 136 /************************************************************************
 137  *              MINIMUM LOCKS
 138  */
 139 
 140 #if defined(lint)
 141 
 142 /*
 143  * lock_try(lp), ulock_try(lp)
 144  *      - returns non-zero on success.
 145  *      - doesn't block interrupts so don't use this to spin on a lock.
 146  *      - uses "0xFF is busy, anything else is free" model.
 147  *
 148  *      ulock_try() is for a lock in the user address space.
 149  *      For all V7/V8 sparc systems they are same since the kernel and
 150  *      user are mapped in a user' context.
 151  *      For V9 platforms the lock_try and ulock_try are different impl.
 152  */
 153 
 154 int
 155 lock_try(lock_t *lp)
 156 {
 157         return (0xFF ^ ldstub(lp));
 158 }
 159 
 160 int
 161 lock_spin_try(lock_t *lp)
 162 {
 163         return (0xFF ^ ldstub(lp));
 164 }
 165 
 166 void
 167 lock_set(lock_t *lp)
 168 {
 169         extern void lock_set_spin(lock_t *);
 170 
 171         if (!lock_try(lp))
 172                 lock_set_spin(lp);
 173         membar_enter();
 174 }
 175 
 176 void
 177 lock_clear(lock_t *lp)
 178 {
 179         membar_exit();
 180         *lp = 0;
 181 }
 182 
 183 int
 184 ulock_try(lock_t *lp)
 185 {
 186         return (0xFF ^ ldstub(lp));
 187 }
 188 
 189 void
 190 ulock_clear(lock_t *lp)
 191 {
 192         membar_exit();
 193         *lp = 0;
 194 }
 195 
 196 #else   /* lint */
 197 
 198         .align  32
 199         ENTRY(lock_try)
 200         ldstub  [%o0], %o1              ! try to set lock, get value in %o1
 201         brnz,pn %o1, 1f
 202         membar  #LoadLoad
 203 .lock_try_lockstat_patch_point:
 204         retl
 205         or      %o0, 1, %o0             ! ensure lo32 != 0
 206 1:
 207         retl
 208         clr     %o0
 209         SET_SIZE(lock_try)
 210 
 211         .align  32
 212         ENTRY(lock_spin_try)
 213         ldstub  [%o0], %o1              ! try to set lock, get value in %o1
 214         brnz,pn %o1, 1f
 215         membar  #LoadLoad
 216         retl
 217         or      %o0, 1, %o0             ! ensure lo32 != 0
 218 1:
 219         retl
 220         clr     %o0
 221         SET_SIZE(lock_spin_try)
 222 
 223         .align  32
 224         ENTRY(lock_set)
 225         ldstub  [%o0], %o1
 226         brnz,pn %o1, 1f                 ! go to C for the hard case
 227         membar  #LoadLoad
 228 .lock_set_lockstat_patch_point:
 229         retl
 230         nop
 231 1:
 232         sethi   %hi(lock_set_spin), %o2 ! load up for jump to C
 233         jmp     %o2 + %lo(lock_set_spin)
 234         nop                             ! delay: do nothing
 235         SET_SIZE(lock_set)
 236 
 237         ENTRY(lock_clear)
 238         membar  #LoadStore|#StoreStore
 239 .lock_clear_lockstat_patch_point:
 240         retl
 241         clrb    [%o0]
 242         SET_SIZE(lock_clear)
 243 
 244         .align  32
 245         ENTRY(ulock_try)
 246         ldstuba [%o0]ASI_USER, %o1      ! try to set lock, get value in %o1
 247         xor     %o1, 0xff, %o0          ! delay - return non-zero if success
 248         retl
 249           membar        #LoadLoad
 250         SET_SIZE(ulock_try)
 251 
 252         ENTRY(ulock_clear)
 253         membar  #LoadStore|#StoreStore
 254         retl
 255           stba  %g0, [%o0]ASI_USER      ! clear lock
 256         SET_SIZE(ulock_clear)
 257 
 258 #endif  /* lint */
 259 
 260 
 261 /*
 262  * lock_set_spl(lp, new_pil, *old_pil_addr)
 263  *      Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
 264  */
 265 
 266 #if defined(lint)
 267 
 268 /* ARGSUSED */
 269 void
 270 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil_addr)
 271 {
 272         extern int splr(int);
 273         extern void lock_set_spl_spin(lock_t *, int, u_short *, int);
 274         int old_pil;
 275 
 276         old_pil = splr(new_pil);
 277         if (!lock_try(lp)) {
 278                 lock_set_spl_spin(lp, new_pil, old_pil_addr, old_pil);
 279         } else {
 280                 *old_pil_addr = (u_short)old_pil;
 281                 membar_enter();
 282         }
 283 }
 284 
 285 #else   /* lint */
 286 
 287         ENTRY(lock_set_spl)
 288         rdpr    %pil, %o3                       ! %o3 = current pil
 289         cmp     %o3, %o1                        ! is current pil high enough?
 290         bl,a,pt %icc, 1f                        ! if not, write %pil in delay
 291         wrpr    %g0, %o1, %pil
 292 1:
 293         ldstub  [%o0], %o4                      ! try the lock
 294         brnz,pn %o4, 2f                         ! go to C for the miss case
 295         membar  #LoadLoad
 296 .lock_set_spl_lockstat_patch_point:
 297         retl
 298         sth     %o3, [%o2]                      ! delay - save original pil
 299 2:
 300         sethi   %hi(lock_set_spl_spin), %o5     ! load up jmp to C
 301         jmp     %o5 + %lo(lock_set_spl_spin)    ! jmp to lock_set_spl_spin
 302         nop                                     ! delay: do nothing
 303         SET_SIZE(lock_set_spl)
 304 
 305 #endif  /* lint */
 306 
 307 /*
 308  * lock_clear_splx(lp, s)
 309  */
 310 
 311 #if defined(lint)
 312 
 313 void
 314 lock_clear_splx(lock_t *lp, int s)
 315 {
 316         extern void splx(int);
 317 
 318         lock_clear(lp);
 319         splx(s);
 320 }
 321 
 322 #else   /* lint */
 323 
 324         ENTRY(lock_clear_splx)
 325         ldn     [THREAD_REG + T_CPU], %o2       ! get CPU pointer
 326         membar  #LoadStore|#StoreStore
 327         ld      [%o2 + CPU_BASE_SPL], %o2
 328         clrb    [%o0]                           ! clear lock
 329         cmp     %o2, %o1                        ! compare new to base
 330         movl    %xcc, %o1, %o2                  ! use new pri if base is less
 331 .lock_clear_splx_lockstat_patch_point:
 332         retl
 333         wrpr    %g0, %o2, %pil
 334         SET_SIZE(lock_clear_splx)
 335 
 336 #endif  /* lint */
 337 
 338 /*
 339  * mutex_enter() and mutex_exit().
 340  * 
 341  * These routines handle the simple cases of mutex_enter() (adaptive
 342  * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
 343  * If anything complicated is going on we punt to mutex_vector_enter().
 344  *
 345  * mutex_tryenter() is similar to mutex_enter() but returns zero if
 346  * the lock cannot be acquired, nonzero on success.
 347  *
 348  * If mutex_exit() gets preempted in the window between checking waiters
 349  * and clearing the lock, we can miss wakeups.  Disabling preemption
 350  * in the mutex code is prohibitively expensive, so instead we detect
 351  * mutex preemption by examining the trapped PC in the interrupt path.
 352  * If we interrupt a thread in mutex_exit() that has not yet cleared
 353  * the lock, pil_interrupt() resets its PC back to the beginning of
 354  * mutex_exit() so it will check again for waiters when it resumes.
 355  *
 356  * The lockstat code below is activated when the lockstat driver
 357  * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
 358  * Note that we don't need to test lockstat_event_mask here -- we won't
 359  * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
 360  */
 361 
 362 #if defined (lint)
 363 
 364 /* ARGSUSED */
 365 void
 366 mutex_enter(kmutex_t *lp)
 367 {}
 368 
 369 /* ARGSUSED */
 370 int
 371 mutex_tryenter(kmutex_t *lp)
 372 { return (0); }
 373 
 374 /* ARGSUSED */
 375 void
 376 mutex_exit(kmutex_t *lp)
 377 {}
 378 
 379 /* ARGSUSED */
 380 void *
 381 mutex_owner_running(mutex_impl_t *lp)
 382 { return (NULL); }
 383 
 384 #else
 385         .align  32
 386         ENTRY(mutex_enter)
 387         mov     THREAD_REG, %o1
 388         casx    [%o0], %g0, %o1                 ! try to acquire as adaptive
 389         brnz,pn %o1, 1f                         ! locked or wrong type
 390         membar  #LoadLoad
 391 .mutex_enter_lockstat_patch_point:
 392         retl
 393         nop
 394 1:
 395         sethi   %hi(mutex_vector_enter), %o2    ! load up for jump to C
 396         jmp     %o2 + %lo(mutex_vector_enter)
 397         nop
 398         SET_SIZE(mutex_enter)
 399 
 400         ENTRY(mutex_tryenter)
 401         mov     THREAD_REG, %o1
 402         casx    [%o0], %g0, %o1                 ! try to acquire as adaptive
 403         brnz,pn %o1, 1f                         ! locked or wrong type continue
 404         membar  #LoadLoad
 405 .mutex_tryenter_lockstat_patch_point:
 406         retl
 407         or      %o0, 1, %o0                     ! ensure lo32 != 0
 408 1:
 409         sethi   %hi(mutex_vector_tryenter), %o2         ! hi bits
 410         jmp     %o2 + %lo(mutex_vector_tryenter)        ! go to C
 411         nop
 412         SET_SIZE(mutex_tryenter)
 413 
 414         ENTRY(mutex_adaptive_tryenter)
 415         mov     THREAD_REG, %o1
 416         casx    [%o0], %g0, %o1                 ! try to acquire as adaptive
 417         brnz,pn %o1, 0f                         ! locked or wrong type
 418         membar  #LoadLoad
 419         retl
 420         or      %o0, 1, %o0                     ! ensure lo32 != 0
 421 0:
 422         retl
 423         mov     %g0, %o0
 424         SET_SIZE(mutex_adaptive_tryenter)
 425 
 426         ! these need to be together and cache aligned for performance.
 427         .align 64
 428         .global mutex_exit_critical_size
 429         .global mutex_exit_critical_start
 430         .global mutex_owner_running_critical_size
 431         .global mutex_owner_running_critical_start
 432 
 433 mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start
 434 
 435         .align  32
 436 
 437         ENTRY(mutex_exit)
 438 mutex_exit_critical_start:              ! If we are interrupted, restart here
 439         ldn     [%o0], %o1              ! get the owner field
 440         membar  #LoadStore|#StoreStore
 441         cmp     THREAD_REG, %o1         ! do we own lock with no waiters?
 442         be,a,pt %ncc, 1f                ! if so, drive on ...
 443         stn     %g0, [%o0]              ! delay: clear lock if we owned it
 444 .mutex_exit_critical_end:               ! for pil_interrupt() hook
 445         ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases
 446 1:
 447 .mutex_exit_lockstat_patch_point:
 448         retl
 449         nop
 450         SET_SIZE(mutex_exit)
 451 
 452 mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start
 453 
 454         .align  32
 455 
 456         ENTRY(mutex_owner_running)
 457 mutex_owner_running_critical_start:     ! If interrupted restart here
 458         ldn     [%o0], %o1              ! get the owner field
 459         and     %o1, MUTEX_THREAD, %o1  ! remove the waiters bit if any
 460         brz,pn  %o1, 1f                 ! if so, drive on ...
 461         nop
 462         ldn     [%o1+T_CPU], %o2        ! get owner->t_cpu
 463         ldn     [%o2+CPU_THREAD], %o3   ! get owner->t_cpu->cpu_thread
 464 .mutex_owner_running_critical_end:      ! for pil_interrupt() hook
 465         cmp     %o1, %o3                ! owner == running thread?
 466         be,a,pt %xcc, 2f                ! yes, go return cpu
 467         nop
 468 1:
 469         retl
 470         mov     %g0, %o0                ! return 0 (owner not running)
 471 2:
 472         retl
 473         mov     %o2, %o0                ! owner running, return cpu
 474         SET_SIZE(mutex_owner_running)
 475 
 476 #endif  /* lint */
 477 
 478 /*
 479  * rw_enter() and rw_exit().
 480  * 
 481  * These routines handle the simple cases of rw_enter (write-locking an unheld
 482  * lock or read-locking a lock that's neither write-locked nor write-wanted)
 483  * and rw_exit (no waiters or not the last reader).  If anything complicated
 484  * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
 485  */
 486 #if defined(lint)
 487 
 488 /* ARGSUSED */
 489 void
 490 rw_enter(krwlock_t *lp, krw_t rw)
 491 {}
 492 
 493 /* ARGSUSED */
 494 void
 495 rw_exit(krwlock_t *lp)
 496 {}
 497 
 498 #else
 499 
 500         .align  16
 501         ENTRY(rw_enter)
 502         cmp     %o1, RW_WRITER                  ! entering as writer?
 503         be,a,pn %icc, 2f                        ! if so, go do it ...
 504         or      THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
 505         ld      [THREAD_REG + T_KPRI_REQ], %o3  ! begin THREAD_KPRI_REQUEST()
 506         ldn     [%o0], %o4                      ! %o4 = old lock value
 507         inc     %o3                             ! bump kpri
 508         st      %o3, [THREAD_REG + T_KPRI_REQ]  ! store new kpri
 509 1:
 510         andcc   %o4, RW_WRITE_CLAIMED, %g0      ! write-locked or write-wanted?
 511         bz,pt   %xcc, 3f                        ! if so, prepare to block
 512         add     %o4, RW_READ_LOCK, %o5          ! delay: increment hold count
 513         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 514         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 515         nop                                     ! delay: do nothing
 516 3:
 517         casx    [%o0], %o4, %o5                 ! try to grab read lock
 518         cmp     %o4, %o5                        ! did we get it?
 519 #ifdef sun4v
 520         be,a,pt %xcc, 0f
 521         membar  #LoadLoad
 522         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 523         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 524         nop                                     ! delay: do nothing
 525 0:
 526 #else /* sun4v */
 527         bne,pn  %xcc, 1b                        ! if not, try again
 528         mov     %o5, %o4                        ! delay: %o4 = old lock value
 529         membar  #LoadLoad
 530 #endif /* sun4v */
 531 .rw_read_enter_lockstat_patch_point:
 532         retl
 533         nop
 534 2:
 535         casx    [%o0], %g0, %o5                 ! try to grab write lock
 536         brz,pt %o5, 4f                          ! branch around if we got it
 537         membar  #LoadLoad                       ! done regardless of where we go
 538         sethi   %hi(rw_enter_sleep), %o2
 539         jmp     %o2 + %lo(rw_enter_sleep)       ! jump to rw_enter_sleep if not
 540         nop                                     ! delay: do nothing
 541 4:
 542 .rw_write_enter_lockstat_patch_point:
 543         retl
 544         nop
 545         SET_SIZE(rw_enter)
 546 
 547         .align  16
 548         ENTRY(rw_exit)
 549         ldn     [%o0], %o4                      ! %o4 = old lock value
 550         membar  #LoadStore|#StoreStore          ! membar_exit()
 551         subcc   %o4, RW_READ_LOCK, %o5          ! %o5 = new lock value if reader
 552         bnz,pn  %xcc, 2f                        ! single reader, no waiters?
 553         clr     %o1
 554 1:
 555         ld      [THREAD_REG + T_KPRI_REQ], %g1  ! begin THREAD_KPRI_RELEASE()
 556         srl     %o4, RW_HOLD_COUNT_SHIFT, %o3   ! %o3 = hold count (lockstat)
 557         casx    [%o0], %o4, %o5                 ! try to drop lock
 558         cmp     %o4, %o5                        ! did we succeed?
 559         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 560         dec     %g1                             ! delay: drop kpri
 561 .rw_read_exit_lockstat_patch_point:
 562         retl
 563         st      %g1, [THREAD_REG + T_KPRI_REQ]  ! delay: store new kpri
 564 2:
 565         andcc   %o4, RW_WRITE_LOCKED, %g0       ! are we a writer?
 566         bnz,a,pt %xcc, 3f
 567         or      THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
 568         cmp     %o5, RW_READ_LOCK               ! would lock still be held?
 569         bge,pt  %xcc, 1b                        ! if so, go ahead and drop it
 570         nop
 571         ba,pt   %xcc, rw_exit_wakeup            ! otherwise, wake waiters
 572         nop
 573 3:
 574         casx    [%o0], %o4, %o1                 ! try to drop write lock
 575         cmp     %o4, %o1                        ! did we succeed?
 576         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 577         nop
 578 .rw_write_exit_lockstat_patch_point:
 579         retl
 580         nop
 581         SET_SIZE(rw_exit)
 582 
 583 #endif
 584 
 585 #if defined(lint)
 586 
 587 void
 588 lockstat_hot_patch(void)
 589 {}
 590 
 591 #else
 592 
 593 #define RETL                    0x81c3e008
 594 #define NOP                     0x01000000
 595 #define BA                      0x10800000
 596 
 597 #define DISP22                  ((1 << 22) - 1)
 598 #define ANNUL                   0x20000000
 599 
 600 #define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs)          \
 601         ba      1f;                                                     \
 602         rd      %pc, %o0;                                               \
 603         save    %sp, -SA(MINFRAME), %sp;                                \
 604         set     lockstat_probemap, %l1;                                 \
 605         ld      [%l1 + (event * DTRACE_IDSIZE)], %o0;                   \
 606         brz,pn  %o0, 0f;                                                \
 607         ldub    [THREAD_REG + T_LOCKSTAT], %l0;                         \
 608         add     %l0, 1, %l2;                                            \
 609         stub    %l2, [THREAD_REG + T_LOCKSTAT];                         \
 610         set     lockstat_probe, %g1;                                    \
 611         ld      [%l1 + (event * DTRACE_IDSIZE)], %o0;                   \
 612         brz,a,pn %o0, 0f;                                               \
 613         stub    %l0, [THREAD_REG + T_LOCKSTAT];                         \
 614         ldn     [%g1], %g2;                                             \
 615         mov     rs, %o2;                                                \
 616         jmpl    %g2, %o7;                                               \
 617         mov     %i0, %o1;                                               \
 618         stub    %l0, [THREAD_REG + T_LOCKSTAT];                         \
 619 0:      ret;                                                            \
 620         restore %g0, 1, %o0;    /* for mutex_tryenter / lock_try */     \
 621 1:      set     addr, %o1;                                              \
 622         sub     %o0, %o1, %o0;                                          \
 623         srl     %o0, 2, %o0;                                            \
 624         inc     %o0;                                                    \
 625         set     DISP22, %o1;                                            \
 626         and     %o1, %o0, %o0;                                          \
 627         set     BA, %o1;                                                \
 628         or      %o1, %o0, %o0;                                          \
 629         sethi   %hi(annul), %o2;                                        \
 630         add     %o0, %o2, %o2;                                          \
 631         set     addr, %o0;                                              \
 632         set     normal_instr, %o1;                                      \
 633         ld      [%i0 + (event * DTRACE_IDSIZE)], %o3;                   \
 634         tst     %o3;                                                    \
 635         movnz   %icc, %o2, %o1;                                         \
 636         call    hot_patch_kernel_text;                                  \
 637         mov     4, %o2;                                                 \
 638         membar  #Sync
 639 
 640 #define HOT_PATCH(addr, event, normal_instr)    \
 641         HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)
 642 
 643 #define HOT_PATCH_ARG(addr, event, normal_instr, arg)   \
 644         HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)
 645 
 646 #define HOT_PATCH_ANNULLED(addr, event, normal_instr)   \
 647         HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)
 648 
 649         ENTRY(lockstat_hot_patch)
 650         save    %sp, -SA(MINFRAME), %sp
 651         set     lockstat_probemap, %i0
 652         HOT_PATCH(.mutex_enter_lockstat_patch_point,
 653                 LS_MUTEX_ENTER_ACQUIRE, RETL)
 654         HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
 655                 LS_MUTEX_TRYENTER_ACQUIRE, RETL)
 656         HOT_PATCH(.mutex_exit_lockstat_patch_point,
 657                 LS_MUTEX_EXIT_RELEASE, RETL)
 658         HOT_PATCH(.rw_write_enter_lockstat_patch_point,
 659                 LS_RW_ENTER_ACQUIRE, RETL)
 660         HOT_PATCH(.rw_read_enter_lockstat_patch_point,
 661                 LS_RW_ENTER_ACQUIRE, RETL)
 662         HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
 663                 LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
 664         HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
 665                 LS_RW_EXIT_RELEASE, RETL, RW_READER)
 666         HOT_PATCH(.lock_set_lockstat_patch_point,
 667                 LS_LOCK_SET_ACQUIRE, RETL)
 668         HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
 669                 LS_LOCK_TRY_ACQUIRE, RETL)
 670         HOT_PATCH(.lock_clear_lockstat_patch_point,
 671                 LS_LOCK_CLEAR_RELEASE, RETL)
 672         HOT_PATCH(.lock_set_spl_lockstat_patch_point,
 673                 LS_LOCK_SET_SPL_ACQUIRE, RETL)
 674         HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
 675                 LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
 676         ret
 677         restore
 678         SET_SIZE(lockstat_hot_patch)
 679 
 680 #endif  /* lint */
 681 
 682 /*
 683  * asm_mutex_spin_enter(mutex_t *)
 684  *
 685  * For use by assembly interrupt handler only.
 686  * Does not change spl, since the interrupt handler is assumed to be
 687  * running at high level already.
 688  * Traps may be off, so cannot panic.
 689  * Does not keep statistics on the lock.
 690  *
 691  * Entry:       %l6 - points to mutex
 692  *              %l7 - address of call (returns to %l7+8)
 693  * Uses:        %l6, %l5
 694  */
 695 #ifndef lint
 696         .align 16
 697         ENTRY_NP(asm_mutex_spin_enter)
 698         ldstub  [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5
 699 1:
 700         tst     %l5
 701         bnz     3f                      ! lock already held - go spin
 702         nop
 703 2:      
 704         jmp     %l7 + 8                 ! return
 705         membar  #LoadLoad
 706         !
 707         ! Spin on lock without using an atomic operation to prevent the caches
 708         ! from unnecessarily moving ownership of the line around.
 709         !
 710 3:
 711         ldub    [%l6 + M_SPINLOCK], %l5
 712 4:
 713         tst     %l5
 714         bz,a    1b                      ! lock appears to be free, try again
 715         ldstub  [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock
 716 
 717         sethi   %hi(panicstr) , %l5
 718         ldn     [%l5 + %lo(panicstr)], %l5
 719         tst     %l5
 720         bnz     2b                      ! after panic, feign success
 721         nop
 722         b       4b
 723         ldub    [%l6 + M_SPINLOCK], %l5 ! delay - reload lock
 724         SET_SIZE(asm_mutex_spin_enter)
 725 #endif /* lint */
 726 
 727 /*
 728  * asm_mutex_spin_exit(mutex_t *)
 729  *
 730  * For use by assembly interrupt handler only.
 731  * Does not change spl, since the interrupt handler is assumed to be
 732  * running at high level already.
 733  *
 734  * Entry:       %l6 - points to mutex
 735  *              %l7 - address of call (returns to %l7+8)
 736  * Uses:        none
 737  */
 738 #ifndef lint
 739         ENTRY_NP(asm_mutex_spin_exit)
 740         membar  #LoadStore|#StoreStore
 741         jmp     %l7 + 8                 ! return
 742         clrb    [%l6 + M_SPINLOCK]      ! delay - clear lock
 743         SET_SIZE(asm_mutex_spin_exit)
 744 #endif /* lint */
 745 
 746 /*
 747  * thread_onproc()
 748  * Set thread in onproc state for the specified CPU.
 749  * Also set the thread lock pointer to the CPU's onproc lock.
 750  * Since the new lock isn't held, the store ordering is important.
 751  * If not done in assembler, the compiler could reorder the stores.
 752  */
 753 #if defined(lint)
 754 
 755 void
 756 thread_onproc(kthread_id_t t, cpu_t *cp)
 757 {
 758         t->t_state = TS_ONPROC;
 759         t->t_lockp = &cp->cpu_thread_lock;
 760 }
 761 
 762 #else   /* lint */
 763 
 764         ENTRY(thread_onproc)
 765         set     TS_ONPROC, %o2          ! TS_ONPROC state
 766         st      %o2, [%o0 + T_STATE]    ! store state
 767         add     %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
 768         retl                            ! return
 769         stn     %o3, [%o0 + T_LOCKP]    ! delay - store new lock pointer
 770         SET_SIZE(thread_onproc)
 771 
 772 #endif  /* lint */
 773 
 774 /* delay function used in some mutex code - just do 3 nop cas ops */
 775 #if defined(lint)
 776 
 777 /* ARGSUSED */
 778 void
 779 cas_delay(void *addr)
 780 {}
 781 #else   /* lint */
 782         ENTRY(cas_delay)
 783         casx [%o0], %g0, %g0
 784         casx [%o0], %g0, %g0
 785         retl
 786         casx [%o0], %g0, %g0
 787         SET_SIZE(cas_delay)
 788 #endif  /* lint */
 789 
 790 #if defined(lint)
 791 
 792 /*
 793  * alternative delay function for some niagara processors.   The rd
 794  * instruction uses less resources than casx on those cpus.
 795  */
 796 /* ARGSUSED */
 797 void
 798 rdccr_delay(void)
 799 {}
 800 #else   /* lint */
 801         ENTRY(rdccr_delay)
 802         rd      %ccr, %g0
 803         rd      %ccr, %g0
 804         retl
 805         rd      %ccr, %g0
 806         SET_SIZE(rdccr_delay)
 807 #endif  /* lint */
 808 
 809 /*
 810  * mutex_delay_default(void)
 811  * Spins for approx a few hundred processor cycles and returns to caller.
 812  */
 813 #if defined(lint)
 814 
 815 void
 816 mutex_delay_default(void)
 817 {}
 818 
 819 #else   /* lint */
 820 
 821         ENTRY(mutex_delay_default)
 822         mov     72,%o0
 823 1:      brgz    %o0, 1b
 824         dec     %o0
 825         retl
 826         nop
 827         SET_SIZE(mutex_delay_default)
 828 
 829 #endif  /* lint */