1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27 
  28 #if defined(lint) || defined(__lint)
  29 #include <sys/types.h>
  30 #include <sys/thread.h>
  31 #include <sys/cpuvar.h>
  32 #include <vm/page.h>
  33 #else   /* __lint */
  34 #include "assym.h"
  35 #endif  /* __lint */
  36 
  37 #include <sys/mutex_impl.h>
  38 #include <sys/asm_linkage.h>
  39 #include <sys/asm_misc.h>
  40 #include <sys/regset.h>
  41 #include <sys/rwlock_impl.h>
  42 #include <sys/lockstat.h>
  43 
  44 /*
  45  * lock_try(lp), ulock_try(lp)
  46  *      - returns non-zero on success.
  47  *      - doesn't block interrupts so don't use this to spin on a lock.
  48  *
  49  * ulock_try() is for a lock in the user address space.
  50  */
  51 
  52 #if defined(lint) || defined(__lint)
  53 
  54 /* ARGSUSED */
  55 int
  56 lock_try(lock_t *lp)
  57 { return (0); }
  58 
  59 /* ARGSUSED */
  60 int
  61 lock_spin_try(lock_t *lp)
  62 { return (0); }
  63 
  64 /* ARGSUSED */
  65 int
  66 ulock_try(lock_t *lp)
  67 { return (0); }
  68 
  69 #else   /* __lint */
  70         .globl  kernelbase
  71 
  72 #if defined(__amd64)
  73 
  74         ENTRY(lock_try)
  75         movb    $-1, %dl
  76         movzbq  %dl, %rax
  77         xchgb   %dl, (%rdi)
  78         xorb    %dl, %al
  79 .lock_try_lockstat_patch_point:
  80         ret
  81         testb   %al, %al
  82         jnz     0f
  83         ret
  84 0:
  85         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
  86         movq    %rdi, %rsi              /* rsi = lock addr */
  87         movl    $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
  88         jmp     lockstat_wrapper
  89         SET_SIZE(lock_try)
  90 
  91         ENTRY(lock_spin_try)
  92         movb    $-1, %dl
  93         movzbq  %dl, %rax
  94         xchgb   %dl, (%rdi)
  95         xorb    %dl, %al
  96         ret
  97         SET_SIZE(lock_spin_try)
  98 
  99         ENTRY(ulock_try)
 100 #ifdef DEBUG
 101         movq    kernelbase(%rip), %rax
 102         cmpq    %rax, %rdi              /* test uaddr < kernelbase */
 103         jb      ulock_pass              /*      uaddr < kernelbase, proceed */
 104 
 105         movq    %rdi, %r12              /* preserve lock ptr for debugging */
 106         leaq    .ulock_panic_msg(%rip), %rdi
 107         pushq   %rbp                    /* align stack properly */
 108         movq    %rsp, %rbp
 109         xorl    %eax, %eax              /* clear for varargs */
 110         call    panic
 111 
 112 #endif /* DEBUG */
 113 
 114 ulock_pass:
 115         movl    $1, %eax
 116         xchgb   %al, (%rdi)
 117         xorb    $1, %al
 118         ret
 119         SET_SIZE(ulock_try)
 120 
 121 #else
 122 
 123         ENTRY(lock_try)
 124         movl    $1,%edx
 125         movl    4(%esp),%ecx            /* ecx = lock addr */
 126         xorl    %eax,%eax
 127         xchgb   %dl, (%ecx)             /* using dl will avoid partial */
 128         testb   %dl,%dl                 /* stalls on P6 ? */
 129         setz    %al
 130 .lock_try_lockstat_patch_point:
 131         ret
 132         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 133         testl   %eax, %eax
 134         jz      0f
 135         movl    $LS_LOCK_TRY_ACQUIRE, %eax
 136         jmp     lockstat_wrapper
 137 0:
 138         ret
 139         SET_SIZE(lock_try)
 140 
 141         ENTRY(lock_spin_try)
 142         movl    $-1,%edx
 143         movl    4(%esp),%ecx            /* ecx = lock addr */
 144         xorl    %eax,%eax
 145         xchgb   %dl, (%ecx)             /* using dl will avoid partial */
 146         testb   %dl,%dl                 /* stalls on P6 ? */
 147         setz    %al
 148         ret
 149         SET_SIZE(lock_spin_try)
 150 
 151         ENTRY(ulock_try)
 152 #ifdef DEBUG
 153         movl    kernelbase, %eax
 154         cmpl    %eax, 4(%esp)           /* test uaddr < kernelbase */
 155         jb      ulock_pass              /* uaddr < kernelbase, proceed */
 156 
 157         pushl   $.ulock_panic_msg
 158         call    panic
 159 
 160 #endif /* DEBUG */
 161 
 162 ulock_pass:
 163         movl    $1,%eax
 164         movl    4(%esp),%ecx
 165         xchgb   %al, (%ecx)
 166         xorb    $1, %al
 167         ret
 168         SET_SIZE(ulock_try)
 169 
 170 #endif  /* !__amd64 */
 171 
 172 #ifdef DEBUG
 173         .data
 174 .ulock_panic_msg:
 175         .string "ulock_try: Argument is above kernelbase"
 176         .text
 177 #endif  /* DEBUG */
 178 
 179 #endif  /* __lint */
 180 
 181 /*
 182  * lock_clear(lp)
 183  *      - unlock lock without changing interrupt priority level.
 184  */
 185 
 186 #if defined(lint) || defined(__lint)
 187 
 188 /* ARGSUSED */
 189 void
 190 lock_clear(lock_t *lp)
 191 {}
 192 
 193 /* ARGSUSED */
 194 void
 195 ulock_clear(lock_t *lp)
 196 {}
 197 
 198 #else   /* __lint */
 199 
 200 #if defined(__amd64)
 201 
 202         ENTRY(lock_clear)
 203         movb    $0, (%rdi)
 204 .lock_clear_lockstat_patch_point:
 205         ret
 206         movq    %rdi, %rsi                      /* rsi = lock addr */
 207         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread addr */
 208         movl    $LS_LOCK_CLEAR_RELEASE, %edi    /* edi = event */
 209         jmp     lockstat_wrapper
 210         SET_SIZE(lock_clear)
 211 
 212         ENTRY(ulock_clear)
 213 #ifdef DEBUG
 214         movq    kernelbase(%rip), %rcx
 215         cmpq    %rcx, %rdi              /* test uaddr < kernelbase */
 216         jb      ulock_clr               /*       uaddr < kernelbase, proceed */
 217 
 218         leaq    .ulock_clear_msg(%rip), %rdi
 219         pushq   %rbp                    /* align stack properly */
 220         movq    %rsp, %rbp
 221         xorl    %eax, %eax              /* clear for varargs */
 222         call    panic
 223 #endif
 224 
 225 ulock_clr:
 226         movb    $0, (%rdi)
 227         ret
 228         SET_SIZE(ulock_clear)
 229 
 230 #else
 231 
 232         ENTRY(lock_clear)
 233         movl    4(%esp), %eax
 234         movb    $0, (%eax)
 235 .lock_clear_lockstat_patch_point:
 236         ret
 237         movl    %gs:CPU_THREAD, %edx            /* edx = thread addr */
 238         movl    %eax, %ecx                      /* ecx = lock pointer */
 239         movl    $LS_LOCK_CLEAR_RELEASE, %eax
 240         jmp     lockstat_wrapper
 241         SET_SIZE(lock_clear)
 242 
 243         ENTRY(ulock_clear)
 244 #ifdef DEBUG
 245         movl    kernelbase, %ecx
 246         cmpl    %ecx, 4(%esp)           /* test uaddr < kernelbase */
 247         jb      ulock_clr               /* uaddr < kernelbase, proceed */
 248 
 249         pushl   $.ulock_clear_msg
 250         call    panic
 251 #endif
 252 
 253 ulock_clr:
 254         movl    4(%esp),%eax
 255         xorl    %ecx,%ecx
 256         movb    %cl, (%eax)
 257         ret
 258         SET_SIZE(ulock_clear)
 259 
 260 #endif  /* !__amd64 */
 261 
 262 #ifdef DEBUG
 263         .data
 264 .ulock_clear_msg:
 265         .string "ulock_clear: Argument is above kernelbase"
 266         .text
 267 #endif  /* DEBUG */
 268 
 269 
 270 #endif  /* __lint */
 271 
 272 /*
 273  * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
 274  * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
 275  */
 276 
 277 #if defined(lint) || defined(__lint)
 278 
 279 /* ARGSUSED */
 280 void
 281 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
 282 {}
 283 
 284 #else   /* __lint */
 285 
 286 #if defined(__amd64)
 287 
 288         ENTRY(lock_set_spl)
 289         pushq   %rbp
 290         movq    %rsp, %rbp
 291         subq    $32, %rsp
 292         movl    %esi, 8(%rsp)           /* save priority level */
 293         movq    %rdx, 16(%rsp)          /* save old pil ptr */
 294         movq    %rdi, 24(%rsp)          /* save lock pointer */
 295         movl    %esi, %edi              /* pass priority level */
 296         call    splr                    /* raise priority level */
 297         movq    24(%rsp), %rdi          /* rdi = lock addr */
 298         movb    $-1, %dl
 299         xchgb   %dl, (%rdi)             /* try to set lock */
 300         testb   %dl, %dl                /* did we get the lock? ... */
 301         jnz     .lss_miss               /* ... no, go to C for the hard case */
 302         movq    16(%rsp), %rdx          /* rdx = old pil addr */
 303         movw    %ax, (%rdx)             /* store old pil */
 304         leave
 305 .lock_set_spl_lockstat_patch_point:
 306         ret
 307         movq    %rdi, %rsi              /* rsi = lock addr */
 308         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 309         movl    $LS_LOCK_SET_SPL_ACQUIRE, %edi
 310         jmp     lockstat_wrapper
 311 .lss_miss:
 312         movl    8(%rsp), %esi           /* new_pil */
 313         movq    16(%rsp), %rdx          /* old_pil_addr */
 314         movl    %eax, %ecx              /* original pil */
 315         leave                           /* unwind stack */
 316         jmp     lock_set_spl_spin
 317         SET_SIZE(lock_set_spl)
 318 
 319 #else
 320 
 321         ENTRY(lock_set_spl)
 322         movl    8(%esp), %eax           /* get priority level */
 323         pushl   %eax
 324         call    splr                    /* raise priority level */
 325         movl    8(%esp), %ecx           /* ecx = lock addr */
 326         movl    $-1, %edx
 327         addl    $4, %esp
 328         xchgb   %dl, (%ecx)             /* try to set lock */
 329         testb   %dl, %dl                /* did we get the lock? ... */
 330         movl    12(%esp), %edx          /* edx = olp pil addr (ZF unaffected) */
 331         jnz     .lss_miss               /* ... no, go to C for the hard case */
 332         movw    %ax, (%edx)             /* store old pil */
 333 .lock_set_spl_lockstat_patch_point:
 334         ret
 335         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr*/
 336         movl    $LS_LOCK_SET_SPL_ACQUIRE, %eax
 337         jmp     lockstat_wrapper
 338 .lss_miss:
 339         pushl   %eax                    /* original pil */
 340         pushl   %edx                    /* old_pil addr */
 341         pushl   16(%esp)                /* new_pil */
 342         pushl   %ecx                    /* lock addr */
 343         call    lock_set_spl_spin
 344         addl    $16, %esp
 345         ret
 346         SET_SIZE(lock_set_spl)
 347 
 348 #endif  /* !__amd64 */
 349 
 350 #endif  /* __lint */
 351 
 352 /*
 353  * void
 354  * lock_init(lp)
 355  */
 356 
 357 #if defined(__lint)
 358 
 359 /* ARGSUSED */
 360 void
 361 lock_init(lock_t *lp)
 362 {}
 363 
 364 #else   /* __lint */
 365 
 366 #if defined(__amd64)
 367 
 368         ENTRY(lock_init)
 369         movb    $0, (%rdi)
 370         ret
 371         SET_SIZE(lock_init)
 372 
 373 #else
 374 
 375         ENTRY(lock_init)
 376         movl    4(%esp), %eax
 377         movb    $0, (%eax)
 378         ret
 379         SET_SIZE(lock_init)
 380 
 381 #endif  /* !__amd64 */
 382 
 383 #endif  /* __lint */
 384 
 385 /*
 386  * void
 387  * lock_set(lp)
 388  */
 389 
 390 #if defined(lint) || defined(__lint)
 391 
 392 /* ARGSUSED */
 393 void
 394 lock_set(lock_t *lp)
 395 {}
 396 
 397 #else   /* __lint */
 398 
 399 #if defined(__amd64)
 400 
 401         ENTRY(lock_set)
 402         movb    $-1, %dl
 403         xchgb   %dl, (%rdi)             /* try to set lock */
 404         testb   %dl, %dl                /* did we get it? */
 405         jnz     lock_set_spin           /* no, go to C for the hard case */
 406 .lock_set_lockstat_patch_point:
 407         ret
 408         movq    %rdi, %rsi              /* rsi = lock addr */
 409         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 410         movl    $LS_LOCK_SET_ACQUIRE, %edi
 411         jmp     lockstat_wrapper
 412         SET_SIZE(lock_set)
 413 
 414 #else
 415 
 416         ENTRY(lock_set)
 417         movl    4(%esp), %ecx           /* ecx = lock addr */
 418         movl    $-1, %edx
 419         xchgb   %dl, (%ecx)             /* try to set lock */
 420         testb   %dl, %dl                /* did we get it? */
 421         jnz     lock_set_spin           /* no, go to C for the hard case */
 422 .lock_set_lockstat_patch_point:
 423         ret
 424         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 425         movl    $LS_LOCK_SET_ACQUIRE, %eax
 426         jmp     lockstat_wrapper
 427         SET_SIZE(lock_set)
 428 
 429 #endif  /* !__amd64 */
 430 
 431 #endif  /* __lint */
 432 
 433 /*
 434  * lock_clear_splx(lp, s)
 435  */
 436 
 437 #if defined(lint) || defined(__lint)
 438 
 439 /* ARGSUSED */
 440 void
 441 lock_clear_splx(lock_t *lp, int s)
 442 {}
 443 
 444 #else   /* __lint */
 445 
 446 #if defined(__amd64)
 447 
 448         ENTRY(lock_clear_splx)
 449         movb    $0, (%rdi)              /* clear lock */
 450 .lock_clear_splx_lockstat_patch_point:
 451         jmp     0f
 452 0:
 453         movl    %esi, %edi              /* arg for splx */
 454         jmp     splx                    /* let splx do its thing */
 455 .lock_clear_splx_lockstat:
 456         pushq   %rbp                    /* align stack properly */
 457         movq    %rsp, %rbp
 458         subq    $16, %rsp               /* space to save args across splx */
 459         movq    %rdi, 8(%rsp)           /* save lock ptr across splx call */
 460         movl    %esi, %edi              /* arg for splx */
 461         call    splx                    /* lower the priority */
 462         movq    8(%rsp), %rsi           /* rsi = lock ptr */
 463         leave                           /* unwind stack */
 464         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 465         movl    $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
 466         jmp     lockstat_wrapper
 467         SET_SIZE(lock_clear_splx)
 468 
 469 #else
 470 
 471         ENTRY(lock_clear_splx)
 472         movl    4(%esp), %eax           /* eax = lock addr */
 473         movb    $0, (%eax)              /* clear lock */
 474 .lock_clear_splx_lockstat_patch_point:
 475         jmp     0f
 476 0:
 477         movl    8(%esp), %edx           /* edx = desired pil */
 478         movl    %edx, 4(%esp)           /* set spl arg up for splx */
 479         jmp     splx                    /* let splx do it's thing */
 480 .lock_clear_splx_lockstat:
 481         movl    8(%esp), %edx           /* edx = desired pil */
 482         pushl   %ebp                    /* set up stack frame */
 483         movl    %esp, %ebp
 484         pushl   %edx
 485         call    splx
 486         leave                           /* unwind stack */
 487         movl    4(%esp), %ecx           /* ecx = lock pointer */
 488         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 489         movl    $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
 490         jmp     lockstat_wrapper
 491         SET_SIZE(lock_clear_splx)
 492 
 493 #endif  /* !__amd64 */
 494 
 495 #if defined(__GNUC_AS__)
 496 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL      \
 497         (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
 498 
 499 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT    \
 500         (.lock_clear_splx_lockstat_patch_point + 1)
 501 #else
 502 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL      \
 503         [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
 504 
 505 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT    \
 506         [.lock_clear_splx_lockstat_patch_point + 1]
 507 #endif
 508 
 509 #endif  /* __lint */
 510 
 511 /*
 512  * mutex_enter() and mutex_exit().
 513  *
 514  * These routines handle the simple cases of mutex_enter() (adaptive
 515  * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
 516  * If anything complicated is going on we punt to mutex_vector_enter().
 517  *
 518  * mutex_tryenter() is similar to mutex_enter() but returns zero if
 519  * the lock cannot be acquired, nonzero on success.
 520  *
 521  * If mutex_exit() gets preempted in the window between checking waiters
 522  * and clearing the lock, we can miss wakeups.  Disabling preemption
 523  * in the mutex code is prohibitively expensive, so instead we detect
 524  * mutex preemption by examining the trapped PC in the interrupt path.
 525  * If we interrupt a thread in mutex_exit() that has not yet cleared
 526  * the lock, cmnint() resets its PC back to the beginning of
 527  * mutex_exit() so it will check again for waiters when it resumes.
 528  *
 529  * The lockstat code below is activated when the lockstat driver
 530  * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
 531  * Note that we don't need to test lockstat_event_mask here -- we won't
 532  * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
 533  */
 534 #if defined(lint) || defined(__lint)
 535 
 536 /* ARGSUSED */
 537 void
 538 mutex_enter(kmutex_t *lp)
 539 {}
 540 
 541 /* ARGSUSED */
 542 int
 543 mutex_tryenter(kmutex_t *lp)
 544 { return (0); }
 545 
 546 /* ARGSUSED */
 547 int
 548 mutex_adaptive_tryenter(mutex_impl_t *lp)
 549 { return (0); }
 550 
 551 /* ARGSUSED */
 552 void
 553 mutex_exit(kmutex_t *lp)
 554 {}
 555 
 556 #else
 557 
 558 #if defined(__amd64)
 559 
 560         ENTRY_NP(mutex_enter)
 561         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 562         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 563         lock
 564         cmpxchgq %rdx, (%rdi)
 565         jnz     mutex_vector_enter
 566 .mutex_enter_lockstat_patch_point:
 567 #if defined(OPTERON_WORKAROUND_6323525)
 568 .mutex_enter_6323525_patch_point:
 569         ret                                     /* nop space for lfence */
 570         nop
 571         nop
 572 .mutex_enter_lockstat_6323525_patch_point:      /* new patch point if lfence */ 
 573         nop
 574 #else   /* OPTERON_WORKAROUND_6323525 */
 575         ret
 576 #endif  /* OPTERON_WORKAROUND_6323525 */
 577         movq    %rdi, %rsi
 578         movl    $LS_MUTEX_ENTER_ACQUIRE, %edi
 579 /*
 580  * expects %rdx=thread, %rsi=lock, %edi=lockstat event
 581  */
 582         ALTENTRY(lockstat_wrapper)
 583         incb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat++ */
 584         leaq    lockstat_probemap(%rip), %rax
 585         movl    (%rax, %rdi, DTRACE_IDSIZE), %eax
 586         testl   %eax, %eax                      /* check for non-zero probe */
 587         jz      1f
 588         pushq   %rbp                            /* align stack properly */
 589         movq    %rsp, %rbp
 590         movl    %eax, %edi
 591         movq    lockstat_probe, %rax
 592         INDIRECT_CALL_REG(rax)
 593         leave                                   /* unwind stack */
 594 1:
 595         movq    %gs:CPU_THREAD, %rdx            /* reload thread ptr */
 596         decb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat-- */
 597         movl    $1, %eax                        /* return success if tryenter */
 598         ret
 599         SET_SIZE(lockstat_wrapper)
 600         SET_SIZE(mutex_enter)
 601 
 602 /*
 603  * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
 604  */
 605         ENTRY(lockstat_wrapper_arg)
 606         incb    T_LOCKSTAT(%rcx)                /* curthread->t_lockstat++ */
 607         leaq    lockstat_probemap(%rip), %rax
 608         movl    (%rax, %rdi, DTRACE_IDSIZE), %eax
 609         testl   %eax, %eax                      /* check for non-zero probe */
 610         jz      1f
 611         pushq   %rbp                            /* align stack properly */
 612         movq    %rsp, %rbp
 613         movl    %eax, %edi
 614         movq    lockstat_probe, %rax
 615         INDIRECT_CALL_REG(rax)
 616         leave                                   /* unwind stack */
 617 1:
 618         movq    %gs:CPU_THREAD, %rdx            /* reload thread ptr */
 619         decb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat-- */
 620         movl    $1, %eax                        /* return success if tryenter */
 621         ret
 622         SET_SIZE(lockstat_wrapper_arg)
 623 
 624 
 625         ENTRY(mutex_tryenter)
 626         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 627         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 628         lock
 629         cmpxchgq %rdx, (%rdi)
 630         jnz     mutex_vector_tryenter
 631         not     %eax                            /* return success (nonzero) */
 632 #if defined(OPTERON_WORKAROUND_6323525)
 633 .mutex_tryenter_lockstat_patch_point:
 634 .mutex_tryenter_6323525_patch_point:
 635         ret                                     /* nop space for lfence */
 636         nop
 637         nop
 638 .mutex_tryenter_lockstat_6323525_patch_point:   /* new patch point if lfence */
 639         nop
 640 #else   /* OPTERON_WORKAROUND_6323525 */
 641 .mutex_tryenter_lockstat_patch_point:
 642         ret
 643 #endif  /* OPTERON_WORKAROUND_6323525 */
 644         movq    %rdi, %rsi
 645         movl    $LS_MUTEX_ENTER_ACQUIRE, %edi
 646         jmp     lockstat_wrapper
 647         SET_SIZE(mutex_tryenter)
 648 
 649         ENTRY(mutex_adaptive_tryenter)
 650         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 651         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 652         lock
 653         cmpxchgq %rdx, (%rdi)
 654         jnz     0f
 655         not     %eax                            /* return success (nonzero) */
 656 #if defined(OPTERON_WORKAROUND_6323525)
 657 .mutex_atryenter_6323525_patch_point:
 658         ret                                     /* nop space for lfence */
 659         nop
 660         nop
 661         nop
 662 #else   /* OPTERON_WORKAROUND_6323525 */
 663         ret
 664 #endif  /* OPTERON_WORKAROUND_6323525 */
 665 0:
 666         xorl    %eax, %eax                      /* return failure */
 667         ret
 668         SET_SIZE(mutex_adaptive_tryenter)
 669 
 670         .globl  mutex_owner_running_critical_start
 671 
 672         ENTRY(mutex_owner_running)
 673 mutex_owner_running_critical_start:
 674         movq    (%rdi), %r11            /* get owner field */
 675         andq    $MUTEX_THREAD, %r11     /* remove waiters bit */
 676         cmpq    $0, %r11                /* if free, skip */
 677         je      1f                      /* go return 0 */
 678         movq    T_CPU(%r11), %r8        /* get owner->t_cpu */
 679         movq    CPU_THREAD(%r8), %r9    /* get t_cpu->cpu_thread */
 680 .mutex_owner_running_critical_end:
 681         cmpq    %r11, %r9       /* owner == running thread? */
 682         je      2f              /* yes, go return cpu */
 683 1:
 684         xorq    %rax, %rax      /* return 0 */
 685         ret
 686 2:
 687         movq    %r8, %rax               /* return cpu */
 688         ret
 689         SET_SIZE(mutex_owner_running)
 690 
 691         .globl  mutex_owner_running_critical_size
 692         .type   mutex_owner_running_critical_size, @object
 693         .align  CPTRSIZE
 694 mutex_owner_running_critical_size:
 695         .quad   .mutex_owner_running_critical_end - mutex_owner_running_critical_start
 696         SET_SIZE(mutex_owner_running_critical_size)
 697 
 698         .globl  mutex_exit_critical_start
 699 
 700         ENTRY(mutex_exit)
 701 mutex_exit_critical_start:              /* If interrupted, restart here */
 702         movq    %gs:CPU_THREAD, %rdx
 703         cmpq    %rdx, (%rdi)
 704         jne     mutex_vector_exit               /* wrong type or wrong owner */
 705         movq    $0, (%rdi)                      /* clear owner AND lock */
 706 .mutex_exit_critical_end:
 707 .mutex_exit_lockstat_patch_point:
 708         ret
 709         movq    %rdi, %rsi
 710         movl    $LS_MUTEX_EXIT_RELEASE, %edi
 711         jmp     lockstat_wrapper
 712         SET_SIZE(mutex_exit)
 713 
 714         .globl  mutex_exit_critical_size
 715         .type   mutex_exit_critical_size, @object
 716         .align  CPTRSIZE
 717 mutex_exit_critical_size:
 718         .quad   .mutex_exit_critical_end - mutex_exit_critical_start
 719         SET_SIZE(mutex_exit_critical_size)
 720 
 721 #else
 722 
 723         ENTRY_NP(mutex_enter)
 724         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 725         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 726         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 727         lock
 728         cmpxchgl %edx, (%ecx)
 729         jnz     mutex_vector_enter
 730 #if defined(OPTERON_WORKAROUND_6323525)
 731 .mutex_enter_lockstat_patch_point:
 732 .mutex_enter_6323525_patch_point:
 733         ret                                     /* nop space for lfence */
 734         nop
 735         nop
 736 .mutex_enter_lockstat_6323525_patch_point:      /* new patch point if lfence */
 737         nop
 738 #else   /* OPTERON_WORKAROUND_6323525 */
 739 .mutex_enter_lockstat_patch_point:
 740         ret
 741 #endif  /* OPTERON_WORKAROUND_6323525 */
 742         movl    $LS_MUTEX_ENTER_ACQUIRE, %eax
 743         ALTENTRY(lockstat_wrapper)      /* expects edx=thread, ecx=lock, */
 744                                         /*   eax=lockstat event */
 745         pushl   %ebp                            /* buy a frame */
 746         movl    %esp, %ebp
 747         incb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat++ */
 748         pushl   %edx                            /* save thread pointer   */
 749         movl    $lockstat_probemap, %edx
 750         movl    (%edx, %eax, DTRACE_IDSIZE), %eax
 751         testl   %eax, %eax                      /* check for non-zero probe */
 752         jz      1f
 753         pushl   %ecx                            /* push lock */
 754         pushl   %eax                            /* push probe ID */
 755         call    *lockstat_probe
 756         addl    $8, %esp
 757 1:
 758         popl    %edx                            /* restore thread pointer */
 759         decb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat-- */
 760         movl    $1, %eax                        /* return success if tryenter */
 761         popl    %ebp                            /* pop off frame */
 762         ret
 763         SET_SIZE(lockstat_wrapper)
 764         SET_SIZE(mutex_enter)
 765 
 766         ENTRY(lockstat_wrapper_arg)     /* expects edx=thread, ecx=lock, */
 767                                         /* eax=lockstat event, pushed arg */
 768         incb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat++ */
 769         pushl   %edx                            /* save thread pointer   */
 770         movl    $lockstat_probemap, %edx
 771         movl    (%edx, %eax, DTRACE_IDSIZE), %eax
 772         testl   %eax, %eax                      /* check for non-zero probe */
 773         jz      1f
 774         pushl   %ebp                            /* save %ebp */
 775         pushl   8(%esp)                         /* push arg1 */
 776         movl    %ebp, 12(%esp)                  /* fake up the stack frame */
 777         movl    %esp, %ebp                      /* fake up base pointer */
 778         addl    $12, %ebp                       /* adjust faked base pointer */
 779         pushl   %ecx                            /* push lock */
 780         pushl   %eax                            /* push probe ID */
 781         call    *lockstat_probe
 782         addl    $12, %esp                       /* adjust for arguments */
 783         popl    %ebp                            /* pop frame */
 784 1:
 785         popl    %edx                            /* restore thread pointer */
 786         decb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat-- */
 787         movl    $1, %eax                        /* return success if tryenter */
 788         addl    $4, %esp                        /* pop argument */
 789         ret
 790         SET_SIZE(lockstat_wrapper_arg)
 791 
 792 
 793         ENTRY(mutex_tryenter)
 794         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 795         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 796         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 797         lock
 798         cmpxchgl %edx, (%ecx)
 799         jnz     mutex_vector_tryenter
 800         movl    %ecx, %eax
 801 #if defined(OPTERON_WORKAROUND_6323525)
 802 .mutex_tryenter_lockstat_patch_point:
 803 .mutex_tryenter_6323525_patch_point:
 804         ret                                     /* nop space for lfence */
 805         nop
 806         nop
 807 .mutex_tryenter_lockstat_6323525_patch_point:   /* new patch point if lfence */
 808         nop
 809 #else   /* OPTERON_WORKAROUND_6323525 */
 810 .mutex_tryenter_lockstat_patch_point:
 811         ret
 812 #endif  /* OPTERON_WORKAROUND_6323525 */
 813         movl    $LS_MUTEX_ENTER_ACQUIRE, %eax
 814         jmp     lockstat_wrapper
 815         SET_SIZE(mutex_tryenter)
 816 
 817         ENTRY(mutex_adaptive_tryenter)
 818         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 819         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 820         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 821         lock
 822         cmpxchgl %edx, (%ecx)
 823         jnz     0f
 824         movl    %ecx, %eax
 825 #if defined(OPTERON_WORKAROUND_6323525)
 826 .mutex_atryenter_6323525_patch_point:
 827         ret                                     /* nop space for lfence */
 828         nop
 829         nop
 830         nop
 831 #else   /* OPTERON_WORKAROUND_6323525 */
 832         ret
 833 #endif  /* OPTERON_WORKAROUND_6323525 */
 834 0:
 835         xorl    %eax, %eax
 836         ret
 837         SET_SIZE(mutex_adaptive_tryenter)
 838 
 839         .globl  mutex_owner_running_critical_start
 840 
 841         ENTRY(mutex_owner_running)
 842 mutex_owner_running_critical_start:
 843         movl    4(%esp), %eax           /* get owner field */
 844         movl    (%eax), %eax
 845         andl    $MUTEX_THREAD, %eax     /* remove waiters bit */
 846         cmpl    $0, %eax                /* if free, skip */
 847         je      1f                      /* go return 0 */
 848         movl    T_CPU(%eax), %ecx       /* get owner->t_cpu */
 849         movl    CPU_THREAD(%ecx), %edx  /* get t_cpu->cpu_thread */
 850 .mutex_owner_running_critical_end:
 851         cmpl    %eax, %edx      /* owner == running thread? */
 852         je      2f              /* yes, go return cpu */
 853 1:
 854         xorl    %eax, %eax      /* return 0 */
 855         ret
 856 2:
 857         movl    %ecx, %eax      /* return cpu */
 858         ret
 859 
 860         SET_SIZE(mutex_owner_running)
 861 
 862         .globl  mutex_owner_running_critical_size
 863         .type   mutex_owner_running_critical_size, @object
 864         .align  CPTRSIZE
 865 mutex_owner_running_critical_size:
 866         .long   .mutex_owner_running_critical_end - mutex_owner_running_critical_start
 867         SET_SIZE(mutex_owner_running_critical_size)
 868 
 869         .globl  mutex_exit_critical_start
 870 
 871         ENTRY(mutex_exit)
 872 mutex_exit_critical_start:              /* If interrupted, restart here */
 873         movl    %gs:CPU_THREAD, %edx
 874         movl    4(%esp), %ecx
 875         cmpl    %edx, (%ecx)
 876         jne     mutex_vector_exit               /* wrong type or wrong owner */
 877         movl    $0, (%ecx)                      /* clear owner AND lock */
 878 .mutex_exit_critical_end:
 879 .mutex_exit_lockstat_patch_point:
 880         ret
 881         movl    $LS_MUTEX_EXIT_RELEASE, %eax
 882         jmp     lockstat_wrapper
 883         SET_SIZE(mutex_exit)
 884 
 885         .globl  mutex_exit_critical_size
 886         .type   mutex_exit_critical_size, @object
 887         .align  CPTRSIZE
 888 mutex_exit_critical_size:
 889         .long   .mutex_exit_critical_end - mutex_exit_critical_start
 890         SET_SIZE(mutex_exit_critical_size)
 891 
 892 #endif  /* !__amd64 */
 893 
 894 #endif  /* __lint */
 895 
 896 /*
 897  * rw_enter() and rw_exit().
 898  *
 899  * These routines handle the simple cases of rw_enter (write-locking an unheld
 900  * lock or read-locking a lock that's neither write-locked nor write-wanted)
 901  * and rw_exit (no waiters or not the last reader).  If anything complicated
 902  * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
 903  */
 904 #if defined(lint) || defined(__lint)
 905 
 906 /* ARGSUSED */
 907 void
 908 rw_enter(krwlock_t *lp, krw_t rw)
 909 {}
 910 
 911 /* ARGSUSED */
 912 void
 913 rw_exit(krwlock_t *lp)
 914 {}
 915 
 916 #else   /* __lint */
 917 
 918 #if defined(__amd64)
 919 
 920         ENTRY(rw_enter)
 921         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 922         cmpl    $RW_WRITER, %esi
 923         je      .rw_write_enter
 924         incl    T_KPRI_REQ(%rdx)                /* THREAD_KPRI_REQUEST() */
 925         movq    (%rdi), %rax                    /* rax = old rw_wwwh value */
 926         testl   $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
 927         jnz     rw_enter_sleep
 928         leaq    RW_READ_LOCK(%rax), %rdx        /* rdx = new rw_wwwh value */
 929         lock
 930         cmpxchgq %rdx, (%rdi)                   /* try to grab read lock */
 931         jnz     rw_enter_sleep
 932 .rw_read_enter_lockstat_patch_point:
 933         ret
 934         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 935         movq    %rdi, %rsi                      /* rsi = lock ptr */
 936         movl    $LS_RW_ENTER_ACQUIRE, %edi
 937         movl    $RW_READER, %edx
 938         jmp     lockstat_wrapper_arg
 939 .rw_write_enter:
 940         orq     $RW_WRITE_LOCKED, %rdx          /* rdx = write-locked value */
 941         xorl    %eax, %eax                      /* rax = unheld value */
 942         lock
 943         cmpxchgq %rdx, (%rdi)                   /* try to grab write lock */
 944         jnz     rw_enter_sleep
 945 
 946 #if defined(OPTERON_WORKAROUND_6323525)
 947 .rw_write_enter_lockstat_patch_point:
 948 .rw_write_enter_6323525_patch_point:
 949         ret
 950         nop
 951         nop
 952 .rw_write_enter_lockstat_6323525_patch_point:
 953         nop
 954 #else   /* OPTERON_WORKAROUND_6323525 */
 955 .rw_write_enter_lockstat_patch_point:
 956         ret
 957 #endif  /* OPTERON_WORKAROUND_6323525 */
 958 
 959         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 960         movq    %rdi, %rsi                      /* rsi = lock ptr */
 961         movl    $LS_RW_ENTER_ACQUIRE, %edi
 962         movl    $RW_WRITER, %edx
 963         jmp     lockstat_wrapper_arg
 964         SET_SIZE(rw_enter)
 965 
 966         ENTRY(rw_exit)
 967         movq    (%rdi), %rax                    /* rax = old rw_wwwh value */
 968         cmpl    $RW_READ_LOCK, %eax             /* single-reader, no waiters? */
 969         jne     .rw_not_single_reader
 970         xorl    %edx, %edx                      /* rdx = new value (unheld) */
 971 .rw_read_exit:
 972         lock
 973         cmpxchgq %rdx, (%rdi)                   /* try to drop read lock */
 974         jnz     rw_exit_wakeup
 975         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 976         decl    T_KPRI_REQ(%rcx)                /* THREAD_KPRI_RELEASE() */
 977 .rw_read_exit_lockstat_patch_point:
 978         ret
 979         movq    %rdi, %rsi                      /* rsi = lock ptr */
 980         movl    $LS_RW_EXIT_RELEASE, %edi
 981         movl    $RW_READER, %edx
 982         jmp     lockstat_wrapper_arg
 983 .rw_not_single_reader:
 984         testl   $RW_WRITE_LOCKED, %eax  /* write-locked or write-wanted? */
 985         jnz     .rw_write_exit
 986         leaq    -RW_READ_LOCK(%rax), %rdx       /* rdx = new value */
 987         cmpl    $RW_READ_LOCK, %edx
 988         jge     .rw_read_exit           /* not last reader, safe to drop */
 989         jmp     rw_exit_wakeup                  /* last reader with waiters */
 990 .rw_write_exit:
 991         movq    %gs:CPU_THREAD, %rax            /* rax = thread ptr */
 992         xorl    %edx, %edx                      /* rdx = new value (unheld) */
 993         orq     $RW_WRITE_LOCKED, %rax          /* eax = write-locked value */
 994         lock
 995         cmpxchgq %rdx, (%rdi)                   /* try to drop read lock */
 996         jnz     rw_exit_wakeup
 997 .rw_write_exit_lockstat_patch_point:
 998         ret
 999         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
1000         movq    %rdi, %rsi                      /* rsi - lock ptr */
1001         movl    $LS_RW_EXIT_RELEASE, %edi
1002         movl    $RW_WRITER, %edx
1003         jmp     lockstat_wrapper_arg
1004         SET_SIZE(rw_exit)
1005 
1006 #else
1007 
1008         ENTRY(rw_enter)
1009         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1010         movl    4(%esp), %ecx                   /* ecx = lock ptr */
1011         cmpl    $RW_WRITER, 8(%esp)
1012         je      .rw_write_enter
1013         incl    T_KPRI_REQ(%edx)                /* THREAD_KPRI_REQUEST() */
1014         movl    (%ecx), %eax                    /* eax = old rw_wwwh value */
1015         testl   $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1016         jnz     rw_enter_sleep
1017         leal    RW_READ_LOCK(%eax), %edx        /* edx = new rw_wwwh value */
1018         lock
1019         cmpxchgl %edx, (%ecx)                   /* try to grab read lock */
1020         jnz     rw_enter_sleep
1021 .rw_read_enter_lockstat_patch_point:
1022         ret
1023         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1024         movl    $LS_RW_ENTER_ACQUIRE, %eax
1025         pushl   $RW_READER
1026         jmp     lockstat_wrapper_arg
1027 .rw_write_enter:
1028         orl     $RW_WRITE_LOCKED, %edx          /* edx = write-locked value */
1029         xorl    %eax, %eax                      /* eax = unheld value */
1030         lock
1031         cmpxchgl %edx, (%ecx)                   /* try to grab write lock */
1032         jnz     rw_enter_sleep
1033 
1034 #if defined(OPTERON_WORKAROUND_6323525)
1035 .rw_write_enter_lockstat_patch_point:
1036 .rw_write_enter_6323525_patch_point:
1037         ret
1038         nop
1039         nop
1040 .rw_write_enter_lockstat_6323525_patch_point:
1041         nop
1042 #else   /* OPTERON_WORKAROUND_6323525 */
1043 .rw_write_enter_lockstat_patch_point:
1044         ret
1045 #endif  /* OPTERON_WORKAROUND_6323525 */
1046 
1047         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1048         movl    $LS_RW_ENTER_ACQUIRE, %eax
1049         pushl   $RW_WRITER
1050         jmp     lockstat_wrapper_arg
1051         SET_SIZE(rw_enter)
1052 
1053         ENTRY(rw_exit)
1054         movl    4(%esp), %ecx                   /* ecx = lock ptr */
1055         movl    (%ecx), %eax                    /* eax = old rw_wwwh value */
1056         cmpl    $RW_READ_LOCK, %eax             /* single-reader, no waiters? */
1057         jne     .rw_not_single_reader
1058         xorl    %edx, %edx                      /* edx = new value (unheld) */
1059 .rw_read_exit:
1060         lock
1061         cmpxchgl %edx, (%ecx)                   /* try to drop read lock */
1062         jnz     rw_exit_wakeup
1063         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1064         decl    T_KPRI_REQ(%edx)                /* THREAD_KPRI_RELEASE() */
1065 .rw_read_exit_lockstat_patch_point:
1066         ret
1067         movl    $LS_RW_EXIT_RELEASE, %eax
1068         pushl   $RW_READER
1069         jmp     lockstat_wrapper_arg
1070 .rw_not_single_reader:
1071         testl   $RW_WRITE_LOCKED, %eax  /* write-locked or write-wanted? */
1072         jnz     .rw_write_exit
1073         leal    -RW_READ_LOCK(%eax), %edx       /* edx = new value */
1074         cmpl    $RW_READ_LOCK, %edx
1075         jge     .rw_read_exit           /* not last reader, safe to drop */
1076         jmp     rw_exit_wakeup                  /* last reader with waiters */
1077 .rw_write_exit:
1078         movl    %gs:CPU_THREAD, %eax            /* eax = thread ptr */
1079         xorl    %edx, %edx                      /* edx = new value (unheld) */
1080         orl     $RW_WRITE_LOCKED, %eax          /* eax = write-locked value */
1081         lock
1082         cmpxchgl %edx, (%ecx)                   /* try to drop read lock */
1083         jnz     rw_exit_wakeup
1084 .rw_write_exit_lockstat_patch_point:
1085         ret
1086         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1087         movl    $LS_RW_EXIT_RELEASE, %eax
1088         pushl   $RW_WRITER
1089         jmp     lockstat_wrapper_arg
1090         SET_SIZE(rw_exit)
1091 
1092 #endif  /* !__amd64 */
1093 
1094 #endif  /* __lint */
1095 
1096 #if defined(OPTERON_WORKAROUND_6323525)
1097 #if defined(lint) || defined(__lint)
1098 
1099 int     workaround_6323525_patched;
1100 
1101 void
1102 patch_workaround_6323525(void)
1103 {}
1104 
1105 #else   /* lint */
1106 
1107 /*
1108  * If it is necessary to patch the lock enter routines with the lfence
1109  * workaround, workaround_6323525_patched is set to a non-zero value so that
1110  * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1111  * instruction.
1112  */
1113         DGDEF3(workaround_6323525_patched, 4, 4)
1114         .long   0
1115 
1116 #if defined(__amd64)
1117 
1118 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1119         movq    $size, %rbx;                    \
1120         movq    $dstaddr, %r13;                 \
1121         addq    %rbx, %r13;                     \
1122         movq    $srcaddr, %r12;                 \
1123         addq    %rbx, %r12;                     \
1124 0:                                              \
1125         decq    %r13;                           \
1126         decq    %r12;                           \
1127         movzbl  (%r12), %esi;                   \
1128         movq    $1, %rdx;                       \
1129         movq    %r13, %rdi;                     \
1130         call    hot_patch_kernel_text;          \
1131         decq    %rbx;                           \
1132         testq   %rbx, %rbx;                     \
1133         jg      0b;
1134 
1135 /*
1136  * patch_workaround_6323525: provide workaround for 6323525
1137  *
1138  * The workaround is to place a fencing instruction (lfence) between the
1139  * mutex operation and the subsequent read-modify-write instruction.
1140  *
1141  * This routine hot patches the lfence instruction on top of the space
1142  * reserved by nops in the lock enter routines.
1143  */
1144         ENTRY_NP(patch_workaround_6323525)
1145         pushq   %rbp
1146         movq    %rsp, %rbp
1147         pushq   %r12
1148         pushq   %r13
1149         pushq   %rbx
1150 
1151         /*
1152          * lockstat_hot_patch() to use the alternate lockstat workaround
1153          * 6323525 patch points (points past the lfence instruction to the
1154          * new ret) when workaround_6323525_patched is set.
1155          */
1156         movl    $1, workaround_6323525_patched
1157 
1158         /*
1159          * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
1160          * routines. The 4 bytes are patched in reverse order so that the
1161          * the existing ret is overwritten last. This provides lock enter
1162          * sanity during the intermediate patching stages.
1163          */
1164         HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1165         HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1166         HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1167         HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1168 
1169         popq    %rbx
1170         popq    %r13
1171         popq    %r12
1172         movq    %rbp, %rsp
1173         popq    %rbp
1174         ret
1175 _lfence_insn:
1176         lfence
1177         ret
1178         SET_SIZE(patch_workaround_6323525)
1179 
1180 
1181 #else   /* __amd64 */
1182 
1183 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1184         movl    $size, %ebx;                    \
1185         movl    $srcaddr, %esi;                 \
1186         addl    %ebx, %esi;                     \
1187         movl    $dstaddr, %edi;                 \
1188         addl    %ebx, %edi;                     \
1189 0:                                              \
1190         decl    %esi;                           \
1191         decl    %edi;                           \
1192         pushl   $1;                             \
1193         movzbl  (%esi), %eax;                   \
1194         pushl   %eax;                           \
1195         pushl   %edi;                           \
1196         call    hot_patch_kernel_text;          \
1197         addl    $12, %esp;                      \
1198         decl    %ebx;                           \
1199         testl   %ebx, %ebx;                     \
1200         jg      0b;
1201 
1202 
1203         /* see comments above */
1204         ENTRY_NP(patch_workaround_6323525)
1205         pushl   %ebp
1206         movl    %esp, %ebp
1207         pushl   %ebx
1208         pushl   %esi
1209         pushl   %edi
1210 
1211         movl    $1, workaround_6323525_patched
1212 
1213         HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1214         HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1215         HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1216         HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1217 
1218         popl    %edi
1219         popl    %esi
1220         popl    %ebx
1221         movl    %ebp, %esp
1222         popl    %ebp
1223         ret
1224 _lfence_insn:
1225         .byte   0xf, 0xae, 0xe8         / [lfence instruction]
1226         ret
1227         SET_SIZE(patch_workaround_6323525)
1228 
1229 #endif  /* !__amd64 */
1230 #endif  /* !lint */
1231 #endif  /* OPTERON_WORKAROUND_6323525 */
1232 
1233 
1234 #if defined(lint) || defined(__lint)
1235 
1236 void
1237 lockstat_hot_patch(void)
1238 {}
1239 
1240 #else
1241 
1242 #if defined(__amd64)
1243 
1244 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1245         movq    $normal_instr, %rsi;            \
1246         movq    $active_instr, %rdi;            \
1247         leaq    lockstat_probemap(%rip), %rax;  \
1248         movl    _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1249         testl   %eax, %eax;                     \
1250         jz      9f;                             \
1251         movq    %rdi, %rsi;                     \
1252 9:                                              \
1253         movq    $len, %rdx;                     \
1254         movq    $addr, %rdi;                    \
1255         call    hot_patch_kernel_text
1256 
1257 #else
1258 
1259 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1260         movl    $normal_instr, %ecx;            \
1261         movl    $active_instr, %edx;            \
1262         movl    $lockstat_probemap, %eax;       \
1263         movl    _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1264         testl   %eax, %eax;                     \
1265         jz      . + 4;                          \
1266         movl    %edx, %ecx;                     \
1267         pushl   $len;                           \
1268         pushl   %ecx;                           \
1269         pushl   $addr;                          \
1270         call    hot_patch_kernel_text;          \
1271         addl    $12, %esp;
1272 
1273 #endif  /* !__amd64 */
1274 
1275         ENTRY(lockstat_hot_patch)
1276 #if defined(__amd64)
1277         pushq   %rbp                    /* align stack properly */
1278         movq    %rsp, %rbp
1279 #endif  /* __amd64 */
1280 
1281 #if defined(OPTERON_WORKAROUND_6323525)
1282         cmpl    $0, workaround_6323525_patched
1283         je      1f
1284         HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1285                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1286         HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1287                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288         HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1289                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1290         jmp     2f
1291 1:
1292         HOT_PATCH(.mutex_enter_lockstat_patch_point,
1293                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1294         HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1295                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296         HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1297                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1298 2:
1299 #else   /* OPTERON_WORKAROUND_6323525 */
1300         HOT_PATCH(.mutex_enter_lockstat_patch_point,
1301                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1302         HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1303                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1304         HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1305                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1306 #endif  /* !OPTERON_WORKAROUND_6323525 */
1307         HOT_PATCH(.mutex_exit_lockstat_patch_point,
1308                 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1309         HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1310                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1311         HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1312                 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1313         HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1314                 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1315         HOT_PATCH(.lock_set_lockstat_patch_point,
1316                 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317         HOT_PATCH(.lock_try_lockstat_patch_point,
1318                 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1319         HOT_PATCH(.lock_clear_lockstat_patch_point,
1320                 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1321         HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1322                 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1323 
1324         HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1325                 LS_LOCK_CLEAR_SPLX_RELEASE,
1326                 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1327 #if defined(__amd64)
1328         leave                   /* unwind stack */
1329 #endif  /* __amd64 */
1330         ret
1331         SET_SIZE(lockstat_hot_patch)
1332 
1333 #endif  /* __lint */
1334 
1335 #if defined(lint) || defined(__lint)
1336 
1337 /* XX64 membar_*() should be inlines */
1338 
1339 void
1340 membar_sync(void)
1341 {}
1342 
1343 void
1344 membar_enter(void)
1345 {}
1346 
1347 void
1348 membar_exit(void)
1349 {}
1350 
1351 void
1352 membar_producer(void)
1353 {}
1354 
1355 void
1356 membar_consumer(void)
1357 {}
1358 
1359 #else   /* __lint */
1360 
1361 #if defined(__amd64)
1362 
1363         ENTRY(membar_enter)
1364         ALTENTRY(membar_exit)
1365         ALTENTRY(membar_sync)
1366         mfence                  /* lighter weight than lock; xorq $0,(%rsp) */
1367         ret
1368         SET_SIZE(membar_sync)
1369         SET_SIZE(membar_exit)
1370         SET_SIZE(membar_enter)
1371 
1372         ENTRY(membar_producer)
1373         sfence
1374         ret
1375         SET_SIZE(membar_producer)
1376 
1377         ENTRY(membar_consumer)
1378         lfence
1379         ret
1380         SET_SIZE(membar_consumer)
1381 
1382 #else
1383 
1384         ENTRY(membar_enter)
1385         ALTENTRY(membar_exit)
1386         ALTENTRY(membar_sync)
1387         lock
1388         xorl    $0, (%esp)
1389         ret
1390         SET_SIZE(membar_sync)
1391         SET_SIZE(membar_exit)
1392         SET_SIZE(membar_enter)
1393 
1394 /*
1395  * On machines that support sfence and lfence, these
1396  * memory barriers can be more precisely implemented
1397  * without causing the whole world to stop
1398  */
1399         ENTRY(membar_producer)
1400         .globl  _patch_sfence_ret
1401 _patch_sfence_ret:                      /* c.f. membar #StoreStore */
1402         lock
1403         xorl    $0, (%esp)
1404         ret
1405         SET_SIZE(membar_producer)
1406 
1407         ENTRY(membar_consumer)
1408         .globl  _patch_lfence_ret
1409 _patch_lfence_ret:                      /* c.f. membar #LoadLoad */
1410         lock
1411         xorl    $0, (%esp)
1412         ret
1413         SET_SIZE(membar_consumer)
1414 
1415 #endif  /* !__amd64 */
1416 
1417 #endif  /* __lint */
1418 
1419 /*
1420  * thread_onproc()
1421  * Set thread in onproc state for the specified CPU.
1422  * Also set the thread lock pointer to the CPU's onproc lock.
1423  * Since the new lock isn't held, the store ordering is important.
1424  * If not done in assembler, the compiler could reorder the stores.
1425  */
1426 #if defined(lint) || defined(__lint)
1427 
1428 void
1429 thread_onproc(kthread_id_t t, cpu_t *cp)
1430 {
1431         t->t_state = TS_ONPROC;
1432         t->t_lockp = &cp->cpu_thread_lock;
1433 }
1434 
1435 #else   /* __lint */
1436 
1437 #if defined(__amd64)
1438 
1439         ENTRY(thread_onproc)
1440         addq    $CPU_THREAD_LOCK, %rsi  /* pointer to disp_lock while running */
1441         movl    $ONPROC_THREAD, T_STATE(%rdi)   /* set state to TS_ONPROC */
1442         movq    %rsi, T_LOCKP(%rdi)     /* store new lock pointer */
1443         ret
1444         SET_SIZE(thread_onproc)
1445 
1446 #else
1447 
1448         ENTRY(thread_onproc)
1449         movl    4(%esp), %eax
1450         movl    8(%esp), %ecx
1451         addl    $CPU_THREAD_LOCK, %ecx  /* pointer to disp_lock while running */
1452         movl    $ONPROC_THREAD, T_STATE(%eax)   /* set state to TS_ONPROC */
1453         movl    %ecx, T_LOCKP(%eax)     /* store new lock pointer */
1454         ret
1455         SET_SIZE(thread_onproc)
1456 
1457 #endif  /* !__amd64 */
1458 
1459 #endif  /* __lint */
1460 
1461 /*
1462  * mutex_delay_default(void)
1463  * Spins for approx a few hundred processor cycles and returns to caller.
1464  */
1465 
1466 #if defined(lint) || defined(__lint)
1467 
1468 void
1469 mutex_delay_default(void)
1470 {}
1471 
1472 #else   /* __lint */
1473 
1474 #if defined(__amd64)
1475 
1476         ENTRY(mutex_delay_default)
1477         movq    $92,%r11
1478 0:      decq    %r11
1479         jg      0b
1480         ret
1481         SET_SIZE(mutex_delay_default)
1482 
1483 #else
1484 
1485         ENTRY(mutex_delay_default)
1486         push    %ebp
1487         movl    %esp,%ebp
1488         andl    $-16,%esp
1489         push    %ebx
1490         movl    $93,%ebx
1491 0:      decl    %ebx
1492         jg      0b
1493         pop     %ebx
1494         leave
1495         ret
1496         SET_SIZE(mutex_delay_default)
1497 
1498 #endif  /* !__amd64 */
1499 #endif  /* __lint */