1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #if defined(lint) || defined(__lint)
28 #include <sys/types.h>
29 #include <sys/thread.h>
30 #include <sys/cpuvar.h>
31 #include <vm/page.h>
32 #else /* __lint */
33 #include "assym.h"
34 #endif /* __lint */
35
36 #include <sys/mutex_impl.h>
37 #include <sys/asm_linkage.h>
38 #include <sys/asm_misc.h>
39 #include <sys/regset.h>
40 #include <sys/rwlock_impl.h>
41 #include <sys/lockstat.h>
42
43 /*
44 * lock_try(lp), ulock_try(lp)
45 * - returns non-zero on success.
46 * - doesn't block interrupts so don't use this to spin on a lock.
47 *
48 * ulock_try() is for a lock in the user address space.
49 */
50
51 #if defined(lint) || defined(__lint)
52
53 /* ARGSUSED */
54 int
55 lock_try(lock_t *lp)
56 { return (0); }
57
58 /* ARGSUSED */
59 int
60 lock_spin_try(lock_t *lp)
61 { return (0); }
62
63 /* ARGSUSED */
64 int
65 ulock_try(lock_t *lp)
66 { return (0); }
67
68 #else /* __lint */
69 .globl kernelbase
70
71 #if defined(__amd64)
72
73 ENTRY(lock_try)
74 movb $-1, %dl
75 movzbq %dl, %rax
76 xchgb %dl, (%rdi)
77 xorb %dl, %al
78 .lock_try_lockstat_patch_point:
79 ret
80 testb %al, %al
81 jnz 0f
82 ret
83 0:
84 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
85 movq %rdi, %rsi /* rsi = lock addr */
86 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
87 jmp lockstat_wrapper
88 SET_SIZE(lock_try)
89
90 ENTRY(lock_spin_try)
91 movb $-1, %dl
92 movzbq %dl, %rax
93 xchgb %dl, (%rdi)
94 xorb %dl, %al
95 ret
96 SET_SIZE(lock_spin_try)
97
98 ENTRY(ulock_try)
99 #ifdef DEBUG
100 movq kernelbase(%rip), %rax
101 cmpq %rax, %rdi /* test uaddr < kernelbase */
102 jb ulock_pass /* uaddr < kernelbase, proceed */
103
104 movq %rdi, %r12 /* preserve lock ptr for debugging */
105 leaq .ulock_panic_msg(%rip), %rdi
106 pushq %rbp /* align stack properly */
107 movq %rsp, %rbp
108 xorl %eax, %eax /* clear for varargs */
109 call panic
110
111 #endif /* DEBUG */
112
113 ulock_pass:
114 movl $1, %eax
115 xchgb %al, (%rdi)
116 xorb $1, %al
117 ret
118 SET_SIZE(ulock_try)
119
120 #else
121
122 ENTRY(lock_try)
123 movl $1,%edx
124 movl 4(%esp),%ecx /* ecx = lock addr */
125 xorl %eax,%eax
126 xchgb %dl, (%ecx) /* using dl will avoid partial */
127 testb %dl,%dl /* stalls on P6 ? */
128 setz %al
129 .lock_try_lockstat_patch_point:
130 ret
131 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
132 testl %eax, %eax
133 jz 0f
134 movl $LS_LOCK_TRY_ACQUIRE, %eax
135 jmp lockstat_wrapper
136 0:
137 ret
138 SET_SIZE(lock_try)
139
140 ENTRY(lock_spin_try)
141 movl $-1,%edx
142 movl 4(%esp),%ecx /* ecx = lock addr */
143 xorl %eax,%eax
144 xchgb %dl, (%ecx) /* using dl will avoid partial */
145 testb %dl,%dl /* stalls on P6 ? */
146 setz %al
147 ret
148 SET_SIZE(lock_spin_try)
149
150 ENTRY(ulock_try)
151 #ifdef DEBUG
152 movl kernelbase, %eax
153 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */
154 jb ulock_pass /* uaddr < kernelbase, proceed */
155
156 pushl $.ulock_panic_msg
157 call panic
158
159 #endif /* DEBUG */
160
161 ulock_pass:
162 movl $1,%eax
163 movl 4(%esp),%ecx
164 xchgb %al, (%ecx)
165 xorb $1, %al
166 ret
167 SET_SIZE(ulock_try)
168
169 #endif /* !__amd64 */
170
171 #ifdef DEBUG
172 .data
173 .ulock_panic_msg:
174 .string "ulock_try: Argument is above kernelbase"
175 .text
176 #endif /* DEBUG */
177
178 #endif /* __lint */
179
180 /*
181 * lock_clear(lp)
182 * - unlock lock without changing interrupt priority level.
183 */
184
185 #if defined(lint) || defined(__lint)
186
187 /* ARGSUSED */
188 void
189 lock_clear(lock_t *lp)
190 {}
191
192 /* ARGSUSED */
193 void
194 ulock_clear(lock_t *lp)
195 {}
196
197 #else /* __lint */
198
199 #if defined(__amd64)
200
201 ENTRY(lock_clear)
202 movb $0, (%rdi)
203 .lock_clear_lockstat_patch_point:
204 ret
205 movq %rdi, %rsi /* rsi = lock addr */
206 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
207 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
208 jmp lockstat_wrapper
209 SET_SIZE(lock_clear)
210
211 ENTRY(ulock_clear)
212 #ifdef DEBUG
213 movq kernelbase(%rip), %rcx
214 cmpq %rcx, %rdi /* test uaddr < kernelbase */
215 jb ulock_clr /* uaddr < kernelbase, proceed */
216
217 leaq .ulock_clear_msg(%rip), %rdi
218 pushq %rbp /* align stack properly */
219 movq %rsp, %rbp
220 xorl %eax, %eax /* clear for varargs */
221 call panic
222 #endif
223
224 ulock_clr:
225 movb $0, (%rdi)
226 ret
227 SET_SIZE(ulock_clear)
228
229 #else
230
231 ENTRY(lock_clear)
232 movl 4(%esp), %eax
233 movb $0, (%eax)
234 .lock_clear_lockstat_patch_point:
235 ret
236 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
237 movl %eax, %ecx /* ecx = lock pointer */
238 movl $LS_LOCK_CLEAR_RELEASE, %eax
239 jmp lockstat_wrapper
240 SET_SIZE(lock_clear)
241
242 ENTRY(ulock_clear)
243 #ifdef DEBUG
244 movl kernelbase, %ecx
245 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */
246 jb ulock_clr /* uaddr < kernelbase, proceed */
247
248 pushl $.ulock_clear_msg
249 call panic
250 #endif
251
252 ulock_clr:
253 movl 4(%esp),%eax
254 xorl %ecx,%ecx
255 movb %cl, (%eax)
256 ret
257 SET_SIZE(ulock_clear)
258
259 #endif /* !__amd64 */
260
261 #ifdef DEBUG
262 .data
263 .ulock_clear_msg:
264 .string "ulock_clear: Argument is above kernelbase"
265 .text
266 #endif /* DEBUG */
267
268
269 #endif /* __lint */
270
271 /*
272 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
273 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
274 */
275
276 #if defined(lint) || defined(__lint)
277
278 /* ARGSUSED */
279 void
280 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
281 {}
282
283 #else /* __lint */
284
285 #if defined(__amd64)
286
287 ENTRY(lock_set_spl)
288 pushq %rbp
289 movq %rsp, %rbp
290 subq $32, %rsp
291 movl %esi, 8(%rsp) /* save priority level */
292 movq %rdx, 16(%rsp) /* save old pil ptr */
293 movq %rdi, 24(%rsp) /* save lock pointer */
294 movl %esi, %edi /* pass priority level */
295 call splr /* raise priority level */
296 movq 24(%rsp), %rdi /* rdi = lock addr */
297 movb $-1, %dl
298 xchgb %dl, (%rdi) /* try to set lock */
299 testb %dl, %dl /* did we get the lock? ... */
300 jnz .lss_miss /* ... no, go to C for the hard case */
301 movq 16(%rsp), %rdx /* rdx = old pil addr */
302 movw %ax, (%rdx) /* store old pil */
303 leave
304 .lock_set_spl_lockstat_patch_point:
305 ret
306 movq %rdi, %rsi /* rsi = lock addr */
307 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
308 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
309 jmp lockstat_wrapper
310 .lss_miss:
311 movl 8(%rsp), %esi /* new_pil */
312 movq 16(%rsp), %rdx /* old_pil_addr */
313 movl %eax, %ecx /* original pil */
314 leave /* unwind stack */
315 jmp lock_set_spl_spin
316 SET_SIZE(lock_set_spl)
317
318 #else
319
320 ENTRY(lock_set_spl)
321 movl 8(%esp), %eax /* get priority level */
322 pushl %eax
323 call splr /* raise priority level */
324 movl 8(%esp), %ecx /* ecx = lock addr */
325 movl $-1, %edx
326 addl $4, %esp
327 xchgb %dl, (%ecx) /* try to set lock */
328 testb %dl, %dl /* did we get the lock? ... */
329 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */
330 jnz .lss_miss /* ... no, go to C for the hard case */
331 movw %ax, (%edx) /* store old pil */
332 .lock_set_spl_lockstat_patch_point:
333 ret
334 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/
335 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax
336 jmp lockstat_wrapper
337 .lss_miss:
338 pushl %eax /* original pil */
339 pushl %edx /* old_pil addr */
340 pushl 16(%esp) /* new_pil */
341 pushl %ecx /* lock addr */
342 call lock_set_spl_spin
343 addl $16, %esp
344 ret
345 SET_SIZE(lock_set_spl)
346
347 #endif /* !__amd64 */
348
349 #endif /* __lint */
350
351 /*
352 * void
353 * lock_init(lp)
354 */
355
356 #if defined(__lint)
357
358 /* ARGSUSED */
359 void
360 lock_init(lock_t *lp)
361 {}
362
363 #else /* __lint */
364
365 #if defined(__amd64)
366
367 ENTRY(lock_init)
368 movb $0, (%rdi)
369 ret
370 SET_SIZE(lock_init)
371
372 #else
373
374 ENTRY(lock_init)
375 movl 4(%esp), %eax
376 movb $0, (%eax)
377 ret
378 SET_SIZE(lock_init)
379
380 #endif /* !__amd64 */
381
382 #endif /* __lint */
383
384 /*
385 * void
386 * lock_set(lp)
387 */
388
389 #if defined(lint) || defined(__lint)
390
391 /* ARGSUSED */
392 void
393 lock_set(lock_t *lp)
394 {}
395
396 #else /* __lint */
397
398 #if defined(__amd64)
399
400 ENTRY(lock_set)
401 movb $-1, %dl
402 xchgb %dl, (%rdi) /* try to set lock */
403 testb %dl, %dl /* did we get it? */
404 jnz lock_set_spin /* no, go to C for the hard case */
405 .lock_set_lockstat_patch_point:
406 ret
407 movq %rdi, %rsi /* rsi = lock addr */
408 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
409 movl $LS_LOCK_SET_ACQUIRE, %edi
410 jmp lockstat_wrapper
411 SET_SIZE(lock_set)
412
413 #else
414
415 ENTRY(lock_set)
416 movl 4(%esp), %ecx /* ecx = lock addr */
417 movl $-1, %edx
418 xchgb %dl, (%ecx) /* try to set lock */
419 testb %dl, %dl /* did we get it? */
420 jnz lock_set_spin /* no, go to C for the hard case */
421 .lock_set_lockstat_patch_point:
422 ret
423 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
424 movl $LS_LOCK_SET_ACQUIRE, %eax
425 jmp lockstat_wrapper
426 SET_SIZE(lock_set)
427
428 #endif /* !__amd64 */
429
430 #endif /* __lint */
431
432 /*
433 * lock_clear_splx(lp, s)
434 */
435
436 #if defined(lint) || defined(__lint)
437
438 /* ARGSUSED */
439 void
440 lock_clear_splx(lock_t *lp, int s)
441 {}
442
443 #else /* __lint */
444
445 #if defined(__amd64)
446
447 ENTRY(lock_clear_splx)
448 movb $0, (%rdi) /* clear lock */
449 .lock_clear_splx_lockstat_patch_point:
450 jmp 0f
451 0:
452 movl %esi, %edi /* arg for splx */
453 jmp splx /* let splx do its thing */
454 .lock_clear_splx_lockstat:
455 pushq %rbp /* align stack properly */
456 movq %rsp, %rbp
457 subq $16, %rsp /* space to save args across splx */
458 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
459 movl %esi, %edi /* arg for splx */
460 call splx /* lower the priority */
461 movq 8(%rsp), %rsi /* rsi = lock ptr */
462 leave /* unwind stack */
463 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
464 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
465 jmp lockstat_wrapper
466 SET_SIZE(lock_clear_splx)
467
468 #else
469
470 ENTRY(lock_clear_splx)
471 movl 4(%esp), %eax /* eax = lock addr */
472 movb $0, (%eax) /* clear lock */
473 .lock_clear_splx_lockstat_patch_point:
474 jmp 0f
475 0:
476 movl 8(%esp), %edx /* edx = desired pil */
477 movl %edx, 4(%esp) /* set spl arg up for splx */
478 jmp splx /* let splx do it's thing */
479 .lock_clear_splx_lockstat:
480 movl 8(%esp), %edx /* edx = desired pil */
481 pushl %ebp /* set up stack frame */
482 movl %esp, %ebp
483 pushl %edx
484 call splx
485 leave /* unwind stack */
486 movl 4(%esp), %ecx /* ecx = lock pointer */
487 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
488 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
489 jmp lockstat_wrapper
490 SET_SIZE(lock_clear_splx)
491
492 #endif /* !__amd64 */
493
494 #if defined(__GNUC_AS__)
495 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
496 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
497
498 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
499 (.lock_clear_splx_lockstat_patch_point + 1)
500 #else
501 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
502 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
503
504 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
505 [.lock_clear_splx_lockstat_patch_point + 1]
506 #endif
507
508 #endif /* __lint */
509
510 /*
511 * mutex_enter() and mutex_exit().
512 *
513 * These routines handle the simple cases of mutex_enter() (adaptive
514 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
515 * If anything complicated is going on we punt to mutex_vector_enter().
516 *
517 * mutex_tryenter() is similar to mutex_enter() but returns zero if
518 * the lock cannot be acquired, nonzero on success.
519 *
520 * If mutex_exit() gets preempted in the window between checking waiters
521 * and clearing the lock, we can miss wakeups. Disabling preemption
522 * in the mutex code is prohibitively expensive, so instead we detect
523 * mutex preemption by examining the trapped PC in the interrupt path.
524 * If we interrupt a thread in mutex_exit() that has not yet cleared
525 * the lock, cmnint() resets its PC back to the beginning of
526 * mutex_exit() so it will check again for waiters when it resumes.
527 *
528 * The lockstat code below is activated when the lockstat driver
529 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
530 * Note that we don't need to test lockstat_event_mask here -- we won't
531 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
532 */
533 #if defined(lint) || defined(__lint)
534
535 /* ARGSUSED */
536 void
537 mutex_enter(kmutex_t *lp)
538 {}
539
540 /* ARGSUSED */
541 int
542 mutex_tryenter(kmutex_t *lp)
543 { return (0); }
544
545 /* ARGSUSED */
546 int
547 mutex_adaptive_tryenter(mutex_impl_t *lp)
548 { return (0); }
549
550 /* ARGSUSED */
551 void
552 mutex_exit(kmutex_t *lp)
553 {}
554
555 #else
556
557 #if defined(__amd64)
558
559 ENTRY_NP(mutex_enter)
560 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
561 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
562 lock
563 cmpxchgq %rdx, (%rdi)
564 jnz mutex_vector_enter
565 .mutex_enter_lockstat_patch_point:
566 #if defined(OPTERON_WORKAROUND_6323525)
567 .mutex_enter_6323525_patch_point:
568 ret /* nop space for lfence */
569 nop
570 nop
571 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
572 nop
573 #else /* OPTERON_WORKAROUND_6323525 */
574 ret
575 #endif /* OPTERON_WORKAROUND_6323525 */
576 movq %rdi, %rsi
577 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
578 /*
579 * expects %rdx=thread, %rsi=lock, %edi=lockstat event
580 */
581 ALTENTRY(lockstat_wrapper)
582 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */
583 leaq lockstat_probemap(%rip), %rax
584 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
585 testl %eax, %eax /* check for non-zero probe */
586 jz 1f
587 pushq %rbp /* align stack properly */
588 movq %rsp, %rbp
589 movl %eax, %edi
590 movq lockstat_probe, %rax
591 INDIRECT_CALL_REG(rax)
592 leave /* unwind stack */
593 1:
594 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
595 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
596 movl $1, %eax /* return success if tryenter */
597 ret
598 SET_SIZE(lockstat_wrapper)
599 SET_SIZE(mutex_enter)
600
601 /*
602 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
603 */
604 ENTRY(lockstat_wrapper_arg)
605 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */
606 leaq lockstat_probemap(%rip), %rax
607 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
608 testl %eax, %eax /* check for non-zero probe */
609 jz 1f
610 pushq %rbp /* align stack properly */
611 movq %rsp, %rbp
612 movl %eax, %edi
613 movq lockstat_probe, %rax
614 INDIRECT_CALL_REG(rax)
615 leave /* unwind stack */
616 1:
617 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
618 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
619 movl $1, %eax /* return success if tryenter */
620 ret
621 SET_SIZE(lockstat_wrapper_arg)
622
623
624 ENTRY(mutex_tryenter)
625 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
626 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
627 lock
628 cmpxchgq %rdx, (%rdi)
629 jnz mutex_vector_tryenter
630 not %eax /* return success (nonzero) */
631 #if defined(OPTERON_WORKAROUND_6323525)
632 .mutex_tryenter_lockstat_patch_point:
633 .mutex_tryenter_6323525_patch_point:
634 ret /* nop space for lfence */
635 nop
636 nop
637 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
638 nop
639 #else /* OPTERON_WORKAROUND_6323525 */
640 .mutex_tryenter_lockstat_patch_point:
641 ret
642 #endif /* OPTERON_WORKAROUND_6323525 */
643 movq %rdi, %rsi
644 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
645 jmp lockstat_wrapper
646 SET_SIZE(mutex_tryenter)
647
648 ENTRY(mutex_adaptive_tryenter)
649 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
650 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
651 lock
652 cmpxchgq %rdx, (%rdi)
653 jnz 0f
654 not %eax /* return success (nonzero) */
655 #if defined(OPTERON_WORKAROUND_6323525)
656 .mutex_atryenter_6323525_patch_point:
657 ret /* nop space for lfence */
658 nop
659 nop
660 nop
661 #else /* OPTERON_WORKAROUND_6323525 */
662 ret
663 #endif /* OPTERON_WORKAROUND_6323525 */
664 0:
665 xorl %eax, %eax /* return failure */
666 ret
667 SET_SIZE(mutex_adaptive_tryenter)
668
669 .globl mutex_owner_running_critical_start
670
671 ENTRY(mutex_owner_running)
672 mutex_owner_running_critical_start:
673 movq (%rdi), %r11 /* get owner field */
674 andq $MUTEX_THREAD, %r11 /* remove waiters bit */
675 cmpq $0, %r11 /* if free, skip */
676 je 1f /* go return 0 */
677 movq T_CPU(%r11), %r8 /* get owner->t_cpu */
678 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */
679 .mutex_owner_running_critical_end:
680 cmpq %r11, %r9 /* owner == running thread? */
681 je 2f /* yes, go return cpu */
682 1:
683 xorq %rax, %rax /* return 0 */
684 ret
685 2:
686 movq %r8, %rax /* return cpu */
687 ret
688 SET_SIZE(mutex_owner_running)
689
690 .globl mutex_owner_running_critical_size
691 .type mutex_owner_running_critical_size, @object
692 .align CPTRSIZE
693 mutex_owner_running_critical_size:
694 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start
695 SET_SIZE(mutex_owner_running_critical_size)
696
697 .globl mutex_exit_critical_start
698
699 ENTRY(mutex_exit)
700 mutex_exit_critical_start: /* If interrupted, restart here */
701 movq %gs:CPU_THREAD, %rdx
702 cmpq %rdx, (%rdi)
703 jne mutex_vector_exit /* wrong type or wrong owner */
704 movq $0, (%rdi) /* clear owner AND lock */
705 .mutex_exit_critical_end:
706 .mutex_exit_lockstat_patch_point:
707 ret
708 movq %rdi, %rsi
709 movl $LS_MUTEX_EXIT_RELEASE, %edi
710 jmp lockstat_wrapper
711 SET_SIZE(mutex_exit)
712
713 .globl mutex_exit_critical_size
714 .type mutex_exit_critical_size, @object
715 .align CPTRSIZE
716 mutex_exit_critical_size:
717 .quad .mutex_exit_critical_end - mutex_exit_critical_start
718 SET_SIZE(mutex_exit_critical_size)
719
720 #else
721
722 ENTRY_NP(mutex_enter)
723 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
724 movl 4(%esp), %ecx /* ecx = lock ptr */
725 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
726 lock
727 cmpxchgl %edx, (%ecx)
728 jnz mutex_vector_enter
729 #if defined(OPTERON_WORKAROUND_6323525)
730 .mutex_enter_lockstat_patch_point:
731 .mutex_enter_6323525_patch_point:
732 ret /* nop space for lfence */
733 nop
734 nop
735 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
736 nop
737 #else /* OPTERON_WORKAROUND_6323525 */
738 .mutex_enter_lockstat_patch_point:
739 ret
740 #endif /* OPTERON_WORKAROUND_6323525 */
741 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
742 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */
743 /* eax=lockstat event */
744 pushl %ebp /* buy a frame */
745 movl %esp, %ebp
746 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
747 pushl %edx /* save thread pointer */
748 movl $lockstat_probemap, %edx
749 movl (%edx, %eax, DTRACE_IDSIZE), %eax
750 testl %eax, %eax /* check for non-zero probe */
751 jz 1f
752 pushl %ecx /* push lock */
753 pushl %eax /* push probe ID */
754 call *lockstat_probe
755 addl $8, %esp
756 1:
757 popl %edx /* restore thread pointer */
758 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
759 movl $1, %eax /* return success if tryenter */
760 popl %ebp /* pop off frame */
761 ret
762 SET_SIZE(lockstat_wrapper)
763 SET_SIZE(mutex_enter)
764
765 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */
766 /* eax=lockstat event, pushed arg */
767 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
768 pushl %edx /* save thread pointer */
769 movl $lockstat_probemap, %edx
770 movl (%edx, %eax, DTRACE_IDSIZE), %eax
771 testl %eax, %eax /* check for non-zero probe */
772 jz 1f
773 pushl %ebp /* save %ebp */
774 pushl 8(%esp) /* push arg1 */
775 movl %ebp, 12(%esp) /* fake up the stack frame */
776 movl %esp, %ebp /* fake up base pointer */
777 addl $12, %ebp /* adjust faked base pointer */
778 pushl %ecx /* push lock */
779 pushl %eax /* push probe ID */
780 call *lockstat_probe
781 addl $12, %esp /* adjust for arguments */
782 popl %ebp /* pop frame */
783 1:
784 popl %edx /* restore thread pointer */
785 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
786 movl $1, %eax /* return success if tryenter */
787 addl $4, %esp /* pop argument */
788 ret
789 SET_SIZE(lockstat_wrapper_arg)
790
791
792 ENTRY(mutex_tryenter)
793 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
794 movl 4(%esp), %ecx /* ecx = lock ptr */
795 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
796 lock
797 cmpxchgl %edx, (%ecx)
798 jnz mutex_vector_tryenter
799 movl %ecx, %eax
800 #if defined(OPTERON_WORKAROUND_6323525)
801 .mutex_tryenter_lockstat_patch_point:
802 .mutex_tryenter_6323525_patch_point:
803 ret /* nop space for lfence */
804 nop
805 nop
806 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
807 nop
808 #else /* OPTERON_WORKAROUND_6323525 */
809 .mutex_tryenter_lockstat_patch_point:
810 ret
811 #endif /* OPTERON_WORKAROUND_6323525 */
812 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
813 jmp lockstat_wrapper
814 SET_SIZE(mutex_tryenter)
815
816 ENTRY(mutex_adaptive_tryenter)
817 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
818 movl 4(%esp), %ecx /* ecx = lock ptr */
819 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
820 lock
821 cmpxchgl %edx, (%ecx)
822 jnz 0f
823 movl %ecx, %eax
824 #if defined(OPTERON_WORKAROUND_6323525)
825 .mutex_atryenter_6323525_patch_point:
826 ret /* nop space for lfence */
827 nop
828 nop
829 nop
830 #else /* OPTERON_WORKAROUND_6323525 */
831 ret
832 #endif /* OPTERON_WORKAROUND_6323525 */
833 0:
834 xorl %eax, %eax
835 ret
836 SET_SIZE(mutex_adaptive_tryenter)
837
838 .globl mutex_owner_running_critical_start
839
840 ENTRY(mutex_owner_running)
841 mutex_owner_running_critical_start:
842 movl 4(%esp), %eax /* get owner field */
843 movl (%eax), %eax
844 andl $MUTEX_THREAD, %eax /* remove waiters bit */
845 cmpl $0, %eax /* if free, skip */
846 je 1f /* go return 0 */
847 movl T_CPU(%eax), %ecx /* get owner->t_cpu */
848 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */
849 .mutex_owner_running_critical_end:
850 cmpl %eax, %edx /* owner == running thread? */
851 je 2f /* yes, go return cpu */
852 1:
853 xorl %eax, %eax /* return 0 */
854 ret
855 2:
856 movl %ecx, %eax /* return cpu */
857 ret
858
859 SET_SIZE(mutex_owner_running)
860
861 .globl mutex_owner_running_critical_size
862 .type mutex_owner_running_critical_size, @object
863 .align CPTRSIZE
864 mutex_owner_running_critical_size:
865 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start
866 SET_SIZE(mutex_owner_running_critical_size)
867
868 .globl mutex_exit_critical_start
869
870 ENTRY(mutex_exit)
871 mutex_exit_critical_start: /* If interrupted, restart here */
872 movl %gs:CPU_THREAD, %edx
873 movl 4(%esp), %ecx
874 cmpl %edx, (%ecx)
875 jne mutex_vector_exit /* wrong type or wrong owner */
876 movl $0, (%ecx) /* clear owner AND lock */
877 .mutex_exit_critical_end:
878 .mutex_exit_lockstat_patch_point:
879 ret
880 movl $LS_MUTEX_EXIT_RELEASE, %eax
881 jmp lockstat_wrapper
882 SET_SIZE(mutex_exit)
883
884 .globl mutex_exit_critical_size
885 .type mutex_exit_critical_size, @object
886 .align CPTRSIZE
887 mutex_exit_critical_size:
888 .long .mutex_exit_critical_end - mutex_exit_critical_start
889 SET_SIZE(mutex_exit_critical_size)
890
891 #endif /* !__amd64 */
892
893 #endif /* __lint */
894
895 /*
896 * rw_enter() and rw_exit().
897 *
898 * These routines handle the simple cases of rw_enter (write-locking an unheld
899 * lock or read-locking a lock that's neither write-locked nor write-wanted)
900 * and rw_exit (no waiters or not the last reader). If anything complicated
901 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
902 */
903 #if defined(lint) || defined(__lint)
904
905 /* ARGSUSED */
906 void
907 rw_enter(krwlock_t *lp, krw_t rw)
908 {}
909
910 /* ARGSUSED */
911 void
912 rw_exit(krwlock_t *lp)
913 {}
914
915 #else /* __lint */
916
917 #if defined(__amd64)
918
919 ENTRY(rw_enter)
920 cmpl $RW_WRITER, %esi
921 je .rw_write_enter
922 movq (%rdi), %rax /* rax = old rw_wwwh value */
923 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
924 jnz rw_enter_sleep
925 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
926 lock
927 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
928 jnz rw_enter_sleep
929 .rw_read_enter_lockstat_patch_point:
930 ret
931 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
932 movq %rdi, %rsi /* rsi = lock ptr */
933 movl $LS_RW_ENTER_ACQUIRE, %edi
934 movl $RW_READER, %edx
935 jmp lockstat_wrapper_arg
936 .rw_write_enter:
937 movq %gs:CPU_THREAD, %rdx
938 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
939 xorl %eax, %eax /* rax = unheld value */
940 lock
941 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
942 jnz rw_enter_sleep
943
944 #if defined(OPTERON_WORKAROUND_6323525)
945 .rw_write_enter_lockstat_patch_point:
946 .rw_write_enter_6323525_patch_point:
947 ret
948 nop
949 nop
950 .rw_write_enter_lockstat_6323525_patch_point:
951 nop
952 #else /* OPTERON_WORKAROUND_6323525 */
953 .rw_write_enter_lockstat_patch_point:
954 ret
955 #endif /* OPTERON_WORKAROUND_6323525 */
956
957 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
958 movq %rdi, %rsi /* rsi = lock ptr */
959 movl $LS_RW_ENTER_ACQUIRE, %edi
960 movl $RW_WRITER, %edx
961 jmp lockstat_wrapper_arg
962 SET_SIZE(rw_enter)
963
964 ENTRY(rw_exit)
965 movq (%rdi), %rax /* rax = old rw_wwwh value */
966 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
967 jne .rw_not_single_reader
968 xorl %edx, %edx /* rdx = new value (unheld) */
969 .rw_read_exit:
970 lock
971 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
972 jnz rw_exit_wakeup
973 .rw_read_exit_lockstat_patch_point:
974 ret
975 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
976 movq %rdi, %rsi /* rsi = lock ptr */
977 movl $LS_RW_EXIT_RELEASE, %edi
978 movl $RW_READER, %edx
979 jmp lockstat_wrapper_arg
980 .rw_not_single_reader:
981 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
982 jnz .rw_write_exit
983 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
984 cmpl $RW_READ_LOCK, %edx
985 jge .rw_read_exit /* not last reader, safe to drop */
986 jmp rw_exit_wakeup /* last reader with waiters */
987 .rw_write_exit:
988 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
989 xorl %edx, %edx /* rdx = new value (unheld) */
990 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
991 lock
992 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
993 jnz rw_exit_wakeup
994 .rw_write_exit_lockstat_patch_point:
995 ret
996 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
997 movq %rdi, %rsi /* rsi - lock ptr */
998 movl $LS_RW_EXIT_RELEASE, %edi
999 movl $RW_WRITER, %edx
1000 jmp lockstat_wrapper_arg
1001 SET_SIZE(rw_exit)
1002
1003 #else
1004
1005 ENTRY(rw_enter)
1006 movl 4(%esp), %ecx /* ecx = lock ptr */
1007 cmpl $RW_WRITER, 8(%esp)
1008 je .rw_write_enter
1009 movl (%ecx), %eax /* eax = old rw_wwwh value */
1010 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1011 jnz rw_enter_sleep
1012 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1013 lock
1014 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1015 jnz rw_enter_sleep
1016 .rw_read_enter_lockstat_patch_point:
1017 ret
1018 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1019 movl $LS_RW_ENTER_ACQUIRE, %eax
1020 pushl $RW_READER
1021 jmp lockstat_wrapper_arg
1022 .rw_write_enter:
1023 movl %gs:CPU_THREAD, %edx
1024 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1025 xorl %eax, %eax /* eax = unheld value */
1026 lock
1027 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1028 jnz rw_enter_sleep
1029
1030 #if defined(OPTERON_WORKAROUND_6323525)
1031 .rw_write_enter_lockstat_patch_point:
1032 .rw_write_enter_6323525_patch_point:
1033 ret
1034 nop
1035 nop
1036 .rw_write_enter_lockstat_6323525_patch_point:
1037 nop
1038 #else /* OPTERON_WORKAROUND_6323525 */
1039 .rw_write_enter_lockstat_patch_point:
1040 ret
1041 #endif /* OPTERON_WORKAROUND_6323525 */
1042
1043 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1044 movl $LS_RW_ENTER_ACQUIRE, %eax
1045 pushl $RW_WRITER
1046 jmp lockstat_wrapper_arg
1047 SET_SIZE(rw_enter)
1048
1049 ENTRY(rw_exit)
1050 movl 4(%esp), %ecx /* ecx = lock ptr */
1051 movl (%ecx), %eax /* eax = old rw_wwwh value */
1052 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1053 jne .rw_not_single_reader
1054 xorl %edx, %edx /* edx = new value (unheld) */
1055 .rw_read_exit:
1056 lock
1057 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1058 jnz rw_exit_wakeup
1059 .rw_read_exit_lockstat_patch_point:
1060 ret
1061 movl $LS_RW_EXIT_RELEASE, %eax
1062 pushl $RW_READER
1063 jmp lockstat_wrapper_arg
1064 .rw_not_single_reader:
1065 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1066 jnz .rw_write_exit
1067 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1068 cmpl $RW_READ_LOCK, %edx
1069 jge .rw_read_exit /* not last reader, safe to drop */
1070 jmp rw_exit_wakeup /* last reader with waiters */
1071 .rw_write_exit:
1072 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1073 xorl %edx, %edx /* edx = new value (unheld) */
1074 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1075 lock
1076 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1077 jnz rw_exit_wakeup
1078 .rw_write_exit_lockstat_patch_point:
1079 ret
1080 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1081 movl $LS_RW_EXIT_RELEASE, %eax
1082 pushl $RW_WRITER
1083 jmp lockstat_wrapper_arg
1084 SET_SIZE(rw_exit)
1085
1086 #endif /* !__amd64 */
1087
1088 #endif /* __lint */
1089
1090 #if defined(OPTERON_WORKAROUND_6323525)
1091 #if defined(lint) || defined(__lint)
1092
1093 int workaround_6323525_patched;
1094
1095 void
1096 patch_workaround_6323525(void)
1097 {}
1098
1099 #else /* lint */
1100
1101 /*
1102 * If it is necessary to patch the lock enter routines with the lfence
1103 * workaround, workaround_6323525_patched is set to a non-zero value so that
1104 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1105 * instruction.
1106 */
1107 DGDEF3(workaround_6323525_patched, 4, 4)
1108 .long 0
1109
1110 #if defined(__amd64)
1111
1112 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1113 movq $size, %rbx; \
1114 movq $dstaddr, %r13; \
1115 addq %rbx, %r13; \
1116 movq $srcaddr, %r12; \
1117 addq %rbx, %r12; \
1118 0: \
1119 decq %r13; \
1120 decq %r12; \
1121 movzbl (%r12), %esi; \
1122 movq $1, %rdx; \
1123 movq %r13, %rdi; \
1124 call hot_patch_kernel_text; \
1125 decq %rbx; \
1126 testq %rbx, %rbx; \
1127 jg 0b;
1128
1129 /*
1130 * patch_workaround_6323525: provide workaround for 6323525
1131 *
1132 * The workaround is to place a fencing instruction (lfence) between the
1133 * mutex operation and the subsequent read-modify-write instruction.
1134 *
1135 * This routine hot patches the lfence instruction on top of the space
1136 * reserved by nops in the lock enter routines.
1137 */
1138 ENTRY_NP(patch_workaround_6323525)
1139 pushq %rbp
1140 movq %rsp, %rbp
1141 pushq %r12
1142 pushq %r13
1143 pushq %rbx
1144
1145 /*
1146 * lockstat_hot_patch() to use the alternate lockstat workaround
1147 * 6323525 patch points (points past the lfence instruction to the
1148 * new ret) when workaround_6323525_patched is set.
1149 */
1150 movl $1, workaround_6323525_patched
1151
1152 /*
1153 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
1154 * routines. The 4 bytes are patched in reverse order so that the
1155 * the existing ret is overwritten last. This provides lock enter
1156 * sanity during the intermediate patching stages.
1157 */
1158 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1159 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1160 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1161 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1162
1163 popq %rbx
1164 popq %r13
1165 popq %r12
1166 movq %rbp, %rsp
1167 popq %rbp
1168 ret
1169 _lfence_insn:
1170 lfence
1171 ret
1172 SET_SIZE(patch_workaround_6323525)
1173
1174
1175 #else /* __amd64 */
1176
1177 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1178 movl $size, %ebx; \
1179 movl $srcaddr, %esi; \
1180 addl %ebx, %esi; \
1181 movl $dstaddr, %edi; \
1182 addl %ebx, %edi; \
1183 0: \
1184 decl %esi; \
1185 decl %edi; \
1186 pushl $1; \
1187 movzbl (%esi), %eax; \
1188 pushl %eax; \
1189 pushl %edi; \
1190 call hot_patch_kernel_text; \
1191 addl $12, %esp; \
1192 decl %ebx; \
1193 testl %ebx, %ebx; \
1194 jg 0b;
1195
1196
1197 /* see comments above */
1198 ENTRY_NP(patch_workaround_6323525)
1199 pushl %ebp
1200 movl %esp, %ebp
1201 pushl %ebx
1202 pushl %esi
1203 pushl %edi
1204
1205 movl $1, workaround_6323525_patched
1206
1207 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1208 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1209 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1210 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1211
1212 popl %edi
1213 popl %esi
1214 popl %ebx
1215 movl %ebp, %esp
1216 popl %ebp
1217 ret
1218 _lfence_insn:
1219 .byte 0xf, 0xae, 0xe8 / [lfence instruction]
1220 ret
1221 SET_SIZE(patch_workaround_6323525)
1222
1223 #endif /* !__amd64 */
1224 #endif /* !lint */
1225 #endif /* OPTERON_WORKAROUND_6323525 */
1226
1227
1228 #if defined(lint) || defined(__lint)
1229
1230 void
1231 lockstat_hot_patch(void)
1232 {}
1233
1234 #else
1235
1236 #if defined(__amd64)
1237
1238 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1239 movq $normal_instr, %rsi; \
1240 movq $active_instr, %rdi; \
1241 leaq lockstat_probemap(%rip), %rax; \
1242 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1243 testl %eax, %eax; \
1244 jz 9f; \
1245 movq %rdi, %rsi; \
1246 9: \
1247 movq $len, %rdx; \
1248 movq $addr, %rdi; \
1249 call hot_patch_kernel_text
1250
1251 #else
1252
1253 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1254 movl $normal_instr, %ecx; \
1255 movl $active_instr, %edx; \
1256 movl $lockstat_probemap, %eax; \
1257 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1258 testl %eax, %eax; \
1259 jz . + 4; \
1260 movl %edx, %ecx; \
1261 pushl $len; \
1262 pushl %ecx; \
1263 pushl $addr; \
1264 call hot_patch_kernel_text; \
1265 addl $12, %esp;
1266
1267 #endif /* !__amd64 */
1268
1269 ENTRY(lockstat_hot_patch)
1270 #if defined(__amd64)
1271 pushq %rbp /* align stack properly */
1272 movq %rsp, %rbp
1273 #endif /* __amd64 */
1274
1275 #if defined(OPTERON_WORKAROUND_6323525)
1276 cmpl $0, workaround_6323525_patched
1277 je 1f
1278 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1279 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1280 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1281 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1282 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1283 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1284 jmp 2f
1285 1:
1286 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1287 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1289 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1290 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1291 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1292 2:
1293 #else /* OPTERON_WORKAROUND_6323525 */
1294 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1295 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1297 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1298 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1299 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1300 #endif /* !OPTERON_WORKAROUND_6323525 */
1301 HOT_PATCH(.mutex_exit_lockstat_patch_point,
1302 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1303 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1304 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1305 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1306 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1307 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1308 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1309 HOT_PATCH(.lock_set_lockstat_patch_point,
1310 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1311 HOT_PATCH(.lock_try_lockstat_patch_point,
1312 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1313 HOT_PATCH(.lock_clear_lockstat_patch_point,
1314 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1315 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1316 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317
1318 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1319 LS_LOCK_CLEAR_SPLX_RELEASE,
1320 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1321 #if defined(__amd64)
1322 leave /* unwind stack */
1323 #endif /* __amd64 */
1324 ret
1325 SET_SIZE(lockstat_hot_patch)
1326
1327 #endif /* __lint */
1328
1329 #if defined(lint) || defined(__lint)
1330
1331 /* XX64 membar_*() should be inlines */
1332
1333 void
1334 membar_sync(void)
1335 {}
1336
1337 void
1338 membar_enter(void)
1339 {}
1340
1341 void
1342 membar_exit(void)
1343 {}
1344
1345 void
1346 membar_producer(void)
1347 {}
1348
1349 void
1350 membar_consumer(void)
1351 {}
1352
1353 #else /* __lint */
1354
1355 #if defined(__amd64)
1356
1357 ENTRY(membar_enter)
1358 ALTENTRY(membar_exit)
1359 ALTENTRY(membar_sync)
1360 mfence /* lighter weight than lock; xorq $0,(%rsp) */
1361 ret
1362 SET_SIZE(membar_sync)
1363 SET_SIZE(membar_exit)
1364 SET_SIZE(membar_enter)
1365
1366 ENTRY(membar_producer)
1367 sfence
1368 ret
1369 SET_SIZE(membar_producer)
1370
1371 ENTRY(membar_consumer)
1372 lfence
1373 ret
1374 SET_SIZE(membar_consumer)
1375
1376 #else
1377
1378 ENTRY(membar_enter)
1379 ALTENTRY(membar_exit)
1380 ALTENTRY(membar_sync)
1381 lock
1382 xorl $0, (%esp)
1383 ret
1384 SET_SIZE(membar_sync)
1385 SET_SIZE(membar_exit)
1386 SET_SIZE(membar_enter)
1387
1388 /*
1389 * On machines that support sfence and lfence, these
1390 * memory barriers can be more precisely implemented
1391 * without causing the whole world to stop
1392 */
1393 ENTRY(membar_producer)
1394 .globl _patch_sfence_ret
1395 _patch_sfence_ret: /* c.f. membar #StoreStore */
1396 lock
1397 xorl $0, (%esp)
1398 ret
1399 SET_SIZE(membar_producer)
1400
1401 ENTRY(membar_consumer)
1402 .globl _patch_lfence_ret
1403 _patch_lfence_ret: /* c.f. membar #LoadLoad */
1404 lock
1405 xorl $0, (%esp)
1406 ret
1407 SET_SIZE(membar_consumer)
1408
1409 #endif /* !__amd64 */
1410
1411 #endif /* __lint */
1412
1413 /*
1414 * thread_onproc()
1415 * Set thread in onproc state for the specified CPU.
1416 * Also set the thread lock pointer to the CPU's onproc lock.
1417 * Since the new lock isn't held, the store ordering is important.
1418 * If not done in assembler, the compiler could reorder the stores.
1419 */
1420 #if defined(lint) || defined(__lint)
1421
1422 void
1423 thread_onproc(kthread_id_t t, cpu_t *cp)
1424 {
1425 t->t_state = TS_ONPROC;
1426 t->t_lockp = &cp->cpu_thread_lock;
1427 }
1428
1429 #else /* __lint */
1430
1431 #if defined(__amd64)
1432
1433 ENTRY(thread_onproc)
1434 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
1435 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
1436 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
1437 ret
1438 SET_SIZE(thread_onproc)
1439
1440 #else
1441
1442 ENTRY(thread_onproc)
1443 movl 4(%esp), %eax
1444 movl 8(%esp), %ecx
1445 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */
1446 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */
1447 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */
1448 ret
1449 SET_SIZE(thread_onproc)
1450
1451 #endif /* !__amd64 */
1452
1453 #endif /* __lint */
1454
1455 /*
1456 * mutex_delay_default(void)
1457 * Spins for approx a few hundred processor cycles and returns to caller.
1458 */
1459
1460 #if defined(lint) || defined(__lint)
1461
1462 void
1463 mutex_delay_default(void)
1464 {}
1465
1466 #else /* __lint */
1467
1468 #if defined(__amd64)
1469
1470 ENTRY(mutex_delay_default)
1471 movq $92,%r11
1472 0: decq %r11
1473 jg 0b
1474 ret
1475 SET_SIZE(mutex_delay_default)
1476
1477 #else
1478
1479 ENTRY(mutex_delay_default)
1480 push %ebp
1481 movl %esp,%ebp
1482 andl $-16,%esp
1483 push %ebx
1484 movl $93,%ebx
1485 0: decl %ebx
1486 jg 0b
1487 pop %ebx
1488 leave
1489 ret
1490 SET_SIZE(mutex_delay_default)
1491
1492 #endif /* !__amd64 */
1493 #endif /* __lint */