6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #if defined(lint) || defined(__lint)
29 #include <sys/types.h>
30 #include <sys/thread.h>
31 #include <sys/cpuvar.h>
32 #include <vm/page.h>
33 #else /* __lint */
34 #include "assym.h"
35 #endif /* __lint */
36
37 #include <sys/mutex_impl.h>
38 #include <sys/asm_linkage.h>
39 #include <sys/asm_misc.h>
40 #include <sys/regset.h>
41 #include <sys/rwlock_impl.h>
42 #include <sys/lockstat.h>
43
44 /*
45 * lock_try(lp), ulock_try(lp)
46 * - returns non-zero on success.
47 * - doesn't block interrupts so don't use this to spin on a lock.
48 *
49 * ulock_try() is for a lock in the user address space.
50 */
51
52 #if defined(lint) || defined(__lint)
53
54 /* ARGSUSED */
55 int
56 lock_try(lock_t *lp)
57 { return (0); }
58
59 /* ARGSUSED */
60 int
61 lock_spin_try(lock_t *lp)
62 { return (0); }
63
64 /* ARGSUSED */
65 int
66 ulock_try(lock_t *lp)
67 { return (0); }
68
69 #else /* __lint */
70 .globl kernelbase
71
72 #if defined(__amd64)
73
74 ENTRY(lock_try)
75 movb $-1, %dl
76 movzbq %dl, %rax
77 xchgb %dl, (%rdi)
78 xorb %dl, %al
79 .lock_try_lockstat_patch_point:
80 ret
81 testb %al, %al
82 jnz 0f
83 ret
84 0:
85 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
86 movq %rdi, %rsi /* rsi = lock addr */
87 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
88 jmp lockstat_wrapper
89 SET_SIZE(lock_try)
90
91 ENTRY(lock_spin_try)
92 movb $-1, %dl
93 movzbq %dl, %rax
101 movq kernelbase(%rip), %rax
102 cmpq %rax, %rdi /* test uaddr < kernelbase */
103 jb ulock_pass /* uaddr < kernelbase, proceed */
104
105 movq %rdi, %r12 /* preserve lock ptr for debugging */
106 leaq .ulock_panic_msg(%rip), %rdi
107 pushq %rbp /* align stack properly */
108 movq %rsp, %rbp
109 xorl %eax, %eax /* clear for varargs */
110 call panic
111
112 #endif /* DEBUG */
113
114 ulock_pass:
115 movl $1, %eax
116 xchgb %al, (%rdi)
117 xorb $1, %al
118 ret
119 SET_SIZE(ulock_try)
120
121 #else
122
123 ENTRY(lock_try)
124 movl $1,%edx
125 movl 4(%esp),%ecx /* ecx = lock addr */
126 xorl %eax,%eax
127 xchgb %dl, (%ecx) /* using dl will avoid partial */
128 testb %dl,%dl /* stalls on P6 ? */
129 setz %al
130 .lock_try_lockstat_patch_point:
131 ret
132 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
133 testl %eax, %eax
134 jz 0f
135 movl $LS_LOCK_TRY_ACQUIRE, %eax
136 jmp lockstat_wrapper
137 0:
138 ret
139 SET_SIZE(lock_try)
140
141 ENTRY(lock_spin_try)
142 movl $-1,%edx
143 movl 4(%esp),%ecx /* ecx = lock addr */
144 xorl %eax,%eax
145 xchgb %dl, (%ecx) /* using dl will avoid partial */
146 testb %dl,%dl /* stalls on P6 ? */
147 setz %al
148 ret
149 SET_SIZE(lock_spin_try)
150
151 ENTRY(ulock_try)
152 #ifdef DEBUG
153 movl kernelbase, %eax
154 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */
155 jb ulock_pass /* uaddr < kernelbase, proceed */
156
157 pushl $.ulock_panic_msg
158 call panic
159
160 #endif /* DEBUG */
161
162 ulock_pass:
163 movl $1,%eax
164 movl 4(%esp),%ecx
165 xchgb %al, (%ecx)
166 xorb $1, %al
167 ret
168 SET_SIZE(ulock_try)
169
170 #endif /* !__amd64 */
171
172 #ifdef DEBUG
173 .data
174 .ulock_panic_msg:
175 .string "ulock_try: Argument is above kernelbase"
176 .text
177 #endif /* DEBUG */
178
179 #endif /* __lint */
180
181 /*
182 * lock_clear(lp)
183 * - unlock lock without changing interrupt priority level.
184 */
185
186 #if defined(lint) || defined(__lint)
187
188 /* ARGSUSED */
189 void
190 lock_clear(lock_t *lp)
191 {}
192
193 /* ARGSUSED */
194 void
195 ulock_clear(lock_t *lp)
196 {}
197
198 #else /* __lint */
199
200 #if defined(__amd64)
201
202 ENTRY(lock_clear)
203 movb $0, (%rdi)
204 .lock_clear_lockstat_patch_point:
205 ret
206 movq %rdi, %rsi /* rsi = lock addr */
207 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
208 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
209 jmp lockstat_wrapper
210 SET_SIZE(lock_clear)
211
212 ENTRY(ulock_clear)
213 #ifdef DEBUG
214 movq kernelbase(%rip), %rcx
215 cmpq %rcx, %rdi /* test uaddr < kernelbase */
216 jb ulock_clr /* uaddr < kernelbase, proceed */
217
218 leaq .ulock_clear_msg(%rip), %rdi
219 pushq %rbp /* align stack properly */
220 movq %rsp, %rbp
221 xorl %eax, %eax /* clear for varargs */
222 call panic
223 #endif
224
225 ulock_clr:
226 movb $0, (%rdi)
227 ret
228 SET_SIZE(ulock_clear)
229
230 #else
231
232 ENTRY(lock_clear)
233 movl 4(%esp), %eax
234 movb $0, (%eax)
235 .lock_clear_lockstat_patch_point:
236 ret
237 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
238 movl %eax, %ecx /* ecx = lock pointer */
239 movl $LS_LOCK_CLEAR_RELEASE, %eax
240 jmp lockstat_wrapper
241 SET_SIZE(lock_clear)
242
243 ENTRY(ulock_clear)
244 #ifdef DEBUG
245 movl kernelbase, %ecx
246 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */
247 jb ulock_clr /* uaddr < kernelbase, proceed */
248
249 pushl $.ulock_clear_msg
250 call panic
251 #endif
252
253 ulock_clr:
254 movl 4(%esp),%eax
255 xorl %ecx,%ecx
256 movb %cl, (%eax)
257 ret
258 SET_SIZE(ulock_clear)
259
260 #endif /* !__amd64 */
261
262 #ifdef DEBUG
263 .data
264 .ulock_clear_msg:
265 .string "ulock_clear: Argument is above kernelbase"
266 .text
267 #endif /* DEBUG */
268
269
270 #endif /* __lint */
271
272 /*
273 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
274 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
275 */
276
277 #if defined(lint) || defined(__lint)
278
279 /* ARGSUSED */
280 void
281 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
282 {}
283
284 #else /* __lint */
285
286 #if defined(__amd64)
287
288 ENTRY(lock_set_spl)
289 pushq %rbp
290 movq %rsp, %rbp
291 subq $32, %rsp
292 movl %esi, 8(%rsp) /* save priority level */
293 movq %rdx, 16(%rsp) /* save old pil ptr */
294 movq %rdi, 24(%rsp) /* save lock pointer */
295 movl %esi, %edi /* pass priority level */
296 call splr /* raise priority level */
297 movq 24(%rsp), %rdi /* rdi = lock addr */
298 movb $-1, %dl
299 xchgb %dl, (%rdi) /* try to set lock */
300 testb %dl, %dl /* did we get the lock? ... */
301 jnz .lss_miss /* ... no, go to C for the hard case */
302 movq 16(%rsp), %rdx /* rdx = old pil addr */
303 movw %ax, (%rdx) /* store old pil */
304 leave
305 .lock_set_spl_lockstat_patch_point:
306 ret
307 movq %rdi, %rsi /* rsi = lock addr */
308 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
309 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
310 jmp lockstat_wrapper
311 .lss_miss:
312 movl 8(%rsp), %esi /* new_pil */
313 movq 16(%rsp), %rdx /* old_pil_addr */
314 movl %eax, %ecx /* original pil */
315 leave /* unwind stack */
316 jmp lock_set_spl_spin
317 SET_SIZE(lock_set_spl)
318
319 #else
320
321 ENTRY(lock_set_spl)
322 movl 8(%esp), %eax /* get priority level */
323 pushl %eax
324 call splr /* raise priority level */
325 movl 8(%esp), %ecx /* ecx = lock addr */
326 movl $-1, %edx
327 addl $4, %esp
328 xchgb %dl, (%ecx) /* try to set lock */
329 testb %dl, %dl /* did we get the lock? ... */
330 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */
331 jnz .lss_miss /* ... no, go to C for the hard case */
332 movw %ax, (%edx) /* store old pil */
333 .lock_set_spl_lockstat_patch_point:
334 ret
335 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/
336 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax
337 jmp lockstat_wrapper
338 .lss_miss:
339 pushl %eax /* original pil */
340 pushl %edx /* old_pil addr */
341 pushl 16(%esp) /* new_pil */
342 pushl %ecx /* lock addr */
343 call lock_set_spl_spin
344 addl $16, %esp
345 ret
346 SET_SIZE(lock_set_spl)
347
348 #endif /* !__amd64 */
349
350 #endif /* __lint */
351
352 /*
353 * void
354 * lock_init(lp)
355 */
356
357 #if defined(__lint)
358
359 /* ARGSUSED */
360 void
361 lock_init(lock_t *lp)
362 {}
363
364 #else /* __lint */
365
366 #if defined(__amd64)
367
368 ENTRY(lock_init)
369 movb $0, (%rdi)
370 ret
371 SET_SIZE(lock_init)
372
373 #else
374
375 ENTRY(lock_init)
376 movl 4(%esp), %eax
377 movb $0, (%eax)
378 ret
379 SET_SIZE(lock_init)
380
381 #endif /* !__amd64 */
382
383 #endif /* __lint */
384
385 /*
386 * void
387 * lock_set(lp)
388 */
389
390 #if defined(lint) || defined(__lint)
391
392 /* ARGSUSED */
393 void
394 lock_set(lock_t *lp)
395 {}
396
397 #else /* __lint */
398
399 #if defined(__amd64)
400
401 ENTRY(lock_set)
402 movb $-1, %dl
403 xchgb %dl, (%rdi) /* try to set lock */
404 testb %dl, %dl /* did we get it? */
405 jnz lock_set_spin /* no, go to C for the hard case */
406 .lock_set_lockstat_patch_point:
407 ret
408 movq %rdi, %rsi /* rsi = lock addr */
409 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
410 movl $LS_LOCK_SET_ACQUIRE, %edi
411 jmp lockstat_wrapper
412 SET_SIZE(lock_set)
413
414 #else
415
416 ENTRY(lock_set)
417 movl 4(%esp), %ecx /* ecx = lock addr */
418 movl $-1, %edx
419 xchgb %dl, (%ecx) /* try to set lock */
420 testb %dl, %dl /* did we get it? */
421 jnz lock_set_spin /* no, go to C for the hard case */
422 .lock_set_lockstat_patch_point:
423 ret
424 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
425 movl $LS_LOCK_SET_ACQUIRE, %eax
426 jmp lockstat_wrapper
427 SET_SIZE(lock_set)
428
429 #endif /* !__amd64 */
430
431 #endif /* __lint */
432
433 /*
434 * lock_clear_splx(lp, s)
435 */
436
437 #if defined(lint) || defined(__lint)
438
439 /* ARGSUSED */
440 void
441 lock_clear_splx(lock_t *lp, int s)
442 {}
443
444 #else /* __lint */
445
446 #if defined(__amd64)
447
448 ENTRY(lock_clear_splx)
449 movb $0, (%rdi) /* clear lock */
450 .lock_clear_splx_lockstat_patch_point:
451 jmp 0f
452 0:
453 movl %esi, %edi /* arg for splx */
454 jmp splx /* let splx do its thing */
455 .lock_clear_splx_lockstat:
456 pushq %rbp /* align stack properly */
457 movq %rsp, %rbp
458 subq $16, %rsp /* space to save args across splx */
459 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
460 movl %esi, %edi /* arg for splx */
461 call splx /* lower the priority */
462 movq 8(%rsp), %rsi /* rsi = lock ptr */
463 leave /* unwind stack */
464 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
465 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
466 jmp lockstat_wrapper
467 SET_SIZE(lock_clear_splx)
468
469 #else
470
471 ENTRY(lock_clear_splx)
472 movl 4(%esp), %eax /* eax = lock addr */
473 movb $0, (%eax) /* clear lock */
474 .lock_clear_splx_lockstat_patch_point:
475 jmp 0f
476 0:
477 movl 8(%esp), %edx /* edx = desired pil */
478 movl %edx, 4(%esp) /* set spl arg up for splx */
479 jmp splx /* let splx do it's thing */
480 .lock_clear_splx_lockstat:
481 movl 8(%esp), %edx /* edx = desired pil */
482 pushl %ebp /* set up stack frame */
483 movl %esp, %ebp
484 pushl %edx
485 call splx
486 leave /* unwind stack */
487 movl 4(%esp), %ecx /* ecx = lock pointer */
488 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
489 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
490 jmp lockstat_wrapper
491 SET_SIZE(lock_clear_splx)
492
493 #endif /* !__amd64 */
494
495 #if defined(__GNUC_AS__)
496 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
497 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
498
499 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
500 (.lock_clear_splx_lockstat_patch_point + 1)
501 #else
502 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
503 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
504
505 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
506 [.lock_clear_splx_lockstat_patch_point + 1]
507 #endif
508
509 #endif /* __lint */
510
511 /*
512 * mutex_enter() and mutex_exit().
513 *
514 * These routines handle the simple cases of mutex_enter() (adaptive
515 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
516 * If anything complicated is going on we punt to mutex_vector_enter().
517 *
518 * mutex_tryenter() is similar to mutex_enter() but returns zero if
519 * the lock cannot be acquired, nonzero on success.
520 *
521 * If mutex_exit() gets preempted in the window between checking waiters
522 * and clearing the lock, we can miss wakeups. Disabling preemption
523 * in the mutex code is prohibitively expensive, so instead we detect
524 * mutex preemption by examining the trapped PC in the interrupt path.
525 * If we interrupt a thread in mutex_exit() that has not yet cleared
526 * the lock, cmnint() resets its PC back to the beginning of
527 * mutex_exit() so it will check again for waiters when it resumes.
528 *
529 * The lockstat code below is activated when the lockstat driver
530 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
531 * Note that we don't need to test lockstat_event_mask here -- we won't
532 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
533 */
534 #if defined(lint) || defined(__lint)
535
536 /* ARGSUSED */
537 void
538 mutex_enter(kmutex_t *lp)
539 {}
540
541 /* ARGSUSED */
542 int
543 mutex_tryenter(kmutex_t *lp)
544 { return (0); }
545
546 /* ARGSUSED */
547 int
548 mutex_adaptive_tryenter(mutex_impl_t *lp)
549 { return (0); }
550
551 /* ARGSUSED */
552 void
553 mutex_exit(kmutex_t *lp)
554 {}
555
556 #else
557
558 #if defined(__amd64)
559
560 ENTRY_NP(mutex_enter)
561 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
562 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
563 lock
564 cmpxchgq %rdx, (%rdi)
565 jnz mutex_vector_enter
566 .mutex_enter_lockstat_patch_point:
567 #if defined(OPTERON_WORKAROUND_6323525)
568 .mutex_enter_6323525_patch_point:
569 ret /* nop space for lfence */
570 nop
571 nop
572 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
573 nop
574 #else /* OPTERON_WORKAROUND_6323525 */
575 ret
576 #endif /* OPTERON_WORKAROUND_6323525 */
577 movq %rdi, %rsi
578 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
579 /*
701 mutex_exit_critical_start: /* If interrupted, restart here */
702 movq %gs:CPU_THREAD, %rdx
703 cmpq %rdx, (%rdi)
704 jne mutex_vector_exit /* wrong type or wrong owner */
705 movq $0, (%rdi) /* clear owner AND lock */
706 .mutex_exit_critical_end:
707 .mutex_exit_lockstat_patch_point:
708 ret
709 movq %rdi, %rsi
710 movl $LS_MUTEX_EXIT_RELEASE, %edi
711 jmp lockstat_wrapper
712 SET_SIZE(mutex_exit)
713
714 .globl mutex_exit_critical_size
715 .type mutex_exit_critical_size, @object
716 .align CPTRSIZE
717 mutex_exit_critical_size:
718 .quad .mutex_exit_critical_end - mutex_exit_critical_start
719 SET_SIZE(mutex_exit_critical_size)
720
721 #else
722
723 ENTRY_NP(mutex_enter)
724 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
725 movl 4(%esp), %ecx /* ecx = lock ptr */
726 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
727 lock
728 cmpxchgl %edx, (%ecx)
729 jnz mutex_vector_enter
730 #if defined(OPTERON_WORKAROUND_6323525)
731 .mutex_enter_lockstat_patch_point:
732 .mutex_enter_6323525_patch_point:
733 ret /* nop space for lfence */
734 nop
735 nop
736 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
737 nop
738 #else /* OPTERON_WORKAROUND_6323525 */
739 .mutex_enter_lockstat_patch_point:
740 ret
741 #endif /* OPTERON_WORKAROUND_6323525 */
742 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
743 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */
744 /* eax=lockstat event */
745 pushl %ebp /* buy a frame */
746 movl %esp, %ebp
747 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
748 pushl %edx /* save thread pointer */
749 movl $lockstat_probemap, %edx
750 movl (%edx, %eax, DTRACE_IDSIZE), %eax
751 testl %eax, %eax /* check for non-zero probe */
752 jz 1f
753 pushl %ecx /* push lock */
754 pushl %eax /* push probe ID */
755 call *lockstat_probe
756 addl $8, %esp
757 1:
758 popl %edx /* restore thread pointer */
759 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
760 movl $1, %eax /* return success if tryenter */
761 popl %ebp /* pop off frame */
762 ret
763 SET_SIZE(lockstat_wrapper)
764 SET_SIZE(mutex_enter)
765
766 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */
767 /* eax=lockstat event, pushed arg */
768 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
769 pushl %edx /* save thread pointer */
770 movl $lockstat_probemap, %edx
771 movl (%edx, %eax, DTRACE_IDSIZE), %eax
772 testl %eax, %eax /* check for non-zero probe */
773 jz 1f
774 pushl %ebp /* save %ebp */
775 pushl 8(%esp) /* push arg1 */
776 movl %ebp, 12(%esp) /* fake up the stack frame */
777 movl %esp, %ebp /* fake up base pointer */
778 addl $12, %ebp /* adjust faked base pointer */
779 pushl %ecx /* push lock */
780 pushl %eax /* push probe ID */
781 call *lockstat_probe
782 addl $12, %esp /* adjust for arguments */
783 popl %ebp /* pop frame */
784 1:
785 popl %edx /* restore thread pointer */
786 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
787 movl $1, %eax /* return success if tryenter */
788 addl $4, %esp /* pop argument */
789 ret
790 SET_SIZE(lockstat_wrapper_arg)
791
792
793 ENTRY(mutex_tryenter)
794 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
795 movl 4(%esp), %ecx /* ecx = lock ptr */
796 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
797 lock
798 cmpxchgl %edx, (%ecx)
799 jnz mutex_vector_tryenter
800 movl %ecx, %eax
801 #if defined(OPTERON_WORKAROUND_6323525)
802 .mutex_tryenter_lockstat_patch_point:
803 .mutex_tryenter_6323525_patch_point:
804 ret /* nop space for lfence */
805 nop
806 nop
807 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
808 nop
809 #else /* OPTERON_WORKAROUND_6323525 */
810 .mutex_tryenter_lockstat_patch_point:
811 ret
812 #endif /* OPTERON_WORKAROUND_6323525 */
813 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
814 jmp lockstat_wrapper
815 SET_SIZE(mutex_tryenter)
816
817 ENTRY(mutex_adaptive_tryenter)
818 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
819 movl 4(%esp), %ecx /* ecx = lock ptr */
820 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
821 lock
822 cmpxchgl %edx, (%ecx)
823 jnz 0f
824 movl %ecx, %eax
825 #if defined(OPTERON_WORKAROUND_6323525)
826 .mutex_atryenter_6323525_patch_point:
827 ret /* nop space for lfence */
828 nop
829 nop
830 nop
831 #else /* OPTERON_WORKAROUND_6323525 */
832 ret
833 #endif /* OPTERON_WORKAROUND_6323525 */
834 0:
835 xorl %eax, %eax
836 ret
837 SET_SIZE(mutex_adaptive_tryenter)
838
839 .globl mutex_owner_running_critical_start
840
841 ENTRY(mutex_owner_running)
842 mutex_owner_running_critical_start:
843 movl 4(%esp), %eax /* get owner field */
844 movl (%eax), %eax
845 andl $MUTEX_THREAD, %eax /* remove waiters bit */
846 cmpl $0, %eax /* if free, skip */
847 je 1f /* go return 0 */
848 movl T_CPU(%eax), %ecx /* get owner->t_cpu */
849 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */
850 .mutex_owner_running_critical_end:
851 cmpl %eax, %edx /* owner == running thread? */
852 je 2f /* yes, go return cpu */
853 1:
854 xorl %eax, %eax /* return 0 */
855 ret
856 2:
857 movl %ecx, %eax /* return cpu */
858 ret
859
860 SET_SIZE(mutex_owner_running)
861
862 .globl mutex_owner_running_critical_size
863 .type mutex_owner_running_critical_size, @object
864 .align CPTRSIZE
865 mutex_owner_running_critical_size:
866 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start
867 SET_SIZE(mutex_owner_running_critical_size)
868
869 .globl mutex_exit_critical_start
870
871 ENTRY(mutex_exit)
872 mutex_exit_critical_start: /* If interrupted, restart here */
873 movl %gs:CPU_THREAD, %edx
874 movl 4(%esp), %ecx
875 cmpl %edx, (%ecx)
876 jne mutex_vector_exit /* wrong type or wrong owner */
877 movl $0, (%ecx) /* clear owner AND lock */
878 .mutex_exit_critical_end:
879 .mutex_exit_lockstat_patch_point:
880 ret
881 movl $LS_MUTEX_EXIT_RELEASE, %eax
882 jmp lockstat_wrapper
883 SET_SIZE(mutex_exit)
884
885 .globl mutex_exit_critical_size
886 .type mutex_exit_critical_size, @object
887 .align CPTRSIZE
888 mutex_exit_critical_size:
889 .long .mutex_exit_critical_end - mutex_exit_critical_start
890 SET_SIZE(mutex_exit_critical_size)
891
892 #endif /* !__amd64 */
893
894 #endif /* __lint */
895
896 /*
897 * rw_enter() and rw_exit().
898 *
899 * These routines handle the simple cases of rw_enter (write-locking an unheld
900 * lock or read-locking a lock that's neither write-locked nor write-wanted)
901 * and rw_exit (no waiters or not the last reader). If anything complicated
902 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
903 */
904 #if defined(lint) || defined(__lint)
905
906 /* ARGSUSED */
907 void
908 rw_enter(krwlock_t *lp, krw_t rw)
909 {}
910
911 /* ARGSUSED */
912 void
913 rw_exit(krwlock_t *lp)
914 {}
915
916 #else /* __lint */
917
918 #if defined(__amd64)
919
920 ENTRY(rw_enter)
921 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
922 cmpl $RW_WRITER, %esi
923 je .rw_write_enter
924 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */
925 movq (%rdi), %rax /* rax = old rw_wwwh value */
926 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
927 jnz rw_enter_sleep
928 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
929 lock
930 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
931 jnz rw_enter_sleep
932 .rw_read_enter_lockstat_patch_point:
933 ret
934 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
935 movq %rdi, %rsi /* rsi = lock ptr */
936 movl $LS_RW_ENTER_ACQUIRE, %edi
937 movl $RW_READER, %edx
938 jmp lockstat_wrapper_arg
939 .rw_write_enter:
986 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
987 cmpl $RW_READ_LOCK, %edx
988 jge .rw_read_exit /* not last reader, safe to drop */
989 jmp rw_exit_wakeup /* last reader with waiters */
990 .rw_write_exit:
991 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
992 xorl %edx, %edx /* rdx = new value (unheld) */
993 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
994 lock
995 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
996 jnz rw_exit_wakeup
997 .rw_write_exit_lockstat_patch_point:
998 ret
999 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
1000 movq %rdi, %rsi /* rsi - lock ptr */
1001 movl $LS_RW_EXIT_RELEASE, %edi
1002 movl $RW_WRITER, %edx
1003 jmp lockstat_wrapper_arg
1004 SET_SIZE(rw_exit)
1005
1006 #else
1007
1008 ENTRY(rw_enter)
1009 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1010 movl 4(%esp), %ecx /* ecx = lock ptr */
1011 cmpl $RW_WRITER, 8(%esp)
1012 je .rw_write_enter
1013 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */
1014 movl (%ecx), %eax /* eax = old rw_wwwh value */
1015 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1016 jnz rw_enter_sleep
1017 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1018 lock
1019 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1020 jnz rw_enter_sleep
1021 .rw_read_enter_lockstat_patch_point:
1022 ret
1023 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1024 movl $LS_RW_ENTER_ACQUIRE, %eax
1025 pushl $RW_READER
1026 jmp lockstat_wrapper_arg
1027 .rw_write_enter:
1028 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1029 xorl %eax, %eax /* eax = unheld value */
1030 lock
1031 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1032 jnz rw_enter_sleep
1033
1034 #if defined(OPTERON_WORKAROUND_6323525)
1035 .rw_write_enter_lockstat_patch_point:
1036 .rw_write_enter_6323525_patch_point:
1037 ret
1038 nop
1039 nop
1040 .rw_write_enter_lockstat_6323525_patch_point:
1041 nop
1042 #else /* OPTERON_WORKAROUND_6323525 */
1043 .rw_write_enter_lockstat_patch_point:
1044 ret
1045 #endif /* OPTERON_WORKAROUND_6323525 */
1046
1047 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1048 movl $LS_RW_ENTER_ACQUIRE, %eax
1049 pushl $RW_WRITER
1050 jmp lockstat_wrapper_arg
1051 SET_SIZE(rw_enter)
1052
1053 ENTRY(rw_exit)
1054 movl 4(%esp), %ecx /* ecx = lock ptr */
1055 movl (%ecx), %eax /* eax = old rw_wwwh value */
1056 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1057 jne .rw_not_single_reader
1058 xorl %edx, %edx /* edx = new value (unheld) */
1059 .rw_read_exit:
1060 lock
1061 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1062 jnz rw_exit_wakeup
1063 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1064 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */
1065 .rw_read_exit_lockstat_patch_point:
1066 ret
1067 movl $LS_RW_EXIT_RELEASE, %eax
1068 pushl $RW_READER
1069 jmp lockstat_wrapper_arg
1070 .rw_not_single_reader:
1071 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1072 jnz .rw_write_exit
1073 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1074 cmpl $RW_READ_LOCK, %edx
1075 jge .rw_read_exit /* not last reader, safe to drop */
1076 jmp rw_exit_wakeup /* last reader with waiters */
1077 .rw_write_exit:
1078 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1079 xorl %edx, %edx /* edx = new value (unheld) */
1080 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1081 lock
1082 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1083 jnz rw_exit_wakeup
1084 .rw_write_exit_lockstat_patch_point:
1085 ret
1086 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1087 movl $LS_RW_EXIT_RELEASE, %eax
1088 pushl $RW_WRITER
1089 jmp lockstat_wrapper_arg
1090 SET_SIZE(rw_exit)
1091
1092 #endif /* !__amd64 */
1093
1094 #endif /* __lint */
1095
1096 #if defined(OPTERON_WORKAROUND_6323525)
1097 #if defined(lint) || defined(__lint)
1098
1099 int workaround_6323525_patched;
1100
1101 void
1102 patch_workaround_6323525(void)
1103 {}
1104
1105 #else /* lint */
1106
1107 /*
1108 * If it is necessary to patch the lock enter routines with the lfence
1109 * workaround, workaround_6323525_patched is set to a non-zero value so that
1110 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1111 * instruction.
1112 */
1113 DGDEF3(workaround_6323525_patched, 4, 4)
1114 .long 0
1115
1116 #if defined(__amd64)
1117
1118 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1119 movq $size, %rbx; \
1120 movq $dstaddr, %r13; \
1121 addq %rbx, %r13; \
1122 movq $srcaddr, %r12; \
1123 addq %rbx, %r12; \
1124 0: \
1125 decq %r13; \
1126 decq %r12; \
1127 movzbl (%r12), %esi; \
1128 movq $1, %rdx; \
1129 movq %r13, %rdi; \
1130 call hot_patch_kernel_text; \
1131 decq %rbx; \
1132 testq %rbx, %rbx; \
1133 jg 0b;
1134
1135 /*
1136 * patch_workaround_6323525: provide workaround for 6323525
1137 *
1161 * the existing ret is overwritten last. This provides lock enter
1162 * sanity during the intermediate patching stages.
1163 */
1164 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1165 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1166 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1167 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1168
1169 popq %rbx
1170 popq %r13
1171 popq %r12
1172 movq %rbp, %rsp
1173 popq %rbp
1174 ret
1175 _lfence_insn:
1176 lfence
1177 ret
1178 SET_SIZE(patch_workaround_6323525)
1179
1180
1181 #else /* __amd64 */
1182
1183 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1184 movl $size, %ebx; \
1185 movl $srcaddr, %esi; \
1186 addl %ebx, %esi; \
1187 movl $dstaddr, %edi; \
1188 addl %ebx, %edi; \
1189 0: \
1190 decl %esi; \
1191 decl %edi; \
1192 pushl $1; \
1193 movzbl (%esi), %eax; \
1194 pushl %eax; \
1195 pushl %edi; \
1196 call hot_patch_kernel_text; \
1197 addl $12, %esp; \
1198 decl %ebx; \
1199 testl %ebx, %ebx; \
1200 jg 0b;
1201
1202
1203 /* see comments above */
1204 ENTRY_NP(patch_workaround_6323525)
1205 pushl %ebp
1206 movl %esp, %ebp
1207 pushl %ebx
1208 pushl %esi
1209 pushl %edi
1210
1211 movl $1, workaround_6323525_patched
1212
1213 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1214 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1215 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1216 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1217
1218 popl %edi
1219 popl %esi
1220 popl %ebx
1221 movl %ebp, %esp
1222 popl %ebp
1223 ret
1224 _lfence_insn:
1225 .byte 0xf, 0xae, 0xe8 / [lfence instruction]
1226 ret
1227 SET_SIZE(patch_workaround_6323525)
1228
1229 #endif /* !__amd64 */
1230 #endif /* !lint */
1231 #endif /* OPTERON_WORKAROUND_6323525 */
1232
1233
1234 #if defined(lint) || defined(__lint)
1235
1236 void
1237 lockstat_hot_patch(void)
1238 {}
1239
1240 #else
1241
1242 #if defined(__amd64)
1243
1244 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1245 movq $normal_instr, %rsi; \
1246 movq $active_instr, %rdi; \
1247 leaq lockstat_probemap(%rip), %rax; \
1248 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1249 testl %eax, %eax; \
1250 jz 9f; \
1251 movq %rdi, %rsi; \
1252 9: \
1253 movq $len, %rdx; \
1254 movq $addr, %rdi; \
1255 call hot_patch_kernel_text
1256
1257 #else
1258
1259 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1260 movl $normal_instr, %ecx; \
1261 movl $active_instr, %edx; \
1262 movl $lockstat_probemap, %eax; \
1263 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1264 testl %eax, %eax; \
1265 jz . + 4; \
1266 movl %edx, %ecx; \
1267 pushl $len; \
1268 pushl %ecx; \
1269 pushl $addr; \
1270 call hot_patch_kernel_text; \
1271 addl $12, %esp;
1272
1273 #endif /* !__amd64 */
1274
1275 ENTRY(lockstat_hot_patch)
1276 #if defined(__amd64)
1277 pushq %rbp /* align stack properly */
1278 movq %rsp, %rbp
1279 #endif /* __amd64 */
1280
1281 #if defined(OPTERON_WORKAROUND_6323525)
1282 cmpl $0, workaround_6323525_patched
1283 je 1f
1284 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1285 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1286 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1287 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1289 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1290 jmp 2f
1291 1:
1292 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1293 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1294 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1295 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1297 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1298 2:
1299 #else /* OPTERON_WORKAROUND_6323525 */
1307 HOT_PATCH(.mutex_exit_lockstat_patch_point,
1308 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1309 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1310 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1311 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1312 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1313 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1314 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1315 HOT_PATCH(.lock_set_lockstat_patch_point,
1316 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317 HOT_PATCH(.lock_try_lockstat_patch_point,
1318 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1319 HOT_PATCH(.lock_clear_lockstat_patch_point,
1320 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1321 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1322 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1323
1324 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1325 LS_LOCK_CLEAR_SPLX_RELEASE,
1326 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1327 #if defined(__amd64)
1328 leave /* unwind stack */
1329 #endif /* __amd64 */
1330 ret
1331 SET_SIZE(lockstat_hot_patch)
1332
1333 #endif /* __lint */
1334
1335 #if defined(lint) || defined(__lint)
1336
1337 /* XX64 membar_*() should be inlines */
1338
1339 void
1340 membar_sync(void)
1341 {}
1342
1343 void
1344 membar_enter(void)
1345 {}
1346
1347 void
1348 membar_exit(void)
1349 {}
1350
1351 void
1352 membar_producer(void)
1353 {}
1354
1355 void
1356 membar_consumer(void)
1357 {}
1358
1359 #else /* __lint */
1360
1361 #if defined(__amd64)
1362
1363 ENTRY(membar_enter)
1364 ALTENTRY(membar_exit)
1365 ALTENTRY(membar_sync)
1366 mfence /* lighter weight than lock; xorq $0,(%rsp) */
1367 ret
1368 SET_SIZE(membar_sync)
1369 SET_SIZE(membar_exit)
1370 SET_SIZE(membar_enter)
1371
1372 ENTRY(membar_producer)
1373 sfence
1374 ret
1375 SET_SIZE(membar_producer)
1376
1377 ENTRY(membar_consumer)
1378 lfence
1379 ret
1380 SET_SIZE(membar_consumer)
1381
1382 #else
1383
1384 ENTRY(membar_enter)
1385 ALTENTRY(membar_exit)
1386 ALTENTRY(membar_sync)
1387 lock
1388 xorl $0, (%esp)
1389 ret
1390 SET_SIZE(membar_sync)
1391 SET_SIZE(membar_exit)
1392 SET_SIZE(membar_enter)
1393
1394 /*
1395 * On machines that support sfence and lfence, these
1396 * memory barriers can be more precisely implemented
1397 * without causing the whole world to stop
1398 */
1399 ENTRY(membar_producer)
1400 .globl _patch_sfence_ret
1401 _patch_sfence_ret: /* c.f. membar #StoreStore */
1402 lock
1403 xorl $0, (%esp)
1404 ret
1405 SET_SIZE(membar_producer)
1406
1407 ENTRY(membar_consumer)
1408 .globl _patch_lfence_ret
1409 _patch_lfence_ret: /* c.f. membar #LoadLoad */
1410 lock
1411 xorl $0, (%esp)
1412 ret
1413 SET_SIZE(membar_consumer)
1414
1415 #endif /* !__amd64 */
1416
1417 #endif /* __lint */
1418
1419 /*
1420 * thread_onproc()
1421 * Set thread in onproc state for the specified CPU.
1422 * Also set the thread lock pointer to the CPU's onproc lock.
1423 * Since the new lock isn't held, the store ordering is important.
1424 * If not done in assembler, the compiler could reorder the stores.
1425 */
1426 #if defined(lint) || defined(__lint)
1427
1428 void
1429 thread_onproc(kthread_id_t t, cpu_t *cp)
1430 {
1431 t->t_state = TS_ONPROC;
1432 t->t_lockp = &cp->cpu_thread_lock;
1433 }
1434
1435 #else /* __lint */
1436
1437 #if defined(__amd64)
1438
1439 ENTRY(thread_onproc)
1440 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
1441 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
1442 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
1443 ret
1444 SET_SIZE(thread_onproc)
1445
1446 #else
1447
1448 ENTRY(thread_onproc)
1449 movl 4(%esp), %eax
1450 movl 8(%esp), %ecx
1451 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */
1452 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */
1453 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */
1454 ret
1455 SET_SIZE(thread_onproc)
1456
1457 #endif /* !__amd64 */
1458
1459 #endif /* __lint */
1460
1461 /*
1462 * mutex_delay_default(void)
1463 * Spins for approx a few hundred processor cycles and returns to caller.
1464 */
1465
1466 #if defined(lint) || defined(__lint)
1467
1468 void
1469 mutex_delay_default(void)
1470 {}
1471
1472 #else /* __lint */
1473
1474 #if defined(__amd64)
1475
1476 ENTRY(mutex_delay_default)
1477 movq $92,%r11
1478 0: decq %r11
1479 jg 0b
1480 ret
1481 SET_SIZE(mutex_delay_default)
1482
1483 #else
1484
1485 ENTRY(mutex_delay_default)
1486 push %ebp
1487 movl %esp,%ebp
1488 andl $-16,%esp
1489 push %ebx
1490 movl $93,%ebx
1491 0: decl %ebx
1492 jg 0b
1493 pop %ebx
1494 leave
1495 ret
1496 SET_SIZE(mutex_delay_default)
1497
1498 #endif /* !__amd64 */
1499 #endif /* __lint */
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright 2019 Joyent, Inc.
28 */
29
30 #include "assym.h"
31
32 #include <sys/mutex_impl.h>
33 #include <sys/asm_linkage.h>
34 #include <sys/asm_misc.h>
35 #include <sys/regset.h>
36 #include <sys/rwlock_impl.h>
37 #include <sys/lockstat.h>
38
39 /*
40 * lock_try(lp), ulock_try(lp)
41 * - returns non-zero on success.
42 * - doesn't block interrupts so don't use this to spin on a lock.
43 *
44 * ulock_try() is for a lock in the user address space.
45 */
46
47 .globl kernelbase
48
49 ENTRY(lock_try)
50 movb $-1, %dl
51 movzbq %dl, %rax
52 xchgb %dl, (%rdi)
53 xorb %dl, %al
54 .lock_try_lockstat_patch_point:
55 ret
56 testb %al, %al
57 jnz 0f
58 ret
59 0:
60 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
61 movq %rdi, %rsi /* rsi = lock addr */
62 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
63 jmp lockstat_wrapper
64 SET_SIZE(lock_try)
65
66 ENTRY(lock_spin_try)
67 movb $-1, %dl
68 movzbq %dl, %rax
76 movq kernelbase(%rip), %rax
77 cmpq %rax, %rdi /* test uaddr < kernelbase */
78 jb ulock_pass /* uaddr < kernelbase, proceed */
79
80 movq %rdi, %r12 /* preserve lock ptr for debugging */
81 leaq .ulock_panic_msg(%rip), %rdi
82 pushq %rbp /* align stack properly */
83 movq %rsp, %rbp
84 xorl %eax, %eax /* clear for varargs */
85 call panic
86
87 #endif /* DEBUG */
88
89 ulock_pass:
90 movl $1, %eax
91 xchgb %al, (%rdi)
92 xorb $1, %al
93 ret
94 SET_SIZE(ulock_try)
95
96 #ifdef DEBUG
97 .data
98 .ulock_panic_msg:
99 .string "ulock_try: Argument is above kernelbase"
100 .text
101 #endif /* DEBUG */
102
103 /*
104 * lock_clear(lp)
105 * - unlock lock without changing interrupt priority level.
106 */
107
108 ENTRY(lock_clear)
109 movb $0, (%rdi)
110 .lock_clear_lockstat_patch_point:
111 ret
112 movq %rdi, %rsi /* rsi = lock addr */
113 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
114 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
115 jmp lockstat_wrapper
116 SET_SIZE(lock_clear)
117
118 ENTRY(ulock_clear)
119 #ifdef DEBUG
120 movq kernelbase(%rip), %rcx
121 cmpq %rcx, %rdi /* test uaddr < kernelbase */
122 jb ulock_clr /* uaddr < kernelbase, proceed */
123
124 leaq .ulock_clear_msg(%rip), %rdi
125 pushq %rbp /* align stack properly */
126 movq %rsp, %rbp
127 xorl %eax, %eax /* clear for varargs */
128 call panic
129 #endif
130
131 ulock_clr:
132 movb $0, (%rdi)
133 ret
134 SET_SIZE(ulock_clear)
135
136 #ifdef DEBUG
137 .data
138 .ulock_clear_msg:
139 .string "ulock_clear: Argument is above kernelbase"
140 .text
141 #endif /* DEBUG */
142
143
144 /*
145 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
146 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
147 */
148
149 ENTRY(lock_set_spl)
150 pushq %rbp
151 movq %rsp, %rbp
152 subq $32, %rsp
153 movl %esi, 8(%rsp) /* save priority level */
154 movq %rdx, 16(%rsp) /* save old pil ptr */
155 movq %rdi, 24(%rsp) /* save lock pointer */
156 movl %esi, %edi /* pass priority level */
157 call splr /* raise priority level */
158 movq 24(%rsp), %rdi /* rdi = lock addr */
159 movb $-1, %dl
160 xchgb %dl, (%rdi) /* try to set lock */
161 testb %dl, %dl /* did we get the lock? ... */
162 jnz .lss_miss /* ... no, go to C for the hard case */
163 movq 16(%rsp), %rdx /* rdx = old pil addr */
164 movw %ax, (%rdx) /* store old pil */
165 leave
166 .lock_set_spl_lockstat_patch_point:
167 ret
168 movq %rdi, %rsi /* rsi = lock addr */
169 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
170 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
171 jmp lockstat_wrapper
172 .lss_miss:
173 movl 8(%rsp), %esi /* new_pil */
174 movq 16(%rsp), %rdx /* old_pil_addr */
175 movl %eax, %ecx /* original pil */
176 leave /* unwind stack */
177 jmp lock_set_spl_spin
178 SET_SIZE(lock_set_spl)
179
180 /*
181 * void
182 * lock_init(lp)
183 */
184
185 ENTRY(lock_init)
186 movb $0, (%rdi)
187 ret
188 SET_SIZE(lock_init)
189
190 /*
191 * void
192 * lock_set(lp)
193 */
194
195 ENTRY(lock_set)
196 movb $-1, %dl
197 xchgb %dl, (%rdi) /* try to set lock */
198 testb %dl, %dl /* did we get it? */
199 jnz lock_set_spin /* no, go to C for the hard case */
200 .lock_set_lockstat_patch_point:
201 ret
202 movq %rdi, %rsi /* rsi = lock addr */
203 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
204 movl $LS_LOCK_SET_ACQUIRE, %edi
205 jmp lockstat_wrapper
206 SET_SIZE(lock_set)
207
208 /*
209 * lock_clear_splx(lp, s)
210 */
211
212 ENTRY(lock_clear_splx)
213 movb $0, (%rdi) /* clear lock */
214 .lock_clear_splx_lockstat_patch_point:
215 jmp 0f
216 0:
217 movl %esi, %edi /* arg for splx */
218 jmp splx /* let splx do its thing */
219 .lock_clear_splx_lockstat:
220 pushq %rbp /* align stack properly */
221 movq %rsp, %rbp
222 subq $16, %rsp /* space to save args across splx */
223 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
224 movl %esi, %edi /* arg for splx */
225 call splx /* lower the priority */
226 movq 8(%rsp), %rsi /* rsi = lock ptr */
227 leave /* unwind stack */
228 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
229 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
230 jmp lockstat_wrapper
231 SET_SIZE(lock_clear_splx)
232
233 #if defined(__GNUC_AS__)
234 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
235 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
236
237 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
238 (.lock_clear_splx_lockstat_patch_point + 1)
239 #else
240 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
241 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
242
243 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
244 [.lock_clear_splx_lockstat_patch_point + 1]
245 #endif
246
247 /*
248 * mutex_enter() and mutex_exit().
249 *
250 * These routines handle the simple cases of mutex_enter() (adaptive
251 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
252 * If anything complicated is going on we punt to mutex_vector_enter().
253 *
254 * mutex_tryenter() is similar to mutex_enter() but returns zero if
255 * the lock cannot be acquired, nonzero on success.
256 *
257 * If mutex_exit() gets preempted in the window between checking waiters
258 * and clearing the lock, we can miss wakeups. Disabling preemption
259 * in the mutex code is prohibitively expensive, so instead we detect
260 * mutex preemption by examining the trapped PC in the interrupt path.
261 * If we interrupt a thread in mutex_exit() that has not yet cleared
262 * the lock, cmnint() resets its PC back to the beginning of
263 * mutex_exit() so it will check again for waiters when it resumes.
264 *
265 * The lockstat code below is activated when the lockstat driver
266 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
267 * Note that we don't need to test lockstat_event_mask here -- we won't
268 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
269 */
270
271 ENTRY_NP(mutex_enter)
272 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
273 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
274 lock
275 cmpxchgq %rdx, (%rdi)
276 jnz mutex_vector_enter
277 .mutex_enter_lockstat_patch_point:
278 #if defined(OPTERON_WORKAROUND_6323525)
279 .mutex_enter_6323525_patch_point:
280 ret /* nop space for lfence */
281 nop
282 nop
283 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
284 nop
285 #else /* OPTERON_WORKAROUND_6323525 */
286 ret
287 #endif /* OPTERON_WORKAROUND_6323525 */
288 movq %rdi, %rsi
289 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
290 /*
412 mutex_exit_critical_start: /* If interrupted, restart here */
413 movq %gs:CPU_THREAD, %rdx
414 cmpq %rdx, (%rdi)
415 jne mutex_vector_exit /* wrong type or wrong owner */
416 movq $0, (%rdi) /* clear owner AND lock */
417 .mutex_exit_critical_end:
418 .mutex_exit_lockstat_patch_point:
419 ret
420 movq %rdi, %rsi
421 movl $LS_MUTEX_EXIT_RELEASE, %edi
422 jmp lockstat_wrapper
423 SET_SIZE(mutex_exit)
424
425 .globl mutex_exit_critical_size
426 .type mutex_exit_critical_size, @object
427 .align CPTRSIZE
428 mutex_exit_critical_size:
429 .quad .mutex_exit_critical_end - mutex_exit_critical_start
430 SET_SIZE(mutex_exit_critical_size)
431
432 /*
433 * rw_enter() and rw_exit().
434 *
435 * These routines handle the simple cases of rw_enter (write-locking an unheld
436 * lock or read-locking a lock that's neither write-locked nor write-wanted)
437 * and rw_exit (no waiters or not the last reader). If anything complicated
438 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
439 */
440
441 ENTRY(rw_enter)
442 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
443 cmpl $RW_WRITER, %esi
444 je .rw_write_enter
445 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */
446 movq (%rdi), %rax /* rax = old rw_wwwh value */
447 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
448 jnz rw_enter_sleep
449 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
450 lock
451 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
452 jnz rw_enter_sleep
453 .rw_read_enter_lockstat_patch_point:
454 ret
455 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
456 movq %rdi, %rsi /* rsi = lock ptr */
457 movl $LS_RW_ENTER_ACQUIRE, %edi
458 movl $RW_READER, %edx
459 jmp lockstat_wrapper_arg
460 .rw_write_enter:
507 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
508 cmpl $RW_READ_LOCK, %edx
509 jge .rw_read_exit /* not last reader, safe to drop */
510 jmp rw_exit_wakeup /* last reader with waiters */
511 .rw_write_exit:
512 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
513 xorl %edx, %edx /* rdx = new value (unheld) */
514 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
515 lock
516 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
517 jnz rw_exit_wakeup
518 .rw_write_exit_lockstat_patch_point:
519 ret
520 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
521 movq %rdi, %rsi /* rsi - lock ptr */
522 movl $LS_RW_EXIT_RELEASE, %edi
523 movl $RW_WRITER, %edx
524 jmp lockstat_wrapper_arg
525 SET_SIZE(rw_exit)
526
527 #if defined(OPTERON_WORKAROUND_6323525)
528
529 /*
530 * If it is necessary to patch the lock enter routines with the lfence
531 * workaround, workaround_6323525_patched is set to a non-zero value so that
532 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
533 * instruction.
534 */
535 DGDEF3(workaround_6323525_patched, 4, 4)
536 .long 0
537
538 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
539 movq $size, %rbx; \
540 movq $dstaddr, %r13; \
541 addq %rbx, %r13; \
542 movq $srcaddr, %r12; \
543 addq %rbx, %r12; \
544 0: \
545 decq %r13; \
546 decq %r12; \
547 movzbl (%r12), %esi; \
548 movq $1, %rdx; \
549 movq %r13, %rdi; \
550 call hot_patch_kernel_text; \
551 decq %rbx; \
552 testq %rbx, %rbx; \
553 jg 0b;
554
555 /*
556 * patch_workaround_6323525: provide workaround for 6323525
557 *
581 * the existing ret is overwritten last. This provides lock enter
582 * sanity during the intermediate patching stages.
583 */
584 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
585 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
586 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
587 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
588
589 popq %rbx
590 popq %r13
591 popq %r12
592 movq %rbp, %rsp
593 popq %rbp
594 ret
595 _lfence_insn:
596 lfence
597 ret
598 SET_SIZE(patch_workaround_6323525)
599
600
601 #endif /* OPTERON_WORKAROUND_6323525 */
602
603
604 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
605 movq $normal_instr, %rsi; \
606 movq $active_instr, %rdi; \
607 leaq lockstat_probemap(%rip), %rax; \
608 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
609 testl %eax, %eax; \
610 jz 9f; \
611 movq %rdi, %rsi; \
612 9: \
613 movq $len, %rdx; \
614 movq $addr, %rdi; \
615 call hot_patch_kernel_text
616
617 ENTRY(lockstat_hot_patch)
618 pushq %rbp /* align stack properly */
619 movq %rsp, %rbp
620
621 #if defined(OPTERON_WORKAROUND_6323525)
622 cmpl $0, workaround_6323525_patched
623 je 1f
624 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
625 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
626 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
627 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
628 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
629 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
630 jmp 2f
631 1:
632 HOT_PATCH(.mutex_enter_lockstat_patch_point,
633 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
634 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
635 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
636 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
637 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
638 2:
639 #else /* OPTERON_WORKAROUND_6323525 */
647 HOT_PATCH(.mutex_exit_lockstat_patch_point,
648 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
649 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
650 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
651 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
652 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
653 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
654 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
655 HOT_PATCH(.lock_set_lockstat_patch_point,
656 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
657 HOT_PATCH(.lock_try_lockstat_patch_point,
658 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
659 HOT_PATCH(.lock_clear_lockstat_patch_point,
660 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
661 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
662 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
663
664 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
665 LS_LOCK_CLEAR_SPLX_RELEASE,
666 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
667 leave /* unwind stack */
668 ret
669 SET_SIZE(lockstat_hot_patch)
670
671 ENTRY(membar_enter)
672 ALTENTRY(membar_exit)
673 ALTENTRY(membar_sync)
674 mfence /* lighter weight than lock; xorq $0,(%rsp) */
675 ret
676 SET_SIZE(membar_sync)
677 SET_SIZE(membar_exit)
678 SET_SIZE(membar_enter)
679
680 ENTRY(membar_producer)
681 sfence
682 ret
683 SET_SIZE(membar_producer)
684
685 ENTRY(membar_consumer)
686 lfence
687 ret
688 SET_SIZE(membar_consumer)
689
690 /*
691 * thread_onproc()
692 * Set thread in onproc state for the specified CPU.
693 * Also set the thread lock pointer to the CPU's onproc lock.
694 * Since the new lock isn't held, the store ordering is important.
695 * If not done in assembler, the compiler could reorder the stores.
696 */
697
698 ENTRY(thread_onproc)
699 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
700 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
701 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
702 ret
703 SET_SIZE(thread_onproc)
704
705 /*
706 * mutex_delay_default(void)
707 * Spins for approx a few hundred processor cycles and returns to caller.
708 */
709
710 ENTRY(mutex_delay_default)
711 movq $92,%r11
712 0: decq %r11
713 jg 0b
714 ret
715 SET_SIZE(mutex_delay_default)
716
|