4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #if defined(lint) || defined(__lint)
29 #include <sys/types.h>
30 #include <sys/thread.h>
31 #include <sys/cpuvar.h>
32 #include <vm/page.h>
33 #else /* __lint */
34 #include "assym.h"
35 #endif /* __lint */
36
37 #include <sys/mutex_impl.h>
38 #include <sys/asm_linkage.h>
39 #include <sys/asm_misc.h>
40 #include <sys/regset.h>
41 #include <sys/rwlock_impl.h>
42 #include <sys/lockstat.h>
43
44 /*
45 * lock_try(lp), ulock_try(lp)
46 * - returns non-zero on success.
47 * - doesn't block interrupts so don't use this to spin on a lock.
901 * and rw_exit (no waiters or not the last reader). If anything complicated
902 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
903 */
904 #if defined(lint) || defined(__lint)
905
906 /* ARGSUSED */
907 void
908 rw_enter(krwlock_t *lp, krw_t rw)
909 {}
910
911 /* ARGSUSED */
912 void
913 rw_exit(krwlock_t *lp)
914 {}
915
916 #else /* __lint */
917
918 #if defined(__amd64)
919
920 ENTRY(rw_enter)
921 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
922 cmpl $RW_WRITER, %esi
923 je .rw_write_enter
924 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */
925 movq (%rdi), %rax /* rax = old rw_wwwh value */
926 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
927 jnz rw_enter_sleep
928 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
929 lock
930 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
931 jnz rw_enter_sleep
932 .rw_read_enter_lockstat_patch_point:
933 ret
934 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
935 movq %rdi, %rsi /* rsi = lock ptr */
936 movl $LS_RW_ENTER_ACQUIRE, %edi
937 movl $RW_READER, %edx
938 jmp lockstat_wrapper_arg
939 .rw_write_enter:
940 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
941 xorl %eax, %eax /* rax = unheld value */
942 lock
943 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
944 jnz rw_enter_sleep
945
946 #if defined(OPTERON_WORKAROUND_6323525)
947 .rw_write_enter_lockstat_patch_point:
948 .rw_write_enter_6323525_patch_point:
949 ret
950 nop
951 nop
952 .rw_write_enter_lockstat_6323525_patch_point:
953 nop
954 #else /* OPTERON_WORKAROUND_6323525 */
955 .rw_write_enter_lockstat_patch_point:
956 ret
957 #endif /* OPTERON_WORKAROUND_6323525 */
958
959 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
960 movq %rdi, %rsi /* rsi = lock ptr */
961 movl $LS_RW_ENTER_ACQUIRE, %edi
962 movl $RW_WRITER, %edx
963 jmp lockstat_wrapper_arg
964 SET_SIZE(rw_enter)
965
966 ENTRY(rw_exit)
967 movq (%rdi), %rax /* rax = old rw_wwwh value */
968 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
969 jne .rw_not_single_reader
970 xorl %edx, %edx /* rdx = new value (unheld) */
971 .rw_read_exit:
972 lock
973 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
974 jnz rw_exit_wakeup
975 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
976 decl T_KPRI_REQ(%rcx) /* THREAD_KPRI_RELEASE() */
977 .rw_read_exit_lockstat_patch_point:
978 ret
979 movq %rdi, %rsi /* rsi = lock ptr */
980 movl $LS_RW_EXIT_RELEASE, %edi
981 movl $RW_READER, %edx
982 jmp lockstat_wrapper_arg
983 .rw_not_single_reader:
984 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
985 jnz .rw_write_exit
986 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
987 cmpl $RW_READ_LOCK, %edx
988 jge .rw_read_exit /* not last reader, safe to drop */
989 jmp rw_exit_wakeup /* last reader with waiters */
990 .rw_write_exit:
991 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
992 xorl %edx, %edx /* rdx = new value (unheld) */
993 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
994 lock
995 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
996 jnz rw_exit_wakeup
997 .rw_write_exit_lockstat_patch_point:
998 ret
999 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
1000 movq %rdi, %rsi /* rsi - lock ptr */
1001 movl $LS_RW_EXIT_RELEASE, %edi
1002 movl $RW_WRITER, %edx
1003 jmp lockstat_wrapper_arg
1004 SET_SIZE(rw_exit)
1005
1006 #else
1007
1008 ENTRY(rw_enter)
1009 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1010 movl 4(%esp), %ecx /* ecx = lock ptr */
1011 cmpl $RW_WRITER, 8(%esp)
1012 je .rw_write_enter
1013 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */
1014 movl (%ecx), %eax /* eax = old rw_wwwh value */
1015 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1016 jnz rw_enter_sleep
1017 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1018 lock
1019 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1020 jnz rw_enter_sleep
1021 .rw_read_enter_lockstat_patch_point:
1022 ret
1023 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1024 movl $LS_RW_ENTER_ACQUIRE, %eax
1025 pushl $RW_READER
1026 jmp lockstat_wrapper_arg
1027 .rw_write_enter:
1028 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1029 xorl %eax, %eax /* eax = unheld value */
1030 lock
1031 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1032 jnz rw_enter_sleep
1033
1034 #if defined(OPTERON_WORKAROUND_6323525)
1035 .rw_write_enter_lockstat_patch_point:
1036 .rw_write_enter_6323525_patch_point:
1037 ret
1038 nop
1039 nop
1040 .rw_write_enter_lockstat_6323525_patch_point:
1041 nop
1042 #else /* OPTERON_WORKAROUND_6323525 */
1043 .rw_write_enter_lockstat_patch_point:
1044 ret
1045 #endif /* OPTERON_WORKAROUND_6323525 */
1046
1047 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1048 movl $LS_RW_ENTER_ACQUIRE, %eax
1049 pushl $RW_WRITER
1050 jmp lockstat_wrapper_arg
1051 SET_SIZE(rw_enter)
1052
1053 ENTRY(rw_exit)
1054 movl 4(%esp), %ecx /* ecx = lock ptr */
1055 movl (%ecx), %eax /* eax = old rw_wwwh value */
1056 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1057 jne .rw_not_single_reader
1058 xorl %edx, %edx /* edx = new value (unheld) */
1059 .rw_read_exit:
1060 lock
1061 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1062 jnz rw_exit_wakeup
1063 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1064 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */
1065 .rw_read_exit_lockstat_patch_point:
1066 ret
1067 movl $LS_RW_EXIT_RELEASE, %eax
1068 pushl $RW_READER
1069 jmp lockstat_wrapper_arg
1070 .rw_not_single_reader:
1071 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1072 jnz .rw_write_exit
1073 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1074 cmpl $RW_READ_LOCK, %edx
1075 jge .rw_read_exit /* not last reader, safe to drop */
1076 jmp rw_exit_wakeup /* last reader with waiters */
1077 .rw_write_exit:
1078 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1079 xorl %edx, %edx /* edx = new value (unheld) */
1080 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1081 lock
1082 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1083 jnz rw_exit_wakeup
1084 .rw_write_exit_lockstat_patch_point:
|
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #if defined(lint) || defined(__lint)
28 #include <sys/types.h>
29 #include <sys/thread.h>
30 #include <sys/cpuvar.h>
31 #include <vm/page.h>
32 #else /* __lint */
33 #include "assym.h"
34 #endif /* __lint */
35
36 #include <sys/mutex_impl.h>
37 #include <sys/asm_linkage.h>
38 #include <sys/asm_misc.h>
39 #include <sys/regset.h>
40 #include <sys/rwlock_impl.h>
41 #include <sys/lockstat.h>
42
43 /*
44 * lock_try(lp), ulock_try(lp)
45 * - returns non-zero on success.
46 * - doesn't block interrupts so don't use this to spin on a lock.
900 * and rw_exit (no waiters or not the last reader). If anything complicated
901 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
902 */
903 #if defined(lint) || defined(__lint)
904
905 /* ARGSUSED */
906 void
907 rw_enter(krwlock_t *lp, krw_t rw)
908 {}
909
910 /* ARGSUSED */
911 void
912 rw_exit(krwlock_t *lp)
913 {}
914
915 #else /* __lint */
916
917 #if defined(__amd64)
918
919 ENTRY(rw_enter)
920 cmpl $RW_WRITER, %esi
921 je .rw_write_enter
922 movq (%rdi), %rax /* rax = old rw_wwwh value */
923 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
924 jnz rw_enter_sleep
925 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
926 lock
927 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
928 jnz rw_enter_sleep
929 .rw_read_enter_lockstat_patch_point:
930 ret
931 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
932 movq %rdi, %rsi /* rsi = lock ptr */
933 movl $LS_RW_ENTER_ACQUIRE, %edi
934 movl $RW_READER, %edx
935 jmp lockstat_wrapper_arg
936 .rw_write_enter:
937 movq %gs:CPU_THREAD, %rdx
938 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
939 xorl %eax, %eax /* rax = unheld value */
940 lock
941 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
942 jnz rw_enter_sleep
943
944 #if defined(OPTERON_WORKAROUND_6323525)
945 .rw_write_enter_lockstat_patch_point:
946 .rw_write_enter_6323525_patch_point:
947 ret
948 nop
949 nop
950 .rw_write_enter_lockstat_6323525_patch_point:
951 nop
952 #else /* OPTERON_WORKAROUND_6323525 */
953 .rw_write_enter_lockstat_patch_point:
954 ret
955 #endif /* OPTERON_WORKAROUND_6323525 */
956
957 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
958 movq %rdi, %rsi /* rsi = lock ptr */
959 movl $LS_RW_ENTER_ACQUIRE, %edi
960 movl $RW_WRITER, %edx
961 jmp lockstat_wrapper_arg
962 SET_SIZE(rw_enter)
963
964 ENTRY(rw_exit)
965 movq (%rdi), %rax /* rax = old rw_wwwh value */
966 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
967 jne .rw_not_single_reader
968 xorl %edx, %edx /* rdx = new value (unheld) */
969 .rw_read_exit:
970 lock
971 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
972 jnz rw_exit_wakeup
973 .rw_read_exit_lockstat_patch_point:
974 ret
975 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
976 movq %rdi, %rsi /* rsi = lock ptr */
977 movl $LS_RW_EXIT_RELEASE, %edi
978 movl $RW_READER, %edx
979 jmp lockstat_wrapper_arg
980 .rw_not_single_reader:
981 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
982 jnz .rw_write_exit
983 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
984 cmpl $RW_READ_LOCK, %edx
985 jge .rw_read_exit /* not last reader, safe to drop */
986 jmp rw_exit_wakeup /* last reader with waiters */
987 .rw_write_exit:
988 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
989 xorl %edx, %edx /* rdx = new value (unheld) */
990 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
991 lock
992 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
993 jnz rw_exit_wakeup
994 .rw_write_exit_lockstat_patch_point:
995 ret
996 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
997 movq %rdi, %rsi /* rsi - lock ptr */
998 movl $LS_RW_EXIT_RELEASE, %edi
999 movl $RW_WRITER, %edx
1000 jmp lockstat_wrapper_arg
1001 SET_SIZE(rw_exit)
1002
1003 #else
1004
1005 ENTRY(rw_enter)
1006 movl 4(%esp), %ecx /* ecx = lock ptr */
1007 cmpl $RW_WRITER, 8(%esp)
1008 je .rw_write_enter
1009 movl (%ecx), %eax /* eax = old rw_wwwh value */
1010 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1011 jnz rw_enter_sleep
1012 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1013 lock
1014 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1015 jnz rw_enter_sleep
1016 .rw_read_enter_lockstat_patch_point:
1017 ret
1018 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1019 movl $LS_RW_ENTER_ACQUIRE, %eax
1020 pushl $RW_READER
1021 jmp lockstat_wrapper_arg
1022 .rw_write_enter:
1023 movl %gs:CPU_THREAD, %edx
1024 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1025 xorl %eax, %eax /* eax = unheld value */
1026 lock
1027 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1028 jnz rw_enter_sleep
1029
1030 #if defined(OPTERON_WORKAROUND_6323525)
1031 .rw_write_enter_lockstat_patch_point:
1032 .rw_write_enter_6323525_patch_point:
1033 ret
1034 nop
1035 nop
1036 .rw_write_enter_lockstat_6323525_patch_point:
1037 nop
1038 #else /* OPTERON_WORKAROUND_6323525 */
1039 .rw_write_enter_lockstat_patch_point:
1040 ret
1041 #endif /* OPTERON_WORKAROUND_6323525 */
1042
1043 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1044 movl $LS_RW_ENTER_ACQUIRE, %eax
1045 pushl $RW_WRITER
1046 jmp lockstat_wrapper_arg
1047 SET_SIZE(rw_enter)
1048
1049 ENTRY(rw_exit)
1050 movl 4(%esp), %ecx /* ecx = lock ptr */
1051 movl (%ecx), %eax /* eax = old rw_wwwh value */
1052 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1053 jne .rw_not_single_reader
1054 xorl %edx, %edx /* edx = new value (unheld) */
1055 .rw_read_exit:
1056 lock
1057 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1058 jnz rw_exit_wakeup
1059 .rw_read_exit_lockstat_patch_point:
1060 ret
1061 movl $LS_RW_EXIT_RELEASE, %eax
1062 pushl $RW_READER
1063 jmp lockstat_wrapper_arg
1064 .rw_not_single_reader:
1065 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1066 jnz .rw_write_exit
1067 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1068 cmpl $RW_READ_LOCK, %edx
1069 jge .rw_read_exit /* not last reader, safe to drop */
1070 jmp rw_exit_wakeup /* last reader with waiters */
1071 .rw_write_exit:
1072 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1073 xorl %edx, %edx /* edx = new value (unheld) */
1074 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1075 lock
1076 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1077 jnz rw_exit_wakeup
1078 .rw_write_exit_lockstat_patch_point:
|