4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #if defined(lint)
29 #include <sys/types.h>
30 #include <sys/thread.h>
31 #include <sys/cpuvar.h>
32 #else /* lint */
33 #include "assym.h"
34 #endif /* lint */
35
36 #include <sys/t_lock.h>
37 #include <sys/mutex.h>
38 #include <sys/mutex_impl.h>
39 #include <sys/rwlock_impl.h>
40 #include <sys/asm_linkage.h>
41 #include <sys/machlock.h>
42 #include <sys/machthread.h>
43 #include <sys/lockstat.h>
44
45 /* #define DEBUG */
46
47 #ifdef DEBUG
485 */
486 #if defined(lint)
487
488 /* ARGSUSED */
489 void
490 rw_enter(krwlock_t *lp, krw_t rw)
491 {}
492
493 /* ARGSUSED */
494 void
495 rw_exit(krwlock_t *lp)
496 {}
497
498 #else
499
500 .align 16
501 ENTRY(rw_enter)
502 cmp %o1, RW_WRITER ! entering as writer?
503 be,a,pn %icc, 2f ! if so, go do it ...
504 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
505 ld [THREAD_REG + T_KPRI_REQ], %o3 ! begin THREAD_KPRI_REQUEST()
506 ldn [%o0], %o4 ! %o4 = old lock value
507 inc %o3 ! bump kpri
508 st %o3, [THREAD_REG + T_KPRI_REQ] ! store new kpri
509 1:
510 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted?
511 bz,pt %xcc, 3f ! if so, prepare to block
512 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count
513 sethi %hi(rw_enter_sleep), %o2 ! load up jump
514 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
515 nop ! delay: do nothing
516 3:
517 casx [%o0], %o4, %o5 ! try to grab read lock
518 cmp %o4, %o5 ! did we get it?
519 #ifdef sun4v
520 be,a,pt %xcc, 0f
521 membar #LoadLoad
522 sethi %hi(rw_enter_sleep), %o2 ! load up jump
523 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
524 nop ! delay: do nothing
525 0:
526 #else /* sun4v */
527 bne,pn %xcc, 1b ! if not, try again
528 mov %o5, %o4 ! delay: %o4 = old lock value
535 casx [%o0], %g0, %o5 ! try to grab write lock
536 brz,pt %o5, 4f ! branch around if we got it
537 membar #LoadLoad ! done regardless of where we go
538 sethi %hi(rw_enter_sleep), %o2
539 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not
540 nop ! delay: do nothing
541 4:
542 .rw_write_enter_lockstat_patch_point:
543 retl
544 nop
545 SET_SIZE(rw_enter)
546
547 .align 16
548 ENTRY(rw_exit)
549 ldn [%o0], %o4 ! %o4 = old lock value
550 membar #LoadStore|#StoreStore ! membar_exit()
551 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader
552 bnz,pn %xcc, 2f ! single reader, no waiters?
553 clr %o1
554 1:
555 ld [THREAD_REG + T_KPRI_REQ], %g1 ! begin THREAD_KPRI_RELEASE()
556 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat)
557 casx [%o0], %o4, %o5 ! try to drop lock
558 cmp %o4, %o5 ! did we succeed?
559 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
560 dec %g1 ! delay: drop kpri
561 .rw_read_exit_lockstat_patch_point:
562 retl
563 st %g1, [THREAD_REG + T_KPRI_REQ] ! delay: store new kpri
564 2:
565 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer?
566 bnz,a,pt %xcc, 3f
567 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
568 cmp %o5, RW_READ_LOCK ! would lock still be held?
569 bge,pt %xcc, 1b ! if so, go ahead and drop it
570 nop
571 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters
572 nop
573 3:
574 casx [%o0], %o4, %o1 ! try to drop write lock
575 cmp %o4, %o1 ! did we succeed?
576 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
577 nop
578 .rw_write_exit_lockstat_patch_point:
579 retl
580 nop
581 SET_SIZE(rw_exit)
582
583 #endif
|
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #if defined(lint)
28 #include <sys/types.h>
29 #include <sys/thread.h>
30 #include <sys/cpuvar.h>
31 #else /* lint */
32 #include "assym.h"
33 #endif /* lint */
34
35 #include <sys/t_lock.h>
36 #include <sys/mutex.h>
37 #include <sys/mutex_impl.h>
38 #include <sys/rwlock_impl.h>
39 #include <sys/asm_linkage.h>
40 #include <sys/machlock.h>
41 #include <sys/machthread.h>
42 #include <sys/lockstat.h>
43
44 /* #define DEBUG */
45
46 #ifdef DEBUG
484 */
485 #if defined(lint)
486
487 /* ARGSUSED */
488 void
489 rw_enter(krwlock_t *lp, krw_t rw)
490 {}
491
492 /* ARGSUSED */
493 void
494 rw_exit(krwlock_t *lp)
495 {}
496
497 #else
498
499 .align 16
500 ENTRY(rw_enter)
501 cmp %o1, RW_WRITER ! entering as writer?
502 be,a,pn %icc, 2f ! if so, go do it ...
503 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
504 ldn [%o0], %o4 ! %o4 = old lock value
505 1:
506 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted?
507 bz,pt %xcc, 3f ! if so, prepare to block
508 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count
509 sethi %hi(rw_enter_sleep), %o2 ! load up jump
510 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
511 nop ! delay: do nothing
512 3:
513 casx [%o0], %o4, %o5 ! try to grab read lock
514 cmp %o4, %o5 ! did we get it?
515 #ifdef sun4v
516 be,a,pt %xcc, 0f
517 membar #LoadLoad
518 sethi %hi(rw_enter_sleep), %o2 ! load up jump
519 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
520 nop ! delay: do nothing
521 0:
522 #else /* sun4v */
523 bne,pn %xcc, 1b ! if not, try again
524 mov %o5, %o4 ! delay: %o4 = old lock value
531 casx [%o0], %g0, %o5 ! try to grab write lock
532 brz,pt %o5, 4f ! branch around if we got it
533 membar #LoadLoad ! done regardless of where we go
534 sethi %hi(rw_enter_sleep), %o2
535 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not
536 nop ! delay: do nothing
537 4:
538 .rw_write_enter_lockstat_patch_point:
539 retl
540 nop
541 SET_SIZE(rw_enter)
542
543 .align 16
544 ENTRY(rw_exit)
545 ldn [%o0], %o4 ! %o4 = old lock value
546 membar #LoadStore|#StoreStore ! membar_exit()
547 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader
548 bnz,pn %xcc, 2f ! single reader, no waiters?
549 clr %o1
550 1:
551 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat)
552 casx [%o0], %o4, %o5 ! try to drop lock
553 cmp %o4, %o5 ! did we succeed?
554 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
555 nop ! delay: do nothing
556 .rw_read_exit_lockstat_patch_point:
557 retl
558 nop ! delay: do nothing
559 2:
560 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer?
561 bnz,a,pt %xcc, 3f
562 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
563 cmp %o5, RW_READ_LOCK ! would lock still be held?
564 bge,pt %xcc, 1b ! if so, go ahead and drop it
565 nop
566 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters
567 nop
568 3:
569 casx [%o0], %o4, %o1 ! try to drop write lock
570 cmp %o4, %o1 ! did we succeed?
571 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
572 nop
573 .rw_write_exit_lockstat_patch_point:
574 retl
575 nop
576 SET_SIZE(rw_exit)
577
578 #endif
|