Print this page
OS-7753 THREAD_KPRI_RELEASE does nothing of the sort
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>


   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.

  24  */
  25 
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27 
  28 #if defined(lint)
  29 #include <sys/types.h>
  30 #include <sys/thread.h>
  31 #include <sys/cpuvar.h>
  32 #else   /* lint */
  33 #include "assym.h"
  34 #endif  /* lint */
  35 
  36 #include <sys/t_lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/mutex_impl.h>
  39 #include <sys/rwlock_impl.h>
  40 #include <sys/asm_linkage.h>
  41 #include <sys/machlock.h>
  42 #include <sys/machthread.h>
  43 #include <sys/lockstat.h>
  44 
  45 /* #define DEBUG */
  46 
  47 #ifdef DEBUG


 485  */
 486 #if defined(lint)
 487 
 488 /* ARGSUSED */
 489 void
 490 rw_enter(krwlock_t *lp, krw_t rw)
 491 {}
 492 
 493 /* ARGSUSED */
 494 void
 495 rw_exit(krwlock_t *lp)
 496 {}
 497 
 498 #else
 499 
 500         .align  16
 501         ENTRY(rw_enter)
 502         cmp     %o1, RW_WRITER                  ! entering as writer?
 503         be,a,pn %icc, 2f                        ! if so, go do it ...
 504         or      THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
 505         ld      [THREAD_REG + T_KPRI_REQ], %o3  ! begin THREAD_KPRI_REQUEST()
 506         ldn     [%o0], %o4                      ! %o4 = old lock value
 507         inc     %o3                             ! bump kpri
 508         st      %o3, [THREAD_REG + T_KPRI_REQ]  ! store new kpri
 509 1:
 510         andcc   %o4, RW_WRITE_CLAIMED, %g0      ! write-locked or write-wanted?
 511         bz,pt   %xcc, 3f                        ! if so, prepare to block
 512         add     %o4, RW_READ_LOCK, %o5          ! delay: increment hold count
 513         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 514         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 515         nop                                     ! delay: do nothing
 516 3:
 517         casx    [%o0], %o4, %o5                 ! try to grab read lock
 518         cmp     %o4, %o5                        ! did we get it?
 519 #ifdef sun4v
 520         be,a,pt %xcc, 0f
 521         membar  #LoadLoad
 522         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 523         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 524         nop                                     ! delay: do nothing
 525 0:
 526 #else /* sun4v */
 527         bne,pn  %xcc, 1b                        ! if not, try again
 528         mov     %o5, %o4                        ! delay: %o4 = old lock value


 535         casx    [%o0], %g0, %o5                 ! try to grab write lock
 536         brz,pt %o5, 4f                          ! branch around if we got it
 537         membar  #LoadLoad                       ! done regardless of where we go
 538         sethi   %hi(rw_enter_sleep), %o2
 539         jmp     %o2 + %lo(rw_enter_sleep)       ! jump to rw_enter_sleep if not
 540         nop                                     ! delay: do nothing
 541 4:
 542 .rw_write_enter_lockstat_patch_point:
 543         retl
 544         nop
 545         SET_SIZE(rw_enter)
 546 
 547         .align  16
 548         ENTRY(rw_exit)
 549         ldn     [%o0], %o4                      ! %o4 = old lock value
 550         membar  #LoadStore|#StoreStore          ! membar_exit()
 551         subcc   %o4, RW_READ_LOCK, %o5          ! %o5 = new lock value if reader
 552         bnz,pn  %xcc, 2f                        ! single reader, no waiters?
 553         clr     %o1
 554 1:
 555         ld      [THREAD_REG + T_KPRI_REQ], %g1  ! begin THREAD_KPRI_RELEASE()
 556         srl     %o4, RW_HOLD_COUNT_SHIFT, %o3   ! %o3 = hold count (lockstat)
 557         casx    [%o0], %o4, %o5                 ! try to drop lock
 558         cmp     %o4, %o5                        ! did we succeed?
 559         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 560         dec     %g1                             ! delay: drop kpri
 561 .rw_read_exit_lockstat_patch_point:
 562         retl
 563         st      %g1, [THREAD_REG + T_KPRI_REQ]  ! delay: store new kpri
 564 2:
 565         andcc   %o4, RW_WRITE_LOCKED, %g0       ! are we a writer?
 566         bnz,a,pt %xcc, 3f
 567         or      THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
 568         cmp     %o5, RW_READ_LOCK               ! would lock still be held?
 569         bge,pt  %xcc, 1b                        ! if so, go ahead and drop it
 570         nop
 571         ba,pt   %xcc, rw_exit_wakeup            ! otherwise, wake waiters
 572         nop
 573 3:
 574         casx    [%o0], %o4, %o1                 ! try to drop write lock
 575         cmp     %o4, %o1                        ! did we succeed?
 576         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 577         nop
 578 .rw_write_exit_lockstat_patch_point:
 579         retl
 580         nop
 581         SET_SIZE(rw_exit)
 582 
 583 #endif




   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  * Copyright 2019 Joyent, Inc.
  25  */
  26 


  27 #if defined(lint)
  28 #include <sys/types.h>
  29 #include <sys/thread.h>
  30 #include <sys/cpuvar.h>
  31 #else   /* lint */
  32 #include "assym.h"
  33 #endif  /* lint */
  34 
  35 #include <sys/t_lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/mutex_impl.h>
  38 #include <sys/rwlock_impl.h>
  39 #include <sys/asm_linkage.h>
  40 #include <sys/machlock.h>
  41 #include <sys/machthread.h>
  42 #include <sys/lockstat.h>
  43 
  44 /* #define DEBUG */
  45 
  46 #ifdef DEBUG


 484  */
 485 #if defined(lint)
 486 
 487 /* ARGSUSED */
 488 void
 489 rw_enter(krwlock_t *lp, krw_t rw)
 490 {}
 491 
 492 /* ARGSUSED */
 493 void
 494 rw_exit(krwlock_t *lp)
 495 {}
 496 
 497 #else
 498 
 499         .align  16
 500         ENTRY(rw_enter)
 501         cmp     %o1, RW_WRITER                  ! entering as writer?
 502         be,a,pn %icc, 2f                        ! if so, go do it ...
 503         or      THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner

 504         ldn     [%o0], %o4                      ! %o4 = old lock value


 505 1:
 506         andcc   %o4, RW_WRITE_CLAIMED, %g0      ! write-locked or write-wanted?
 507         bz,pt   %xcc, 3f                        ! if so, prepare to block
 508         add     %o4, RW_READ_LOCK, %o5          ! delay: increment hold count
 509         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 510         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 511         nop                                     ! delay: do nothing
 512 3:
 513         casx    [%o0], %o4, %o5                 ! try to grab read lock
 514         cmp     %o4, %o5                        ! did we get it?
 515 #ifdef sun4v
 516         be,a,pt %xcc, 0f
 517         membar  #LoadLoad
 518         sethi   %hi(rw_enter_sleep), %o2        ! load up jump
 519         jmp     %o2 + %lo(rw_enter_sleep)       ! jmp to rw_enter_sleep
 520         nop                                     ! delay: do nothing
 521 0:
 522 #else /* sun4v */
 523         bne,pn  %xcc, 1b                        ! if not, try again
 524         mov     %o5, %o4                        ! delay: %o4 = old lock value


 531         casx    [%o0], %g0, %o5                 ! try to grab write lock
 532         brz,pt %o5, 4f                          ! branch around if we got it
 533         membar  #LoadLoad                       ! done regardless of where we go
 534         sethi   %hi(rw_enter_sleep), %o2
 535         jmp     %o2 + %lo(rw_enter_sleep)       ! jump to rw_enter_sleep if not
 536         nop                                     ! delay: do nothing
 537 4:
 538 .rw_write_enter_lockstat_patch_point:
 539         retl
 540         nop
 541         SET_SIZE(rw_enter)
 542 
 543         .align  16
 544         ENTRY(rw_exit)
 545         ldn     [%o0], %o4                      ! %o4 = old lock value
 546         membar  #LoadStore|#StoreStore          ! membar_exit()
 547         subcc   %o4, RW_READ_LOCK, %o5          ! %o5 = new lock value if reader
 548         bnz,pn  %xcc, 2f                        ! single reader, no waiters?
 549         clr     %o1
 550 1:

 551         srl     %o4, RW_HOLD_COUNT_SHIFT, %o3   ! %o3 = hold count (lockstat)
 552         casx    [%o0], %o4, %o5                 ! try to drop lock
 553         cmp     %o4, %o5                        ! did we succeed?
 554         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 555         nop                                     ! delay: do nothing
 556 .rw_read_exit_lockstat_patch_point:
 557         retl
 558         nop                                     ! delay: do nothing
 559 2:
 560         andcc   %o4, RW_WRITE_LOCKED, %g0       ! are we a writer?
 561         bnz,a,pt %xcc, 3f
 562         or      THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
 563         cmp     %o5, RW_READ_LOCK               ! would lock still be held?
 564         bge,pt  %xcc, 1b                        ! if so, go ahead and drop it
 565         nop
 566         ba,pt   %xcc, rw_exit_wakeup            ! otherwise, wake waiters
 567         nop
 568 3:
 569         casx    [%o0], %o4, %o1                 ! try to drop write lock
 570         cmp     %o4, %o1                        ! did we succeed?
 571         bne,pn  %xcc, rw_exit_wakeup            ! if not, go to C
 572         nop
 573 .rw_write_exit_lockstat_patch_point:
 574         retl
 575         nop
 576         SET_SIZE(rw_exit)
 577 
 578 #endif