477 #endif /* OPTERON_WORKAROUND_6323525 */
478
479 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
480 movq %rdi, %rsi /* rsi = lock ptr */
481 movl $LS_RW_ENTER_ACQUIRE, %edi
482 movl $RW_WRITER, %edx
483 jmp lockstat_wrapper_arg
484 SET_SIZE(rw_enter)
485
486 ENTRY(rw_exit)
487 movq (%rdi), %rax /* rax = old rw_wwwh value */
488 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
489 jne .rw_not_single_reader
490 xorl %edx, %edx /* rdx = new value (unheld) */
491 .rw_read_exit:
492 lock
493 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
494 jnz rw_exit_wakeup
495 .rw_read_exit_lockstat_patch_point:
496 ret
497 movq %rdi, %rsi /* rsi = lock ptr */
498 movl $LS_RW_EXIT_RELEASE, %edi
499 movl $RW_READER, %edx
500 jmp lockstat_wrapper_arg
501 .rw_not_single_reader:
502 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
503 jnz .rw_write_exit
504 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
505 cmpl $RW_READ_LOCK, %edx
506 jge .rw_read_exit /* not last reader, safe to drop */
507 jmp rw_exit_wakeup /* last reader with waiters */
508 .rw_write_exit:
509 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
510 xorl %edx, %edx /* rdx = new value (unheld) */
511 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
512 lock
513 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
514 jnz rw_exit_wakeup
515 .rw_write_exit_lockstat_patch_point:
516 ret
|
477 #endif /* OPTERON_WORKAROUND_6323525 */
478
479 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
480 movq %rdi, %rsi /* rsi = lock ptr */
481 movl $LS_RW_ENTER_ACQUIRE, %edi
482 movl $RW_WRITER, %edx
483 jmp lockstat_wrapper_arg
484 SET_SIZE(rw_enter)
485
486 ENTRY(rw_exit)
487 movq (%rdi), %rax /* rax = old rw_wwwh value */
488 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
489 jne .rw_not_single_reader
490 xorl %edx, %edx /* rdx = new value (unheld) */
491 .rw_read_exit:
492 lock
493 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
494 jnz rw_exit_wakeup
495 .rw_read_exit_lockstat_patch_point:
496 ret
497 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
498 movq %rdi, %rsi /* rsi = lock ptr */
499 movl $LS_RW_EXIT_RELEASE, %edi
500 movl $RW_READER, %edx
501 jmp lockstat_wrapper_arg
502 .rw_not_single_reader:
503 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
504 jnz .rw_write_exit
505 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
506 cmpl $RW_READ_LOCK, %edx
507 jge .rw_read_exit /* not last reader, safe to drop */
508 jmp rw_exit_wakeup /* last reader with waiters */
509 .rw_write_exit:
510 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
511 xorl %edx, %edx /* rdx = new value (unheld) */
512 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
513 lock
514 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
515 jnz rw_exit_wakeup
516 .rw_write_exit_lockstat_patch_point:
517 ret
|