mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-13 00:30:07 +00:00
Support requeueing for condvars using PI mutex. x86-64 only.
Add support for the new FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI options of futex.
This commit is contained in:
Ulrich Drepper
parent
515a8908ce
commit
42e69bcf11
@ -1,5 +1,14 @@
|
||||
2009-07-19 Ulrich Drepper <drepper@redhat.com>
|
||||
|
||||
* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
|
||||
FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: If mutex
|
||||
is a PI mutex, then use FUTEX_CMP_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
|
||||
* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: If mutex
|
||||
is a PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
|
||||
|
||||
* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
|
||||
(__pthread_cond_timedwait): Make more robust.
|
||||
|
||||
|
||||
@ -54,6 +54,8 @@
|
||||
#define FUTEX_TRYLOCK_PI 8
|
||||
#define FUTEX_WAIT_BITSET 9
|
||||
#define FUTEX_WAKE_BITSET 10
|
||||
#define FUTEX_WAIT_REQUEUE_PI 11
|
||||
#define FUTEX_CMP_REQUEUE_PI 12
|
||||
#define FUTEX_PRIVATE_FLAG 128
|
||||
#define FUTEX_CLOCK_REALTIME 256
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007
|
||||
/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009
|
||||
Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
|
||||
@ -70,12 +70,14 @@ __pthread_cond_broadcast:
|
||||
8: cmpq $-1, %r8
|
||||
je 9f
|
||||
|
||||
/* XXX: The kernel so far doesn't support requeue to PI futex. */
|
||||
/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
|
||||
type of futex (private resp. shared). */
|
||||
testl $(PI_BIT | PS_BIT), MUTEX_KIND(%r8)
|
||||
/* Do not use requeue for pshared condvars. */
|
||||
testl $PS_BIT, MUTEX_KIND(%r8)
|
||||
jne 9f
|
||||
|
||||
/* Requeue to a PI mutex if the PI bit is set. */
|
||||
testl $PI_BIT, MUTEX_KIND(%r8)
|
||||
jne 81f
|
||||
|
||||
/* Wake up all threads. */
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
|
||||
@ -97,6 +99,20 @@ __pthread_cond_broadcast:
|
||||
10: xorl %eax, %eax
|
||||
retq
|
||||
|
||||
/* Wake up all threads. */
|
||||
81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
|
||||
movl $SYS_futex, %eax
|
||||
movl $1, %edx
|
||||
movl $0x7fffffff, %r10d
|
||||
syscall
|
||||
|
||||
/* For any kind of error, which mainly is EAGAIN, we try again
|
||||
with WAKE. The general test also covers running on old
|
||||
kernels. */
|
||||
cmpq $-4095, %rax
|
||||
jb 10b
|
||||
jmp 9f
|
||||
|
||||
.align 16
|
||||
/* Unlock. */
|
||||
4: LOCK
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
|
||||
/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
#include <shlib-compat.h>
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
#include <kernel-features.h>
|
||||
#include <pthread-errnos.h>
|
||||
|
||||
@ -56,19 +57,23 @@ __pthread_cond_signal:
|
||||
|
||||
/* Wake up one thread. */
|
||||
cmpq $-1, dep_mutex(%r8)
|
||||
movl $FUTEX_WAKE_OP, %esi
|
||||
movl $1, %edx
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
movl $FUTEX_WAKE_OP, %eax
|
||||
movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
|
||||
cmove %eax, %esi
|
||||
#else
|
||||
movl $0, %eax
|
||||
movl %fs:PRIVATE_FUTEX, %esi
|
||||
cmove %eax, %esi
|
||||
orl $FUTEX_WAKE_OP, %esi
|
||||
#endif
|
||||
movl $1, %r10d
|
||||
movl $SYS_futex, %eax
|
||||
je 8f
|
||||
|
||||
/* Get the address of the mutex used. */
|
||||
movq dep_mutex(%r8), %rcx
|
||||
testl $PI_BIT, MUTEX_KIND(%rcx)
|
||||
jne 9f
|
||||
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
|
||||
#else
|
||||
orl %fs:PRIVATE_FUTEX, %esi
|
||||
#endif
|
||||
|
||||
8: movl $1, %r10d
|
||||
#if cond_lock != 0
|
||||
addq $cond_lock, %r8
|
||||
#endif
|
||||
@ -85,9 +90,27 @@ __pthread_cond_signal:
|
||||
xorl %eax, %eax
|
||||
retq
|
||||
|
||||
7: /* %esi should be either FUTEX_WAKE_OP or
|
||||
FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
|
||||
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi
|
||||
/* Wake up one thread and requeue none in the PI Mutex case. */
|
||||
9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
|
||||
movq %rcx, %r8
|
||||
xorq %r10, %r10
|
||||
movl (%rdi), %r9d // XXX Can this be right?
|
||||
syscall
|
||||
|
||||
leaq -cond_futex(%rdi), %r8
|
||||
|
||||
/* For any kind of error, we try again with WAKE.
|
||||
The general test also covers running on old kernels. */
|
||||
cmpq $-4095, %rax
|
||||
jb 4f
|
||||
|
||||
7:
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
andl $FUTEX_PRIVATE_FLAG, %esi
|
||||
#else
|
||||
andl %fs:PRIVATE_FUTEX, %esi
|
||||
#endif
|
||||
orl $FUTEX_WAKE, %esi
|
||||
movl $SYS_futex, %eax
|
||||
/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
|
||||
movl $1, %edx */
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
#include <shlib-compat.h>
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
#include <pthread-errnos.h>
|
||||
|
||||
#include <kernel-features.h>
|
||||
@ -58,6 +59,9 @@ __pthread_cond_timedwait:
|
||||
pushq %r14
|
||||
cfi_adjust_cfa_offset(8)
|
||||
cfi_rel_offset(%r14, 0)
|
||||
pushq %r15
|
||||
cfi_adjust_cfa_offset(8)
|
||||
cfi_rel_offset(%r15, 0)
|
||||
#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
|
||||
# define FRAME_SIZE 32
|
||||
#else
|
||||
@ -160,9 +164,41 @@ __pthread_cond_timedwait:
|
||||
movl $FUTEX_WAIT_BITSET, %eax
|
||||
movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
|
||||
cmove %eax, %esi
|
||||
je 60f
|
||||
|
||||
movq dep_mutex(%rdi), %r8
|
||||
/* Requeue to a PI mutex if the PI bit is set. */
|
||||
testl $PI_BIT, MUTEX_KIND(%r8)
|
||||
je 60f
|
||||
|
||||
movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
|
||||
xorl %eax, %eax
|
||||
/* The following only works like this because we only support
|
||||
two clocks, represented using a single bit. */
|
||||
testl $1, cond_nwaiters(%rdi)
|
||||
movl $FUTEX_CLOCK_REALTIME, %edx
|
||||
cmove %edx, %eax
|
||||
orl %eax, %esi
|
||||
movq %r12, %rdx
|
||||
addq $cond_futex, %rdi
|
||||
movl $SYS_futex, %eax
|
||||
syscall
|
||||
|
||||
movl $1, %r15d
|
||||
#ifdef __ASSUME_REQUEUE_PI
|
||||
jmp 62f
|
||||
#else
|
||||
cmpq $-4095, %rax
|
||||
jnae 62f
|
||||
|
||||
movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
|
||||
subq $cond_futex, %rdi
|
||||
#endif
|
||||
|
||||
60: xorl %r15d, %r15d
|
||||
xorl %eax, %eax
|
||||
/* The following only works like this because we only support
|
||||
two clocks, represented using a single bit. */
|
||||
testl $1, cond_nwaiters(%rdi)
|
||||
movl $FUTEX_CLOCK_REALTIME, %edx
|
||||
movl $0xffffffff, %r9d
|
||||
@ -172,7 +208,7 @@ __pthread_cond_timedwait:
|
||||
addq $cond_futex, %rdi
|
||||
movl $SYS_futex, %eax
|
||||
syscall
|
||||
movq %rax, %r14
|
||||
62: movq %rax, %r14
|
||||
|
||||
movl (%rsp), %edi
|
||||
callq __pthread_disable_asynccancel
|
||||
@ -253,14 +289,23 @@ __pthread_cond_timedwait:
|
||||
#endif
|
||||
jne 40f
|
||||
|
||||
41: movq 16(%rsp), %rdi
|
||||
/* If requeue_pi is used the kernel performs the locking of the
|
||||
mutex. */
|
||||
41: xorl %eax, %eax
|
||||
testl %r15d, %r15d
|
||||
jnz 63f
|
||||
|
||||
movq 16(%rsp), %rdi
|
||||
callq __pthread_mutex_cond_lock
|
||||
|
||||
testq %rax, %rax
|
||||
63: testq %rax, %rax
|
||||
cmoveq %r14, %rax
|
||||
|
||||
48: addq $FRAME_SIZE, %rsp
|
||||
cfi_adjust_cfa_offset(-FRAME_SIZE)
|
||||
popq %r15
|
||||
cfi_adjust_cfa_offset(-8)
|
||||
cfi_restore(%r15)
|
||||
popq %r14
|
||||
cfi_adjust_cfa_offset(-8)
|
||||
cfi_restore(%r14)
|
||||
@ -274,10 +319,11 @@ __pthread_cond_timedwait:
|
||||
retq
|
||||
|
||||
/* Initial locking failed. */
|
||||
31: cfi_adjust_cfa_offset(3 * 8 + FRAME_SIZE)
|
||||
cfi_rel_offset(%r12, FRAME_SIZE + 16)
|
||||
cfi_rel_offset(%r13, FRAME_SIZE + 8)
|
||||
cfi_rel_offset(%r14, FRAME_SIZE)
|
||||
31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE)
|
||||
cfi_rel_offset(%r12, FRAME_SIZE + 24)
|
||||
cfi_rel_offset(%r13, FRAME_SIZE + 16)
|
||||
cfi_rel_offset(%r14, FRAME_SIZE + 8)
|
||||
cfi_rel_offset(%r15, FRAME_SIZE)
|
||||
#if cond_lock != 0
|
||||
addq $cond_lock, %rdi
|
||||
#endif
|
||||
@ -353,6 +399,8 @@ __pthread_cond_timedwait:
|
||||
|
||||
#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
|
||||
.Lreltmo:
|
||||
xorl %r15d, %r15d
|
||||
|
||||
/* Get internal lock. */
|
||||
movl $1, %esi
|
||||
xorl %eax, %eax
|
||||
@ -716,9 +764,10 @@ __condvar_cleanup2:
|
||||
callq __pthread_mutex_cond_lock
|
||||
|
||||
movq 24(%rsp), %rdi
|
||||
movq FRAME_SIZE(%rsp), %r14
|
||||
movq FRAME_SIZE+8(%rsp), %r13
|
||||
movq FRAME_SIZE+16(%rsp), %r12
|
||||
movq FRAME_SIZE(%rsp), %r15
|
||||
movq FRAME_SIZE+8(%rsp), %r14
|
||||
movq FRAME_SIZE+16(%rsp), %r13
|
||||
movq FRAME_SIZE+24(%rsp), %r12
|
||||
.LcallUR:
|
||||
call _Unwind_Resume@PLT
|
||||
hlt
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <tcb-offsets.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
|
||||
#include <kernel-features.h>
|
||||
|
||||
@ -47,6 +48,9 @@ __pthread_cond_wait:
|
||||
pushq %r12
|
||||
cfi_adjust_cfa_offset(8)
|
||||
cfi_rel_offset(%r12, 0)
|
||||
pushq %r13
|
||||
cfi_adjust_cfa_offset(8)
|
||||
cfi_rel_offset(%r13, 0)
|
||||
#define FRAME_SIZE 32
|
||||
subq $FRAME_SIZE, %rsp
|
||||
cfi_adjust_cfa_offset(FRAME_SIZE)
|
||||
@ -124,24 +128,48 @@ __pthread_cond_wait:
|
||||
movq 8(%rsp), %rdi
|
||||
xorq %r10, %r10
|
||||
movq %r12, %rdx
|
||||
addq $cond_futex-cond_lock, %rdi
|
||||
// XXX reverse + lea
|
||||
addq $cond_futex, %rdi
|
||||
cmpq $-1, dep_mutex-cond_futex(%rdi)
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
movl $FUTEX_WAIT, %eax
|
||||
movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
|
||||
cmove %eax, %esi
|
||||
#else
|
||||
movl $FUTEX_WAIT, %eax
|
||||
movl $0, %eax
|
||||
movl %fs:PRIVATE_FUTEX, %esi
|
||||
cmove %eax, %esi
|
||||
# if FUTEX_WAIT != 0
|
||||
# error "cc destroyed by following orl"
|
||||
orl $FUTEX_WAIT, %esi
|
||||
# endif
|
||||
#endif
|
||||
je 60f
|
||||
|
||||
movq dep_mutex-cond_futex(%rdi), %r8
|
||||
/* Requeue to a PI mutex if the PI bit is set. */
|
||||
testl $PI_BIT, MUTEX_KIND(%r8)
|
||||
je 60f
|
||||
|
||||
movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
|
||||
movl $SYS_futex, %eax
|
||||
syscall
|
||||
|
||||
movl (%rsp), %edi
|
||||
movl $1, %r13d
|
||||
#ifdef __ASSUME_REQUEUE_PI
|
||||
jmp 62f
|
||||
#else
|
||||
cmpq $-4095, %rax
|
||||
jnae 62f
|
||||
|
||||
movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
|
||||
#endif
|
||||
|
||||
60: xorl %r13d, %r13d
|
||||
movl $SYS_futex, %eax
|
||||
syscall
|
||||
|
||||
62: movl (%rsp), %edi
|
||||
callq __pthread_disable_asynccancel
|
||||
.LcleanupEND:
|
||||
|
||||
@ -209,11 +237,21 @@ __pthread_cond_wait:
|
||||
#endif
|
||||
jne 10f
|
||||
|
||||
11: movq 16(%rsp), %rdi
|
||||
/* If requeue_pi is used the kernel performs the locking of the
|
||||
mutex. */
|
||||
11: xorl %eax, %eax
|
||||
testl %r13d, %r13d
|
||||
jnz 14f
|
||||
|
||||
movq 16(%rsp), %rdi
|
||||
callq __pthread_mutex_cond_lock
|
||||
|
||||
14: addq $FRAME_SIZE, %rsp
|
||||
cfi_adjust_cfa_offset(-FRAME_SIZE)
|
||||
|
||||
popq %r13
|
||||
cfi_adjust_cfa_offset(-8)
|
||||
cfi_restore(%r13)
|
||||
popq %r12
|
||||
cfi_adjust_cfa_offset(-8)
|
||||
cfi_restore(%r12)
|
||||
@ -223,8 +261,9 @@ __pthread_cond_wait:
|
||||
|
||||
/* Initial locking failed. */
|
||||
1:
|
||||
cfi_adjust_cfa_offset(8 + FRAME_SIZE)
|
||||
cfi_rel_offset(%r12, FRAME_SIZE)
|
||||
cfi_adjust_cfa_offset(16 + FRAME_SIZE)
|
||||
cfi_rel_offset(%r12, FRAME_SIZE + 8)
|
||||
cfi_rel_offset(%r13, FRAME_SIZE)
|
||||
#if cond_lock != 0
|
||||
addq $cond_lock, %rdi
|
||||
#endif
|
||||
@ -308,9 +347,11 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
|
||||
__condvar_cleanup1:
|
||||
/* Stack frame:
|
||||
|
||||
rsp + 40
|
||||
rsp + 48
|
||||
+--------------------------+
|
||||
rsp + 32 | %r12 |
|
||||
rsp + 40 | %r12 |
|
||||
+--------------------------+
|
||||
rsp + 32 | %r13 |
|
||||
+--------------------------+
|
||||
rsp + 24 | unused |
|
||||
+--------------------------+
|
||||
@ -431,7 +472,8 @@ __condvar_cleanup1:
|
||||
callq __pthread_mutex_cond_lock
|
||||
|
||||
movq 24(%rsp), %rdi
|
||||
movq 32(%rsp), %r12
|
||||
movq 40(%rsp), %r12
|
||||
movq 32(%rsp), %r13
|
||||
.LcallUR:
|
||||
call _Unwind_Resume@PLT
|
||||
hlt
|
||||
|
||||
Loading…
Reference in New Issue
Block a user