AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-09 23:00:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

FUTEX_*_REQUEUE_PI support for non-x86 code

Browse Source 
Add FUTEX_*_REQUEUE_PI support for the default C code and also add
implementations for s-390 and ppc.
This commit is contained in:
Siddhesh Poyarekar 2013-02-18 16:07:10 +05:30
parent f78b5caa6e
commit 8313cb997d
10 changed files with 261 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
NEWS
View File

@ -9,9 +9,9 @@ Version 2.18
* The following bugs are resolved with this release:
11561, 13951, 14142, 14200, 14317, 14327, 14496, 14964, 14981, 14982,
14985, 14994, 14996, 15003, 15006, 15020, 15023, 15036, 15054, 15062,
15078.
11561, 13951, 14142, 14200, 14317, 14327, 14496, 14920, 14964, 14981,
14982, 14985, 14994, 14996, 15003, 15006, 15020, 15023, 15036, 15054,
15062, 15078.
Version 2.17

30
nptl/ChangeLog
View File

@ -1,3 +1,33 @@
2013-02-18 Siddhesh Poyarekar <siddhesh@redhat.com>
[BZ #14920]
* pthreadP.h (USE_REQUEUE_PI): New macro to check if mutex is
PI-aware.
* pthread_cond_broadcast.c (__pthread_cond_broadcast): Use
PI-aware futex operations if available and mutex is PI-aware.
* pthread_cond_signal.c (__pthread_cond_signal): Likewise.
* nptl/pthread_cond_timedwait.c (__pthread_cond_timedwait):
Likewise.
* pthread_cond_wait.c (__condvar_cleanup): Adjust lock if
cancellation occurred just after futex returned successfully
from a PI operation with the mutex held.
(__pthread_cond_wait): Use PI-aware futex operations if
available and mutex is PI-aware.
* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
(FUTEX_WAIT_REQUEUE_PI): Define.
(FUTEX_CMP_REQUEUE_PI): Likewise.
(lll_futex_wait_requeue_pi): Likewise.
(lll_futex_timed_wait_requeue_pi): Likewise.
(lll_futex_cmp_requeue_pi): Likewise.
* nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
(FUTEX_WAIT_REQUEUE_PI): Define.
(FUTEX_CMP_REQUEUE_PI): Likewise.
(lll_futex_wait_requeue_pi): Likewise.
(lll_futex_timed_wait_requeue_pi): Likewise.
(lll_futex_cmp_requeue_pi): Likewise.
* sysdeps/unix/sysv/linux/kernel-features.h: Define
__ASSUME_REQUEUE_PI for Linux version higher than 2.6.31.
2013-02-04 Andreas Schwab <schwab@suse.de>
[BZ #14142]

12
nptl/pthreadP.h
View File

@ -577,4 +577,16 @@ extern void __wait_lookup_done (void) attribute_hidden;
# define PTHREAD_STATIC_FN_REQUIRE(name) __asm (".globl " #name);
#endif
/* Test if the mutex is suitable for the FUTEX_WAIT_REQUEUE_PI operation. */
#if (defined lll_futex_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
# define USE_REQUEUE_PI(mut) \
((mut) && (mut) != (void *) ~0l \
&& (((mut)->__data.__kind \
& (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) \
== PTHREAD_MUTEX_PRIO_INHERIT_NP))
#else
# define USE_REQUEUE_PI(mut) 0
#endif
#endif /* pthreadP.h */

47
nptl/pthread_cond_broadcast.c
View File

@ -53,34 +53,37 @@ __pthread_cond_broadcast (cond)
/* We are done. */
lll_unlock (cond->__data.__lock, pshared);
/* Do not use requeue for pshared condvars. */
if (cond->__data.__mutex == (void *) ~0l)
goto wake_all;
/* Wake everybody. */
pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
/* XXX: Kernel so far doesn't support requeue to PI futex. */
/* XXX: Kernel so far can only requeue to the same type of futex,
in this case private (we don't requeue for pshared condvars). */
if (__builtin_expect (mut->__data.__kind
& (PTHREAD_MUTEX_PRIO_INHERIT_NP
| PTHREAD_MUTEX_PSHARED_BIT), 0))
/* Do not use requeue for pshared condvars. */
if (mut == (void *) ~0l
|| PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT)
goto wake_all;
/* lll_futex_requeue returns 0 for success and non-zero
for errors. */
if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
INT_MAX, &mut->__data.__lock,
futex_val, LLL_PRIVATE), 0))
{
/* The requeue functionality is not available. */
wake_all:
lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
}
#if (defined lll_futex_cmp_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
pi_flag &= mut->__data.__kind;
/* That's all. */
return 0;
if (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP)
{
if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX,
&mut->__data.__lock, futex_val,
LLL_PRIVATE) == 0)
return 0;
}
else
#endif
/* lll_futex_requeue returns 0 for success and non-zero
for errors. */
if (!__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
INT_MAX, &mut->__data.__lock,
futex_val, LLL_PRIVATE), 0))
return 0;
wake_all:
lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
}
/* We are done. */

33
nptl/pthread_cond_signal.c
View File

@ -47,12 +47,35 @@ __pthread_cond_signal (cond)
++cond->__data.__wakeup_seq;
++cond->__data.__futex;
/* Wake one. */
if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
1, &cond->__data.__lock,
pshared), 0))
return 0;
#if (defined lll_futex_cmp_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
pthread_mutex_t *mut = cond->__data.__mutex;
/* Do not use requeue for pshared condvars. */
if (mut != (void *) ~0l)
pi_flag &= mut->__data.__kind;
if (__builtin_expect (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP, 0)
/* This can only really fail with a ENOSYS, since nobody can modify
futex while we have the cond_lock. */
&& lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
&mut->__data.__lock,
cond->__data.__futex, pshared) == 0)
{
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
else
#endif
/* Wake one. */
if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
1, 1,
&cond->__data.__lock,
pshared), 0))
return 0;
/* Fallback if neither of them work. */
lll_futex_wake (&cond->__data.__futex, 1, pshared);
}

59
nptl/pthread_cond_timedwait.c
View File

@ -64,6 +64,11 @@ __pthread_cond_timedwait (cond, mutex, abstime)
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
#if (defined lll_futex_timed_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
int pi_flag = 0;
#endif
/* Make sure we are alone. */
lll_lock (cond->__data.__lock, pshared);
@ -155,17 +160,46 @@ __pthread_cond_timedwait (cond, mutex, abstime)
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
/* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient
to check just the former. */
#if (defined lll_futex_timed_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
/* If pi_flag remained 1 then it means that we had the lock and the mutex
but a spurious waker raced ahead of us. Give back the mutex before
going into wait again. */
if (pi_flag)
{
__pthread_mutex_cond_lock_adjust (mutex);
__pthread_mutex_unlock_usercnt (mutex, 0);
}
pi_flag = USE_REQUEUE_PI (mutex);
if (pi_flag)
{
unsigned int clockbit = (cond->__data.__nwaiters & 1
? 0 : FUTEX_CLOCK_REALTIME);
err = lll_futex_timed_wait_requeue_pi (&cond->__data.__futex,
futex_val, abstime, clockbit,
&mutex->__data.__lock,
pshared);
pi_flag = (err == 0);
}
else
#endif
{
#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
|| !defined lll_futex_timed_wait_bitset)
/* Wait until woken by signal or broadcast. */
err = lll_futex_timed_wait (&cond->__data.__futex,
futex_val, &rt, pshared);
/* Wait until woken by signal or broadcast. */
err = lll_futex_timed_wait (&cond->__data.__futex,
futex_val, &rt, pshared);
#else
unsigned int clockbit = (cond->__data.__nwaiters & 1
? 0 : FUTEX_CLOCK_REALTIME);
err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
abstime, clockbit, pshared);
unsigned int clockbit = (cond->__data.__nwaiters & 1
? 0 : FUTEX_CLOCK_REALTIME);
err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
abstime, clockbit, pshared);
#endif
}
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
@ -217,7 +251,16 @@ __pthread_cond_timedwait (cond, mutex, abstime)
__pthread_cleanup_pop (&buffer, 0);
/* Get the mutex before returning. */
err = __pthread_mutex_cond_lock (mutex);
#if (defined lll_futex_timed_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
if (pi_flag)
{
__pthread_mutex_cond_lock_adjust (mutex);
err = 0;
}
else
#endif
err = __pthread_mutex_cond_lock (mutex);
return err ?: result;
}

57
nptl/pthread_cond_wait.c
View File

@ -26,7 +26,6 @@
#include <shlib-compat.h>
#include <stap-probe.h>
struct _condvar_cleanup_buffer
{
int oldtype;
@ -85,8 +84,15 @@ __condvar_cleanup (void *arg)
lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared);
/* Get the mutex before returning unless asynchronous cancellation
is in effect. */
__pthread_mutex_cond_lock (cbuffer->mutex);
is in effect. We don't try to get the mutex if we already own it. */
if (!(USE_REQUEUE_PI (cbuffer->mutex))
|| ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK)
!= THREAD_GETMEM (THREAD_SELF, tid)))
{
__pthread_mutex_cond_lock (cbuffer->mutex);
}
else
__pthread_mutex_cond_lock_adjust (cbuffer->mutex);
}
@ -101,6 +107,11 @@ __pthread_cond_wait (cond, mutex)
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
#if (defined lll_futex_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
int pi_flag = 0;
#endif
LIBC_PROBE (cond_wait, 2, cond, mutex);
/* Make sure we are alone. */
@ -144,15 +155,36 @@ __pthread_cond_wait (cond, mutex)
do
{
unsigned int futex_val = cond->__data.__futex;
/* Prepare to wait. Release the condvar futex. */
lll_unlock (cond->__data.__lock, pshared);
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
/* Wait until woken by signal or broadcast. */
lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
#if (defined lll_futex_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
/* If pi_flag remained 1 then it means that we had the lock and the mutex
but a spurious waker raced ahead of us. Give back the mutex before
going into wait again. */
if (pi_flag)
{
__pthread_mutex_cond_lock_adjust (mutex);
__pthread_mutex_unlock_usercnt (mutex, 0);
}
pi_flag = USE_REQUEUE_PI (mutex);
if (pi_flag)
{
err = lll_futex_wait_requeue_pi (&cond->__data.__futex,
futex_val, &mutex->__data.__lock,
pshared);
pi_flag = (err == 0);
}
else
#endif
/* Wait until woken by signal or broadcast. */
lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
@ -189,8 +221,17 @@ __pthread_cond_wait (cond, mutex)
/* The cancellation handling is back to normal, remove the handler. */
__pthread_cleanup_pop (&buffer, 0);
/* Get the mutex before returning. */
return __pthread_mutex_cond_lock (mutex);
/* Get the mutex before returning. Not needed for PI. */
#if (defined lll_futex_wait_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
if (pi_flag)
{
__pthread_mutex_cond_lock_adjust (mutex);
return 0;
}
else
#endif
return __pthread_mutex_cond_lock (mutex);
}
versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,

30
nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
View File

@ -39,6 +39,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
#define FUTEX_WAIT_REQUEUE_PI 11
#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
@ -149,6 +151,34 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
/* Priority Inheritance support. */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
mutex, private) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
int __op = FUTEX_WAIT_REQUEUE_PI | clockbit; \
\
__ret = INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
__lll_private_flag (__op, private), \
(val), (timespec), mutex); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \
})
#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
\
__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
__lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
(nr_wake), (nr_move), (mutex), (val)); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
#ifdef UP
# define __lll_acq_instr ""

28
nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
View File

@ -37,6 +37,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
#define FUTEX_WAIT_REQUEUE_PI 11
#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
@ -141,6 +143,32 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
/* Priority Inheritance support. */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
mutex, private) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
int __op = FUTEX_WAIT_REQUEUE_PI | clockbit; \
\
INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
__lll_private_flag (__op, private), \
(val), (timespec), mutex); \
})
#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
\
__ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
__lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
(nr_wake), (nr_move), (mutex), (val)); \
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
#define lll_compare_and_swap(futex, oldval, newval, operation) \
do { \
__typeof (futex) __futex = (futex); \

5
sysdeps/unix/sysv/linux/kernel-features.h
View File

@ -187,6 +187,11 @@
# define __ASSUME_PWRITEV 1
#endif
/* Support for FUTEX_*_REQUEUE_PI was added in 2.6.31. */
#if __LINUX_KERNEL_VERSION >= 0x02061f
# define __ASSUME_REQUEUE_PI 1
#endif
/* Support for F_GETOWN_EX was introduced in 2.6.32. */
#if __LINUX_KERNEL_VERSION >= 0x020620
# define __ASSUME_F_GETOWN_EX 1