glibc/sysdeps/nptl/lowlevellock.h

194 lines
9.1 KiB
C
Raw Normal View History

/* Low-level lock implementation. Generic futex-based version.
Copyright (C) 2005-2020 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library. If not, see
Prefer https to http for gnu.org and fsf.org URLs Also, change sources.redhat.com to sourceware.org. This patch was automatically generated by running the following shell script, which uses GNU sed, and which avoids modifying files imported from upstream: sed -ri ' s,(http|ftp)(://(.*\.)?(gnu|fsf|sourceware)\.org($|[^.]|\.[^a-z])),https\2,g s,(http|ftp)(://(.*\.)?)sources\.redhat\.com($|[^.]|\.[^a-z]),https\2sourceware.org\4,g ' \ $(find $(git ls-files) -prune -type f \ ! -name '*.po' \ ! -name 'ChangeLog*' \ ! -path COPYING ! -path COPYING.LIB \ ! -path manual/fdl-1.3.texi ! -path manual/lgpl-2.1.texi \ ! -path manual/texinfo.tex ! -path scripts/config.guess \ ! -path scripts/config.sub ! -path scripts/install-sh \ ! -path scripts/mkinstalldirs ! -path scripts/move-if-change \ ! -path INSTALL ! -path locale/programs/charmap-kw.h \ ! -path po/libc.pot ! -path sysdeps/gnu/errlist.c \ ! '(' -name configure \ -execdir test -f configure.ac -o -f configure.in ';' ')' \ ! '(' -name preconfigure \ -execdir test -f preconfigure.ac ';' ')' \ -print) and then by running 'make dist-prepare' to regenerate files built from the altered files, and then executing the following to cleanup: chmod a+x sysdeps/unix/sysv/linux/riscv/configure # Omit irrelevant whitespace and comment-only changes, # perhaps from a slightly-different Autoconf version. git checkout -f \ sysdeps/csky/configure \ sysdeps/hppa/configure \ sysdeps/riscv/configure \ sysdeps/unix/sysv/linux/csky/configure # Omit changes that caused a pre-commit check to fail like this: # remote: *** error: sysdeps/powerpc/powerpc64/ppc-mcount.S: trailing lines git checkout -f \ sysdeps/powerpc/powerpc64/ppc-mcount.S \ sysdeps/unix/sysv/linux/s390/s390-64/syscall.S # Omit change that caused a pre-commit check to fail like this: # remote: *** error: sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: last line does not end in newline git checkout -f sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
2019-09-07 05:40:42 +00:00
<https://www.gnu.org/licenses/>. */
#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H 1
#include <atomic.h>
#include <lowlevellock-futex.h>
#include <time.h>
/* Low-level locks use a combination of atomic operations (to acquire and
release lock ownership) and futex operations (to block until the state
of a lock changes). A lock can be in one of three states:
0: not acquired,
1: acquired with no waiters; no other threads are blocked or about to block
for changes to the lock state,
>1: acquired, possibly with waiters; there may be other threads blocked or
about to block for changes to the lock state.
We expect that the common case is an uncontended lock, so we just need
to transition the lock between states 0 and 1; releasing the lock does
not need to wake any other blocked threads. If the lock is contended
and a thread decides to block using a futex operation, then this thread
needs to first change the state to >1; if this state is observed during
lock release, the releasing thread will wake one of the potentially
blocked threads.
Much of this code takes a 'private' parameter. This may be:
LLL_PRIVATE: lock only shared within a process
LLL_SHARED: lock may be shared across processes.
Condition variables contain an optimization for broadcasts that requeues
waiting threads on a lock's futex. Therefore, there is a special
variant of the locks (whose name contains "cond") that makes sure to
always set the lock state to >1 and not just 1.
Robust locks set the lock to the id of the owner. This allows detection
of the case where the owner exits without releasing the lock. Flags are
OR'd with the owner id to record additional information about lock state.
Therefore the states of robust locks are:
0: not acquired
id: acquired (by user identified by id & FUTEX_TID_MASK)
The following flags may be set in the robust lock value:
FUTEX_WAITERS - possibly has waiters
FUTEX_OWNER_DIED - owning user has exited without releasing the futex. */
/* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return
0. Otherwise leave lock unchanged and return non-zero to indicate that the
lock was not acquired. */
x86: Remove arch-specific low level lock implementation This patch removes the arch-specific x86 assembly implementation for low level locking and consolidate both 64 bits and 32 bits in a single implementation. Different than other architectures, x86 lll_trylock, lll_lock, and lll_unlock implements a single-thread optimization to avoid atomic operation, using cmpxchgl instead. This patch implements by using the new single-thread.h definitions in a generic way, although using the previous semantic. The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use atomic operations plus calls to lll_lock_wait*. For __lll_lock_wait_private and __lll_lock_wait the generic implemtation there is no indication that assembly implementation is required performance-wise. Checked on x86_64-linux-gnu and i686-linux-gnu. * sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro. (lll_trylock): Call __lll_trylock. * sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file. * sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise. * sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise. * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file. * sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include lowlevellock-futex.h.
2019-02-21 16:33:54 +00:00
#define __lll_trylock(lock) \
__glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0))
#define lll_trylock(lock) \
x86: Remove arch-specific low level lock implementation This patch removes the arch-specific x86 assembly implementation for low level locking and consolidate both 64 bits and 32 bits in a single implementation. Different than other architectures, x86 lll_trylock, lll_lock, and lll_unlock implements a single-thread optimization to avoid atomic operation, using cmpxchgl instead. This patch implements by using the new single-thread.h definitions in a generic way, although using the previous semantic. The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use atomic operations plus calls to lll_lock_wait*. For __lll_lock_wait_private and __lll_lock_wait the generic implemtation there is no indication that assembly implementation is required performance-wise. Checked on x86_64-linux-gnu and i686-linux-gnu. * sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro. (lll_trylock): Call __lll_trylock. * sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file. * sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise. * sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise. * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file. * sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include lowlevellock-futex.h.
2019-02-21 16:33:54 +00:00
__lll_trylock (&(lock))
/* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and
return 0. Otherwise leave lock unchanged and return non-zero to indicate
that the lock was not acquired. */
#define lll_cond_trylock(lock) \
__glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 2, 0))
extern void __lll_lock_wait_private (int *futex) attribute_hidden;
extern void __lll_lock_wait (int *futex, int private) attribute_hidden;
/* This is an expression rather than a statement even though its value is
void, so that it can be used in a comma expression or as an expression
that's cast to void. */
/* The inner conditional compiles to a call to __lll_lock_wait_private if
private is known at compile time to be LLL_PRIVATE, and to a call to
__lll_lock_wait otherwise. */
/* If FUTEX is 0 (not acquired), set to 1 (acquired with no waiters) and
return. Otherwise, ensure that it is >1 (acquired, possibly with waiters)
and then block until we acquire the lock, at which point FUTEX will still be
>1. The lock is always acquired on return. */
#define __lll_lock(futex, private) \
((void) \
({ \
int *__futex = (futex); \
if (__glibc_unlikely \
(atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \
{ \
if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
__lll_lock_wait_private (__futex); \
else \
__lll_lock_wait (__futex, private); \
} \
}))
#define lll_lock(futex, private) \
__lll_lock (&(futex), private)
/* This is an expression rather than a statement even though its value is
void, so that it can be used in a comma expression or as an expression
that's cast to void. */
/* Unconditionally set FUTEX to 2 (acquired, possibly with waiters). If FUTEX
was 0 (not acquired) then return. Otherwise, block until the lock is
acquired, at which point FUTEX is 2 (acquired, possibly with waiters). The
lock is always acquired on return. */
#define __lll_cond_lock(futex, private) \
((void) \
({ \
int *__futex = (futex); \
if (__glibc_unlikely (atomic_exchange_acq (__futex, 2) != 0)) \
__lll_lock_wait (__futex, private); \
}))
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
extern int __lll_clocklock_wait (int *futex, int val, clockid_t,
nptl: Rename lll_timedlock to lll_clocklock and add clockid parameter Rename lll_timedlock to lll_clocklock and add clockid parameter to indicate the clock that the abstime parameter should be measured against in preparation for adding pthread_mutex_clocklock. The name change mirrors the naming for the exposed pthread functions: timed => absolute timeout measured against CLOCK_REALTIME (or clock specified by attribute in the case of pthread_cond_timedwait.) clock => absolute timeout measured against clock specified in preceding parameter. * sysdeps/nptl/lowlevellock.h (lll_clocklock): Rename from lll_timedlock and add clockid parameter. (__lll_clocklock): Rename from __lll_timedlock and add clockid parameter. * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_clocklock): Likewise. * nptl/lll_timedlock_wait.c (__lll_clocklock_wait): Rename from __lll_timedlock_wait and add clockid parameter. Use __clock_gettime rather than __gettimeofday so that clockid can be used. This means that conversion from struct timeval is no longer required. * sysdeps/sparc/sparc32/lowlevellock.c (lll_clocklock_wait): Likewise. * sysdeps/sparc/sparc32/lll_timedlock_wait.c: Update comment to refer to __lll_clocklock_wait rather than __lll_timedlock_wait. * nptl/pthread_mutex_timedlock.c (lll_clocklock_elision): Rename from lll_timedlock_elision, add clockid parameter and use meaningful names for other parameters. (__pthread_mutex_timedlock): Pass CLOCK_REALTIME where necessary to lll_clocklock and lll_clocklock_elision. * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_clocklock_elision): Rename from lll_timedlock_elision and add clockid parameter. (__lll_clocklock_elision): Rename from __lll_timedlock_elision and add clockid parameter. * sysdeps/unix/sysv/linux/s390/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/powerpc/elision-timed.c (__lll_lock_elision): Call __lll_clocklock_elision rather than __lll_timedlock_elision. (EXTRAARG): Add clockid parameter. (LLL_LOCK): Likewise. * sysdeps/unix/sysv/linux/s390/elision-timed.c: Likewise. * sysdeps/unix/sysv/linux/x86/elision-timed.c: Likewise. Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
2019-06-24 13:08:25 +00:00
const struct timespec *,
int private) attribute_hidden;
#define lll_timedwait(futex, val, clockid, abstime, private) \
__lll_clocklock_wait (futex, val, clockid, abstime, private)
nptl: Rename lll_timedlock to lll_clocklock and add clockid parameter Rename lll_timedlock to lll_clocklock and add clockid parameter to indicate the clock that the abstime parameter should be measured against in preparation for adding pthread_mutex_clocklock. The name change mirrors the naming for the exposed pthread functions: timed => absolute timeout measured against CLOCK_REALTIME (or clock specified by attribute in the case of pthread_cond_timedwait.) clock => absolute timeout measured against clock specified in preceding parameter. * sysdeps/nptl/lowlevellock.h (lll_clocklock): Rename from lll_timedlock and add clockid parameter. (__lll_clocklock): Rename from __lll_timedlock and add clockid parameter. * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_clocklock): Likewise. * nptl/lll_timedlock_wait.c (__lll_clocklock_wait): Rename from __lll_timedlock_wait and add clockid parameter. Use __clock_gettime rather than __gettimeofday so that clockid can be used. This means that conversion from struct timeval is no longer required. * sysdeps/sparc/sparc32/lowlevellock.c (lll_clocklock_wait): Likewise. * sysdeps/sparc/sparc32/lll_timedlock_wait.c: Update comment to refer to __lll_clocklock_wait rather than __lll_timedlock_wait. * nptl/pthread_mutex_timedlock.c (lll_clocklock_elision): Rename from lll_timedlock_elision, add clockid parameter and use meaningful names for other parameters. (__pthread_mutex_timedlock): Pass CLOCK_REALTIME where necessary to lll_clocklock and lll_clocklock_elision. * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_clocklock_elision): Rename from lll_timedlock_elision and add clockid parameter. (__lll_clocklock_elision): Rename from __lll_timedlock_elision and add clockid parameter. * sysdeps/unix/sysv/linux/s390/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/powerpc/elision-timed.c (__lll_lock_elision): Call __lll_clocklock_elision rather than __lll_timedlock_elision. (EXTRAARG): Add clockid parameter. (LLL_LOCK): Likewise. * sysdeps/unix/sysv/linux/s390/elision-timed.c: Likewise. * sysdeps/unix/sysv/linux/x86/elision-timed.c: Likewise. Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
2019-06-24 13:08:25 +00:00
/* As __lll_lock, but with an absolute timeout measured against the clock
specified in CLOCKID. If the timeout occurs then return ETIMEDOUT. If
ABSTIME is invalid, return EINVAL. */
#define __lll_clocklock(futex, clockid, abstime, private) \
({ \
int *__futex = (futex); \
int __val = 0; \
\
if (__glibc_unlikely \
(atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \
{ \
while (atomic_exchange_acq (futex, 2) != 0) \
{ \
__val = __lll_clocklock_wait (__futex, 2, clockid, \
abstime, private); \
if (__val == EINVAL || __val == ETIMEDOUT) \
break; \
} \
} \
__val; \
})
nptl: Rename lll_timedlock to lll_clocklock and add clockid parameter Rename lll_timedlock to lll_clocklock and add clockid parameter to indicate the clock that the abstime parameter should be measured against in preparation for adding pthread_mutex_clocklock. The name change mirrors the naming for the exposed pthread functions: timed => absolute timeout measured against CLOCK_REALTIME (or clock specified by attribute in the case of pthread_cond_timedwait.) clock => absolute timeout measured against clock specified in preceding parameter. * sysdeps/nptl/lowlevellock.h (lll_clocklock): Rename from lll_timedlock and add clockid parameter. (__lll_clocklock): Rename from __lll_timedlock and add clockid parameter. * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_clocklock): Likewise. * nptl/lll_timedlock_wait.c (__lll_clocklock_wait): Rename from __lll_timedlock_wait and add clockid parameter. Use __clock_gettime rather than __gettimeofday so that clockid can be used. This means that conversion from struct timeval is no longer required. * sysdeps/sparc/sparc32/lowlevellock.c (lll_clocklock_wait): Likewise. * sysdeps/sparc/sparc32/lll_timedlock_wait.c: Update comment to refer to __lll_clocklock_wait rather than __lll_timedlock_wait. * nptl/pthread_mutex_timedlock.c (lll_clocklock_elision): Rename from lll_timedlock_elision, add clockid parameter and use meaningful names for other parameters. (__pthread_mutex_timedlock): Pass CLOCK_REALTIME where necessary to lll_clocklock and lll_clocklock_elision. * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_clocklock_elision): Rename from lll_timedlock_elision and add clockid parameter. (__lll_clocklock_elision): Rename from __lll_timedlock_elision and add clockid parameter. * sysdeps/unix/sysv/linux/s390/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/x86/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/powerpc/elision-timed.c (__lll_lock_elision): Call __lll_clocklock_elision rather than __lll_timedlock_elision. (EXTRAARG): Add clockid parameter. (LLL_LOCK): Likewise. * sysdeps/unix/sysv/linux/s390/elision-timed.c: Likewise. * sysdeps/unix/sysv/linux/x86/elision-timed.c: Likewise. Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
2019-06-24 13:08:25 +00:00
#define lll_clocklock(futex, clockid, abstime, private) \
__lll_clocklock (&(futex), clockid, abstime, private)
/* This is an expression rather than a statement even though its value is
void, so that it can be used in a comma expression or as an expression
that's cast to void. */
/* Unconditionally set FUTEX to 0 (not acquired), releasing the lock. If FUTEX
was >1 (acquired, possibly with waiters), then wake any waiters. The waiter
that acquires the lock will set FUTEX to >1.
Evaluate PRIVATE before releasing the lock so that we do not violate the
mutex destruction requirements. Specifically, we need to ensure that
another thread can destroy the mutex (and reuse its memory) once it
acquires the lock and when there will be no further lock acquisitions;
thus, we must not access the lock after releasing it, or those accesses
could be concurrent with mutex destruction or reuse of the memory. */
#define __lll_unlock(futex, private) \
((void) \
({ \
int *__futex = (futex); \
int __private = (private); \
int __oldval = atomic_exchange_rel (__futex, 0); \
if (__glibc_unlikely (__oldval > 1)) \
lll_futex_wake (__futex, 1, __private); \
}))
#define lll_unlock(futex, private) \
__lll_unlock (&(futex), private)
#define lll_islocked(futex) \
((futex) != LLL_LOCK_INITIALIZER)
/* Our internal lock implementation is identical to the binary-compatible
mutex implementation. */
/* Initializers for lock. */
#define LLL_LOCK_INITIALIZER (0)
#define LLL_LOCK_INITIALIZER_LOCKED (1)
#endif /* lowlevellock.h */