AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-24 14:00:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

nptl: Move elision implementations into libc

Browse Source 
The elision interfaces are closely aligned between the targets that
implement them, so declare them in the generic <lowlevellock.h>
file.

Empty .c stubs are provided, so that fewer makefile updates
under sysdeps are needed.  Also simplify initialization via
__libc_early_init.

The symbols __lll_clocklock_elision, __lll_lock_elision,
__lll_trylock_elision, __lll_unlock_elision, __pthread_force_elision
move into libc.  For the time being, non-hidden references are used
from libpthread to access them, but once that part of libpthread
is moved into libc, hidden symbols will be used again.  (Hidden
references seem desirable to reduce the likelihood of transactions
aborts.)
This commit is contained in:
Florian Weimer 2021-02-23 14:59:34 +01:00
parent 597d0267b5
commit 5a664d7ae8
46 changed files with 185 additions and 646 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
elf/libc_early_init.c
View File

@ -17,8 +17,10 @@
<https://www.gnu.org/licenses/>. */
#include <ctype.h>
#include <elision-conf.h>
#include <libc-early-init.h>
#include <libc-internal.h>
#include <lowlevellock.h>
#include <sys/single_threaded.h>
#ifdef SHARED
@ -37,4 +39,8 @@ __libc_early_init (_Bool initial)
#ifdef SHARED
__libc_initial = initial;
#endif
#if ENABLE_ELISION_SUPPORT
__lll_elision_init ();
#endif
}

5
nptl/Makefile
View File

@ -30,6 +30,11 @@ extra-libs-others := $(extra-libs)
routines = \
alloca_cutoff \
elision-conf \
elision-lock \
elision-timed \
elision-trylock \
elision-unlock \
forward \
futex-internal \
libc-cancellation \

5
nptl/Versions
View File

@ -79,8 +79,12 @@ libc {
__libc_current_sigrtmin_private;
__libc_dl_error_tsd;
__libc_pthread_init;
__lll_clocklock_elision;
__lll_lock_elision;
__lll_lock_wait;
__lll_lock_wait_private;
__lll_trylock_elision;
__lll_unlock_elision;
__pthread_attr_copy;
__pthread_attr_destroy;
__pthread_attr_init;
@ -88,6 +92,7 @@ libc {
__pthread_attr_setsigmask_internal;
__pthread_cond_destroy; # Used by the C11 threads.
__pthread_cond_init; # Used by the C11 threads.
__pthread_force_elision;
__pthread_getattr_default_np;
}
}

9
sysdeps/unix/sysv/linux/s390/pthread_mutex_lock.c → nptl/elision-conf.c
View File

@ -1,5 +1,5 @@
/* Elided version of pthread_mutex_lock.
Copyright (C) 2014-2021 Free Software Foundation, Inc.
/* elision-conf.c: Lock elision tunable parameters. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -15,8 +15,3 @@
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#include <nptl/pthread_mutex_lock.c>

9
sysdeps/unix/sysv/linux/powerpc/pthread_mutex_lock.c → nptl/elision-lock.c
View File

@ -1,5 +1,5 @@
/* Elided version of pthread_mutex_lock.
Copyright (C) 2015-2021 Free Software Foundation, Inc.
/* elision-lock.c: Lock elision locking. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -15,8 +15,3 @@
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#include <nptl/pthread_mutex_lock.c>

9
sysdeps/unix/sysv/linux/powerpc/pthread_mutex_trylock.c → nptl/elision-timed.c
View File

@ -1,5 +1,5 @@
/* Elided version of pthread_mutex_trylock.
Copyright (C) 2015-2021 Free Software Foundation, Inc.
/* elision-lock.c: Lock elision timed locking. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -15,8 +15,3 @@
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#include <nptl/pthread_mutex_trylock.c>

9
sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c → nptl/elision-trylock.c
View File

@ -1,5 +1,5 @@
/* Elided version of pthread_mutex_lock.
Copyright (C) 2011-2021 Free Software Foundation, Inc.
/* elision-lock.c: Lock elision locking attempts. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -15,8 +15,3 @@
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include "force-elision.h"
#include "nptl/pthread_mutex_lock.c"

17
nptl/elision-unlock.c Normal file
View File

@ -0,0 +1,17 @@
/* elision-lock.c: Lock elision unlocking support. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */

5
nptl/lowlevellock.c
View File

@ -51,3 +51,8 @@ __lll_lock_wait (int *futex, int private)
}
}
libc_hidden_def (__lll_lock_wait)
#if ENABLE_ELISION_SUPPORT
int __pthread_force_elision __attribute__ ((nocommon));
libc_hidden_data_def (__pthread_force_elision)
#endif

15
nptl/pthread_mutex_lock.c
View File

@ -27,15 +27,6 @@
#include <futex-internal.h>
#include <stap-probe.h>
#ifndef lll_lock_elision
#define lll_lock_elision(lock, try_lock, private) ({ \
lll_lock (lock, private); 0; })
#endif
#ifndef lll_trylock_elision
#define lll_trylock_elision(a,t) lll_trylock(a)
#endif
/* Some of the following definitions differ when pthread_mutex_cond_lock.c
includes this file. */
#ifndef LLL_MUTEX_LOCK
@ -52,10 +43,6 @@
PTHREAD_MUTEX_PSHARED (mutex))
#endif
#ifndef FORCE_ELISION
#define FORCE_ELISION(m, s)
#endif
static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
__attribute_noinline__;
@ -80,7 +67,7 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
LLL_MUTEX_LOCK (mutex);
assert (mutex->__data.__owner == 0);
}
#ifdef HAVE_ELISION
#if ENABLE_ELISION_SUPPORT
else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
{
elision: __attribute__((unused))

13
nptl/pthread_mutex_timedlock.c
View File

@ -29,19 +29,6 @@
#include <stap-probe.h>
#ifndef lll_clocklock_elision
#define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \
__futex_clocklock64 (&(futex), clockid, abstime, private)
#endif
#ifndef lll_trylock_elision
#define lll_trylock_elision(a,t) lll_trylock(a)
#endif
#ifndef FORCE_ELISION
#define FORCE_ELISION(m, s)
#endif
int
__pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
clockid_t clockid,

8
nptl/pthread_mutex_trylock.c
View File

@ -23,14 +23,6 @@
#include <lowlevellock.h>
#include <futex-internal.h>
#ifndef lll_trylock_elision
#define lll_trylock_elision(a,t) lll_trylock(a)
#endif
#ifndef FORCE_ELISION
#define FORCE_ELISION(m, s)
#endif
int
__pthread_mutex_trylock (pthread_mutex_t *mutex)
{

4
nptl/pthread_mutex_unlock.c
View File

@ -24,10 +24,6 @@
#include <stap-probe.h>
#include <futex-internal.h>
#ifndef lll_unlock_elision
#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
#endif
static int
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
__attribute_noinline__;

102
sysdeps/nptl/lowlevellock.h
View File

@ -20,6 +20,7 @@
#define _LOWLEVELLOCK_H 1
#include <atomic.h>
#include <elision-conf.h>
#include <lowlevellock-futex.h>
#include <time.h>
@ -160,4 +161,105 @@ libc_hidden_proto (__lll_lock_wait)
#define LLL_LOCK_INITIALIZER (0)
#define LLL_LOCK_INITIALIZER_LOCKED (1)
/* Elision support. */
#if ENABLE_ELISION_SUPPORT
/* Force elision for all new locks. This is used to decide whether
existing DEFAULT locks should be automatically upgraded to elision
in pthread_mutex_lock. Disabled for suid programs. Only used when
elision is available. */
extern int __pthread_force_elision;
libc_hidden_proto (__pthread_force_elision)
extern void __lll_elision_init (void) attribute_hidden;
extern int __lll_clocklock_elision (int *futex, short *adapt_count,
clockid_t clockid,
const struct __timespec64 *timeout,
int private);
libc_hidden_proto (__lll_clocklock_elision)
extern int __lll_lock_elision (int *futex, short *adapt_count, int private);
libc_hidden_proto (__lll_lock_elision)
# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT
extern int __lll_unlock_elision (int *lock, short *adapt_count, int private);
# else
extern int __lll_unlock_elision (int *lock, int private);
# endif
libc_hidden_proto (__lll_unlock_elision)
extern int __lll_trylock_elision (int *lock, short *adapt_count);
libc_hidden_proto (__lll_trylock_elision)
# define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
__lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
# define lll_lock_elision(futex, adapt_count, private) \
__lll_lock_elision (&(futex), &(adapt_count), private)
# define lll_trylock_elision(futex, adapt_count) \
__lll_trylock_elision (&(futex), &(adapt_count))
# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT
# define lll_unlock_elision(futex, adapt_count, private) \
__lll_unlock_elision (&(futex), &(adapt_count), private)
# else
# define lll_unlock_elision(futex, adapt_count, private) \
__lll_unlock_elision (&(futex), private)
# endif
/* Automatically enable elision for existing user lock kinds. */
# define FORCE_ELISION(m, s) \
if (__pthread_force_elision) \
{ \
/* See concurrency notes regarding __kind in \
struct __pthread_mutex_s in \
sysdeps/nptl/bits/thread-shared-types.h. \
\
There are the following cases for the kind of a mutex \
(The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
only one of both flags can be set): \
- both flags are not set: \
This is the first lock operation for this mutex. Enable \
elision as it is not enabled so far. \
Note: It can happen that multiple threads are calling e.g. \
pthread_mutex_lock at the same time as the first lock \
operation for this mutex. Then elision is enabled for this \
mutex by multiple threads. Storing with relaxed MO is enough \
as all threads will store the same new value for the kind of \
the mutex. But we have to ensure that we always use the \
elision path regardless if this thread has enabled elision or \
another one. \
\
- PTHREAD_MUTEX_ELISION_NP flag is set: \
Elision was already enabled for this mutex by a previous lock \
operation. See case above. Just use the elision path. \
\
- PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
Elision was explicitly disabled by pthread_mutexattr_settype. \
Do not use the elision path. \
Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
changed after mutex initialization. */ \
int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
{ \
mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
} \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
{ \
s; \
} \
}
#else /* !ENABLE_ELISION_SUPPORT */
# define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \
__futex_clocklock64 (&(futex), clockid, abstime, private)
# define lll_lock_elision(lock, try_lock, private) \
({ lll_lock (lock, private); 0; })
# define lll_trylock_elision(a,t) lll_trylock(a)
# define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
# define FORCE_ELISION(m, s)
#endif /* !ENABLE_ELISION_SUPPORT */
#endif /* lowlevellock.h */

17
sysdeps/unix/sysv/linux/powerpc/pthread_mutex_timedlock.c → sysdeps/pthread/elision-conf.h
View File

@ -1,5 +1,5 @@
/* Elided version of pthread_mutex_timedlock.
Copyright (C) 2015-2021 Free Software Foundation, Inc.
/* elision-conf.h: Lock elision configuration. Stub version.
Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@ -16,7 +16,14 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#ifndef _ELISION_CONF_H
#define _ELISION_CONF_H 1
#include <nptl/pthread_mutex_timedlock.c>
/* No elision support by default. */
#define ENABLE_ELISION_SUPPORT 0
/* Whether __lll_unlock_elision expects a pointer argument to the
adaptive counter. Here, an unused arbitrary value. */
#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0
#endif

2
sysdeps/unix/sysv/linux/powerpc/Makefile
View File

@ -32,7 +32,5 @@ endif
ifeq ($(subdir),nptl)
libpthread-routines += sysdep
libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
elision-trylock
libpthread-shared-only-routines += sysdep
endif

26
sysdeps/unix/sysv/linux/powerpc/elision-conf.c
View File

@ -51,12 +51,6 @@ struct elision_config __elision_aconf =
.skip_trylock_internal_abort = 3,
};
/* Force elision for all new locks. This is used to decide whether existing
DEFAULT locks should be automatically use elision in pthread_mutex_lock().
Disabled for suid programs. Only used when elision is available. */
int __pthread_force_elision attribute_hidden = 0;
#if HAVE_TUNABLES
static inline void
__always_inline
@ -104,10 +98,8 @@ TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t);
/* Initialize elision. */
static void
elision_init (int argc __attribute__ ((unused)),
char **argv __attribute__ ((unused)),
char **environ)
void
__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
@ -150,17 +142,3 @@ elision_init (int argc __attribute__ ((unused)),
if (!__pthread_force_elision)
__elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */
}
#ifdef SHARED
# define INIT_SECTION ".init_array"
# define MAYBE_CONST
#else
# define INIT_SECTION ".preinit_array"
# define MAYBE_CONST const
#endif
void (*MAYBE_CONST __pthread_init_array []) (int, char **, char **)
__attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
{
&elision_init
};

8
sysdeps/unix/sysv/linux/powerpc/elision-conf.h
View File

@ -22,6 +22,9 @@
#include <pthread.h>
#include <time.h>
#define ENABLE_ELISION_SUPPORT 1
#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1
/* Should make sure there is no false sharing on this. */
struct elision_config
{
@ -34,9 +37,4 @@ struct elision_config
extern struct elision_config __elision_aconf attribute_hidden;
extern int __pthread_force_elision attribute_hidden;
/* Tell the test suite to test elision for this architecture. */
#define HAVE_ELISION 1
#endif

7
sysdeps/unix/sysv/linux/powerpc/elision-lock.c
View File

@ -23,12 +23,6 @@
#include <elision-conf.h>
#include "htm.h"
#if !defined(LLL_LOCK) && !defined(EXTRAARG)
/* Make sure the configuration code is always linked in for static
libraries. */
#include "elision-conf.c"
#endif
#ifndef EXTRAARG
# define EXTRAARG
#endif
@ -84,3 +78,4 @@ __lll_lock_elision (int *lock, short *adapt_count, EXTRAARG int pshared)
use_lock:
return LLL_LOCK ((*lock), pshared);
}
libc_hidden_def (__lll_lock_elision)

1
sysdeps/unix/sysv/linux/powerpc/elision-trylock.c
View File

@ -67,3 +67,4 @@ __lll_trylock_elision (int *futex, short *adapt_count)
use_lock:
return lll_trylock (*futex);
}
libc_hidden_def (__lll_trylock_elision)

1
sysdeps/unix/sysv/linux/powerpc/elision-unlock.c
View File

@ -41,3 +41,4 @@ __lll_unlock_elision (int *lock, short *adapt_count, int pshared)
}
return 0;
}
libc_hidden_def (__lll_unlock_elision)

62
sysdeps/unix/sysv/linux/powerpc/force-elision.h
View File

@ -1,62 +0,0 @@
/* force-elision.h: Automatic enabling of elision for mutexes
Copyright (C) 2015-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* Automatically enable elision for existing user lock kinds. */
#define FORCE_ELISION(m, s) \
if (__pthread_force_elision) \
{ \
/* See concurrency notes regarding __kind in \
struct __pthread_mutex_s in \
sysdeps/nptl/bits/thread-shared-types.h. \
\
There are the following cases for the kind of a mutex \
(The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
only one of both flags can be set): \
- both flags are not set: \
This is the first lock operation for this mutex. Enable \
elision as it is not enabled so far. \
Note: It can happen that multiple threads are calling e.g. \
pthread_mutex_lock at the same time as the first lock \
operation for this mutex. Then elision is enabled for this \
mutex by multiple threads. Storing with relaxed MO is enough \
as all threads will store the same new value for the kind of \
the mutex. But we have to ensure that we always use the \
elision path regardless if this thread has enabled elision or \
another one. \
\
- PTHREAD_MUTEX_ELISION_NP flag is set: \
Elision was already enabled for this mutex by a previous lock \
operation. See case above. Just use the elision path. \
\
- PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
Elision was explicitly disabled by pthread_mutexattr_settype. \
Do not use the elision path. \
Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
changed after mutex initialization. */ \
int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
{ \
mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
} \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
{ \
s; \
} \
}

49
sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
View File

@ -1,49 +0,0 @@
/* PowerPC specific lock definitions.
Copyright (C) 2015-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _POWERPC_LOWLEVELLOCK_H
#define _POWERPC_LOWLEVELLOCK_H 1
#include <sysdeps/nptl/lowlevellock.h>
/* Transactional lock elision definitions. */
extern int __lll_clocklock_elision
(int *futex, short *adapt_count,
clockid_t clockid, const struct __timespec64 *timeout, int private)
attribute_hidden;
#define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
__lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
attribute_hidden;
extern int __lll_unlock_elision (int *lock, short *adapt_count, int private)
attribute_hidden;
extern int __lll_trylock_elision(int *lock, short *adapt_count)
attribute_hidden;
#define lll_lock_elision(futex, adapt_count, private) \
__lll_lock_elision (&(futex), &(adapt_count), private)
#define lll_unlock_elision(futex, adapt_count, private) \
__lll_unlock_elision (&(futex), &(adapt_count), private)
#define lll_trylock_elision(futex, adapt_count) \
__lll_trylock_elision (&(futex), &(adapt_count))
#endif

22
sysdeps/unix/sysv/linux/powerpc/pthread_mutex_cond_lock.c
View File

@ -1,22 +0,0 @@
/* Copyright (C) 2015-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* The cond lock is not actually elided yet, but we still need to handle
already elided locks. */
#include <elision-conf.h>
#include <nptl/pthread_mutex_cond_lock.c>

3
sysdeps/unix/sysv/linux/s390/Makefile
View File

@ -12,9 +12,6 @@ gen-as-const-headers += ucontext_i.sym
endif
ifeq ($(subdir),nptl)
libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
elision-trylock
elision-CFLAGS = -mhtm -msoft-float
CFLAGS-elision-lock.c = $(elision-CFLAGS)
CFLAGS-elision-timed.c = $(elision-CFLAGS)

27
sysdeps/unix/sysv/linux/s390/elision-conf.c
View File

@ -51,13 +51,6 @@ struct elision_config __elision_aconf =
.skip_trylock_internal_abort = 3,
};
/* Force elision for all new locks. This is used to decide whether existing
DEFAULT locks should be automatically upgraded to elision in
pthread_mutex_lock(). Disabled for suid programs. Only used when elision
is available. */
int __pthread_force_elision attribute_hidden = 0;
#if HAVE_TUNABLES
static inline void
__always_inline
@ -104,10 +97,8 @@ TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t);
/* Initialize elison. */
static void
elision_init (int argc __attribute__ ((unused)),
char **argv __attribute__ ((unused)),
char **environ)
void
__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
@ -130,17 +121,3 @@ elision_init (int argc __attribute__ ((unused)),
if (!__pthread_force_elision)
__elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */
}
#ifdef SHARED
# define INIT_SECTION ".init_array"
# define MAYBE_CONST
#else
# define INIT_SECTION ".preinit_array"
# define MAYBE_CONST const
#endif
void (*MAYBE_CONST __pthread_init_array []) (int, char **, char **)
__attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
{
&elision_init
};

8
sysdeps/unix/sysv/linux/s390/elision-conf.h
View File

@ -21,6 +21,9 @@
#include <pthread.h>
#include <time.h>
#define ENABLE_ELISION_SUPPORT 1
#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1
/* Should make sure there is no false sharing on this. */
struct elision_config
@ -34,9 +37,4 @@ struct elision_config
extern struct elision_config __elision_aconf attribute_hidden;
extern int __pthread_force_elision attribute_hidden;
/* Tell the test suite to test elision for this architecture. */
#define HAVE_ELISION 1
#endif

7
sysdeps/unix/sysv/linux/s390/elision-lock.c
View File

@ -23,12 +23,6 @@
#include <elision-conf.h>
#include <stdint.h>
#if !defined(LLL_LOCK) && !defined(EXTRAARG)
/* Make sure the configuration code is always linked in for static
libraries. */
#include "elision-conf.c"
#endif
#ifndef EXTRAARG
#define EXTRAARG
#endif
@ -123,3 +117,4 @@ __lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private)
succeed. */
return LLL_LOCK ((*futex), private);
}
libc_hidden_def (__lll_lock_elision)

1
sysdeps/unix/sysv/linux/s390/elision-trylock.c
View File

@ -95,3 +95,4 @@ __lll_trylock_elision (int *futex, short *adapt_count)
succeed. */
return lll_trylock (*futex);
}
libc_hidden_def (__lll_trylock_elision)

1
sysdeps/unix/sysv/linux/s390/elision-unlock.c
View File

@ -59,3 +59,4 @@ __lll_unlock_elision(int *futex, short *adapt_count, int private)
}
return 0;
}
libc_hidden_def (__lll_unlock_elision)

62
sysdeps/unix/sysv/linux/s390/force-elision.h
View File

@ -1,62 +0,0 @@
/* Automatic enabling of elision for mutexes
Copyright (C) 2014-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* Automatically enable elision for existing user lock kinds. */
#define FORCE_ELISION(m, s) \
if (__pthread_force_elision) \
{ \
/* See concurrency notes regarding __kind in \
struct __pthread_mutex_s in \
sysdeps/nptl/bits/thread-shared-types.h. \
\
There are the following cases for the kind of a mutex \
(The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
only one of both flags can be set): \
- both flags are not set: \
This is the first lock operation for this mutex. Enable \
elision as it is not enabled so far. \
Note: It can happen that multiple threads are calling e.g. \
pthread_mutex_lock at the same time as the first lock \
operation for this mutex. Then elision is enabled for this \
mutex by multiple threads. Storing with relaxed MO is enough \
as all threads will store the same new value for the kind of \
the mutex. But we have to ensure that we always use the \
elision path regardless if this thread has enabled elision or \
another one. \
\
- PTHREAD_MUTEX_ELISION_NP flag is set: \
Elision was already enabled for this mutex by a previous lock \
operation. See case above. Just use the elision path. \
\
- PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
Elision was explicitly disabled by pthread_mutexattr_settype. \
Do not use the elision path. \
Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
changed after mutex initialization. */ \
int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
{ \
mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
} \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
{ \
s; \
} \
}

49
sysdeps/unix/sysv/linux/s390/lowlevellock.h
View File

@ -1,49 +0,0 @@
/* Copyright (C) 2003-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _S390_LOWLEVELLOCK_H
#define _S390_LOWLEVELLOCK_H 1
#include <sysdeps/nptl/lowlevellock.h>
/* Transactional lock elision definitions. */
extern int __lll_clocklock_elision
(int *futex, short *adapt_count,
clockid_t clockid, const struct __timespec64 *timeout, int private)
attribute_hidden;
# define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
__lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
attribute_hidden;
extern int __lll_unlock_elision(int *futex, short *adapt_count, int private)
attribute_hidden;
extern int __lll_trylock_elision(int *futex, short *adapt_count)
attribute_hidden;
# define lll_lock_elision(futex, adapt_count, private) \
__lll_lock_elision (&(futex), &(adapt_count), private)
# define lll_unlock_elision(futex, adapt_count, private) \
__lll_unlock_elision (&(futex), &(adapt_count), private)
# define lll_trylock_elision(futex, adapt_count) \
__lll_trylock_elision(&(futex), &(adapt_count))
#endif /* lowlevellock.h */

22
sysdeps/unix/sysv/linux/s390/pthread_mutex_cond_lock.c
View File

@ -1,22 +0,0 @@
/* Copyright (C) 2014-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* The cond lock is not actually elided yet, but we still need to handle
already elided locks. */
#include <elision-conf.h>
#include <nptl/pthread_mutex_cond_lock.c>

22
sysdeps/unix/sysv/linux/s390/pthread_mutex_timedlock.c
View File

@ -1,22 +0,0 @@
/* Elided version of pthread_mutex_timedlock.
Copyright (C) 2014-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#include <nptl/pthread_mutex_timedlock.c>

22
sysdeps/unix/sysv/linux/s390/pthread_mutex_trylock.c
View File

@ -1,22 +0,0 @@
/* Elided version of pthread_mutex_trylock.
Copyright (C) 2014-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include <force-elision.h>
#include <nptl/pthread_mutex_trylock.c>

2
sysdeps/unix/sysv/linux/x86/Makefile
View File

@ -12,8 +12,6 @@ sysdep_headers += sys/elf.h sys/perm.h sys/reg.h sys/vm86.h sys/debugreg.h sys/i
endif
ifeq ($(subdir),nptl)
libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
elision-trylock
CFLAGS-elision-lock.c += -mrtm
CFLAGS-elision-unlock.c += -mrtm
CFLAGS-elision-timed.c += -mrtm

25
sysdeps/unix/sysv/linux/x86/elision-conf.c
View File

@ -48,13 +48,6 @@ struct elision_config __elision_aconf =
.skip_trylock_internal_abort = 3,
};
/* Force elision for all new locks. This is used to decide whether existing
DEFAULT locks should be automatically upgraded to elision in
pthread_mutex_lock(). Disabled for suid programs. Only used when elision
is available. */
int __pthread_force_elision attribute_hidden = 0;
#if HAVE_TUNABLES
static __always_inline void
do_set_elision_enable (int32_t elision_enable)
@ -98,10 +91,8 @@ TUNABLE_CALLBACK_FNDECL (skip_trylock_internal_abort, int32_t);
/* Initialize elision. */
static void
elision_init (int argc __attribute__ ((unused)),
char **argv __attribute__ ((unused)),
char **environ)
void
__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
@ -122,15 +113,3 @@ elision_init (int argc __attribute__ ((unused)),
if (!__pthread_force_elision)
__elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks. */
}
#ifdef SHARED
# define INIT_SECTION ".init_array"
#else
# define INIT_SECTION ".preinit_array"
#endif
void (*const __pthread_init_array []) (int, char **, char **)
__attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
{
&elision_init
};

8
sysdeps/unix/sysv/linux/x86/elision-conf.h
View File

@ -21,6 +21,9 @@
#include <pthread.h>
#include <time.h>
#define ENABLE_ELISION_SUPPORT 1
#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0
/* Should make sure there is no false sharing on this. */
struct elision_config
@ -33,9 +36,4 @@ struct elision_config
extern struct elision_config __elision_aconf attribute_hidden;
extern int __pthread_force_elision attribute_hidden;
/* Tell the test suite to test elision for this architecture. */
#define HAVE_ELISION 1
#endif

7
sysdeps/unix/sysv/linux/x86/elision-lock.c
View File

@ -22,12 +22,6 @@
#include "hle.h"
#include <elision-conf.h>
#if !defined(LLL_LOCK) && !defined(EXTRAARG)
/* Make sure the configuration code is always linked in for static
libraries. */
#include "elision-conf.c"
#endif
#ifndef EXTRAARG
#define EXTRAARG
#endif
@ -105,3 +99,4 @@ __lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private)
/* Use a normal lock here. */
return LLL_LOCK ((*futex), private);
}
libc_hidden_def (__lll_lock_elision)

1
sysdeps/unix/sysv/linux/x86/elision-trylock.c
View File

@ -73,3 +73,4 @@ __lll_trylock_elision (int *futex, short *adapt_count)
return lll_trylock (*futex);
}
libc_hidden_def (__lll_trylock_elision)

1
sysdeps/unix/sysv/linux/x86/elision-unlock.c
View File

@ -31,3 +31,4 @@ __lll_unlock_elision(int *lock, int private)
lll_unlock ((*lock), private);
return 0;
}
libc_hidden_def (__lll_unlock_elision)

62
sysdeps/unix/sysv/linux/x86/force-elision.h
View File

@ -1,62 +0,0 @@
/* force-elision.h: Automatic enabling of elision for mutexes
Copyright (C) 2013-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* Automatically enable elision for existing user lock kinds. */
#define FORCE_ELISION(m, s) \
if (__pthread_force_elision) \
{ \
/* See concurrency notes regarding __kind in \
struct __pthread_mutex_s in \
sysdeps/nptl/bits/thread-shared-types.h. \
\
There are the following cases for the kind of a mutex \
(The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
only one of both flags can be set): \
- both flags are not set: \
This is the first lock operation for this mutex. Enable \
elision as it is not enabled so far. \
Note: It can happen that multiple threads are calling e.g. \
pthread_mutex_lock at the same time as the first lock \
operation for this mutex. Then elision is enabled for this \
mutex by multiple threads. Storing with relaxed MO is enough \
as all threads will store the same new value for the kind of \
the mutex. But we have to ensure that we always use the \
elision path regardless if this thread has enabled elision or \
another one. \
\
- PTHREAD_MUTEX_ELISION_NP flag is set: \
Elision was already enabled for this mutex by a previous lock \
operation. See case above. Just use the elision path. \
\
- PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
Elision was explicitly disabled by pthread_mutexattr_settype. \
Do not use the elision path. \
Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
changed after mutex initialization. */ \
int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
{ \
mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
} \
if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
{ \
s; \
} \
}

24
sysdeps/unix/sysv/linux/x86/lowlevellock.h
View File

@ -82,30 +82,6 @@ __lll_cas_lock (int *futex)
__lll_unlock (&(lock), private); \
}))
extern int __lll_clocklock_elision (int *futex, short *adapt_count,
clockid_t clockid,
const struct __timespec64 *timeout,
int private) attribute_hidden;
#define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
__lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
attribute_hidden;
extern int __lll_unlock_elision (int *lock, int private)
attribute_hidden;
extern int __lll_trylock_elision (int *lock, short *adapt_count)
attribute_hidden;
#define lll_lock_elision(futex, adapt_count, private) \
__lll_lock_elision (&(futex), &(adapt_count), private)
#define lll_unlock_elision(futex, adapt_count, private) \
__lll_unlock_elision (&(futex), private)
#define lll_trylock_elision(futex, adapt_count) \
__lll_trylock_elision (&(futex), &(adapt_count))
#endif /* !__ASSEMBLER__ */
#endif /* lowlevellock.h */

22
sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c
View File

@ -1,22 +0,0 @@
/* Copyright (C) 2013-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* The cond lock is not actually elided yet, but we still need to handle
already elided locks. */
#include <elision-conf.h>
#include <nptl/pthread_mutex_cond_lock.c>

22
sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c
View File

@ -1,22 +0,0 @@
/* Elided version of pthread_mutex_timedlock.
Copyright (C) 2011-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include "force-elision.h"
#include "nptl/pthread_mutex_timedlock.c"

22
sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c
View File

@ -1,22 +0,0 @@
/* Elided version of pthread_mutex_trylock.
Copyright (C) 2011-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <elision-conf.h>
#include "force-elision.h"
#include "nptl/pthread_mutex_trylock.c"