nptl: Use out-of-line wake function in __libc_lock_unlock slow path

This slightly reduces code size, as can be seen below.
__libc_lock_unlock is usually used along with __libc_lock_lock in
the same function.  __libc_lock_lock already has an out-of-line
slow path, so this change should not introduce many additional
non-leaf functions.

This change also fixes a link failure in 32-bit Arm thumb mode
because commit 1f9c804fbd
("nptl: Use internal low-level lock type for !IS_IN (libc)")
introduced __libc_do_syscall calls outside of libc.

Before x86-64:

   text	   data	    bss	    dec	    hex	filename
1937748	  20456	  54896	2013100	 1eb7ac	libc.so.6
  25601	    856	  12768	  39225	   9939	nss/libnss_db.so.2
  40310	    952	  25144	  66406	  10366	nss/libnss_files.so.2

After x86-64:
   text	   data	    bss	    dec	    hex	filename
1935312	  20456	  54896	2010664	 1eae28	libc.so.6
  25559	    864	  12768	  39191	   9917	nss/libnss_db.so.2
  39764	    960	  25144	  65868	  1014c	nss/libnss_files.so.2

Before i686:

2110961	  11272	  39144	2161377	 20fae1	libc.so.6
  27243	    428	  12652	  40323	   9d83	nss/libnss_db.so.2
  43062	    476	  25028	  68566	  10bd6	nss/libnss_files.so.2

After i686:

2107347	  11272	  39144	2157763	 20ecc3	libc.so.6
  26929	    432	  12652	  40013	   9c4d	nss/libnss_db.so.2
  43132	    480	  25028	  68640	  10c20	nss/libnss_files.so.2

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
This commit is contained in:
Florian Weimer 2021-07-09 10:59:22 +02:00
parent 01d7806282
commit 508ee037a3
3 changed files with 33 additions and 8 deletions

View File

@ -380,6 +380,7 @@ libc {
}
GLIBC_PRIVATE {
__libc_alloca_cutoff;
__lll_lock_wake_private;
__lll_lock_wait_private;
__nptl_create_event;
__nptl_death_event;

View File

@ -52,6 +52,20 @@ __lll_lock_wait (int *futex, int private)
}
libc_hidden_def (__lll_lock_wait)
void
__lll_lock_wake_private (int *futex)
{
lll_futex_wake (futex, 1, LLL_PRIVATE);
}
libc_hidden_def (__lll_lock_wake_private)
void
__lll_lock_wake (int *futex, int private)
{
lll_futex_wake (futex, 1, private);
}
libc_hidden_def (__lll_lock_wake)
#if ENABLE_ELISION_SUPPORT
int __pthread_force_elision __attribute__ ((nocommon));
libc_hidden_data_def (__pthread_force_elision)

View File

@ -125,6 +125,11 @@ libc_hidden_proto (__lll_lock_wait)
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
extern void __lll_lock_wake_private (int *futex);
libc_hidden_proto (__lll_lock_wake_private)
extern void __lll_lock_wake (int *futex, int private);
libc_hidden_proto (__lll_lock_wake)
/* This is an expression rather than a statement even though its value is
void, so that it can be used in a comma expression or as an expression
that's cast to void. */
@ -137,14 +142,19 @@ libc_hidden_proto (__lll_lock_wait)
acquires the lock and when there will be no further lock acquisitions;
thus, we must not access the lock after releasing it, or those accesses
could be concurrent with mutex destruction or reuse of the memory. */
#define __lll_unlock(futex, private) \
((void) \
({ \
int *__futex = (futex); \
int __private = (private); \
int __oldval = atomic_exchange_rel (__futex, 0); \
if (__glibc_unlikely (__oldval > 1)) \
lll_futex_wake (__futex, 1, __private); \
#define __lll_unlock(futex, private) \
((void) \
({ \
int *__futex = (futex); \
int __private = (private); \
int __oldval = atomic_exchange_rel (__futex, 0); \
if (__glibc_unlikely (__oldval > 1)) \
{ \
if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
__lll_lock_wake_private (__futex); \
else \
__lll_lock_wake (__futex, __private); \
} \
}))
#define lll_unlock(futex, private) \
__lll_unlock (&(futex), private)