mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-09 14:50:05 +00:00
nptl: Move stack list variables into _rtld_global
Now __thread_gscope_wait (the function behind THREAD_GSCOPE_WAIT, formerly __wait_lookup_done) can be implemented directly in ld.so, eliminating the unprotected GL (dl_wait_lookup_done) function pointer. Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
This commit is contained in:
parent
aac0f62c47
commit
1daccf403b
@ -24,6 +24,7 @@
|
||||
#include <stdio.h>
|
||||
#include <sys/param.h>
|
||||
#include <array_length.h>
|
||||
#include <list.h>
|
||||
|
||||
#ifdef SHARED
|
||||
#error makefile bug, this file is for static only
|
||||
@ -193,6 +194,12 @@ __libc_setup_tls (void)
|
||||
if (__builtin_expect (lossage != NULL, 0))
|
||||
_startup_fatal (lossage);
|
||||
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
INIT_LIST_HEAD (&_dl_stack_used);
|
||||
INIT_LIST_HEAD (&_dl_stack_user);
|
||||
list_add (&THREAD_SELF->list, &_dl_stack_user);
|
||||
#endif
|
||||
|
||||
/* Update the executable's link map with enough information to make
|
||||
the TLS routines happy. */
|
||||
main_map->l_tls_align = align;
|
||||
|
@ -34,7 +34,8 @@ dl-routines = $(addprefix dl-,load lookup object reloc deps \
|
||||
version profile tls origin scope \
|
||||
execstack open close trampoline \
|
||||
exception sort-maps lookup-direct \
|
||||
call-libc-early-init write)
|
||||
call-libc-early-init write \
|
||||
thread_gscope_wait)
|
||||
ifeq (yes,$(use-ldconfig))
|
||||
dl-routines += dl-cache
|
||||
endif
|
||||
@ -54,7 +55,8 @@ endif
|
||||
all-dl-routines = $(dl-routines) $(sysdep-dl-routines)
|
||||
# But they are absent from the shared libc, because that code is in ld.so.
|
||||
elide-routines.os = $(all-dl-routines) dl-support enbl-secure dl-origin \
|
||||
dl-sysdep dl-exception dl-reloc-static-pie
|
||||
dl-sysdep dl-exception dl-reloc-static-pie \
|
||||
thread_gscope_wait
|
||||
|
||||
# ld.so uses those routines, plus some special stuff for being the program
|
||||
# interpreter and operating independent of libc.
|
||||
|
@ -183,10 +183,11 @@ ElfW(Word) _dl_stack_flags = DEFAULT_STACK_PERMS;
|
||||
int (*_dl_make_stack_executable_hook) (void **) = _dl_make_stack_executable;
|
||||
|
||||
|
||||
/* Function in libpthread to wait for termination of lookups. */
|
||||
void (*_dl_wait_lookup_done) (void);
|
||||
|
||||
#if !THREAD_GSCOPE_IN_TCB
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
list_t _dl_stack_used;
|
||||
list_t _dl_stack_user;
|
||||
int _dl_stack_cache_lock;
|
||||
#else
|
||||
int _dl_thread_gscope_count;
|
||||
#endif
|
||||
struct dl_scope_free_list *_dl_scope_free_list;
|
||||
|
2
elf/dl-thread_gscope_wait.c
Normal file
2
elf/dl-thread_gscope_wait.c
Normal file
@ -0,0 +1,2 @@
|
||||
/* By default, the dynamic linker does not use an out-of-line
|
||||
__thread_gscope_wait function. */
|
12
elf/rtld.c
12
elf/rtld.c
@ -48,6 +48,7 @@
|
||||
#include <array_length.h>
|
||||
#include <libc-early-init.h>
|
||||
#include <dl-main.h>
|
||||
#include <list.h>
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
@ -799,6 +800,9 @@ cannot allocate TLS data structures for initial thread\n");
|
||||
const char *lossage = TLS_INIT_TP (tcbp);
|
||||
if (__glibc_unlikely (lossage != NULL))
|
||||
_dl_fatal_printf ("cannot set up thread-local storage: %s\n", lossage);
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
list_add (&THREAD_SELF->list, &GL (dl_stack_user));
|
||||
#endif
|
||||
tls_init_tp_called = true;
|
||||
|
||||
return tcbp;
|
||||
@ -1139,6 +1143,11 @@ dl_main (const ElfW(Phdr) *phdr,
|
||||
GL(dl_rtld_unlock_recursive) = rtld_lock_default_unlock_recursive;
|
||||
#endif
|
||||
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
INIT_LIST_HEAD (&GL (dl_stack_used));
|
||||
INIT_LIST_HEAD (&GL (dl_stack_user));
|
||||
#endif
|
||||
|
||||
/* The explicit initialization here is cheaper than processing the reloc
|
||||
in the _rtld_local definition's initializer. */
|
||||
GL(dl_make_stack_executable_hook) = &_dl_make_stack_executable;
|
||||
@ -2383,6 +2392,9 @@ dl_main (const ElfW(Phdr) *phdr,
|
||||
if (__glibc_unlikely (lossage != NULL))
|
||||
_dl_fatal_printf ("cannot set up thread-local storage: %s\n",
|
||||
lossage);
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
list_add (&THREAD_SELF->list, &GL (dl_stack_user));
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Make sure no new search directories have been added. */
|
||||
|
@ -106,26 +106,14 @@
|
||||
static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
|
||||
static size_t stack_cache_actsize;
|
||||
|
||||
/* Mutex protecting this variable. */
|
||||
static int stack_cache_lock = LLL_LOCK_INITIALIZER;
|
||||
|
||||
/* List of queued stack frames. */
|
||||
static LIST_HEAD (stack_cache);
|
||||
|
||||
/* List of the stacks in use. */
|
||||
static LIST_HEAD (stack_used);
|
||||
|
||||
/* We need to record what list operations we are going to do so that,
|
||||
in case of an asynchronous interruption due to a fork() call, we
|
||||
can correct for the work. */
|
||||
static uintptr_t in_flight_stack;
|
||||
|
||||
/* List of the threads with user provided stacks in use. No need to
|
||||
initialize this, since it's done in __pthread_initialize_minimal. */
|
||||
list_t __stack_user __attribute__ ((nocommon));
|
||||
hidden_data_def (__stack_user)
|
||||
|
||||
|
||||
/* Check whether the stack is still used or not. */
|
||||
#define FREE_P(descr) ((descr)->tid <= 0)
|
||||
|
||||
@ -173,7 +161,7 @@ get_cached_stack (size_t *sizep, void **memp)
|
||||
struct pthread *result = NULL;
|
||||
list_t *entry;
|
||||
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Search the cache for a matching entry. We search for the
|
||||
smallest stack which has at least the required size. Note that
|
||||
@ -206,7 +194,7 @@ get_cached_stack (size_t *sizep, void **memp)
|
||||
|| __builtin_expect (result->stackblock_size > 4 * size, 0))
|
||||
{
|
||||
/* Release the lock. */
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
@ -218,13 +206,13 @@ get_cached_stack (size_t *sizep, void **memp)
|
||||
stack_list_del (&result->list);
|
||||
|
||||
/* And add to the list of stacks in use. */
|
||||
stack_list_add (&result->list, &stack_used);
|
||||
stack_list_add (&result->list, &GL (dl_stack_used));
|
||||
|
||||
/* And decrease the cache size. */
|
||||
stack_cache_actsize -= result->stackblock_size;
|
||||
|
||||
/* Release the lock early. */
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Report size and location of the stack to the caller. */
|
||||
*sizep = result->stackblock_size;
|
||||
@ -510,12 +498,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
|
||||
|
||||
|
||||
/* Prepare to modify global data. */
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* And add to the list of stacks in use. */
|
||||
list_add (&pd->list, &__stack_user);
|
||||
list_add (&pd->list, &GL (dl_stack_user));
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -644,12 +632,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
|
||||
|
||||
|
||||
/* Prepare to modify global data. */
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* And add to the list of stacks in use. */
|
||||
stack_list_add (&pd->list, &stack_used);
|
||||
stack_list_add (&pd->list, &GL (dl_stack_used));
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
|
||||
/* There might have been a race. Another thread might have
|
||||
@ -690,12 +678,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
|
||||
if (__mprotect (guard, guardsize, PROT_NONE) != 0)
|
||||
{
|
||||
mprot_error:
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Remove the thread from the list. */
|
||||
stack_list_del (&pd->list);
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Get rid of the TLS block we allocated. */
|
||||
_dl_deallocate_tls (TLS_TPADJ (pd), false);
|
||||
@ -799,7 +787,7 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
|
||||
void
|
||||
__deallocate_stack (struct pthread *pd)
|
||||
{
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Remove the thread from the list of threads with user defined
|
||||
stacks. */
|
||||
@ -815,7 +803,7 @@ __deallocate_stack (struct pthread *pd)
|
||||
/* Free the memory associated with the ELF TLS. */
|
||||
_dl_deallocate_tls (TLS_TPADJ (pd), false);
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
}
|
||||
|
||||
|
||||
@ -831,10 +819,10 @@ __make_stacks_executable (void **stack_endp)
|
||||
const size_t pagemask = ~(__getpagesize () - 1);
|
||||
#endif
|
||||
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
list_t *runp;
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
err = change_stack_perm (list_entry (runp, struct pthread, list)
|
||||
#ifdef NEED_SEPARATE_REGISTER_STACK
|
||||
@ -860,7 +848,7 @@ __make_stacks_executable (void **stack_endp)
|
||||
break;
|
||||
}
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
return err;
|
||||
}
|
||||
@ -891,8 +879,8 @@ __reclaim_stacks (void)
|
||||
pointers at the head of the list are inconsistent. */
|
||||
list_t *l = NULL;
|
||||
|
||||
if (stack_used.next->prev != &stack_used)
|
||||
l = &stack_used;
|
||||
if (GL (dl_stack_used).next->prev != &GL (dl_stack_used))
|
||||
l = &GL (dl_stack_used);
|
||||
else if (stack_cache.next->prev != &stack_cache)
|
||||
l = &stack_cache;
|
||||
|
||||
@ -914,7 +902,7 @@ __reclaim_stacks (void)
|
||||
|
||||
/* Mark all stacks except the still running one as free. */
|
||||
list_t *runp;
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *curp = list_entry (runp, struct pthread, list);
|
||||
if (curp != self)
|
||||
@ -948,7 +936,7 @@ __reclaim_stacks (void)
|
||||
}
|
||||
|
||||
/* Add the stack of all running threads to the cache. */
|
||||
list_splice (&stack_used, &stack_cache);
|
||||
list_splice (&GL (dl_stack_used), &stack_cache);
|
||||
|
||||
/* Remove the entry for the current thread to from the cache list
|
||||
and add it to the list of running threads. Which of the two
|
||||
@ -956,13 +944,13 @@ __reclaim_stacks (void)
|
||||
stack_list_del (&self->list);
|
||||
|
||||
/* Re-initialize the lists for all the threads. */
|
||||
INIT_LIST_HEAD (&stack_used);
|
||||
INIT_LIST_HEAD (&__stack_user);
|
||||
INIT_LIST_HEAD (&GL (dl_stack_used));
|
||||
INIT_LIST_HEAD (&GL (dl_stack_user));
|
||||
|
||||
if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
|
||||
list_add (&self->list, &__stack_user);
|
||||
list_add (&self->list, &GL (dl_stack_user));
|
||||
else
|
||||
list_add (&self->list, &stack_used);
|
||||
list_add (&self->list, &GL (dl_stack_used));
|
||||
|
||||
/* There is one thread running. */
|
||||
__nptl_nthreads = 1;
|
||||
@ -970,7 +958,7 @@ __reclaim_stacks (void)
|
||||
in_flight_stack = 0;
|
||||
|
||||
/* Initialize locks. */
|
||||
stack_cache_lock = LLL_LOCK_INITIALIZER;
|
||||
GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
|
||||
__default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
|
||||
}
|
||||
|
||||
@ -1083,7 +1071,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
{
|
||||
int signalled;
|
||||
int result;
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
__xidcmd = cmdp;
|
||||
cmdp->cntr = 0;
|
||||
@ -1093,7 +1081,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
|
||||
/* Iterate over the list with system-allocated threads first. */
|
||||
list_t *runp;
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1103,7 +1091,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
}
|
||||
|
||||
/* Now the list with threads using user-allocated stacks. */
|
||||
list_for_each (runp, &__stack_user)
|
||||
list_for_each (runp, &GL (dl_stack_user))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1119,7 +1107,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
{
|
||||
signalled = 0;
|
||||
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1128,7 +1116,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
signalled += setxid_signal_thread (cmdp, t);
|
||||
}
|
||||
|
||||
list_for_each (runp, &__stack_user)
|
||||
list_for_each (runp, &GL (dl_stack_user))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1149,7 +1137,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
|
||||
/* Clean up flags, so that no thread blocks during exit waiting
|
||||
for a signal which will never come. */
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1158,7 +1146,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
setxid_unmark_thread (cmdp, t);
|
||||
}
|
||||
|
||||
list_for_each (runp, &__stack_user)
|
||||
list_for_each (runp, &GL (dl_stack_user))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self)
|
||||
@ -1180,7 +1168,7 @@ __nptl_setxid (struct xid_command *cmdp)
|
||||
}
|
||||
__nptl_setxid_error (cmdp, error);
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -1204,75 +1192,16 @@ void
|
||||
attribute_hidden
|
||||
__pthread_init_static_tls (struct link_map *map)
|
||||
{
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
/* Iterate over the list with system-allocated threads first. */
|
||||
list_t *runp;
|
||||
list_for_each (runp, &stack_used)
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
init_one_static_tls (list_entry (runp, struct pthread, list), map);
|
||||
|
||||
/* Now the list with threads using user-allocated stacks. */
|
||||
list_for_each (runp, &__stack_user)
|
||||
list_for_each (runp, &GL (dl_stack_user))
|
||||
init_one_static_tls (list_entry (runp, struct pthread, list), map);
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
attribute_hidden
|
||||
__wait_lookup_done (void)
|
||||
{
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
|
||||
struct pthread *self = THREAD_SELF;
|
||||
|
||||
/* Iterate over the list with system-allocated threads first. */
|
||||
list_t *runp;
|
||||
list_for_each (runp, &stack_used)
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
|
||||
continue;
|
||||
|
||||
int *const gscope_flagp = &t->header.gscope_flag;
|
||||
|
||||
/* We have to wait until this thread is done with the global
|
||||
scope. First tell the thread that we are waiting and
|
||||
possibly have to be woken. */
|
||||
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT,
|
||||
THREAD_GSCOPE_FLAG_USED))
|
||||
continue;
|
||||
|
||||
do
|
||||
futex_wait_simple ((unsigned int *) gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
|
||||
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
|
||||
}
|
||||
|
||||
/* Now the list with threads using user-allocated stacks. */
|
||||
list_for_each (runp, &__stack_user)
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
|
||||
continue;
|
||||
|
||||
int *const gscope_flagp = &t->header.gscope_flag;
|
||||
|
||||
/* We have to wait until this thread is done with the global
|
||||
scope. First tell the thread that we are waiting and
|
||||
possibly have to be woken. */
|
||||
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT,
|
||||
THREAD_GSCOPE_FLAG_USED))
|
||||
continue;
|
||||
|
||||
do
|
||||
futex_wait_simple ((unsigned int *) gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
|
||||
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
|
||||
}
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
}
|
||||
|
@ -162,7 +162,8 @@ struct pthread
|
||||
void *__padding[24];
|
||||
};
|
||||
|
||||
/* This descriptor's link on the `stack_used' or `__stack_user' list. */
|
||||
/* This descriptor's link on the GL (dl_stack_used) or
|
||||
GL (dl_stack_user) list. */
|
||||
list_t list;
|
||||
|
||||
/* Thread ID - which is also a 'is this thread descriptor (and
|
||||
|
@ -251,12 +251,9 @@ __pthread_initialize_minimal_internal (void)
|
||||
purposes this is good enough. */
|
||||
THREAD_SETMEM (pd, stackblock_size, (size_t) __libc_stack_end);
|
||||
|
||||
/* Initialize the list of all running threads with the main thread. */
|
||||
INIT_LIST_HEAD (&__stack_user);
|
||||
list_add (&pd->list, &__stack_user);
|
||||
|
||||
/* Before initializing __stack_user, the debugger could not find us and
|
||||
had to set __nptl_initial_report_events. Propagate its setting. */
|
||||
/* Before initializing GL (dl_stack_user), the debugger could not
|
||||
find us and had to set __nptl_initial_report_events. Propagate
|
||||
its setting. */
|
||||
THREAD_SETMEM (pd, report_events, __nptl_initial_report_events);
|
||||
|
||||
struct sigaction sa;
|
||||
@ -336,8 +333,6 @@ __pthread_initialize_minimal_internal (void)
|
||||
|
||||
GL(dl_init_static_tls) = &__pthread_init_static_tls;
|
||||
|
||||
GL(dl_wait_lookup_done) = &__wait_lookup_done;
|
||||
|
||||
/* Register the fork generation counter with the libc. */
|
||||
#ifndef TLS_MULTIPLE_THREADS_IN_TCB
|
||||
__libc_multiple_threads_ptr =
|
||||
|
@ -208,10 +208,6 @@ extern void __default_pthread_attr_freeres (void) attribute_hidden;
|
||||
extern size_t __static_tls_size attribute_hidden;
|
||||
extern size_t __static_tls_align_m1 attribute_hidden;
|
||||
|
||||
/* Thread descriptor handling. */
|
||||
extern list_t __stack_user;
|
||||
hidden_proto (__stack_user)
|
||||
|
||||
/* Attribute handling. */
|
||||
extern struct pthread_attr *__attr_list attribute_hidden;
|
||||
extern int __attr_list_lock attribute_hidden;
|
||||
|
@ -213,9 +213,9 @@ __find_in_stack_list (struct pthread *pd)
|
||||
list_t *entry;
|
||||
struct pthread *result = NULL;
|
||||
|
||||
lll_lock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
list_for_each (entry, &stack_used)
|
||||
list_for_each (entry, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *curp;
|
||||
|
||||
@ -228,7 +228,7 @@ __find_in_stack_list (struct pthread *pd)
|
||||
}
|
||||
|
||||
if (result == NULL)
|
||||
list_for_each (entry, &__stack_user)
|
||||
list_for_each (entry, &GL (dl_stack_user))
|
||||
{
|
||||
struct pthread *curp;
|
||||
|
||||
@ -240,7 +240,7 @@ __find_in_stack_list (struct pthread *pd)
|
||||
}
|
||||
}
|
||||
|
||||
lll_unlock (stack_cache_lock, LLL_PRIVATE);
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
@ -69,8 +69,6 @@ DB_STRUCT (td_eventbuf_t)
|
||||
DB_STRUCT_FIELD (td_eventbuf_t, eventnum)
|
||||
DB_STRUCT_FIELD (td_eventbuf_t, eventdata)
|
||||
|
||||
DB_SYMBOL (stack_used)
|
||||
DB_SYMBOL (__stack_user)
|
||||
DB_SYMBOL (nptl_version)
|
||||
DB_FUNCTION (__nptl_create_event)
|
||||
DB_FUNCTION (__nptl_death_event)
|
||||
@ -106,6 +104,8 @@ DB_STRUCT (rtld_global)
|
||||
DB_RTLD_VARIABLE (_rtld_global)
|
||||
#endif
|
||||
DB_RTLD_GLOBAL_FIELD (dl_tls_dtv_slotinfo_list)
|
||||
DB_RTLD_GLOBAL_FIELD (dl_stack_user)
|
||||
DB_RTLD_GLOBAL_FIELD (dl_stack_used)
|
||||
|
||||
DB_STRUCT (dtv_slotinfo_list)
|
||||
DB_STRUCT_FIELD (dtv_slotinfo_list, len)
|
||||
|
@ -29,3 +29,17 @@ td_init (void)
|
||||
LOG ("td_init");
|
||||
return TD_OK;
|
||||
}
|
||||
|
||||
bool
|
||||
__td_ta_rtld_global (td_thragent_t *ta)
|
||||
{
|
||||
if (ta->ta_addr__rtld_global == 0
|
||||
&& td_mod_lookup (ta->ph, LD_SO, SYM__rtld_global,
|
||||
&ta->ta_addr__rtld_global) != PS_OK)
|
||||
{
|
||||
ta->ta_addr__rtld_global = (void*)-1;
|
||||
return false;
|
||||
}
|
||||
else
|
||||
return ta->ta_addr__rtld_global != (void*)-1;
|
||||
}
|
||||
|
@ -187,7 +187,7 @@ td_ta_map_lwp2thr (const td_thragent_t *ta_arg,
|
||||
fake a special descriptor for the initial thread. */
|
||||
|
||||
psaddr_t list;
|
||||
td_err_e err = DB_GET_SYMBOL (list, ta, __stack_user);
|
||||
td_err_e err = __td_ta_stack_user (ta, &list);
|
||||
if (err != TD_OK)
|
||||
return err;
|
||||
|
||||
|
@ -133,14 +133,14 @@ td_ta_thr_iter (const td_thragent_t *ta_arg, td_thr_iter_f *callback,
|
||||
have to iterate over both lists separately. We start with the
|
||||
list of threads with user-defined stacks. */
|
||||
|
||||
err = DB_GET_SYMBOL (list, ta, __stack_user);
|
||||
err = __td_ta_stack_user (ta, &list);
|
||||
if (err == TD_OK)
|
||||
err = iterate_thread_list (ta, callback, cbdata_p, state, ti_pri,
|
||||
list, true);
|
||||
|
||||
/* And the threads with stacks allocated by the implementation. */
|
||||
if (err == TD_OK)
|
||||
err = DB_GET_SYMBOL (list, ta, stack_used);
|
||||
err = __td_ta_stack_used (ta, &list);
|
||||
if (err == TD_OK)
|
||||
err = iterate_thread_list (ta, callback, cbdata_p, state, ti_pri,
|
||||
list, false);
|
||||
|
@ -28,12 +28,7 @@ dtv_slotinfo_list (td_thragent_t *ta,
|
||||
td_err_e err;
|
||||
psaddr_t head;
|
||||
|
||||
if (ta->ta_addr__rtld_global == 0
|
||||
&& td_mod_lookup (ta->ph, LD_SO, SYM__rtld_global,
|
||||
&ta->ta_addr__rtld_global) != PS_OK)
|
||||
ta->ta_addr__rtld_global = (void*)-1;
|
||||
|
||||
if (ta->ta_addr__rtld_global != (void*)-1)
|
||||
if (__td_ta_rtld_global (ta))
|
||||
{
|
||||
err = DB_GET_FIELD (head, ta, ta->ta_addr__rtld_global,
|
||||
rtld_global, _dl_tls_dtv_slotinfo_list, 0);
|
||||
|
@ -20,6 +20,41 @@
|
||||
#include "thread_dbP.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
td_err_e
|
||||
__td_ta_stack_user (td_thragent_t *ta, psaddr_t *plist)
|
||||
{
|
||||
if (__td_ta_rtld_global (ta))
|
||||
return DB_GET_FIELD_ADDRESS (*plist, ta, ta->ta_addr__rtld_global,
|
||||
rtld_global, _dl_stack_user, 0);
|
||||
else
|
||||
{
|
||||
if (ta->ta_addr__dl_stack_user == 0
|
||||
&& td_mod_lookup (ta->ph, NULL, SYM__dl_stack_user,
|
||||
&ta->ta_addr__dl_stack_user) != PS_OK)
|
||||
return TD_ERR;
|
||||
*plist = ta->ta_addr__dl_stack_user;
|
||||
return TD_OK;
|
||||
}
|
||||
}
|
||||
|
||||
td_err_e
|
||||
__td_ta_stack_used (td_thragent_t *ta, psaddr_t *plist)
|
||||
{
|
||||
|
||||
if (__td_ta_rtld_global (ta))
|
||||
return DB_GET_FIELD_ADDRESS (*plist, ta, ta->ta_addr__rtld_global,
|
||||
rtld_global, _dl_stack_used, 0);
|
||||
else
|
||||
{
|
||||
if (ta->ta_addr__dl_stack_used == 0
|
||||
&& td_mod_lookup (ta->ph, NULL, SYM__dl_stack_used,
|
||||
&ta->ta_addr__dl_stack_used) != PS_OK)
|
||||
return TD_ERR;
|
||||
*plist = ta->ta_addr__dl_stack_used;
|
||||
return TD_OK;
|
||||
}
|
||||
}
|
||||
|
||||
static td_err_e
|
||||
check_thread_list (const td_thrhandle_t *th, psaddr_t head, bool *uninit)
|
||||
{
|
||||
@ -62,7 +97,7 @@ td_thr_validate (const td_thrhandle_t *th)
|
||||
|
||||
/* First check the list with threads using user allocated stacks. */
|
||||
bool uninit = false;
|
||||
err = DB_GET_SYMBOL (list, th->th_ta_p, __stack_user);
|
||||
err = __td_ta_stack_user (th->th_ta_p, &list);
|
||||
if (err == TD_OK)
|
||||
err = check_thread_list (th, list, &uninit);
|
||||
|
||||
@ -70,7 +105,7 @@ td_thr_validate (const td_thrhandle_t *th)
|
||||
using implementation allocated stacks. */
|
||||
if (err == TD_NOTHR)
|
||||
{
|
||||
err = DB_GET_SYMBOL (list, th->th_ta_p, stack_used);
|
||||
err = __td_ta_stack_used (th->th_ta_p, &list);
|
||||
if (err == TD_OK)
|
||||
err = check_thread_list (th, list, &uninit);
|
||||
|
||||
|
@ -269,4 +269,17 @@ extern td_err_e _td_check_sizeof (td_thragent_t *ta, uint32_t *sizep,
|
||||
extern td_err_e __td_ta_lookup_th_unique (const td_thragent_t *ta,
|
||||
lwpid_t lwpid, td_thrhandle_t *th);
|
||||
|
||||
/* Try to initialize TA->ta_addr__rtld_global. Return true on
|
||||
success, false on failure (which may be cached). */
|
||||
bool __td_ta_rtld_global (td_thragent_t *ta) attribute_hidden;
|
||||
|
||||
/* Obtain the address of the list_t fields _dl_stack_user and
|
||||
_dl_stack_used in _rtld_global, or fall back to the global
|
||||
variables of the same name (to support statically linked
|
||||
programs). */
|
||||
td_err_e __td_ta_stack_user (td_thragent_t *ta, psaddr_t *plist)
|
||||
attribute_hidden;
|
||||
td_err_e __td_ta_stack_used (td_thragent_t *ta, psaddr_t *plist)
|
||||
attribute_hidden;
|
||||
|
||||
#endif /* thread_dbP.h */
|
||||
|
@ -129,8 +129,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
# define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
# endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -123,8 +123,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#else /* __ASSEMBLER__ */
|
||||
# include <tcb-offsets.h>
|
||||
|
@ -131,8 +131,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
|
@ -120,8 +120,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -147,8 +147,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
# define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -38,6 +38,7 @@
|
||||
#include <libc-lock.h>
|
||||
#include <hp-timing.h>
|
||||
#include <tls.h>
|
||||
#include <list_t.h>
|
||||
|
||||
__BEGIN_DECLS
|
||||
|
||||
@ -461,15 +462,22 @@ struct rtld_global
|
||||
|
||||
EXTERN void (*_dl_init_static_tls) (struct link_map *);
|
||||
|
||||
EXTERN void (*_dl_wait_lookup_done) (void);
|
||||
|
||||
/* Scopes to free after next THREAD_GSCOPE_WAIT (). */
|
||||
EXTERN struct dl_scope_free_list
|
||||
{
|
||||
size_t count;
|
||||
void *list[50];
|
||||
} *_dl_scope_free_list;
|
||||
#if !THREAD_GSCOPE_IN_TCB
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
/* List of active thread stacks, with memory managed by glibc. */
|
||||
EXTERN list_t _dl_stack_used;
|
||||
|
||||
/* List of thread stacks that were allocated by the application. */
|
||||
EXTERN list_t _dl_stack_user;
|
||||
|
||||
/* Mutex protecting the stack lists. */
|
||||
EXTERN int _dl_stack_cache_lock;
|
||||
#else
|
||||
EXTERN int _dl_thread_gscope_count;
|
||||
#endif
|
||||
#ifdef SHARED
|
||||
@ -1252,6 +1260,11 @@ link_map_audit_state (struct link_map *l, size_t index)
|
||||
}
|
||||
#endif /* SHARED */
|
||||
|
||||
#if THREAD_GSCOPE_IN_TCB
|
||||
void __thread_gscope_wait (void) attribute_hidden;
|
||||
# define THREAD_GSCOPE_WAIT() __thread_gscope_wait ()
|
||||
#endif
|
||||
|
||||
__END_DECLS
|
||||
|
||||
#endif /* ldsodefs.h */
|
||||
|
@ -154,8 +154,6 @@ static inline void __set_cr27(struct pthread *cr27)
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
|
||||
|
@ -387,8 +387,6 @@ tls_fill_user_desc (union user_desc_init *desc,
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_SET_FLAG() \
|
||||
THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -175,8 +175,6 @@ register struct pthread *__thread_self __asm__("r13");
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -153,8 +153,6 @@ extern void * __m68k_read_tp (void);
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -136,8 +136,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
# define THREAD_GSCOPE_WAIT() \
|
||||
GL (dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -178,8 +178,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -157,8 +157,6 @@ register struct pthread *__thread_self __asm__("r23");
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
80
sysdeps/nptl/dl-thread_gscope_wait.c
Normal file
80
sysdeps/nptl/dl-thread_gscope_wait.c
Normal file
@ -0,0 +1,80 @@
|
||||
/* Out-of-line notification function for the GSCOPE locking mechanism.
|
||||
Copyright (C) 2007-2020 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <nptl/descr.h>
|
||||
#include <futex-internal.h>
|
||||
#include <ldsodefs.h>
|
||||
#include <list.h>
|
||||
#include <lowlevellock.h>
|
||||
|
||||
void
|
||||
__thread_gscope_wait (void)
|
||||
{
|
||||
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
|
||||
struct pthread *self = THREAD_SELF;
|
||||
|
||||
/* Iterate over the list with system-allocated threads first. */
|
||||
list_t *runp;
|
||||
list_for_each (runp, &GL (dl_stack_used))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
|
||||
continue;
|
||||
|
||||
int *const gscope_flagp = &t->header.gscope_flag;
|
||||
|
||||
/* We have to wait until this thread is done with the global
|
||||
scope. First tell the thread that we are waiting and
|
||||
possibly have to be woken. */
|
||||
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT,
|
||||
THREAD_GSCOPE_FLAG_USED))
|
||||
continue;
|
||||
|
||||
do
|
||||
futex_wait_simple ((unsigned int *) gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
|
||||
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
|
||||
}
|
||||
|
||||
/* Now the list with threads using user-allocated stacks. */
|
||||
list_for_each (runp, &GL (dl_stack_user))
|
||||
{
|
||||
struct pthread *t = list_entry (runp, struct pthread, list);
|
||||
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
|
||||
continue;
|
||||
|
||||
int *const gscope_flagp = &t->header.gscope_flag;
|
||||
|
||||
/* We have to wait until this thread is done with the global
|
||||
scope. First tell the thread that we are waiting and
|
||||
possibly have to be woken. */
|
||||
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT,
|
||||
THREAD_GSCOPE_FLAG_USED))
|
||||
continue;
|
||||
|
||||
do
|
||||
futex_wait_simple ((unsigned int *) gscope_flagp,
|
||||
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
|
||||
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
|
||||
}
|
||||
|
||||
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
|
||||
}
|
@ -245,8 +245,6 @@ register void *__thread_register __asm__ ("r13");
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -139,8 +139,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
# define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -185,8 +185,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -161,8 +161,6 @@ typedef struct
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
@ -158,8 +158,6 @@ register struct pthread *__thread_self __asm__("%g7");
|
||||
atomic_write_barrier (); \
|
||||
} \
|
||||
while (0)
|
||||
#define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* !ASSEMBLER */
|
||||
|
||||
|
@ -332,8 +332,6 @@ _Static_assert (offsetof (tcbhead_t, __glibc_unused2) == 0x80,
|
||||
while (0)
|
||||
# define THREAD_GSCOPE_SET_FLAG() \
|
||||
THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED)
|
||||
# define THREAD_GSCOPE_WAIT() \
|
||||
GL(dl_wait_lookup_done) ()
|
||||
|
||||
#endif /* __ASSEMBLER__ */
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user