From 542ac2393c1ca0d070fa9cce57ca2b2e562f72cc Mon Sep 17 00:00:00 2001 From: Jakub Jelinek Date: Sat, 7 Jul 2007 17:15:07 +0000 Subject: [PATCH] 2007-06-09 Ulrich Drepper * elf/do-lookup.h (do_lookup_x): Read r_nlist before r_list and make sure gcc doesn't mess around with this. 2007-06-08 Ulrich Drepper * elf/dl-lookup.c (_dl_lookup_symbol_x): Remove use of r_nlist. 2007-06-08 Jakub Jelinek * elf/dl-close.c (_dl_close_worker): Remove all to be removed libraries from the global scope at once and call THREAD_GSCOPE_WAIT 2007-05-18 Ulrich Drepper * elf/dl-close.c (_dl_close_worker): When removing object from global scope, wait for all lookups to finish afterwards. * elf/dl-open.c (add_to_global): When global scope array must grow, allocate a new one and free old array only after all lookups finish. * elf/dl-runtime.c (_dl_fixup): Protect using global scope. (_dl_lookup_symbol_x): Likewise. * elf/dl-support.c: Define _dl_wait_lookup_done. * sysdeps/generic/ldsodefs.h (struct rtld_global): Add _dl_wait_lookup_done. nptl/ 2007-05-28 Jakub Jelinek * sysdeps/i386/tls.h (THREAD_GSCOPE_RESET_FLAG): Use explicit insn suffix. (THREAD_GSCOPE_GET_FLAG): Remove. * sysdeps/x86_64/tls.h (THREAD_GSCOPE_GET_FLAG): Remove. * allocatestack.c (__wait_lookup_done): Revert 2007-05-24 changes. * sysdeps/powerpc/tls.h (tcbhead_t): Remove gscope_flag. (THREAD_GSCOPE_GET_FLAG): Remove. (THREAD_GSCOPE_RESET_FLAG): Use THREAD_SELF->header.gscope_flag instead of THREAD_GSCOPE_GET_FLAG. (THREAD_GSCOPE_SET_FLAG): Likewise. Add atomic_write_barrier after it. * sysdeps/s390/tls.h (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_WAIT): Define. * sysdeps/sparc/tls.h (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_WAIT): Define. * sysdeps/sh/tls.h (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_WAIT): Define. * sysdeps/ia64/tls.h (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_WAIT): Define. 2007-05-24 Richard Henderson * descr.h (struct pthread): Add header.gscope_flag. * sysdeps/alpha/tls.h (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_WAIT): Define. 2007-05-26 Ulrich Drepper * allocatestack.c: Revert last change. * init.c: Likewise. * sysdeps/i386/tls.h: Likewise. * sysdeps/x86_64/tls.h: Likewise. 2007-05-24 Jakub Jelinek * sysdeps/powerpc/tls.h (tcbhead_t): Add gscope_flag. (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT): Define. (THREAD_GSCOPE_GET_FLAG, THREAD_GSCOPE_SET_FLAG, THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_WAIT): Define. * sysdeps/i386/tls.h (THREAD_GSCOPE_WAIT): Don't use PTR_DEMANGLE. (THREAD_GSCOPE_GET_FLAG): Define. * sysdeps/x86_64/tls.h (THREAD_GSCOPE_GET_FLAG): Define. * allocatestack.c (__wait_lookup_done): Use THREAD_GSCOPE_GET_FLAG instead of ->header.gscope_flag directly. 2007-05-21 Ulrich Drepper * sysdeps/pthread/pthread-functions.h (struct pthread_functions): Remove ptr_wait_lookup_done again. * init.c (pthread_functions): Don't add .ptr_wait_lookup_done here. (__pthread_initialize_minimal_internal): Initialize _dl_wait_lookup_done pointer in _rtld_global directly. * sysdeps/unix/sysv/linux/libc_pthread_init.c (__libc_pthread_init): Remove code to code _dl_wait_lookup_done. * sysdeps/x86_64/tls.h (THREAD_GSCOPE_WAIT): The pointer is not encrypted for now. 2007-05-19 Ulrich Drepper * allocatestack.c (__wait_lookup_done): New function. * sysdeps/pthread/pthread-functions.h (struct pthread_functions): Add ptr_wait_lookup_done. * init.c (pthread_functions): Initialize .ptr_wait_lookup_done. * pthreadP.h: Declare __wait_lookup_done. * sysdeps/i386/tls.h (tcbhead_t): Add gscope_flag. Define macros to implement reference handling of global scope. * sysdeps/x86_64/tls.h: Likewise. * sysdeps/unix/sysv/linux/libc_pthread_init.c (__libc_pthread_init): Initialize GL(dl_wait_lookup_done). --- ChangeLog | 33 ++++++++++++++--- elf/dl-close.c | 49 +++++++++++++++++-------- elf/dl-lookup.c | 10 ++--- elf/dl-open.c | 23 ++++++++---- elf/dl-runtime.c | 32 +++++++++++++--- elf/dl-support.c | 5 ++- elf/do-lookup.h | 9 ++++- nptl/ChangeLog | 91 +++++++++++++++++++++++++++++++++++++++++++--- nptl/allocatestack.c | 57 +++++++++++++++++++++++++++++ nptl/descr.h | 1 + nptl/init.c | 2 + nptl/pthreadP.h | 2 + nptl/pthread_create.c | 24 ------------ nptl/sysdeps/alpha/tls.h | 23 ++++++++++++ nptl/sysdeps/i386/tls.h | 21 +++++++++++ nptl/sysdeps/ia64/tls.h | 23 ++++++++++++ nptl/sysdeps/powerpc/tls.h | 23 ++++++++++++ nptl/sysdeps/s390/tls.h | 24 ++++++++++++ nptl/sysdeps/sh/tls.h | 23 ++++++++++++ nptl/sysdeps/sparc/tls.h | 23 ++++++++++++ nptl/sysdeps/x86_64/tls.h | 21 +++++++++++ sysdeps/generic/ldsodefs.h | 2 + 22 files changed, 445 insertions(+), 76 deletions(-) diff --git a/ChangeLog b/ChangeLog index ad7cad416d..3b1946a483 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,30 @@ +2007-06-09 Ulrich Drepper + + * elf/do-lookup.h (do_lookup_x): Read r_nlist before r_list and + make sure gcc doesn't mess around with this. + +2007-06-08 Ulrich Drepper + + * elf/dl-lookup.c (_dl_lookup_symbol_x): Remove use of r_nlist. + +2007-06-08 Jakub Jelinek + + * elf/dl-close.c (_dl_close_worker): Remove all to be removed + libraries from the global scope at once and call THREAD_GSCOPE_WAIT + +2007-05-18 Ulrich Drepper + + * elf/dl-close.c (_dl_close_worker): When removing object from + global scope, wait for all lookups to finish afterwards. + * elf/dl-open.c (add_to_global): When global scope array must + grow, allocate a new one and free old array only after all + lookups finish. + * elf/dl-runtime.c (_dl_fixup): Protect using global scope. + (_dl_lookup_symbol_x): Likewise. + * elf/dl-support.c: Define _dl_wait_lookup_done. + * sysdeps/generic/ldsodefs.h (struct rtld_global): Add + _dl_wait_lookup_done. + 2007-06-05 Jakub Jelinek * sysdeps/ieee754/ldbl-128ibm/mpn2ldbl.c @@ -56,12 +83,6 @@ * sysdeps/x86_64/mempcpy.S: Adjust appropriately. Patch by Evandro Menezes . -2007-01-15 Jakub Jelinek - - * elf/dl-open.c (add_to_global): If the main searchlist is 256 - entries or more, on each reallocation at least double the size - of the search list rather than growing it linearly. - 2007-05-21 Ulrich Drepper * sysdeps/unix/sysv/linux/i386/epoll_pwait.S: New file. diff --git a/elf/dl-close.c b/elf/dl-close.c index e0fe26ad02..2c2b3b6163 100644 --- a/elf/dl-close.c +++ b/elf/dl-close.c @@ -31,6 +31,7 @@ #include #include #include +#include /* Type of the constructor functions. */ @@ -228,6 +229,7 @@ _dl_close_worker (struct link_map *map) bool do_audit = GLRO(dl_naudit) > 0 && !ns->_ns_loaded->l_auditing; #endif bool unload_any = false; + unsigned int unload_global = 0; unsigned int first_loaded = ~0; for (unsigned int i = 0; i < nloaded; ++i) { @@ -292,6 +294,9 @@ _dl_close_worker (struct link_map *map) /* We indeed have an object to remove. */ unload_any = true; + if (imap->l_global) + ++unload_global; + /* Remember where the first dynamically loaded object is. */ if (i < first_loaded) first_loaded = i; @@ -457,6 +462,34 @@ _dl_close_worker (struct link_map *map) r->r_state = RT_DELETE; _dl_debug_state (); + if (unload_global) + { + /* Some objects are in the global scope list. Remove them. */ + struct r_scope_elem *ns_msl = ns->_ns_main_searchlist; + unsigned int i; + unsigned int j = 0; + unsigned int cnt = ns_msl->r_nlist; + + while (cnt > 0 && ns_msl->r_list[cnt - 1]->l_removed) + --cnt; + + if (cnt + unload_global == ns_msl->r_nlist) + /* Speed up removing most recently added objects. */ + j = cnt; + else + for (i = 0; i < cnt; i++) + if (ns_msl->r_list[i]->l_removed == 0) + { + if (i != j) + ns_msl->r_list[j] = ns_msl->r_list[i]; + j++; + } + ns_msl->r_nlist = j; + + if (!RTLD_SINGLE_THREAD_P) + THREAD_GSCOPE_WAIT (); + } + size_t tls_free_start; size_t tls_free_end; tls_free_start = tls_free_end = NO_TLS_OFFSET; @@ -472,22 +505,6 @@ _dl_close_worker (struct link_map *map) /* That was the last reference, and this was a dlopen-loaded object. We can unmap it. */ - if (__builtin_expect (imap->l_global, 0)) - { - /* This object is in the global scope list. Remove it. */ - struct r_scope_elem *ns_msl = ns->_ns_main_searchlist; - unsigned int cnt = ns_msl->r_nlist; - - do - --cnt; - while (ns_msl->r_list[cnt] != imap); - - /* The object was already correctly registered. */ - while (++cnt < ns_msl->r_nlist) - ns_msl->r_list[cnt - 1] = ns_msl->r_list[cnt]; - - --ns_msl->r_nlist; - } /* Remove the object from the dtv slotinfo array if it uses TLS. */ if (__builtin_expect (imap->l_tls_blocksize > 0, 0)) diff --git a/elf/dl-lookup.c b/elf/dl-lookup.c index a6a958419f..dc1b865bc7 100644 --- a/elf/dl-lookup.c +++ b/elf/dl-lookup.c @@ -244,13 +244,9 @@ _dl_lookup_symbol_x (const char *undef_name, struct link_map *undef_map, size_t i = 0; if (__builtin_expect (skip_map != NULL, 0)) - { - /* Search the relevant loaded objects for a definition. */ - while ((*scope)->r_list[i] != skip_map) - ++i; - - assert (i < (*scope)->r_nlist); - } + /* Search the relevant loaded objects for a definition. */ + while ((*scope)->r_list[i] != skip_map) + ++i; /* Search the relevant loaded objects for a definition. */ for (size_t start = i; *scope != NULL; start = 0, ++scope) diff --git a/elf/dl-open.c b/elf/dl-open.c index 0afe94b5bf..a043cf61b6 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -32,6 +32,7 @@ #include #include #include +#include #include @@ -125,19 +126,25 @@ add_to_global (struct link_map *new) { /* We have to extend the existing array of link maps in the main map. */ - size_t new_size = ns->_ns_global_scope_alloc; - if (new_size >= 256 && new_size > to_add + 8) - new_size *= 2; - else - new_size += to_add + 8; + struct link_map **old_global + = GL(dl_ns)[new->l_ns]._ns_main_searchlist->r_list; + size_t new_nalloc = ((ns->_ns_global_scope_alloc + to_add) * 2); + new_global = (struct link_map **) - realloc (ns->_ns_main_searchlist->r_list, - new_size * sizeof (struct link_map *)); + malloc (new_nalloc * sizeof (struct link_map *)); if (new_global == NULL) goto nomem; - ns->_ns_global_scope_alloc = new_size; + memcpy (new_global, old_global, + ns->_ns_global_scope_alloc * sizeof (struct link_map *)); + + ns->_ns_global_scope_alloc = new_nalloc; ns->_ns_main_searchlist->r_list = new_global; + + if (!RTLD_SINGLE_THREAD_P) + THREAD_GSCOPE_WAIT (); + + free (old_global); } /* Now add the new entries. */ diff --git a/elf/dl-runtime.c b/elf/dl-runtime.c index 9ecf62b436..6add5e4fff 100644 --- a/elf/dl-runtime.c +++ b/elf/dl-runtime.c @@ -26,6 +26,8 @@ #include #include #include "dynamic-link.h" +#include + #if (!defined ELF_MACHINE_NO_RELA && !defined ELF_MACHINE_PLT_REL) \ || ELF_MACHINE_NO_REL @@ -97,10 +99,15 @@ _dl_fixup ( not necessary for objects which cannot be unloaded or when we are not using any threads (yet). */ int flags = DL_LOOKUP_ADD_DEPENDENCY; - if (l->l_type == lt_loaded && !RTLD_SINGLE_THREAD_P) + if (!RTLD_SINGLE_THREAD_P) { - __rtld_mrlock_lock (l->l_scope_lock); - flags |= DL_LOOKUP_SCOPE_LOCK; + THREAD_GSCOPE_SET_FLAG (); + + if (l->l_type == lt_loaded) + { + __rtld_mrlock_lock (l->l_scope_lock); + flags |= DL_LOOKUP_SCOPE_LOCK; + } } result = _dl_lookup_symbol_x (strtab + sym->st_name, l, &sym, l->l_scope, @@ -109,6 +116,10 @@ _dl_fixup ( if ((flags & DL_LOOKUP_SCOPE_LOCK) != 0) __rtld_mrlock_unlock (l->l_scope_lock); + /* We are done with the global scope. */ + if (!RTLD_SINGLE_THREAD_P) + THREAD_GSCOPE_RESET_FLAG (); + /* Currently result contains the base load address (or link map) of the object that defines sym. Now add in the symbol offset. */ @@ -191,10 +202,15 @@ _dl_profile_fixup ( not necessary for objects which cannot be unloaded or when we are not using any threads (yet). */ int flags = DL_LOOKUP_ADD_DEPENDENCY; - if (l->l_type == lt_loaded && !RTLD_SINGLE_THREAD_P) + if (!RTLD_SINGLE_THREAD_P) { - __rtld_mrlock_lock (l->l_scope_lock); - flags |= DL_LOOKUP_SCOPE_LOCK; + THREAD_GSCOPE_SET_FLAG (); + + if (l->l_type == lt_loaded) + { + __rtld_mrlock_lock (l->l_scope_lock); + flags |= DL_LOOKUP_SCOPE_LOCK; + } } result = _dl_lookup_symbol_x (strtab + refsym->st_name, l, @@ -204,6 +220,10 @@ _dl_profile_fixup ( if ((flags & DL_LOOKUP_SCOPE_LOCK) != 0) __rtld_mrlock_unlock (l->l_scope_lock); + /* We are done with the global scope. */ + if (!RTLD_SINGLE_THREAD_P) + THREAD_GSCOPE_RESET_FLAG (); + /* Currently result contains the base load address (or link map) of the object that defines sym. Now add in the symbol offset. */ diff --git a/elf/dl-support.c b/elf/dl-support.c index c7479dc639..cecb603ae6 100644 --- a/elf/dl-support.c +++ b/elf/dl-support.c @@ -1,5 +1,5 @@ /* Support for dynamic linking code in static libc. - Copyright (C) 1996-2005, 2006 Free Software Foundation, Inc. + Copyright (C) 1996-2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -132,6 +132,9 @@ int (*_dl_make_stack_executable_hook) (void **) internal_function = _dl_make_stack_executable; +/* Function in libpthread to wait for termination of lookups. */ +void (*_dl_wait_lookup_done) (void); + #ifdef NEED_DL_SYSINFO /* Needed for improved syscall handling on at least x86/Linux. */ uintptr_t _dl_sysinfo = DL_SYSINFO_DEFAULT; diff --git a/elf/do-lookup.h b/elf/do-lookup.h index 2585d83005..ab9a510ba1 100644 --- a/elf/do-lookup.h +++ b/elf/do-lookup.h @@ -1,5 +1,5 @@ /* Look up a symbol in the loaded objects. - Copyright (C) 1995-2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 1995-2004, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -29,8 +29,13 @@ do_lookup_x (const char *undef_name, uint_fast32_t new_hash, const struct r_found_version *const version, int flags, struct link_map *skip, int type_class) { - struct link_map **list = scope->r_list; size_t n = scope->r_nlist; + /* Make sure we read the value before proceeding. Otherwise we + might use r_list pointing to the initial scope and r_nlist being + the value after a resize. That is the only path in dl-open.c not + protected by GSCOPE. A read barrier here might be to expensive. */ + __asm volatile ("" : "+r" (n), "+m" (scope->r_list)); + struct link_map **list = scope->r_list; do { diff --git a/nptl/ChangeLog b/nptl/ChangeLog index ea877d0374..bb7dd528ab 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,88 @@ +2007-05-28 Jakub Jelinek + + * sysdeps/i386/tls.h (THREAD_GSCOPE_RESET_FLAG): Use explicit + insn suffix. + (THREAD_GSCOPE_GET_FLAG): Remove. + * sysdeps/x86_64/tls.h (THREAD_GSCOPE_GET_FLAG): Remove. + * allocatestack.c (__wait_lookup_done): Revert 2007-05-24 + changes. + * sysdeps/powerpc/tls.h (tcbhead_t): Remove gscope_flag. + (THREAD_GSCOPE_GET_FLAG): Remove. + (THREAD_GSCOPE_RESET_FLAG): Use THREAD_SELF->header.gscope_flag + instead of THREAD_GSCOPE_GET_FLAG. + (THREAD_GSCOPE_SET_FLAG): Likewise. Add atomic_write_barrier after + it. + * sysdeps/s390/tls.h (THREAD_GSCOPE_FLAG_UNUSED, + THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_WAIT): Define. + * sysdeps/sparc/tls.h (THREAD_GSCOPE_FLAG_UNUSED, + THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_WAIT): Define. + * sysdeps/sh/tls.h (THREAD_GSCOPE_FLAG_UNUSED, + THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_WAIT): Define. + * sysdeps/ia64/tls.h (THREAD_GSCOPE_FLAG_UNUSED, + THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_WAIT): Define. + +2007-05-24 Richard Henderson + + * descr.h (struct pthread): Add header.gscope_flag. + * sysdeps/alpha/tls.h (THREAD_GSCOPE_FLAG_UNUSED, + THREAD_GSCOPE_FLAG_USED, THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_WAIT): Define. + +2007-05-26 Ulrich Drepper + + * allocatestack.c: Revert last change. + * init.c: Likewise. + * sysdeps/i386/tls.h: Likewise. + * sysdeps/x86_64/tls.h: Likewise. + +2007-05-24 Jakub Jelinek + + * sysdeps/powerpc/tls.h (tcbhead_t): Add gscope_flag. + (THREAD_GSCOPE_FLAG_UNUSED, THREAD_GSCOPE_FLAG_USED, + THREAD_GSCOPE_FLAG_WAIT): Define. + (THREAD_GSCOPE_GET_FLAG, THREAD_GSCOPE_SET_FLAG, + THREAD_GSCOPE_RESET_FLAG, THREAD_GSCOPE_WAIT): Define. + * sysdeps/i386/tls.h (THREAD_GSCOPE_WAIT): Don't use + PTR_DEMANGLE. + (THREAD_GSCOPE_GET_FLAG): Define. + * sysdeps/x86_64/tls.h (THREAD_GSCOPE_GET_FLAG): Define. + * allocatestack.c (__wait_lookup_done): Use THREAD_GSCOPE_GET_FLAG + instead of ->header.gscope_flag directly. + +2007-05-21 Ulrich Drepper + + * sysdeps/pthread/pthread-functions.h (struct pthread_functions): + Remove ptr_wait_lookup_done again. + * init.c (pthread_functions): Don't add .ptr_wait_lookup_done here. + (__pthread_initialize_minimal_internal): Initialize + _dl_wait_lookup_done pointer in _rtld_global directly. + * sysdeps/unix/sysv/linux/libc_pthread_init.c (__libc_pthread_init): + Remove code to code _dl_wait_lookup_done. + * sysdeps/x86_64/tls.h (THREAD_GSCOPE_WAIT): The pointer is not + encrypted for now. + +2007-05-19 Ulrich Drepper + + * allocatestack.c (__wait_lookup_done): New function. + * sysdeps/pthread/pthread-functions.h (struct pthread_functions): + Add ptr_wait_lookup_done. + * init.c (pthread_functions): Initialize .ptr_wait_lookup_done. + * pthreadP.h: Declare __wait_lookup_done. + * sysdeps/i386/tls.h (tcbhead_t): Add gscope_flag. + Define macros to implement reference handling of global scope. + * sysdeps/x86_64/tls.h: Likewise. + * sysdeps/unix/sysv/linux/libc_pthread_init.c (__libc_pthread_init): + Initialize GL(dl_wait_lookup_done). + 2007-05-25 Ulrich Drepper * Makefile (tests): Add tst-sem10. @@ -16,12 +101,6 @@ * tst-robust9.c (do_test): Don't fail if ENABLE_PI and pthread_mutex_init failed with ENOTSUP. -2007-01-15 Jakub Jelinek - - * pthread_create.c (__pthread_create_2_1): On the first pthread_create - in a process make sure main search list can store at least 256 - entries. - 2007-05-17 Ulrich Drepper [BZ #4512] diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index 6b60642042..e556dbac08 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -996,3 +996,60 @@ __pthread_init_static_tls (struct link_map *map) lll_unlock (stack_cache_lock); } + + +void +attribute_hidden +__wait_lookup_done (void) +{ + lll_lock (stack_cache_lock); + + struct pthread *self = THREAD_SELF; + + /* Iterate over the list with system-allocated threads first. */ + list_t *runp; + list_for_each (runp, &stack_used) + { + struct pthread *t = list_entry (runp, struct pthread, list); + if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED) + continue; + + int *const gscope_flagp = &t->header.gscope_flag; + + /* We have to wait until this thread is done with the global + scope. First tell the thread that we are waiting and + possibly have to be woken. */ + if (atomic_compare_and_exchange_bool_acq (gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_FLAG_USED)) + continue; + + do + lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); + while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); + } + + /* Now the list with threads using user-allocated stacks. */ + list_for_each (runp, &__stack_user) + { + struct pthread *t = list_entry (runp, struct pthread, list); + if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED) + continue; + + int *const gscope_flagp = &t->header.gscope_flag; + + /* We have to wait until this thread is done with the global + scope. First tell the thread that we are waiting and + possibly have to be woken. */ + if (atomic_compare_and_exchange_bool_acq (gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, + THREAD_GSCOPE_FLAG_USED)) + continue; + + do + lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); + while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); + } + + lll_unlock (stack_cache_lock); +} diff --git a/nptl/descr.h b/nptl/descr.h index 00cad1aa83..74d8c44140 100644 --- a/nptl/descr.h +++ b/nptl/descr.h @@ -131,6 +131,7 @@ struct pthread struct { int multiple_threads; + int gscope_flag; } header; #endif diff --git a/nptl/init.c b/nptl/init.c index dddc975a5e..827e795c2d 100644 --- a/nptl/init.c +++ b/nptl/init.c @@ -386,6 +386,8 @@ __pthread_initialize_minimal_internal (void) GL(dl_init_static_tls) = &__pthread_init_static_tls; + GL(dl_wait_lookup_done) = &__wait_lookup_done; + /* Register the fork generation counter with the libc. */ #ifndef TLS_MULTIPLE_THREADS_IN_TCB __libc_multiple_threads_ptr = diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index f9634ab0ff..f560f72e4e 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -545,6 +545,8 @@ extern int __nptl_setxid (struct xid_command *cmdp) attribute_hidden; extern void __free_stack_cache (void) attribute_hidden; +extern void __wait_lookup_done (void) attribute_hidden; + #ifdef SHARED # define PTHREAD_STATIC_FN_REQUIRE(name) #else diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index 22e8f01804..79729ced03 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -462,30 +462,6 @@ __pthread_create_2_1 (newthread, attr, start_routine, arg) pd->flags = ((iattr->flags & ~(ATTR_FLAG_SCHED_SET | ATTR_FLAG_POLICY_SET)) | (self->flags & (ATTR_FLAG_SCHED_SET | ATTR_FLAG_POLICY_SET))); - /* Hack: realloc the main search list on the first pthread_create call - to minimize the number of global search scope reallocations. - Wastes at most 1KB on 32-bit and 2KB on 64-bit per process - which calls pthread_create. */ - if (__builtin_expect (self->header.multiple_threads == 0, 0) - && GL(dl_ns)[0]._ns_main_searchlist - && GL(dl_ns)[0]._ns_main_searchlist->r_nlist < 256 - && GL(dl_ns)[0]._ns_global_scope_alloc < 256) - { - struct link_map **new_global = (struct link_map **) - realloc (GL(dl_ns)[0]._ns_global_scope_alloc == 0 - ? NULL : GL(dl_ns)[0]._ns_main_searchlist->r_list, - 256 * sizeof (struct link_map *)); - if (new_global != NULL) - { - if (GL(dl_ns)[0]._ns_global_scope_alloc == 0) - memcpy (new_global, GL(dl_ns)[0]._ns_main_searchlist->r_list, - GL(dl_ns)[0]._ns_main_searchlist->r_nlist - * sizeof (struct link_map *)); - GL(dl_ns)[0]._ns_global_scope_alloc = 256; - GL(dl_ns)[0]._ns_main_searchlist->r_list = new_global; - } - } - /* Initialize the field for the ID of the thread which is waiting for us. This is a self-reference in case the thread is created detached. */ diff --git a/nptl/sysdeps/alpha/tls.h b/nptl/sysdeps/alpha/tls.h index be2430f676..9072e2d6fb 100644 --- a/nptl/sysdeps/alpha/tls.h +++ b/nptl/sysdeps/alpha/tls.h @@ -121,6 +121,29 @@ typedef struct #define THREAD_SETMEM_NC(descr, member, idx, value) \ descr->member[idx] = (value) +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h index d5b3797e69..7c80cd7d14 100644 --- a/nptl/sysdeps/i386/tls.h +++ b/nptl/sysdeps/i386/tls.h @@ -51,6 +51,7 @@ typedef struct uintptr_t sysinfo; uintptr_t stack_guard; uintptr_t pointer_guard; + int gscope_flag; } tcbhead_t; # define TLS_MULTIPLE_THREADS_IN_TCB 1 @@ -431,6 +432,26 @@ union user_desc_init = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) +/* Get and set the global scope generation counter in the TCB head. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res; \ + asm volatile ("xchgl %0, %%gs:%P1" \ + : "=r" (__res) \ + : "i" (offsetof (struct pthread, header.gscope_flag)), \ + "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/ia64/tls.h b/nptl/sysdeps/ia64/tls.h index 22a8b08144..4270723b9a 100644 --- a/nptl/sysdeps/ia64/tls.h +++ b/nptl/sysdeps/ia64/tls.h @@ -163,6 +163,29 @@ register struct pthread *__thread_self __asm__("r13"); (((uintptr_t *) ((char *) (descr) + TLS_PRE_TCB_SIZE))[-2] \ = THREAD_GET_POINTER_GUARD ()) +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/powerpc/tls.h b/nptl/sysdeps/powerpc/tls.h index ddaafe23d0..de822833f9 100644 --- a/nptl/sysdeps/powerpc/tls.h +++ b/nptl/sysdeps/powerpc/tls.h @@ -180,6 +180,29 @@ register void *__thread_register __asm__ ("r13"); different value to mean unset l_tls_offset. */ # define NO_TLS_OFFSET -1 +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/s390/tls.h b/nptl/sysdeps/s390/tls.h index 6f6f17b975..1c86a19676 100644 --- a/nptl/sysdeps/s390/tls.h +++ b/nptl/sysdeps/s390/tls.h @@ -50,6 +50,7 @@ typedef struct int multiple_threads; uintptr_t sysinfo; uintptr_t stack_guard; + int gscope_flag; } tcbhead_t; # ifndef __s390x__ @@ -168,6 +169,29 @@ typedef struct #define THREAD_SET_POINTER_GUARD(value) #define THREAD_COPY_POINTER_GUARD(descr) +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/sh/tls.h b/nptl/sysdeps/sh/tls.h index d9aa1073b8..09522189a6 100644 --- a/nptl/sysdeps/sh/tls.h +++ b/nptl/sysdeps/sh/tls.h @@ -150,6 +150,29 @@ typedef struct __asm __volatile ("stc gbr,%0" : "=r" (__tcbp)); \ ((tcbhead_t *) (descr + 1))->pointer_guard = __tcbp->pointer_guard;}) +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/nptl/sysdeps/sparc/tls.h b/nptl/sysdeps/sparc/tls.h index 4fbe426595..8a0c930b3e 100644 --- a/nptl/sysdeps/sparc/tls.h +++ b/nptl/sysdeps/sparc/tls.h @@ -141,6 +141,29 @@ register struct pthread *__thread_self __asm__("%g7"); # define THREAD_COPY_POINTER_GUARD(descr) \ ((descr)->header.pointer_guard = THREAD_GET_POINTER_GUARD ()) +/* Get and set the global scope generation counter in struct pthread. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res \ + = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ + THREAD_GSCOPE_FLAG_UNUSED); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + do \ + { \ + THREAD_SELF->header.gscope_flag = THREAD_GSCOPE_FLAG_USED; \ + atomic_write_barrier (); \ + } \ + while (0) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* !ASSEMBLER */ #endif /* tls.h */ diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h index 0b5aeb00ff..4a614c02af 100644 --- a/nptl/sysdeps/x86_64/tls.h +++ b/nptl/sysdeps/x86_64/tls.h @@ -47,6 +47,7 @@ typedef struct dtv_t *dtv; void *self; /* Pointer to the thread descriptor. */ int multiple_threads; + int gscope_flag; uintptr_t sysinfo; uintptr_t stack_guard; uintptr_t pointer_guard; @@ -337,6 +338,26 @@ typedef struct = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) +/* Get and set the global scope generation counter in the TCB head. */ +#define THREAD_GSCOPE_FLAG_UNUSED 0 +#define THREAD_GSCOPE_FLAG_USED 1 +#define THREAD_GSCOPE_FLAG_WAIT 2 +#define THREAD_GSCOPE_RESET_FLAG() \ + do \ + { int __res; \ + asm volatile ("xchgl %0, %%fs:%P1" \ + : "=r" (__res) \ + : "i" (offsetof (struct pthread, header.gscope_flag)), \ + "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ + if (__res == THREAD_GSCOPE_FLAG_WAIT) \ + lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + } \ + while (0) +#define THREAD_GSCOPE_SET_FLAG() \ + THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) +#define THREAD_GSCOPE_WAIT() \ + GL(dl_wait_lookup_done) () + #endif /* __ASSEMBLER__ */ #endif /* tls.h */ diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h index a9d20b2150..5205c41493 100644 --- a/sysdeps/generic/ldsodefs.h +++ b/sysdeps/generic/ldsodefs.h @@ -486,6 +486,8 @@ struct rtld_global EXTERN void (*_dl_init_static_tls) (struct link_map *); + EXTERN void (*_dl_wait_lookup_done) (void); + #ifdef SHARED }; # define __rtld_global_attribute__ -- 2.11.4.GIT