Fix memory leak in TLS of loaded objects.
[glibc.git] / nptl / allocatestack.c
blob550a8bec73e01c478636e87ea5b2b46655074d8b
1 /* Copyright (C) 2002-2007, 2009, 2011 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18 02111-1307 USA. */
20 #include <assert.h>
21 #include <errno.h>
22 #include <signal.h>
23 #include <stdint.h>
24 #include <string.h>
25 #include <unistd.h>
26 #include <sys/mman.h>
27 #include <sys/param.h>
28 #include <dl-sysdep.h>
29 #include <dl-tls.h>
30 #include <tls.h>
31 #include <lowlevellock.h>
32 #include <kernel-features.h>
35 #ifndef NEED_SEPARATE_REGISTER_STACK
37 /* Most architectures have exactly one stack pointer. Some have more. */
38 # define STACK_VARIABLES void *stackaddr = NULL
40 /* How to pass the values to the 'create_thread' function. */
41 # define STACK_VARIABLES_ARGS stackaddr
43 /* How to declare function which gets there parameters. */
44 # define STACK_VARIABLES_PARMS void *stackaddr
46 /* How to declare allocate_stack. */
47 # define ALLOCATE_STACK_PARMS void **stack
49 /* This is how the function is called. We do it this way to allow
50 other variants of the function to have more parameters. */
51 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
53 #else
55 /* We need two stacks. The kernel will place them but we have to tell
56 the kernel about the size of the reserved address space. */
57 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
59 /* How to pass the values to the 'create_thread' function. */
60 # define STACK_VARIABLES_ARGS stackaddr, stacksize
62 /* How to declare function which gets there parameters. */
63 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
65 /* How to declare allocate_stack. */
66 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
68 /* This is how the function is called. We do it this way to allow
69 other variants of the function to have more parameters. */
70 # define ALLOCATE_STACK(attr, pd) \
71 allocate_stack (attr, pd, &stackaddr, &stacksize)
73 #endif
76 /* Default alignment of stack. */
77 #ifndef STACK_ALIGN
78 # define STACK_ALIGN __alignof__ (long double)
79 #endif
81 /* Default value for minimal stack size after allocating thread
82 descriptor and guard. */
83 #ifndef MINIMAL_REST_STACK
84 # define MINIMAL_REST_STACK 4096
85 #endif
88 /* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
89 a stack. Use it when possible. */
90 #ifndef MAP_STACK
91 # define MAP_STACK 0
92 #endif
94 /* This yields the pointer that TLS support code calls the thread pointer. */
95 #if TLS_TCB_AT_TP
96 # define TLS_TPADJ(pd) (pd)
97 #elif TLS_DTV_AT_TP
98 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
99 #endif
101 /* Cache handling for not-yet free stacks. */
103 /* Maximum size in kB of cache. */
104 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
105 static size_t stack_cache_actsize;
107 /* Mutex protecting this variable. */
108 static int stack_cache_lock = LLL_LOCK_INITIALIZER;
110 /* List of queued stack frames. */
111 static LIST_HEAD (stack_cache);
113 /* List of the stacks in use. */
114 static LIST_HEAD (stack_used);
116 /* We need to record what list operations we are going to do so that,
117 in case of an asynchronous interruption due to a fork() call, we
118 can correct for the work. */
119 static uintptr_t in_flight_stack;
121 /* List of the threads with user provided stacks in use. No need to
122 initialize this, since it's done in __pthread_initialize_minimal. */
123 list_t __stack_user __attribute__ ((nocommon));
124 hidden_data_def (__stack_user)
126 #if COLORING_INCREMENT != 0
127 /* Number of threads created. */
128 static unsigned int nptl_ncreated;
129 #endif
132 /* Check whether the stack is still used or not. */
133 #define FREE_P(descr) ((descr)->tid <= 0)
136 static void
137 stack_list_del (list_t *elem)
139 in_flight_stack = (uintptr_t) elem;
141 atomic_write_barrier ();
143 list_del (elem);
145 atomic_write_barrier ();
147 in_flight_stack = 0;
151 static void
152 stack_list_add (list_t *elem, list_t *list)
154 in_flight_stack = (uintptr_t) elem | 1;
156 atomic_write_barrier ();
158 list_add (elem, list);
160 atomic_write_barrier ();
162 in_flight_stack = 0;
166 /* We create a double linked list of all cache entries. Double linked
167 because this allows removing entries from the end. */
170 /* Get a stack frame from the cache. We have to match by size since
171 some blocks might be too small or far too large. */
172 static struct pthread *
173 get_cached_stack (size_t *sizep, void **memp)
175 size_t size = *sizep;
176 struct pthread *result = NULL;
177 list_t *entry;
179 lll_lock (stack_cache_lock, LLL_PRIVATE);
181 /* Search the cache for a matching entry. We search for the
182 smallest stack which has at least the required size. Note that
183 in normal situations the size of all allocated stacks is the
184 same. As the very least there are only a few different sizes.
185 Therefore this loop will exit early most of the time with an
186 exact match. */
187 list_for_each (entry, &stack_cache)
189 struct pthread *curr;
191 curr = list_entry (entry, struct pthread, list);
192 if (FREE_P (curr) && curr->stackblock_size >= size)
194 if (curr->stackblock_size == size)
196 result = curr;
197 break;
200 if (result == NULL
201 || result->stackblock_size > curr->stackblock_size)
202 result = curr;
206 if (__builtin_expect (result == NULL, 0)
207 /* Make sure the size difference is not too excessive. In that
208 case we do not use the block. */
209 || __builtin_expect (result->stackblock_size > 4 * size, 0))
211 /* Release the lock. */
212 lll_unlock (stack_cache_lock, LLL_PRIVATE);
214 return NULL;
217 /* Dequeue the entry. */
218 stack_list_del (&result->list);
220 /* And add to the list of stacks in use. */
221 stack_list_add (&result->list, &stack_used);
223 /* And decrease the cache size. */
224 stack_cache_actsize -= result->stackblock_size;
226 /* Release the lock early. */
227 lll_unlock (stack_cache_lock, LLL_PRIVATE);
229 /* Report size and location of the stack to the caller. */
230 *sizep = result->stackblock_size;
231 *memp = result->stackblock;
233 /* Cancellation handling is back to the default. */
234 result->cancelhandling = 0;
235 result->cleanup = NULL;
237 /* No pending event. */
238 result->nextevent = NULL;
240 /* Clear the DTV. */
241 dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
242 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
243 if (! dtv[1 + cnt].pointer.is_static
244 && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
245 free (dtv[1 + cnt].pointer.val);
246 memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
248 /* Re-initialize the TLS. */
249 _dl_allocate_tls_init (TLS_TPADJ (result));
251 return result;
255 /* Free stacks until cache size is lower than LIMIT. */
256 void
257 __free_stacks (size_t limit)
259 /* We reduce the size of the cache. Remove the last entries until
260 the size is below the limit. */
261 list_t *entry;
262 list_t *prev;
264 /* Search from the end of the list. */
265 list_for_each_prev_safe (entry, prev, &stack_cache)
267 struct pthread *curr;
269 curr = list_entry (entry, struct pthread, list);
270 if (FREE_P (curr))
272 /* Unlink the block. */
273 stack_list_del (entry);
275 /* Account for the freed memory. */
276 stack_cache_actsize -= curr->stackblock_size;
278 /* Free the memory associated with the ELF TLS. */
279 _dl_deallocate_tls (TLS_TPADJ (curr), false);
281 /* Remove this block. This should never fail. If it does
282 something is really wrong. */
283 if (munmap (curr->stackblock, curr->stackblock_size) != 0)
284 abort ();
286 /* Maybe we have freed enough. */
287 if (stack_cache_actsize <= limit)
288 break;
294 /* Add a stack frame which is not used anymore to the stack. Must be
295 called with the cache lock held. */
296 static inline void
297 __attribute ((always_inline))
298 queue_stack (struct pthread *stack)
300 /* We unconditionally add the stack to the list. The memory may
301 still be in use but it will not be reused until the kernel marks
302 the stack as not used anymore. */
303 stack_list_add (&stack->list, &stack_cache);
305 stack_cache_actsize += stack->stackblock_size;
306 if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
307 __free_stacks (stack_cache_maxsize);
311 static int
312 internal_function
313 change_stack_perm (struct pthread *pd
314 #ifdef NEED_SEPARATE_REGISTER_STACK
315 , size_t pagemask
316 #endif
319 #ifdef NEED_SEPARATE_REGISTER_STACK
320 void *stack = (pd->stackblock
321 + (((((pd->stackblock_size - pd->guardsize) / 2)
322 & pagemask) + pd->guardsize) & pagemask));
323 size_t len = pd->stackblock + pd->stackblock_size - stack;
324 #elif _STACK_GROWS_DOWN
325 void *stack = pd->stackblock + pd->guardsize;
326 size_t len = pd->stackblock_size - pd->guardsize;
327 #elif _STACK_GROWS_UP
328 void *stack = pd->stackblock;
329 size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
330 #else
331 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
332 #endif
333 if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
334 return errno;
336 return 0;
340 static int
341 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
342 ALLOCATE_STACK_PARMS)
344 struct pthread *pd;
345 size_t size;
346 size_t pagesize_m1 = __getpagesize () - 1;
347 void *stacktop;
349 assert (attr != NULL);
350 assert (powerof2 (pagesize_m1 + 1));
351 assert (TCB_ALIGNMENT >= STACK_ALIGN);
353 /* Get the stack size from the attribute if it is set. Otherwise we
354 use the default we determined at start time. */
355 size = attr->stacksize ?: __default_stacksize;
357 /* Get memory for the stack. */
358 if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
360 uintptr_t adj;
362 /* If the user also specified the size of the stack make sure it
363 is large enough. */
364 if (attr->stacksize != 0
365 && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
366 return EINVAL;
368 /* Adjust stack size for alignment of the TLS block. */
369 #if TLS_TCB_AT_TP
370 adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
371 & __static_tls_align_m1;
372 assert (size > adj + TLS_TCB_SIZE);
373 #elif TLS_DTV_AT_TP
374 adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
375 & __static_tls_align_m1;
376 assert (size > adj);
377 #endif
379 /* The user provided some memory. Let's hope it matches the
380 size... We do not allocate guard pages if the user provided
381 the stack. It is the user's responsibility to do this if it
382 is wanted. */
383 #if TLS_TCB_AT_TP
384 pd = (struct pthread *) ((uintptr_t) attr->stackaddr
385 - TLS_TCB_SIZE - adj);
386 #elif TLS_DTV_AT_TP
387 pd = (struct pthread *) (((uintptr_t) attr->stackaddr
388 - __static_tls_size - adj)
389 - TLS_PRE_TCB_SIZE);
390 #endif
392 /* The user provided stack memory needs to be cleared. */
393 memset (pd, '\0', sizeof (struct pthread));
395 /* The first TSD block is included in the TCB. */
396 pd->specific[0] = pd->specific_1stblock;
398 /* Remember the stack-related values. */
399 pd->stackblock = (char *) attr->stackaddr - size;
400 pd->stackblock_size = size;
402 /* This is a user-provided stack. It will not be queued in the
403 stack cache nor will the memory (except the TLS memory) be freed. */
404 pd->user_stack = true;
406 /* This is at least the second thread. */
407 pd->header.multiple_threads = 1;
408 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
409 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
410 #endif
412 #ifndef __ASSUME_PRIVATE_FUTEX
413 /* The thread must know when private futexes are supported. */
414 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
415 header.private_futex);
416 #endif
418 #ifdef NEED_DL_SYSINFO
419 /* Copy the sysinfo value from the parent. */
420 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
421 #endif
423 /* The process ID is also the same as that of the caller. */
424 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
426 /* Allocate the DTV for this thread. */
427 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
429 /* Something went wrong. */
430 assert (errno == ENOMEM);
431 return EAGAIN;
435 /* Prepare to modify global data. */
436 lll_lock (stack_cache_lock, LLL_PRIVATE);
438 /* And add to the list of stacks in use. */
439 list_add (&pd->list, &__stack_user);
441 lll_unlock (stack_cache_lock, LLL_PRIVATE);
443 else
445 /* Allocate some anonymous memory. If possible use the cache. */
446 size_t guardsize;
447 size_t reqsize;
448 void *mem;
449 const int prot = (PROT_READ | PROT_WRITE
450 | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
452 #if COLORING_INCREMENT != 0
453 /* Add one more page for stack coloring. Don't do it for stacks
454 with 16 times pagesize or larger. This might just cause
455 unnecessary misalignment. */
456 if (size <= 16 * pagesize_m1)
457 size += pagesize_m1 + 1;
458 #endif
460 /* Adjust the stack size for alignment. */
461 size &= ~__static_tls_align_m1;
462 assert (size != 0);
464 /* Make sure the size of the stack is enough for the guard and
465 eventually the thread descriptor. */
466 guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
467 if (__builtin_expect (size < ((guardsize + __static_tls_size
468 + MINIMAL_REST_STACK + pagesize_m1)
469 & ~pagesize_m1),
471 /* The stack is too small (or the guard too large). */
472 return EINVAL;
474 /* Try to get a stack from the cache. */
475 reqsize = size;
476 pd = get_cached_stack (&size, &mem);
477 if (pd == NULL)
479 /* To avoid aliasing effects on a larger scale than pages we
480 adjust the allocated stack size if necessary. This way
481 allocations directly following each other will not have
482 aliasing problems. */
483 #if MULTI_PAGE_ALIASING != 0
484 if ((size % MULTI_PAGE_ALIASING) == 0)
485 size += pagesize_m1 + 1;
486 #endif
488 mem = mmap (NULL, size, prot,
489 MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
491 if (__builtin_expect (mem == MAP_FAILED, 0))
493 if (errno == ENOMEM)
494 __set_errno (EAGAIN);
496 return errno;
499 /* SIZE is guaranteed to be greater than zero.
500 So we can never get a null pointer back from mmap. */
501 assert (mem != NULL);
503 #if COLORING_INCREMENT != 0
504 /* Atomically increment NCREATED. */
505 unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
507 /* We chose the offset for coloring by incrementing it for
508 every new thread by a fixed amount. The offset used
509 module the page size. Even if coloring would be better
510 relative to higher alignment values it makes no sense to
511 do it since the mmap() interface does not allow us to
512 specify any alignment for the returned memory block. */
513 size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
515 /* Make sure the coloring offsets does not disturb the alignment
516 of the TCB and static TLS block. */
517 if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
518 coloring = (((coloring + __static_tls_align_m1)
519 & ~(__static_tls_align_m1))
520 & ~pagesize_m1);
521 #else
522 /* Unless specified we do not make any adjustments. */
523 # define coloring 0
524 #endif
526 /* Place the thread descriptor at the end of the stack. */
527 #if TLS_TCB_AT_TP
528 pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
529 #elif TLS_DTV_AT_TP
530 pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
531 - __static_tls_size)
532 & ~__static_tls_align_m1)
533 - TLS_PRE_TCB_SIZE);
534 #endif
536 /* Remember the stack-related values. */
537 pd->stackblock = mem;
538 pd->stackblock_size = size;
540 /* We allocated the first block thread-specific data array.
541 This address will not change for the lifetime of this
542 descriptor. */
543 pd->specific[0] = pd->specific_1stblock;
545 /* This is at least the second thread. */
546 pd->header.multiple_threads = 1;
547 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
548 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
549 #endif
551 #ifndef __ASSUME_PRIVATE_FUTEX
552 /* The thread must know when private futexes are supported. */
553 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
554 header.private_futex);
555 #endif
557 #ifdef NEED_DL_SYSINFO
558 /* Copy the sysinfo value from the parent. */
559 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
560 #endif
562 /* The process ID is also the same as that of the caller. */
563 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
565 /* Allocate the DTV for this thread. */
566 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
568 /* Something went wrong. */
569 assert (errno == ENOMEM);
571 /* Free the stack memory we just allocated. */
572 (void) munmap (mem, size);
574 return EAGAIN;
578 /* Prepare to modify global data. */
579 lll_lock (stack_cache_lock, LLL_PRIVATE);
581 /* And add to the list of stacks in use. */
582 stack_list_add (&pd->list, &stack_used);
584 lll_unlock (stack_cache_lock, LLL_PRIVATE);
587 /* There might have been a race. Another thread might have
588 caused the stacks to get exec permission while this new
589 stack was prepared. Detect if this was possible and
590 change the permission if necessary. */
591 if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
592 && (prot & PROT_EXEC) == 0, 0))
594 int err = change_stack_perm (pd
595 #ifdef NEED_SEPARATE_REGISTER_STACK
596 , ~pagesize_m1
597 #endif
599 if (err != 0)
601 /* Free the stack memory we just allocated. */
602 (void) munmap (mem, size);
604 return err;
609 /* Note that all of the stack and the thread descriptor is
610 zeroed. This means we do not have to initialize fields
611 with initial value zero. This is specifically true for
612 the 'tid' field which is always set back to zero once the
613 stack is not used anymore and for the 'guardsize' field
614 which will be read next. */
617 /* Create or resize the guard area if necessary. */
618 if (__builtin_expect (guardsize > pd->guardsize, 0))
620 #ifdef NEED_SEPARATE_REGISTER_STACK
621 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
622 #elif _STACK_GROWS_DOWN
623 char *guard = mem;
624 # elif _STACK_GROWS_UP
625 char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
626 #endif
627 if (mprotect (guard, guardsize, PROT_NONE) != 0)
629 int err;
630 mprot_error:
631 err = errno;
633 lll_lock (stack_cache_lock, LLL_PRIVATE);
635 /* Remove the thread from the list. */
636 stack_list_del (&pd->list);
638 lll_unlock (stack_cache_lock, LLL_PRIVATE);
640 /* Get rid of the TLS block we allocated. */
641 _dl_deallocate_tls (TLS_TPADJ (pd), false);
643 /* Free the stack memory regardless of whether the size
644 of the cache is over the limit or not. If this piece
645 of memory caused problems we better do not use it
646 anymore. Uh, and we ignore possible errors. There
647 is nothing we could do. */
648 (void) munmap (mem, size);
650 return err;
653 pd->guardsize = guardsize;
655 else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
658 /* The old guard area is too large. */
660 #ifdef NEED_SEPARATE_REGISTER_STACK
661 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
662 char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
664 if (oldguard < guard
665 && mprotect (oldguard, guard - oldguard, prot) != 0)
666 goto mprot_error;
668 if (mprotect (guard + guardsize,
669 oldguard + pd->guardsize - guard - guardsize,
670 prot) != 0)
671 goto mprot_error;
672 #elif _STACK_GROWS_DOWN
673 if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
674 prot) != 0)
675 goto mprot_error;
676 #elif _STACK_GROWS_UP
677 if (mprotect ((char *) pd - pd->guardsize,
678 pd->guardsize - guardsize, prot) != 0)
679 goto mprot_error;
680 #endif
682 pd->guardsize = guardsize;
684 /* The pthread_getattr_np() calls need to get passed the size
685 requested in the attribute, regardless of how large the
686 actually used guardsize is. */
687 pd->reported_guardsize = guardsize;
690 /* Initialize the lock. We have to do this unconditionally since the
691 stillborn thread could be canceled while the lock is taken. */
692 pd->lock = LLL_LOCK_INITIALIZER;
694 /* The robust mutex lists also need to be initialized
695 unconditionally because the cleanup for the previous stack owner
696 might have happened in the kernel. */
697 pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
698 - offsetof (pthread_mutex_t,
699 __data.__list.__next));
700 pd->robust_head.list_op_pending = NULL;
701 #ifdef __PTHREAD_MUTEX_HAVE_PREV
702 pd->robust_prev = &pd->robust_head;
703 #endif
704 pd->robust_head.list = &pd->robust_head;
706 /* We place the thread descriptor at the end of the stack. */
707 *pdp = pd;
709 #if TLS_TCB_AT_TP
710 /* The stack begins before the TCB and the static TLS block. */
711 stacktop = ((char *) (pd + 1) - __static_tls_size);
712 #elif TLS_DTV_AT_TP
713 stacktop = (char *) (pd - 1);
714 #endif
716 #ifdef NEED_SEPARATE_REGISTER_STACK
717 *stack = pd->stackblock;
718 *stacksize = stacktop - *stack;
719 #elif _STACK_GROWS_DOWN
720 *stack = stacktop;
721 #elif _STACK_GROWS_UP
722 *stack = pd->stackblock;
723 assert (*stack > 0);
724 #endif
726 return 0;
730 void
731 internal_function
732 __deallocate_stack (struct pthread *pd)
734 lll_lock (stack_cache_lock, LLL_PRIVATE);
736 /* Remove the thread from the list of threads with user defined
737 stacks. */
738 stack_list_del (&pd->list);
740 /* Not much to do. Just free the mmap()ed memory. Note that we do
741 not reset the 'used' flag in the 'tid' field. This is done by
742 the kernel. If no thread has been created yet this field is
743 still zero. */
744 if (__builtin_expect (! pd->user_stack, 1))
745 (void) queue_stack (pd);
746 else
747 /* Free the memory associated with the ELF TLS. */
748 _dl_deallocate_tls (TLS_TPADJ (pd), false);
750 lll_unlock (stack_cache_lock, LLL_PRIVATE);
755 internal_function
756 __make_stacks_executable (void **stack_endp)
758 /* First the main thread's stack. */
759 int err = _dl_make_stack_executable (stack_endp);
760 if (err != 0)
761 return err;
763 #ifdef NEED_SEPARATE_REGISTER_STACK
764 const size_t pagemask = ~(__getpagesize () - 1);
765 #endif
767 lll_lock (stack_cache_lock, LLL_PRIVATE);
769 list_t *runp;
770 list_for_each (runp, &stack_used)
772 err = change_stack_perm (list_entry (runp, struct pthread, list)
773 #ifdef NEED_SEPARATE_REGISTER_STACK
774 , pagemask
775 #endif
777 if (err != 0)
778 break;
781 /* Also change the permission for the currently unused stacks. This
782 might be wasted time but better spend it here than adding a check
783 in the fast path. */
784 if (err == 0)
785 list_for_each (runp, &stack_cache)
787 err = change_stack_perm (list_entry (runp, struct pthread, list)
788 #ifdef NEED_SEPARATE_REGISTER_STACK
789 , pagemask
790 #endif
792 if (err != 0)
793 break;
796 lll_unlock (stack_cache_lock, LLL_PRIVATE);
798 return err;
802 /* In case of a fork() call the memory allocation in the child will be
803 the same but only one thread is running. All stacks except that of
804 the one running thread are not used anymore. We have to recycle
805 them. */
806 void
807 __reclaim_stacks (void)
809 struct pthread *self = (struct pthread *) THREAD_SELF;
811 /* No locking necessary. The caller is the only stack in use. But
812 we have to be aware that we might have interrupted a list
813 operation. */
815 if (in_flight_stack != 0)
817 bool add_p = in_flight_stack & 1;
818 list_t *elem = (list_t *) (in_flight_stack & ~UINTMAX_C (1));
820 if (add_p)
822 /* We always add at the beginning of the list. So in this
823 case we only need to check the beginning of these lists. */
824 int check_list (list_t *l)
826 if (l->next->prev != l)
828 assert (l->next->prev == elem);
830 elem->next = l->next;
831 elem->prev = l;
832 l->next = elem;
834 return 1;
837 return 0;
840 if (check_list (&stack_used) == 0)
841 (void) check_list (&stack_cache);
843 else
845 /* We can simply always replay the delete operation. */
846 elem->next->prev = elem->prev;
847 elem->prev->next = elem->next;
851 /* Mark all stacks except the still running one as free. */
852 list_t *runp;
853 list_for_each (runp, &stack_used)
855 struct pthread *curp = list_entry (runp, struct pthread, list);
856 if (curp != self)
858 /* This marks the stack as free. */
859 curp->tid = 0;
861 /* The PID field must be initialized for the new process. */
862 curp->pid = self->pid;
864 /* Account for the size of the stack. */
865 stack_cache_actsize += curp->stackblock_size;
867 if (curp->specific_used)
869 /* Clear the thread-specific data. */
870 memset (curp->specific_1stblock, '\0',
871 sizeof (curp->specific_1stblock));
873 curp->specific_used = false;
875 for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
876 if (curp->specific[cnt] != NULL)
878 memset (curp->specific[cnt], '\0',
879 sizeof (curp->specific_1stblock));
881 /* We have allocated the block which we do not
882 free here so re-set the bit. */
883 curp->specific_used = true;
889 /* Reset the PIDs in any cached stacks. */
890 list_for_each (runp, &stack_cache)
892 struct pthread *curp = list_entry (runp, struct pthread, list);
893 curp->pid = self->pid;
896 /* Add the stack of all running threads to the cache. */
897 list_splice (&stack_used, &stack_cache);
899 /* Remove the entry for the current thread to from the cache list
900 and add it to the list of running threads. Which of the two
901 lists is decided by the user_stack flag. */
902 stack_list_del (&self->list);
904 /* Re-initialize the lists for all the threads. */
905 INIT_LIST_HEAD (&stack_used);
906 INIT_LIST_HEAD (&__stack_user);
908 if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
909 list_add (&self->list, &__stack_user);
910 else
911 list_add (&self->list, &stack_used);
913 /* There is one thread running. */
914 __nptl_nthreads = 1;
916 in_flight_stack = 0;
918 /* Initialize the lock. */
919 stack_cache_lock = LLL_LOCK_INITIALIZER;
923 #if HP_TIMING_AVAIL
924 # undef __find_thread_by_id
925 /* Find a thread given the thread ID. */
926 attribute_hidden
927 struct pthread *
928 __find_thread_by_id (pid_t tid)
930 struct pthread *result = NULL;
932 lll_lock (stack_cache_lock, LLL_PRIVATE);
934 /* Iterate over the list with system-allocated threads first. */
935 list_t *runp;
936 list_for_each (runp, &stack_used)
938 struct pthread *curp;
940 curp = list_entry (runp, struct pthread, list);
942 if (curp->tid == tid)
944 result = curp;
945 goto out;
949 /* Now the list with threads using user-allocated stacks. */
950 list_for_each (runp, &__stack_user)
952 struct pthread *curp;
954 curp = list_entry (runp, struct pthread, list);
956 if (curp->tid == tid)
958 result = curp;
959 goto out;
963 out:
964 lll_unlock (stack_cache_lock, LLL_PRIVATE);
966 return result;
968 #endif
971 static void
972 internal_function
973 setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
975 int ch;
977 /* Don't let the thread exit before the setxid handler runs. */
978 t->setxid_futex = 0;
982 ch = t->cancelhandling;
984 /* If the thread is exiting right now, ignore it. */
985 if ((ch & EXITING_BITMASK) != 0)
986 return;
988 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
989 ch | SETXID_BITMASK, ch));
993 static void
994 internal_function
995 setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t)
997 int ch;
1001 ch = t->cancelhandling;
1002 if ((ch & SETXID_BITMASK) == 0)
1003 return;
1005 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
1006 ch & ~SETXID_BITMASK, ch));
1008 /* Release the futex just in case. */
1009 t->setxid_futex = 1;
1010 lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE);
1014 static int
1015 internal_function
1016 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
1018 if ((t->cancelhandling & SETXID_BITMASK) == 0)
1019 return 0;
1021 int val;
1022 INTERNAL_SYSCALL_DECL (err);
1023 #if __ASSUME_TGKILL
1024 val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
1025 t->tid, SIGSETXID);
1026 #else
1027 # ifdef __NR_tgkill
1028 val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
1029 t->tid, SIGSETXID);
1030 if (INTERNAL_SYSCALL_ERROR_P (val, err)
1031 && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
1032 # endif
1033 val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
1034 #endif
1036 /* If this failed, it must have had not started yet or else exited. */
1037 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
1039 atomic_increment (&cmdp->cntr);
1040 return 1;
1042 else
1043 return 0;
1048 attribute_hidden
1049 __nptl_setxid (struct xid_command *cmdp)
1051 int signalled;
1052 int result;
1053 lll_lock (stack_cache_lock, LLL_PRIVATE);
1055 __xidcmd = cmdp;
1056 cmdp->cntr = 0;
1058 struct pthread *self = THREAD_SELF;
1060 /* Iterate over the list with system-allocated threads first. */
1061 list_t *runp;
1062 list_for_each (runp, &stack_used)
1064 struct pthread *t = list_entry (runp, struct pthread, list);
1065 if (t == self)
1066 continue;
1068 setxid_mark_thread (cmdp, t);
1071 /* Now the list with threads using user-allocated stacks. */
1072 list_for_each (runp, &__stack_user)
1074 struct pthread *t = list_entry (runp, struct pthread, list);
1075 if (t == self)
1076 continue;
1078 setxid_mark_thread (cmdp, t);
1081 /* Iterate until we don't succeed in signalling anyone. That means
1082 we have gotten all running threads, and their children will be
1083 automatically correct once started. */
1086 signalled = 0;
1088 list_for_each (runp, &stack_used)
1090 struct pthread *t = list_entry (runp, struct pthread, list);
1091 if (t == self)
1092 continue;
1094 signalled += setxid_signal_thread (cmdp, t);
1097 list_for_each (runp, &__stack_user)
1099 struct pthread *t = list_entry (runp, struct pthread, list);
1100 if (t == self)
1101 continue;
1103 signalled += setxid_signal_thread (cmdp, t);
1106 int cur = cmdp->cntr;
1107 while (cur != 0)
1109 lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
1110 cur = cmdp->cntr;
1113 while (signalled != 0);
1115 /* Clean up flags, so that no thread blocks during exit waiting
1116 for a signal which will never come. */
1117 list_for_each (runp, &stack_used)
1119 struct pthread *t = list_entry (runp, struct pthread, list);
1120 if (t == self)
1121 continue;
1123 setxid_unmark_thread (cmdp, t);
1126 list_for_each (runp, &__stack_user)
1128 struct pthread *t = list_entry (runp, struct pthread, list);
1129 if (t == self)
1130 continue;
1132 setxid_unmark_thread (cmdp, t);
1135 /* This must be last, otherwise the current thread might not have
1136 permissions to send SIGSETXID syscall to the other threads. */
1137 INTERNAL_SYSCALL_DECL (err);
1138 result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
1139 cmdp->id[0], cmdp->id[1], cmdp->id[2]);
1140 if (INTERNAL_SYSCALL_ERROR_P (result, err))
1142 __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
1143 result = -1;
1146 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1147 return result;
1150 static inline void __attribute__((always_inline))
1151 init_one_static_tls (struct pthread *curp, struct link_map *map)
1153 dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
1154 # if TLS_TCB_AT_TP
1155 void *dest = (char *) curp - map->l_tls_offset;
1156 # elif TLS_DTV_AT_TP
1157 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
1158 # else
1159 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1160 # endif
1162 /* Fill in the DTV slot so that a later LD/GD access will find it. */
1163 dtv[map->l_tls_modid].pointer.val = dest;
1164 dtv[map->l_tls_modid].pointer.is_static = true;
1166 /* Initialize the memory. */
1167 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
1168 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
1171 void
1172 attribute_hidden
1173 __pthread_init_static_tls (struct link_map *map)
1175 lll_lock (stack_cache_lock, LLL_PRIVATE);
1177 /* Iterate over the list with system-allocated threads first. */
1178 list_t *runp;
1179 list_for_each (runp, &stack_used)
1180 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1182 /* Now the list with threads using user-allocated stacks. */
1183 list_for_each (runp, &__stack_user)
1184 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1186 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1190 void
1191 attribute_hidden
1192 __wait_lookup_done (void)
1194 lll_lock (stack_cache_lock, LLL_PRIVATE);
1196 struct pthread *self = THREAD_SELF;
1198 /* Iterate over the list with system-allocated threads first. */
1199 list_t *runp;
1200 list_for_each (runp, &stack_used)
1202 struct pthread *t = list_entry (runp, struct pthread, list);
1203 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1204 continue;
1206 int *const gscope_flagp = &t->header.gscope_flag;
1208 /* We have to wait until this thread is done with the global
1209 scope. First tell the thread that we are waiting and
1210 possibly have to be woken. */
1211 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1212 THREAD_GSCOPE_FLAG_WAIT,
1213 THREAD_GSCOPE_FLAG_USED))
1214 continue;
1217 lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1218 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1221 /* Now the list with threads using user-allocated stacks. */
1222 list_for_each (runp, &__stack_user)
1224 struct pthread *t = list_entry (runp, struct pthread, list);
1225 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1226 continue;
1228 int *const gscope_flagp = &t->header.gscope_flag;
1230 /* We have to wait until this thread is done with the global
1231 scope. First tell the thread that we are waiting and
1232 possibly have to be woken. */
1233 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1234 THREAD_GSCOPE_FLAG_WAIT,
1235 THREAD_GSCOPE_FLAG_USED))
1236 continue;
1239 lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1240 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1243 lll_unlock (stack_cache_lock, LLL_PRIVATE);