1 /* Copyright (C) 2002-2017 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
26 #include <sys/param.h>
27 #include <dl-sysdep.h>
31 #include <lowlevellock.h>
32 #include <futex-internal.h>
33 #include <kernel-features.h>
34 #include <stack-aliasing.h>
37 #ifndef NEED_SEPARATE_REGISTER_STACK
39 /* Most architectures have exactly one stack pointer. Some have more. */
40 # define STACK_VARIABLES void *stackaddr = NULL
42 /* How to pass the values to the 'create_thread' function. */
43 # define STACK_VARIABLES_ARGS stackaddr
45 /* How to declare function which gets there parameters. */
46 # define STACK_VARIABLES_PARMS void *stackaddr
48 /* How to declare allocate_stack. */
49 # define ALLOCATE_STACK_PARMS void **stack
51 /* This is how the function is called. We do it this way to allow
52 other variants of the function to have more parameters. */
53 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
57 /* We need two stacks. The kernel will place them but we have to tell
58 the kernel about the size of the reserved address space. */
59 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
61 /* How to pass the values to the 'create_thread' function. */
62 # define STACK_VARIABLES_ARGS stackaddr, stacksize
64 /* How to declare function which gets there parameters. */
65 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
67 /* How to declare allocate_stack. */
68 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
70 /* This is how the function is called. We do it this way to allow
71 other variants of the function to have more parameters. */
72 # define ALLOCATE_STACK(attr, pd) \
73 allocate_stack (attr, pd, &stackaddr, &stacksize)
78 /* Default alignment of stack. */
80 # define STACK_ALIGN __alignof__ (long double)
83 /* Default value for minimal stack size after allocating thread
84 descriptor and guard. */
85 #ifndef MINIMAL_REST_STACK
86 # define MINIMAL_REST_STACK 4096
90 /* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
91 a stack. Use it when possible. */
96 /* This yields the pointer that TLS support code calls the thread pointer. */
98 # define TLS_TPADJ(pd) (pd)
100 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
103 /* Cache handling for not-yet free stacks. */
105 /* Maximum size in kB of cache. */
106 static size_t stack_cache_maxsize
= 40 * 1024 * 1024; /* 40MiBi by default. */
107 static size_t stack_cache_actsize
;
109 /* Mutex protecting this variable. */
110 static int stack_cache_lock
= LLL_LOCK_INITIALIZER
;
112 /* List of queued stack frames. */
113 static LIST_HEAD (stack_cache
);
115 /* List of the stacks in use. */
116 static LIST_HEAD (stack_used
);
118 /* We need to record what list operations we are going to do so that,
119 in case of an asynchronous interruption due to a fork() call, we
120 can correct for the work. */
121 static uintptr_t in_flight_stack
;
123 /* List of the threads with user provided stacks in use. No need to
124 initialize this, since it's done in __pthread_initialize_minimal. */
125 list_t __stack_user
__attribute__ ((nocommon
));
126 hidden_data_def (__stack_user
)
129 /* Check whether the stack is still used or not. */
130 #define FREE_P(descr) ((descr)->tid <= 0)
134 stack_list_del (list_t
*elem
)
136 in_flight_stack
= (uintptr_t) elem
;
138 atomic_write_barrier ();
142 atomic_write_barrier ();
149 stack_list_add (list_t
*elem
, list_t
*list
)
151 in_flight_stack
= (uintptr_t) elem
| 1;
153 atomic_write_barrier ();
155 list_add (elem
, list
);
157 atomic_write_barrier ();
163 /* We create a double linked list of all cache entries. Double linked
164 because this allows removing entries from the end. */
167 /* Get a stack frame from the cache. We have to match by size since
168 some blocks might be too small or far too large. */
169 static struct pthread
*
170 get_cached_stack (size_t *sizep
, void **memp
)
172 size_t size
= *sizep
;
173 struct pthread
*result
= NULL
;
176 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
178 /* Search the cache for a matching entry. We search for the
179 smallest stack which has at least the required size. Note that
180 in normal situations the size of all allocated stacks is the
181 same. As the very least there are only a few different sizes.
182 Therefore this loop will exit early most of the time with an
184 list_for_each (entry
, &stack_cache
)
186 struct pthread
*curr
;
188 curr
= list_entry (entry
, struct pthread
, list
);
189 if (FREE_P (curr
) && curr
->stackblock_size
>= size
)
191 if (curr
->stackblock_size
== size
)
198 || result
->stackblock_size
> curr
->stackblock_size
)
203 if (__builtin_expect (result
== NULL
, 0)
204 /* Make sure the size difference is not too excessive. In that
205 case we do not use the block. */
206 || __builtin_expect (result
->stackblock_size
> 4 * size
, 0))
208 /* Release the lock. */
209 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
214 /* Don't allow setxid until cloned. */
215 result
->setxid_futex
= -1;
217 /* Dequeue the entry. */
218 stack_list_del (&result
->list
);
220 /* And add to the list of stacks in use. */
221 stack_list_add (&result
->list
, &stack_used
);
223 /* And decrease the cache size. */
224 stack_cache_actsize
-= result
->stackblock_size
;
226 /* Release the lock early. */
227 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
229 /* Report size and location of the stack to the caller. */
230 *sizep
= result
->stackblock_size
;
231 *memp
= result
->stackblock
;
233 /* Cancellation handling is back to the default. */
234 result
->cancelhandling
= 0;
235 result
->cleanup
= NULL
;
237 /* No pending event. */
238 result
->nextevent
= NULL
;
241 dtv_t
*dtv
= GET_DTV (TLS_TPADJ (result
));
242 for (size_t cnt
= 0; cnt
< dtv
[-1].counter
; ++cnt
)
243 free (dtv
[1 + cnt
].pointer
.to_free
);
244 memset (dtv
, '\0', (dtv
[-1].counter
+ 1) * sizeof (dtv_t
));
246 /* Re-initialize the TLS. */
247 _dl_allocate_tls_init (TLS_TPADJ (result
));
253 /* Free stacks until cache size is lower than LIMIT. */
255 __free_stacks (size_t limit
)
257 /* We reduce the size of the cache. Remove the last entries until
258 the size is below the limit. */
262 /* Search from the end of the list. */
263 list_for_each_prev_safe (entry
, prev
, &stack_cache
)
265 struct pthread
*curr
;
267 curr
= list_entry (entry
, struct pthread
, list
);
270 /* Unlink the block. */
271 stack_list_del (entry
);
273 /* Account for the freed memory. */
274 stack_cache_actsize
-= curr
->stackblock_size
;
276 /* Free the memory associated with the ELF TLS. */
277 _dl_deallocate_tls (TLS_TPADJ (curr
), false);
279 /* Remove this block. This should never fail. If it does
280 something is really wrong. */
281 if (munmap (curr
->stackblock
, curr
->stackblock_size
) != 0)
284 /* Maybe we have freed enough. */
285 if (stack_cache_actsize
<= limit
)
292 /* Add a stack frame which is not used anymore to the stack. Must be
293 called with the cache lock held. */
295 __attribute ((always_inline
))
296 queue_stack (struct pthread
*stack
)
298 /* We unconditionally add the stack to the list. The memory may
299 still be in use but it will not be reused until the kernel marks
300 the stack as not used anymore. */
301 stack_list_add (&stack
->list
, &stack_cache
);
303 stack_cache_actsize
+= stack
->stackblock_size
;
304 if (__glibc_unlikely (stack_cache_actsize
> stack_cache_maxsize
))
305 __free_stacks (stack_cache_maxsize
);
311 change_stack_perm (struct pthread
*pd
312 #ifdef NEED_SEPARATE_REGISTER_STACK
317 #ifdef NEED_SEPARATE_REGISTER_STACK
318 void *stack
= (pd
->stackblock
319 + (((((pd
->stackblock_size
- pd
->guardsize
) / 2)
320 & pagemask
) + pd
->guardsize
) & pagemask
));
321 size_t len
= pd
->stackblock
+ pd
->stackblock_size
- stack
;
322 #elif _STACK_GROWS_DOWN
323 void *stack
= pd
->stackblock
+ pd
->guardsize
;
324 size_t len
= pd
->stackblock_size
- pd
->guardsize
;
325 #elif _STACK_GROWS_UP
326 void *stack
= pd
->stackblock
;
327 size_t len
= (uintptr_t) pd
- pd
->guardsize
- (uintptr_t) pd
->stackblock
;
329 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
331 if (mprotect (stack
, len
, PROT_READ
| PROT_WRITE
| PROT_EXEC
) != 0)
338 /* Returns a usable stack for a new thread either by allocating a
339 new stack or reusing a cached stack of sufficient size.
340 ATTR must be non-NULL and point to a valid pthread_attr.
341 PDP must be non-NULL. */
343 allocate_stack (const struct pthread_attr
*attr
, struct pthread
**pdp
,
344 ALLOCATE_STACK_PARMS
)
348 size_t pagesize_m1
= __getpagesize () - 1;
350 assert (powerof2 (pagesize_m1
+ 1));
351 assert (TCB_ALIGNMENT
>= STACK_ALIGN
);
353 /* Get the stack size from the attribute if it is set. Otherwise we
354 use the default we determined at start time. */
355 if (attr
->stacksize
!= 0)
356 size
= attr
->stacksize
;
359 lll_lock (__default_pthread_attr_lock
, LLL_PRIVATE
);
360 size
= __default_pthread_attr
.stacksize
;
361 lll_unlock (__default_pthread_attr_lock
, LLL_PRIVATE
);
364 /* Get memory for the stack. */
365 if (__glibc_unlikely (attr
->flags
& ATTR_FLAG_STACKADDR
))
368 char *stackaddr
= (char *) attr
->stackaddr
;
370 /* Assume the same layout as the _STACK_GROWS_DOWN case, with struct
371 pthread at the top of the stack block. Later we adjust the guard
372 location and stack address to match the _STACK_GROWS_UP case. */
374 stackaddr
+= attr
->stacksize
;
376 /* If the user also specified the size of the stack make sure it
378 if (attr
->stacksize
!= 0
379 && attr
->stacksize
< (__static_tls_size
+ MINIMAL_REST_STACK
))
382 /* Adjust stack size for alignment of the TLS block. */
384 adj
= ((uintptr_t) stackaddr
- TLS_TCB_SIZE
)
385 & __static_tls_align_m1
;
386 assert (size
> adj
+ TLS_TCB_SIZE
);
388 adj
= ((uintptr_t) stackaddr
- __static_tls_size
)
389 & __static_tls_align_m1
;
393 /* The user provided some memory. Let's hope it matches the
394 size... We do not allocate guard pages if the user provided
395 the stack. It is the user's responsibility to do this if it
398 pd
= (struct pthread
*) ((uintptr_t) stackaddr
399 - TLS_TCB_SIZE
- adj
);
401 pd
= (struct pthread
*) (((uintptr_t) stackaddr
402 - __static_tls_size
- adj
)
406 /* The user provided stack memory needs to be cleared. */
407 memset (pd
, '\0', sizeof (struct pthread
));
409 /* The first TSD block is included in the TCB. */
410 pd
->specific
[0] = pd
->specific_1stblock
;
412 /* Remember the stack-related values. */
413 pd
->stackblock
= (char *) stackaddr
- size
;
414 pd
->stackblock_size
= size
;
416 /* This is a user-provided stack. It will not be queued in the
417 stack cache nor will the memory (except the TLS memory) be freed. */
418 pd
->user_stack
= true;
420 /* This is at least the second thread. */
421 pd
->header
.multiple_threads
= 1;
422 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
423 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
426 #ifndef __ASSUME_PRIVATE_FUTEX
427 /* The thread must know when private futexes are supported. */
428 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
429 header
.private_futex
);
432 #ifdef NEED_DL_SYSINFO
433 SETUP_THREAD_SYSINFO (pd
);
436 /* Don't allow setxid until cloned. */
437 pd
->setxid_futex
= -1;
439 /* Allocate the DTV for this thread. */
440 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
442 /* Something went wrong. */
443 assert (errno
== ENOMEM
);
448 /* Prepare to modify global data. */
449 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
451 /* And add to the list of stacks in use. */
452 list_add (&pd
->list
, &__stack_user
);
454 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
458 /* Allocate some anonymous memory. If possible use the cache. */
462 const int prot
= (PROT_READ
| PROT_WRITE
463 | ((GL(dl_stack_flags
) & PF_X
) ? PROT_EXEC
: 0));
465 /* Adjust the stack size for alignment. */
466 size
&= ~__static_tls_align_m1
;
469 /* Make sure the size of the stack is enough for the guard and
470 eventually the thread descriptor. */
471 guardsize
= (attr
->guardsize
+ pagesize_m1
) & ~pagesize_m1
;
472 if (__builtin_expect (size
< ((guardsize
+ __static_tls_size
473 + MINIMAL_REST_STACK
+ pagesize_m1
)
476 /* The stack is too small (or the guard too large). */
479 /* Try to get a stack from the cache. */
481 pd
= get_cached_stack (&size
, &mem
);
484 /* To avoid aliasing effects on a larger scale than pages we
485 adjust the allocated stack size if necessary. This way
486 allocations directly following each other will not have
487 aliasing problems. */
488 #if MULTI_PAGE_ALIASING != 0
489 if ((size
% MULTI_PAGE_ALIASING
) == 0)
490 size
+= pagesize_m1
+ 1;
493 mem
= mmap (NULL
, size
, prot
,
494 MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_STACK
, -1, 0);
496 if (__glibc_unlikely (mem
== MAP_FAILED
))
499 /* SIZE is guaranteed to be greater than zero.
500 So we can never get a null pointer back from mmap. */
501 assert (mem
!= NULL
);
503 /* Place the thread descriptor at the end of the stack. */
505 pd
= (struct pthread
*) ((char *) mem
+ size
) - 1;
507 pd
= (struct pthread
*) ((((uintptr_t) mem
+ size
509 & ~__static_tls_align_m1
)
513 /* Remember the stack-related values. */
514 pd
->stackblock
= mem
;
515 pd
->stackblock_size
= size
;
517 /* We allocated the first block thread-specific data array.
518 This address will not change for the lifetime of this
520 pd
->specific
[0] = pd
->specific_1stblock
;
522 /* This is at least the second thread. */
523 pd
->header
.multiple_threads
= 1;
524 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
525 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
528 #ifndef __ASSUME_PRIVATE_FUTEX
529 /* The thread must know when private futexes are supported. */
530 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
531 header
.private_futex
);
534 #ifdef NEED_DL_SYSINFO
535 SETUP_THREAD_SYSINFO (pd
);
538 /* Don't allow setxid until cloned. */
539 pd
->setxid_futex
= -1;
541 /* Allocate the DTV for this thread. */
542 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
544 /* Something went wrong. */
545 assert (errno
== ENOMEM
);
547 /* Free the stack memory we just allocated. */
548 (void) munmap (mem
, size
);
554 /* Prepare to modify global data. */
555 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
557 /* And add to the list of stacks in use. */
558 stack_list_add (&pd
->list
, &stack_used
);
560 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
563 /* There might have been a race. Another thread might have
564 caused the stacks to get exec permission while this new
565 stack was prepared. Detect if this was possible and
566 change the permission if necessary. */
567 if (__builtin_expect ((GL(dl_stack_flags
) & PF_X
) != 0
568 && (prot
& PROT_EXEC
) == 0, 0))
570 int err
= change_stack_perm (pd
571 #ifdef NEED_SEPARATE_REGISTER_STACK
577 /* Free the stack memory we just allocated. */
578 (void) munmap (mem
, size
);
585 /* Note that all of the stack and the thread descriptor is
586 zeroed. This means we do not have to initialize fields
587 with initial value zero. This is specifically true for
588 the 'tid' field which is always set back to zero once the
589 stack is not used anymore and for the 'guardsize' field
590 which will be read next. */
593 /* Create or resize the guard area if necessary. */
594 if (__glibc_unlikely (guardsize
> pd
->guardsize
))
596 #ifdef NEED_SEPARATE_REGISTER_STACK
597 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
598 #elif _STACK_GROWS_DOWN
600 #elif _STACK_GROWS_UP
601 char *guard
= (char *) (((uintptr_t) pd
- guardsize
) & ~pagesize_m1
);
603 if (mprotect (guard
, guardsize
, PROT_NONE
) != 0)
606 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
608 /* Remove the thread from the list. */
609 stack_list_del (&pd
->list
);
611 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
613 /* Get rid of the TLS block we allocated. */
614 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
616 /* Free the stack memory regardless of whether the size
617 of the cache is over the limit or not. If this piece
618 of memory caused problems we better do not use it
619 anymore. Uh, and we ignore possible errors. There
620 is nothing we could do. */
621 (void) munmap (mem
, size
);
626 pd
->guardsize
= guardsize
;
628 else if (__builtin_expect (pd
->guardsize
- guardsize
> size
- reqsize
,
631 /* The old guard area is too large. */
633 #ifdef NEED_SEPARATE_REGISTER_STACK
634 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
635 char *oldguard
= mem
+ (((size
- pd
->guardsize
) / 2) & ~pagesize_m1
);
638 && mprotect (oldguard
, guard
- oldguard
, prot
) != 0)
641 if (mprotect (guard
+ guardsize
,
642 oldguard
+ pd
->guardsize
- guard
- guardsize
,
645 #elif _STACK_GROWS_DOWN
646 if (mprotect ((char *) mem
+ guardsize
, pd
->guardsize
- guardsize
,
649 #elif _STACK_GROWS_UP
650 if (mprotect ((char *) pd
- pd
->guardsize
,
651 pd
->guardsize
- guardsize
, prot
) != 0)
655 pd
->guardsize
= guardsize
;
657 /* The pthread_getattr_np() calls need to get passed the size
658 requested in the attribute, regardless of how large the
659 actually used guardsize is. */
660 pd
->reported_guardsize
= guardsize
;
663 /* Initialize the lock. We have to do this unconditionally since the
664 stillborn thread could be canceled while the lock is taken. */
665 pd
->lock
= LLL_LOCK_INITIALIZER
;
667 /* The robust mutex lists also need to be initialized
668 unconditionally because the cleanup for the previous stack owner
669 might have happened in the kernel. */
670 pd
->robust_head
.futex_offset
= (offsetof (pthread_mutex_t
, __data
.__lock
)
671 - offsetof (pthread_mutex_t
,
672 __data
.__list
.__next
));
673 pd
->robust_head
.list_op_pending
= NULL
;
674 #ifdef __PTHREAD_MUTEX_HAVE_PREV
675 pd
->robust_prev
= &pd
->robust_head
;
677 pd
->robust_head
.list
= &pd
->robust_head
;
679 /* We place the thread descriptor at the end of the stack. */
682 #if _STACK_GROWS_DOWN
686 /* The stack begins before the TCB and the static TLS block. */
687 stacktop
= ((char *) (pd
+ 1) - __static_tls_size
);
689 stacktop
= (char *) (pd
- 1);
692 # ifdef NEED_SEPARATE_REGISTER_STACK
693 *stack
= pd
->stackblock
;
694 *stacksize
= stacktop
- *stack
;
699 *stack
= pd
->stackblock
;
708 __deallocate_stack (struct pthread
*pd
)
710 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
712 /* Remove the thread from the list of threads with user defined
714 stack_list_del (&pd
->list
);
716 /* Not much to do. Just free the mmap()ed memory. Note that we do
717 not reset the 'used' flag in the 'tid' field. This is done by
718 the kernel. If no thread has been created yet this field is
720 if (__glibc_likely (! pd
->user_stack
))
721 (void) queue_stack (pd
);
723 /* Free the memory associated with the ELF TLS. */
724 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
726 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
732 __make_stacks_executable (void **stack_endp
)
734 /* First the main thread's stack. */
735 int err
= _dl_make_stack_executable (stack_endp
);
739 #ifdef NEED_SEPARATE_REGISTER_STACK
740 const size_t pagemask
= ~(__getpagesize () - 1);
743 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
746 list_for_each (runp
, &stack_used
)
748 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
749 #ifdef NEED_SEPARATE_REGISTER_STACK
757 /* Also change the permission for the currently unused stacks. This
758 might be wasted time but better spend it here than adding a check
761 list_for_each (runp
, &stack_cache
)
763 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
764 #ifdef NEED_SEPARATE_REGISTER_STACK
772 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
778 /* In case of a fork() call the memory allocation in the child will be
779 the same but only one thread is running. All stacks except that of
780 the one running thread are not used anymore. We have to recycle
783 __reclaim_stacks (void)
785 struct pthread
*self
= (struct pthread
*) THREAD_SELF
;
787 /* No locking necessary. The caller is the only stack in use. But
788 we have to be aware that we might have interrupted a list
791 if (in_flight_stack
!= 0)
793 bool add_p
= in_flight_stack
& 1;
794 list_t
*elem
= (list_t
*) (in_flight_stack
& ~(uintptr_t) 1);
798 /* We always add at the beginning of the list. So in this case we
799 only need to check the beginning of these lists to see if the
800 pointers at the head of the list are inconsistent. */
803 if (stack_used
.next
->prev
!= &stack_used
)
805 else if (stack_cache
.next
->prev
!= &stack_cache
)
810 assert (l
->next
->prev
== elem
);
811 elem
->next
= l
->next
;
818 /* We can simply always replay the delete operation. */
819 elem
->next
->prev
= elem
->prev
;
820 elem
->prev
->next
= elem
->next
;
824 /* Mark all stacks except the still running one as free. */
826 list_for_each (runp
, &stack_used
)
828 struct pthread
*curp
= list_entry (runp
, struct pthread
, list
);
831 /* This marks the stack as free. */
834 /* Account for the size of the stack. */
835 stack_cache_actsize
+= curp
->stackblock_size
;
837 if (curp
->specific_used
)
839 /* Clear the thread-specific data. */
840 memset (curp
->specific_1stblock
, '\0',
841 sizeof (curp
->specific_1stblock
));
843 curp
->specific_used
= false;
845 for (size_t cnt
= 1; cnt
< PTHREAD_KEY_1STLEVEL_SIZE
; ++cnt
)
846 if (curp
->specific
[cnt
] != NULL
)
848 memset (curp
->specific
[cnt
], '\0',
849 sizeof (curp
->specific_1stblock
));
851 /* We have allocated the block which we do not
852 free here so re-set the bit. */
853 curp
->specific_used
= true;
859 /* Add the stack of all running threads to the cache. */
860 list_splice (&stack_used
, &stack_cache
);
862 /* Remove the entry for the current thread to from the cache list
863 and add it to the list of running threads. Which of the two
864 lists is decided by the user_stack flag. */
865 stack_list_del (&self
->list
);
867 /* Re-initialize the lists for all the threads. */
868 INIT_LIST_HEAD (&stack_used
);
869 INIT_LIST_HEAD (&__stack_user
);
871 if (__glibc_unlikely (THREAD_GETMEM (self
, user_stack
)))
872 list_add (&self
->list
, &__stack_user
);
874 list_add (&self
->list
, &stack_used
);
876 /* There is one thread running. */
881 /* Initialize locks. */
882 stack_cache_lock
= LLL_LOCK_INITIALIZER
;
883 __default_pthread_attr_lock
= LLL_LOCK_INITIALIZER
;
888 # undef __find_thread_by_id
889 /* Find a thread given the thread ID. */
892 __find_thread_by_id (pid_t tid
)
894 struct pthread
*result
= NULL
;
896 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
898 /* Iterate over the list with system-allocated threads first. */
900 list_for_each (runp
, &stack_used
)
902 struct pthread
*curp
;
904 curp
= list_entry (runp
, struct pthread
, list
);
906 if (curp
->tid
== tid
)
913 /* Now the list with threads using user-allocated stacks. */
914 list_for_each (runp
, &__stack_user
)
916 struct pthread
*curp
;
918 curp
= list_entry (runp
, struct pthread
, list
);
920 if (curp
->tid
== tid
)
928 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
938 setxid_mark_thread (struct xid_command
*cmdp
, struct pthread
*t
)
942 /* Wait until this thread is cloned. */
943 if (t
->setxid_futex
== -1
944 && ! atomic_compare_and_exchange_bool_acq (&t
->setxid_futex
, -2, -1))
946 futex_wait_simple (&t
->setxid_futex
, -2, FUTEX_PRIVATE
);
947 while (t
->setxid_futex
== -2);
949 /* Don't let the thread exit before the setxid handler runs. */
954 ch
= t
->cancelhandling
;
956 /* If the thread is exiting right now, ignore it. */
957 if ((ch
& EXITING_BITMASK
) != 0)
959 /* Release the futex if there is no other setxid in
961 if ((ch
& SETXID_BITMASK
) == 0)
964 futex_wake (&t
->setxid_futex
, 1, FUTEX_PRIVATE
);
969 while (atomic_compare_and_exchange_bool_acq (&t
->cancelhandling
,
970 ch
| SETXID_BITMASK
, ch
));
976 setxid_unmark_thread (struct xid_command
*cmdp
, struct pthread
*t
)
982 ch
= t
->cancelhandling
;
983 if ((ch
& SETXID_BITMASK
) == 0)
986 while (atomic_compare_and_exchange_bool_acq (&t
->cancelhandling
,
987 ch
& ~SETXID_BITMASK
, ch
));
989 /* Release the futex just in case. */
991 futex_wake (&t
->setxid_futex
, 1, FUTEX_PRIVATE
);
997 setxid_signal_thread (struct xid_command
*cmdp
, struct pthread
*t
)
999 if ((t
->cancelhandling
& SETXID_BITMASK
) == 0)
1003 pid_t pid
= __getpid ();
1004 INTERNAL_SYSCALL_DECL (err
);
1005 val
= INTERNAL_SYSCALL_CALL (tgkill
, err
, pid
, t
->tid
, SIGSETXID
);
1007 /* If this failed, it must have had not started yet or else exited. */
1008 if (!INTERNAL_SYSCALL_ERROR_P (val
, err
))
1010 atomic_increment (&cmdp
->cntr
);
1017 /* Check for consistency across set*id system call results. The abort
1018 should not happen as long as all privileges changes happen through
1019 the glibc wrappers. ERROR must be 0 (no error) or an errno
1023 __nptl_setxid_error (struct xid_command
*cmdp
, int error
)
1027 int olderror
= cmdp
->error
;
1028 if (olderror
== error
)
1031 /* Mismatch between current and previous results. */
1034 while (atomic_compare_and_exchange_bool_acq (&cmdp
->error
, error
, -1));
1039 __nptl_setxid (struct xid_command
*cmdp
)
1043 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1049 struct pthread
*self
= THREAD_SELF
;
1051 /* Iterate over the list with system-allocated threads first. */
1053 list_for_each (runp
, &stack_used
)
1055 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1059 setxid_mark_thread (cmdp
, t
);
1062 /* Now the list with threads using user-allocated stacks. */
1063 list_for_each (runp
, &__stack_user
)
1065 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1069 setxid_mark_thread (cmdp
, t
);
1072 /* Iterate until we don't succeed in signalling anyone. That means
1073 we have gotten all running threads, and their children will be
1074 automatically correct once started. */
1079 list_for_each (runp
, &stack_used
)
1081 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1085 signalled
+= setxid_signal_thread (cmdp
, t
);
1088 list_for_each (runp
, &__stack_user
)
1090 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1094 signalled
+= setxid_signal_thread (cmdp
, t
);
1097 int cur
= cmdp
->cntr
;
1100 futex_wait_simple ((unsigned int *) &cmdp
->cntr
, cur
,
1105 while (signalled
!= 0);
1107 /* Clean up flags, so that no thread blocks during exit waiting
1108 for a signal which will never come. */
1109 list_for_each (runp
, &stack_used
)
1111 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1115 setxid_unmark_thread (cmdp
, t
);
1118 list_for_each (runp
, &__stack_user
)
1120 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1124 setxid_unmark_thread (cmdp
, t
);
1127 /* This must be last, otherwise the current thread might not have
1128 permissions to send SIGSETXID syscall to the other threads. */
1129 INTERNAL_SYSCALL_DECL (err
);
1130 result
= INTERNAL_SYSCALL_NCS (cmdp
->syscall_no
, err
, 3,
1131 cmdp
->id
[0], cmdp
->id
[1], cmdp
->id
[2]);
1133 if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result
, err
)))
1135 error
= INTERNAL_SYSCALL_ERRNO (result
, err
);
1136 __set_errno (error
);
1139 __nptl_setxid_error (cmdp
, error
);
1141 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1144 #endif /* SIGSETXID. */
1147 static inline void __attribute__((always_inline
))
1148 init_one_static_tls (struct pthread
*curp
, struct link_map
*map
)
1151 void *dest
= (char *) curp
- map
->l_tls_offset
;
1152 # elif TLS_DTV_AT_TP
1153 void *dest
= (char *) curp
+ map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
1155 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1158 /* Initialize the memory. */
1159 memset (__mempcpy (dest
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
1160 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
1165 __pthread_init_static_tls (struct link_map
*map
)
1167 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1169 /* Iterate over the list with system-allocated threads first. */
1171 list_for_each (runp
, &stack_used
)
1172 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1174 /* Now the list with threads using user-allocated stacks. */
1175 list_for_each (runp
, &__stack_user
)
1176 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1178 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1184 __wait_lookup_done (void)
1186 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1188 struct pthread
*self
= THREAD_SELF
;
1190 /* Iterate over the list with system-allocated threads first. */
1192 list_for_each (runp
, &stack_used
)
1194 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1195 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1198 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1200 /* We have to wait until this thread is done with the global
1201 scope. First tell the thread that we are waiting and
1202 possibly have to be woken. */
1203 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1204 THREAD_GSCOPE_FLAG_WAIT
,
1205 THREAD_GSCOPE_FLAG_USED
))
1209 futex_wait_simple ((unsigned int *) gscope_flagp
,
1210 THREAD_GSCOPE_FLAG_WAIT
, FUTEX_PRIVATE
);
1211 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1214 /* Now the list with threads using user-allocated stacks. */
1215 list_for_each (runp
, &__stack_user
)
1217 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1218 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1221 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1223 /* We have to wait until this thread is done with the global
1224 scope. First tell the thread that we are waiting and
1225 possibly have to be woken. */
1226 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1227 THREAD_GSCOPE_FLAG_WAIT
,
1228 THREAD_GSCOPE_FLAG_USED
))
1232 futex_wait_simple ((unsigned int *) gscope_flagp
,
1233 THREAD_GSCOPE_FLAG_WAIT
, FUTEX_PRIVATE
);
1234 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1237 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);