1 /* Copyright (C) 2002-2007, 2009, 2010, 2011 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
27 #include <sys/param.h>
28 #include <dl-sysdep.h>
32 #include <lowlevellock.h>
33 #include <kernel-features.h>
36 #ifndef NEED_SEPARATE_REGISTER_STACK
38 /* Most architectures have exactly one stack pointer. Some have more. */
39 # define STACK_VARIABLES void *stackaddr = NULL
41 /* How to pass the values to the 'create_thread' function. */
42 # define STACK_VARIABLES_ARGS stackaddr
44 /* How to declare function which gets there parameters. */
45 # define STACK_VARIABLES_PARMS void *stackaddr
47 /* How to declare allocate_stack. */
48 # define ALLOCATE_STACK_PARMS void **stack
50 /* This is how the function is called. We do it this way to allow
51 other variants of the function to have more parameters. */
52 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
56 /* We need two stacks. The kernel will place them but we have to tell
57 the kernel about the size of the reserved address space. */
58 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
60 /* How to pass the values to the 'create_thread' function. */
61 # define STACK_VARIABLES_ARGS stackaddr, stacksize
63 /* How to declare function which gets there parameters. */
64 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
66 /* How to declare allocate_stack. */
67 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
69 /* This is how the function is called. We do it this way to allow
70 other variants of the function to have more parameters. */
71 # define ALLOCATE_STACK(attr, pd) \
72 allocate_stack (attr, pd, &stackaddr, &stacksize)
77 /* Default alignment of stack. */
79 # define STACK_ALIGN __alignof__ (long double)
82 /* Default value for minimal stack size after allocating thread
83 descriptor and guard. */
84 #ifndef MINIMAL_REST_STACK
85 # define MINIMAL_REST_STACK 4096
89 /* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
90 a stack. Use it when possible. */
95 /* This yields the pointer that TLS support code calls the thread pointer. */
97 # define TLS_TPADJ(pd) (pd)
99 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
102 /* Cache handling for not-yet free stacks. */
104 /* Maximum size in kB of cache. */
105 static size_t stack_cache_maxsize
= 40 * 1024 * 1024; /* 40MiBi by default. */
106 static size_t stack_cache_actsize
;
108 /* Mutex protecting this variable. */
109 static int stack_cache_lock
= LLL_LOCK_INITIALIZER
;
111 /* List of queued stack frames. */
112 static LIST_HEAD (stack_cache
);
114 /* List of the stacks in use. */
115 static LIST_HEAD (stack_used
);
117 /* We need to record what list operations we are going to do so that,
118 in case of an asynchronous interruption due to a fork() call, we
119 can correct for the work. */
120 static uintptr_t in_flight_stack
;
122 /* List of the threads with user provided stacks in use. No need to
123 initialize this, since it's done in __pthread_initialize_minimal. */
124 list_t __stack_user
__attribute__ ((nocommon
));
125 hidden_data_def (__stack_user
)
127 #if COLORING_INCREMENT != 0
128 /* Number of threads created. */
129 static unsigned int nptl_ncreated
;
133 /* Check whether the stack is still used or not. */
134 #define FREE_P(descr) ((descr)->tid <= 0)
138 stack_list_del (list_t
*elem
)
140 in_flight_stack
= (uintptr_t) elem
;
142 atomic_write_barrier ();
146 atomic_write_barrier ();
153 stack_list_add (list_t
*elem
, list_t
*list
)
155 in_flight_stack
= (uintptr_t) elem
| 1;
157 atomic_write_barrier ();
159 list_add (elem
, list
);
161 atomic_write_barrier ();
167 /* We create a double linked list of all cache entries. Double linked
168 because this allows removing entries from the end. */
171 /* Get a stack frame from the cache. We have to match by size since
172 some blocks might be too small or far too large. */
173 static struct pthread
*
174 get_cached_stack (size_t *sizep
, void **memp
)
176 size_t size
= *sizep
;
177 struct pthread
*result
= NULL
;
180 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
182 /* Search the cache for a matching entry. We search for the
183 smallest stack which has at least the required size. Note that
184 in normal situations the size of all allocated stacks is the
185 same. As the very least there are only a few different sizes.
186 Therefore this loop will exit early most of the time with an
188 list_for_each (entry
, &stack_cache
)
190 struct pthread
*curr
;
192 curr
= list_entry (entry
, struct pthread
, list
);
193 if (FREE_P (curr
) && curr
->stackblock_size
>= size
)
195 if (curr
->stackblock_size
== size
)
202 || result
->stackblock_size
> curr
->stackblock_size
)
207 if (__builtin_expect (result
== NULL
, 0)
208 /* Make sure the size difference is not too excessive. In that
209 case we do not use the block. */
210 || __builtin_expect (result
->stackblock_size
> 4 * size
, 0))
212 /* Release the lock. */
213 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
218 /* Don't allow setxid until cloned. */
219 result
->setxid_futex
= -1;
221 /* Dequeue the entry. */
222 stack_list_del (&result
->list
);
224 /* And add to the list of stacks in use. */
225 stack_list_add (&result
->list
, &stack_used
);
227 /* And decrease the cache size. */
228 stack_cache_actsize
-= result
->stackblock_size
;
230 /* Release the lock early. */
231 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
233 /* Report size and location of the stack to the caller. */
234 *sizep
= result
->stackblock_size
;
235 *memp
= result
->stackblock
;
237 /* Cancellation handling is back to the default. */
238 result
->cancelhandling
= 0;
239 result
->cleanup
= NULL
;
241 /* No pending event. */
242 result
->nextevent
= NULL
;
245 dtv_t
*dtv
= GET_DTV (TLS_TPADJ (result
));
246 for (size_t cnt
= 0; cnt
< dtv
[-1].counter
; ++cnt
)
247 if (! dtv
[1 + cnt
].pointer
.is_static
248 && dtv
[1 + cnt
].pointer
.val
!= TLS_DTV_UNALLOCATED
)
249 free (dtv
[1 + cnt
].pointer
.val
);
250 memset (dtv
, '\0', (dtv
[-1].counter
+ 1) * sizeof (dtv_t
));
252 /* Re-initialize the TLS. */
253 _dl_allocate_tls_init (TLS_TPADJ (result
));
259 /* Free stacks until cache size is lower than LIMIT. */
261 __free_stacks (size_t limit
)
263 /* We reduce the size of the cache. Remove the last entries until
264 the size is below the limit. */
268 /* Search from the end of the list. */
269 list_for_each_prev_safe (entry
, prev
, &stack_cache
)
271 struct pthread
*curr
;
273 curr
= list_entry (entry
, struct pthread
, list
);
276 /* Unlink the block. */
277 stack_list_del (entry
);
279 /* Account for the freed memory. */
280 stack_cache_actsize
-= curr
->stackblock_size
;
282 /* Free the memory associated with the ELF TLS. */
283 _dl_deallocate_tls (TLS_TPADJ (curr
), false);
285 /* Remove this block. This should never fail. If it does
286 something is really wrong. */
287 if (munmap (curr
->stackblock
, curr
->stackblock_size
) != 0)
290 /* Maybe we have freed enough. */
291 if (stack_cache_actsize
<= limit
)
298 /* Add a stack frame which is not used anymore to the stack. Must be
299 called with the cache lock held. */
301 __attribute ((always_inline
))
302 queue_stack (struct pthread
*stack
)
304 /* We unconditionally add the stack to the list. The memory may
305 still be in use but it will not be reused until the kernel marks
306 the stack as not used anymore. */
307 stack_list_add (&stack
->list
, &stack_cache
);
309 stack_cache_actsize
+= stack
->stackblock_size
;
310 if (__builtin_expect (stack_cache_actsize
> stack_cache_maxsize
, 0))
311 __free_stacks (stack_cache_maxsize
);
317 change_stack_perm (struct pthread
*pd
318 #ifdef NEED_SEPARATE_REGISTER_STACK
323 #ifdef NEED_SEPARATE_REGISTER_STACK
324 void *stack
= (pd
->stackblock
325 + (((((pd
->stackblock_size
- pd
->guardsize
) / 2)
326 & pagemask
) + pd
->guardsize
) & pagemask
));
327 size_t len
= pd
->stackblock
+ pd
->stackblock_size
- stack
;
328 #elif _STACK_GROWS_DOWN
329 void *stack
= pd
->stackblock
+ pd
->guardsize
;
330 size_t len
= pd
->stackblock_size
- pd
->guardsize
;
331 #elif _STACK_GROWS_UP
332 void *stack
= pd
->stackblock
;
333 size_t len
= (uintptr_t) pd
- pd
->guardsize
- (uintptr_t) pd
->stackblock
;
335 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
337 if (mprotect (stack
, len
, PROT_READ
| PROT_WRITE
| PROT_EXEC
) != 0)
345 allocate_stack (const struct pthread_attr
*attr
, struct pthread
**pdp
,
346 ALLOCATE_STACK_PARMS
)
350 size_t pagesize_m1
= __getpagesize () - 1;
353 assert (attr
!= NULL
);
354 assert (powerof2 (pagesize_m1
+ 1));
355 assert (TCB_ALIGNMENT
>= STACK_ALIGN
);
357 /* Get the stack size from the attribute if it is set. Otherwise we
358 use the default we determined at start time. */
359 size
= attr
->stacksize
?: __default_stacksize
;
361 /* Get memory for the stack. */
362 if (__builtin_expect (attr
->flags
& ATTR_FLAG_STACKADDR
, 0))
366 /* If the user also specified the size of the stack make sure it
368 if (attr
->stacksize
!= 0
369 && attr
->stacksize
< (__static_tls_size
+ MINIMAL_REST_STACK
))
372 /* Adjust stack size for alignment of the TLS block. */
374 adj
= ((uintptr_t) attr
->stackaddr
- TLS_TCB_SIZE
)
375 & __static_tls_align_m1
;
376 assert (size
> adj
+ TLS_TCB_SIZE
);
378 adj
= ((uintptr_t) attr
->stackaddr
- __static_tls_size
)
379 & __static_tls_align_m1
;
383 /* The user provided some memory. Let's hope it matches the
384 size... We do not allocate guard pages if the user provided
385 the stack. It is the user's responsibility to do this if it
388 pd
= (struct pthread
*) ((uintptr_t) attr
->stackaddr
389 - TLS_TCB_SIZE
- adj
);
391 pd
= (struct pthread
*) (((uintptr_t) attr
->stackaddr
392 - __static_tls_size
- adj
)
396 /* The user provided stack memory needs to be cleared. */
397 memset (pd
, '\0', sizeof (struct pthread
));
399 /* The first TSD block is included in the TCB. */
400 pd
->specific
[0] = pd
->specific_1stblock
;
402 /* Remember the stack-related values. */
403 pd
->stackblock
= (char *) attr
->stackaddr
- size
;
404 pd
->stackblock_size
= size
;
406 /* This is a user-provided stack. It will not be queued in the
407 stack cache nor will the memory (except the TLS memory) be freed. */
408 pd
->user_stack
= true;
410 /* This is at least the second thread. */
411 pd
->header
.multiple_threads
= 1;
412 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
413 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
416 #ifndef __ASSUME_PRIVATE_FUTEX
417 /* The thread must know when private futexes are supported. */
418 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
419 header
.private_futex
);
422 #ifdef NEED_DL_SYSINFO
423 /* Copy the sysinfo value from the parent. */
424 THREAD_SYSINFO(pd
) = THREAD_SELF_SYSINFO
;
427 /* The process ID is also the same as that of the caller. */
428 pd
->pid
= THREAD_GETMEM (THREAD_SELF
, pid
);
430 /* Don't allow setxid until cloned. */
431 pd
->setxid_futex
= -1;
433 /* Allocate the DTV for this thread. */
434 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
436 /* Something went wrong. */
437 assert (errno
== ENOMEM
);
442 /* Prepare to modify global data. */
443 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
445 /* And add to the list of stacks in use. */
446 list_add (&pd
->list
, &__stack_user
);
448 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
452 /* Allocate some anonymous memory. If possible use the cache. */
456 const int prot
= (PROT_READ
| PROT_WRITE
457 | ((GL(dl_stack_flags
) & PF_X
) ? PROT_EXEC
: 0));
459 #if COLORING_INCREMENT != 0
460 /* Add one more page for stack coloring. Don't do it for stacks
461 with 16 times pagesize or larger. This might just cause
462 unnecessary misalignment. */
463 if (size
<= 16 * pagesize_m1
)
464 size
+= pagesize_m1
+ 1;
467 /* Adjust the stack size for alignment. */
468 size
&= ~__static_tls_align_m1
;
471 /* Make sure the size of the stack is enough for the guard and
472 eventually the thread descriptor. */
473 guardsize
= (attr
->guardsize
+ pagesize_m1
) & ~pagesize_m1
;
474 if (__builtin_expect (size
< ((guardsize
+ __static_tls_size
475 + MINIMAL_REST_STACK
+ pagesize_m1
)
478 /* The stack is too small (or the guard too large). */
481 /* Try to get a stack from the cache. */
483 pd
= get_cached_stack (&size
, &mem
);
486 /* To avoid aliasing effects on a larger scale than pages we
487 adjust the allocated stack size if necessary. This way
488 allocations directly following each other will not have
489 aliasing problems. */
490 #if MULTI_PAGE_ALIASING != 0
491 if ((size
% MULTI_PAGE_ALIASING
) == 0)
492 size
+= pagesize_m1
+ 1;
495 mem
= mmap (NULL
, size
, prot
,
496 MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_STACK
, -1, 0);
498 if (__builtin_expect (mem
== MAP_FAILED
, 0))
501 __set_errno (EAGAIN
);
506 /* SIZE is guaranteed to be greater than zero.
507 So we can never get a null pointer back from mmap. */
508 assert (mem
!= NULL
);
510 #if COLORING_INCREMENT != 0
511 /* Atomically increment NCREATED. */
512 unsigned int ncreated
= atomic_increment_val (&nptl_ncreated
);
514 /* We chose the offset for coloring by incrementing it for
515 every new thread by a fixed amount. The offset used
516 module the page size. Even if coloring would be better
517 relative to higher alignment values it makes no sense to
518 do it since the mmap() interface does not allow us to
519 specify any alignment for the returned memory block. */
520 size_t coloring
= (ncreated
* COLORING_INCREMENT
) & pagesize_m1
;
522 /* Make sure the coloring offsets does not disturb the alignment
523 of the TCB and static TLS block. */
524 if (__builtin_expect ((coloring
& __static_tls_align_m1
) != 0, 0))
525 coloring
= (((coloring
+ __static_tls_align_m1
)
526 & ~(__static_tls_align_m1
))
529 /* Unless specified we do not make any adjustments. */
533 /* Place the thread descriptor at the end of the stack. */
535 pd
= (struct pthread
*) ((char *) mem
+ size
- coloring
) - 1;
537 pd
= (struct pthread
*) ((((uintptr_t) mem
+ size
- coloring
539 & ~__static_tls_align_m1
)
543 /* Remember the stack-related values. */
544 pd
->stackblock
= mem
;
545 pd
->stackblock_size
= size
;
547 /* We allocated the first block thread-specific data array.
548 This address will not change for the lifetime of this
550 pd
->specific
[0] = pd
->specific_1stblock
;
552 /* This is at least the second thread. */
553 pd
->header
.multiple_threads
= 1;
554 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
555 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
558 #ifndef __ASSUME_PRIVATE_FUTEX
559 /* The thread must know when private futexes are supported. */
560 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
561 header
.private_futex
);
564 #ifdef NEED_DL_SYSINFO
565 /* Copy the sysinfo value from the parent. */
566 THREAD_SYSINFO(pd
) = THREAD_SELF_SYSINFO
;
569 /* Don't allow setxid until cloned. */
570 pd
->setxid_futex
= -1;
572 /* The process ID is also the same as that of the caller. */
573 pd
->pid
= THREAD_GETMEM (THREAD_SELF
, pid
);
575 /* Allocate the DTV for this thread. */
576 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
578 /* Something went wrong. */
579 assert (errno
== ENOMEM
);
581 /* Free the stack memory we just allocated. */
582 (void) munmap (mem
, size
);
588 /* Prepare to modify global data. */
589 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
591 /* And add to the list of stacks in use. */
592 stack_list_add (&pd
->list
, &stack_used
);
594 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
597 /* There might have been a race. Another thread might have
598 caused the stacks to get exec permission while this new
599 stack was prepared. Detect if this was possible and
600 change the permission if necessary. */
601 if (__builtin_expect ((GL(dl_stack_flags
) & PF_X
) != 0
602 && (prot
& PROT_EXEC
) == 0, 0))
604 int err
= change_stack_perm (pd
605 #ifdef NEED_SEPARATE_REGISTER_STACK
611 /* Free the stack memory we just allocated. */
612 (void) munmap (mem
, size
);
619 /* Note that all of the stack and the thread descriptor is
620 zeroed. This means we do not have to initialize fields
621 with initial value zero. This is specifically true for
622 the 'tid' field which is always set back to zero once the
623 stack is not used anymore and for the 'guardsize' field
624 which will be read next. */
627 /* Create or resize the guard area if necessary. */
628 if (__builtin_expect (guardsize
> pd
->guardsize
, 0))
630 #ifdef NEED_SEPARATE_REGISTER_STACK
631 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
632 #elif _STACK_GROWS_DOWN
634 # elif _STACK_GROWS_UP
635 char *guard
= (char *) (((uintptr_t) pd
- guardsize
) & ~pagesize_m1
);
637 if (mprotect (guard
, guardsize
, PROT_NONE
) != 0)
641 err
= errno
== ENOMEM
? EAGAIN
: errno
;
643 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
645 /* Remove the thread from the list. */
646 stack_list_del (&pd
->list
);
648 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
650 /* Get rid of the TLS block we allocated. */
651 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
653 /* Free the stack memory regardless of whether the size
654 of the cache is over the limit or not. If this piece
655 of memory caused problems we better do not use it
656 anymore. Uh, and we ignore possible errors. There
657 is nothing we could do. */
658 (void) munmap (mem
, size
);
663 pd
->guardsize
= guardsize
;
665 else if (__builtin_expect (pd
->guardsize
- guardsize
> size
- reqsize
,
668 /* The old guard area is too large. */
670 #ifdef NEED_SEPARATE_REGISTER_STACK
671 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
672 char *oldguard
= mem
+ (((size
- pd
->guardsize
) / 2) & ~pagesize_m1
);
675 && mprotect (oldguard
, guard
- oldguard
, prot
) != 0)
678 if (mprotect (guard
+ guardsize
,
679 oldguard
+ pd
->guardsize
- guard
- guardsize
,
682 #elif _STACK_GROWS_DOWN
683 if (mprotect ((char *) mem
+ guardsize
, pd
->guardsize
- guardsize
,
686 #elif _STACK_GROWS_UP
687 if (mprotect ((char *) pd
- pd
->guardsize
,
688 pd
->guardsize
- guardsize
, prot
) != 0)
692 pd
->guardsize
= guardsize
;
694 /* The pthread_getattr_np() calls need to get passed the size
695 requested in the attribute, regardless of how large the
696 actually used guardsize is. */
697 pd
->reported_guardsize
= guardsize
;
700 /* Initialize the lock. We have to do this unconditionally since the
701 stillborn thread could be canceled while the lock is taken. */
702 pd
->lock
= LLL_LOCK_INITIALIZER
;
704 /* The robust mutex lists also need to be initialized
705 unconditionally because the cleanup for the previous stack owner
706 might have happened in the kernel. */
707 pd
->robust_head
.futex_offset
= (offsetof (pthread_mutex_t
, __data
.__lock
)
708 - offsetof (pthread_mutex_t
,
709 __data
.__list
.__next
));
710 pd
->robust_head
.list_op_pending
= NULL
;
711 #ifdef __PTHREAD_MUTEX_HAVE_PREV
712 pd
->robust_prev
= &pd
->robust_head
;
714 pd
->robust_head
.list
= &pd
->robust_head
;
716 /* We place the thread descriptor at the end of the stack. */
720 /* The stack begins before the TCB and the static TLS block. */
721 stacktop
= ((char *) (pd
+ 1) - __static_tls_size
);
723 stacktop
= (char *) (pd
- 1);
726 #ifdef NEED_SEPARATE_REGISTER_STACK
727 *stack
= pd
->stackblock
;
728 *stacksize
= stacktop
- *stack
;
729 #elif _STACK_GROWS_DOWN
731 #elif _STACK_GROWS_UP
732 *stack
= pd
->stackblock
;
742 __deallocate_stack (struct pthread
*pd
)
744 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
746 /* Remove the thread from the list of threads with user defined
748 stack_list_del (&pd
->list
);
750 /* Not much to do. Just free the mmap()ed memory. Note that we do
751 not reset the 'used' flag in the 'tid' field. This is done by
752 the kernel. If no thread has been created yet this field is
754 if (__builtin_expect (! pd
->user_stack
, 1))
755 (void) queue_stack (pd
);
757 /* Free the memory associated with the ELF TLS. */
758 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
760 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
766 __make_stacks_executable (void **stack_endp
)
768 /* First the main thread's stack. */
769 int err
= _dl_make_stack_executable (stack_endp
);
773 #ifdef NEED_SEPARATE_REGISTER_STACK
774 const size_t pagemask
= ~(__getpagesize () - 1);
777 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
780 list_for_each (runp
, &stack_used
)
782 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
783 #ifdef NEED_SEPARATE_REGISTER_STACK
791 /* Also change the permission for the currently unused stacks. This
792 might be wasted time but better spend it here than adding a check
795 list_for_each (runp
, &stack_cache
)
797 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
798 #ifdef NEED_SEPARATE_REGISTER_STACK
806 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
812 /* In case of a fork() call the memory allocation in the child will be
813 the same but only one thread is running. All stacks except that of
814 the one running thread are not used anymore. We have to recycle
817 __reclaim_stacks (void)
819 struct pthread
*self
= (struct pthread
*) THREAD_SELF
;
821 /* No locking necessary. The caller is the only stack in use. But
822 we have to be aware that we might have interrupted a list
825 if (in_flight_stack
!= 0)
827 bool add_p
= in_flight_stack
& 1;
828 list_t
*elem
= (list_t
*) (in_flight_stack
& ~(uintptr_t) 1);
832 /* We always add at the beginning of the list. So in this
833 case we only need to check the beginning of these lists. */
834 int check_list (list_t
*l
)
836 if (l
->next
->prev
!= l
)
838 assert (l
->next
->prev
== elem
);
840 elem
->next
= l
->next
;
850 if (check_list (&stack_used
) == 0)
851 (void) check_list (&stack_cache
);
855 /* We can simply always replay the delete operation. */
856 elem
->next
->prev
= elem
->prev
;
857 elem
->prev
->next
= elem
->next
;
861 /* Mark all stacks except the still running one as free. */
863 list_for_each (runp
, &stack_used
)
865 struct pthread
*curp
= list_entry (runp
, struct pthread
, list
);
868 /* This marks the stack as free. */
871 /* The PID field must be initialized for the new process. */
872 curp
->pid
= self
->pid
;
874 /* Account for the size of the stack. */
875 stack_cache_actsize
+= curp
->stackblock_size
;
877 if (curp
->specific_used
)
879 /* Clear the thread-specific data. */
880 memset (curp
->specific_1stblock
, '\0',
881 sizeof (curp
->specific_1stblock
));
883 curp
->specific_used
= false;
885 for (size_t cnt
= 1; cnt
< PTHREAD_KEY_1STLEVEL_SIZE
; ++cnt
)
886 if (curp
->specific
[cnt
] != NULL
)
888 memset (curp
->specific
[cnt
], '\0',
889 sizeof (curp
->specific_1stblock
));
891 /* We have allocated the block which we do not
892 free here so re-set the bit. */
893 curp
->specific_used
= true;
899 /* Reset the PIDs in any cached stacks. */
900 list_for_each (runp
, &stack_cache
)
902 struct pthread
*curp
= list_entry (runp
, struct pthread
, list
);
903 curp
->pid
= self
->pid
;
906 /* Add the stack of all running threads to the cache. */
907 list_splice (&stack_used
, &stack_cache
);
909 /* Remove the entry for the current thread to from the cache list
910 and add it to the list of running threads. Which of the two
911 lists is decided by the user_stack flag. */
912 stack_list_del (&self
->list
);
914 /* Re-initialize the lists for all the threads. */
915 INIT_LIST_HEAD (&stack_used
);
916 INIT_LIST_HEAD (&__stack_user
);
918 if (__builtin_expect (THREAD_GETMEM (self
, user_stack
), 0))
919 list_add (&self
->list
, &__stack_user
);
921 list_add (&self
->list
, &stack_used
);
923 /* There is one thread running. */
928 /* Initialize the lock. */
929 stack_cache_lock
= LLL_LOCK_INITIALIZER
;
934 # undef __find_thread_by_id
935 /* Find a thread given the thread ID. */
938 __find_thread_by_id (pid_t tid
)
940 struct pthread
*result
= NULL
;
942 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
944 /* Iterate over the list with system-allocated threads first. */
946 list_for_each (runp
, &stack_used
)
948 struct pthread
*curp
;
950 curp
= list_entry (runp
, struct pthread
, list
);
952 if (curp
->tid
== tid
)
959 /* Now the list with threads using user-allocated stacks. */
960 list_for_each (runp
, &__stack_user
)
962 struct pthread
*curp
;
964 curp
= list_entry (runp
, struct pthread
, list
);
966 if (curp
->tid
== tid
)
974 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
983 setxid_mark_thread (struct xid_command
*cmdp
, struct pthread
*t
)
987 /* Wait until this thread is cloned. */
988 if (t
->setxid_futex
== -1
989 && ! atomic_compare_and_exchange_bool_acq (&t
->setxid_futex
, -2, -1))
991 lll_futex_wait (&t
->setxid_futex
, -2, LLL_PRIVATE
);
992 while (t
->setxid_futex
== -2);
994 /* Don't let the thread exit before the setxid handler runs. */
999 ch
= t
->cancelhandling
;
1001 /* If the thread is exiting right now, ignore it. */
1002 if ((ch
& EXITING_BITMASK
) != 0)
1004 /* Release the futex if there is no other setxid in
1006 if ((ch
& SETXID_BITMASK
) == 0)
1008 t
->setxid_futex
= 1;
1009 lll_futex_wake (&t
->setxid_futex
, 1, LLL_PRIVATE
);
1014 while (atomic_compare_and_exchange_bool_acq (&t
->cancelhandling
,
1015 ch
| SETXID_BITMASK
, ch
));
1021 setxid_unmark_thread (struct xid_command
*cmdp
, struct pthread
*t
)
1027 ch
= t
->cancelhandling
;
1028 if ((ch
& SETXID_BITMASK
) == 0)
1031 while (atomic_compare_and_exchange_bool_acq (&t
->cancelhandling
,
1032 ch
& ~SETXID_BITMASK
, ch
));
1034 /* Release the futex just in case. */
1035 t
->setxid_futex
= 1;
1036 lll_futex_wake (&t
->setxid_futex
, 1, LLL_PRIVATE
);
1042 setxid_signal_thread (struct xid_command
*cmdp
, struct pthread
*t
)
1044 if ((t
->cancelhandling
& SETXID_BITMASK
) == 0)
1048 INTERNAL_SYSCALL_DECL (err
);
1050 val
= INTERNAL_SYSCALL (tgkill
, err
, 3, THREAD_GETMEM (THREAD_SELF
, pid
),
1054 val
= INTERNAL_SYSCALL (tgkill
, err
, 3, THREAD_GETMEM (THREAD_SELF
, pid
),
1056 if (INTERNAL_SYSCALL_ERROR_P (val
, err
)
1057 && INTERNAL_SYSCALL_ERRNO (val
, err
) == ENOSYS
)
1059 val
= INTERNAL_SYSCALL (tkill
, err
, 2, t
->tid
, SIGSETXID
);
1062 /* If this failed, it must have had not started yet or else exited. */
1063 if (!INTERNAL_SYSCALL_ERROR_P (val
, err
))
1065 atomic_increment (&cmdp
->cntr
);
1075 __nptl_setxid (struct xid_command
*cmdp
)
1079 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1084 struct pthread
*self
= THREAD_SELF
;
1086 /* Iterate over the list with system-allocated threads first. */
1088 list_for_each (runp
, &stack_used
)
1090 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1094 setxid_mark_thread (cmdp
, t
);
1097 /* Now the list with threads using user-allocated stacks. */
1098 list_for_each (runp
, &__stack_user
)
1100 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1104 setxid_mark_thread (cmdp
, t
);
1107 /* Iterate until we don't succeed in signalling anyone. That means
1108 we have gotten all running threads, and their children will be
1109 automatically correct once started. */
1114 list_for_each (runp
, &stack_used
)
1116 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1120 signalled
+= setxid_signal_thread (cmdp
, t
);
1123 list_for_each (runp
, &__stack_user
)
1125 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1129 signalled
+= setxid_signal_thread (cmdp
, t
);
1132 int cur
= cmdp
->cntr
;
1135 lll_futex_wait (&cmdp
->cntr
, cur
, LLL_PRIVATE
);
1139 while (signalled
!= 0);
1141 /* Clean up flags, so that no thread blocks during exit waiting
1142 for a signal which will never come. */
1143 list_for_each (runp
, &stack_used
)
1145 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1149 setxid_unmark_thread (cmdp
, t
);
1152 list_for_each (runp
, &__stack_user
)
1154 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1158 setxid_unmark_thread (cmdp
, t
);
1161 /* This must be last, otherwise the current thread might not have
1162 permissions to send SIGSETXID syscall to the other threads. */
1163 INTERNAL_SYSCALL_DECL (err
);
1164 result
= INTERNAL_SYSCALL_NCS (cmdp
->syscall_no
, err
, 3,
1165 cmdp
->id
[0], cmdp
->id
[1], cmdp
->id
[2]);
1166 if (INTERNAL_SYSCALL_ERROR_P (result
, err
))
1168 __set_errno (INTERNAL_SYSCALL_ERRNO (result
, err
));
1172 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1176 static inline void __attribute__((always_inline
))
1177 init_one_static_tls (struct pthread
*curp
, struct link_map
*map
)
1179 dtv_t
*dtv
= GET_DTV (TLS_TPADJ (curp
));
1181 void *dest
= (char *) curp
- map
->l_tls_offset
;
1182 # elif TLS_DTV_AT_TP
1183 void *dest
= (char *) curp
+ map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
1185 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1188 /* Fill in the DTV slot so that a later LD/GD access will find it. */
1189 dtv
[map
->l_tls_modid
].pointer
.val
= dest
;
1190 dtv
[map
->l_tls_modid
].pointer
.is_static
= true;
1192 /* Initialize the memory. */
1193 memset (__mempcpy (dest
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
1194 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
1199 __pthread_init_static_tls (struct link_map
*map
)
1201 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1203 /* Iterate over the list with system-allocated threads first. */
1205 list_for_each (runp
, &stack_used
)
1206 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1208 /* Now the list with threads using user-allocated stacks. */
1209 list_for_each (runp
, &__stack_user
)
1210 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1212 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1218 __wait_lookup_done (void)
1220 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1222 struct pthread
*self
= THREAD_SELF
;
1224 /* Iterate over the list with system-allocated threads first. */
1226 list_for_each (runp
, &stack_used
)
1228 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1229 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1232 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1234 /* We have to wait until this thread is done with the global
1235 scope. First tell the thread that we are waiting and
1236 possibly have to be woken. */
1237 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1238 THREAD_GSCOPE_FLAG_WAIT
,
1239 THREAD_GSCOPE_FLAG_USED
))
1243 lll_futex_wait (gscope_flagp
, THREAD_GSCOPE_FLAG_WAIT
, LLL_PRIVATE
);
1244 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1247 /* Now the list with threads using user-allocated stacks. */
1248 list_for_each (runp
, &__stack_user
)
1250 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1251 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1254 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1256 /* We have to wait until this thread is done with the global
1257 scope. First tell the thread that we are waiting and
1258 possibly have to be woken. */
1259 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1260 THREAD_GSCOPE_FLAG_WAIT
,
1261 THREAD_GSCOPE_FLAG_USED
))
1265 lll_futex_wait (gscope_flagp
, THREAD_GSCOPE_FLAG_WAIT
, LLL_PRIVATE
);
1266 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1269 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);