1 /* Copyright (C) 2002-2007, 2009 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
27 #include <sys/param.h>
28 #include <dl-sysdep.h>
30 #include <lowlevellock.h>
31 #include <kernel-features.h>
34 #ifndef NEED_SEPARATE_REGISTER_STACK
36 /* Most architectures have exactly one stack pointer. Some have more. */
37 # define STACK_VARIABLES void *stackaddr = NULL
39 /* How to pass the values to the 'create_thread' function. */
40 # define STACK_VARIABLES_ARGS stackaddr
42 /* How to declare function which gets there parameters. */
43 # define STACK_VARIABLES_PARMS void *stackaddr
45 /* How to declare allocate_stack. */
46 # define ALLOCATE_STACK_PARMS void **stack
48 /* This is how the function is called. We do it this way to allow
49 other variants of the function to have more parameters. */
50 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
54 /* We need two stacks. The kernel will place them but we have to tell
55 the kernel about the size of the reserved address space. */
56 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
58 /* How to pass the values to the 'create_thread' function. */
59 # define STACK_VARIABLES_ARGS stackaddr, stacksize
61 /* How to declare function which gets there parameters. */
62 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
64 /* How to declare allocate_stack. */
65 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
67 /* This is how the function is called. We do it this way to allow
68 other variants of the function to have more parameters. */
69 # define ALLOCATE_STACK(attr, pd) \
70 allocate_stack (attr, pd, &stackaddr, &stacksize)
75 /* Default alignment of stack. */
77 # define STACK_ALIGN __alignof__ (long double)
80 /* Default value for minimal stack size after allocating thread
81 descriptor and guard. */
82 #ifndef MINIMAL_REST_STACK
83 # define MINIMAL_REST_STACK 4096
87 /* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
88 a stack. Use it when possible. */
93 /* This yields the pointer that TLS support code calls the thread pointer. */
95 # define TLS_TPADJ(pd) (pd)
97 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
100 /* Cache handling for not-yet free stacks. */
102 /* Maximum size in kB of cache. */
103 static size_t stack_cache_maxsize
= 40 * 1024 * 1024; /* 40MiBi by default. */
104 static size_t stack_cache_actsize
;
106 /* Mutex protecting this variable. */
107 static int stack_cache_lock
= LLL_LOCK_INITIALIZER
;
109 /* List of queued stack frames. */
110 static LIST_HEAD (stack_cache
);
112 /* List of the stacks in use. */
113 static LIST_HEAD (stack_used
);
115 /* We need to record what list operations we are going to do so that,
116 in case of an asynchronous interruption due to a fork() call, we
117 can correct for the work. */
118 static uintptr_t *in_flight_stack
;
120 /* List of the threads with user provided stacks in use. No need to
121 initialize this, since it's done in __pthread_initialize_minimal. */
122 list_t __stack_user
__attribute__ ((nocommon
));
123 hidden_data_def (__stack_user
)
125 #if COLORING_INCREMENT != 0
126 /* Number of threads created. */
127 static unsigned int nptl_ncreated
;
131 /* Check whether the stack is still used or not. */
132 #define FREE_P(descr) ((descr)->tid <= 0)
136 stack_list_del (list_t
*elem
)
138 in_flight_stack
= (uintptr_t) elem
;
140 atomic_write_barrier ();
144 atomic_write_barrier ();
151 stack_list_add (list_t
*elem
, list_t
*list
)
153 in_flight_stack
= (uintptr_t) elem
| 1;
155 atomic_write_barrier ();
157 list_add (elem
, list
);
159 atomic_write_barrier ();
165 /* We create a double linked list of all cache entries. Double linked
166 because this allows removing entries from the end. */
169 /* Get a stack frame from the cache. We have to match by size since
170 some blocks might be too small or far too large. */
171 static struct pthread
*
172 get_cached_stack (size_t *sizep
, void **memp
)
174 size_t size
= *sizep
;
175 struct pthread
*result
= NULL
;
178 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
180 /* Search the cache for a matching entry. We search for the
181 smallest stack which has at least the required size. Note that
182 in normal situations the size of all allocated stacks is the
183 same. As the very least there are only a few different sizes.
184 Therefore this loop will exit early most of the time with an
186 list_for_each (entry
, &stack_cache
)
188 struct pthread
*curr
;
190 curr
= list_entry (entry
, struct pthread
, list
);
191 if (FREE_P (curr
) && curr
->stackblock_size
>= size
)
193 if (curr
->stackblock_size
== size
)
200 || result
->stackblock_size
> curr
->stackblock_size
)
205 if (__builtin_expect (result
== NULL
, 0)
206 /* Make sure the size difference is not too excessive. In that
207 case we do not use the block. */
208 || __builtin_expect (result
->stackblock_size
> 4 * size
, 0))
210 /* Release the lock. */
211 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
216 /* Dequeue the entry. */
217 stack_list_del (&result
->list
);
219 /* And add to the list of stacks in use. */
220 stack_list_add (&result
->list
, &stack_used
);
222 /* And decrease the cache size. */
223 stack_cache_actsize
-= result
->stackblock_size
;
225 /* Release the lock early. */
226 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
228 /* Report size and location of the stack to the caller. */
229 *sizep
= result
->stackblock_size
;
230 *memp
= result
->stackblock
;
232 /* Cancellation handling is back to the default. */
233 result
->cancelhandling
= 0;
234 result
->cleanup
= NULL
;
236 /* No pending event. */
237 result
->nextevent
= NULL
;
240 dtv_t
*dtv
= GET_DTV (TLS_TPADJ (result
));
241 memset (dtv
, '\0', (dtv
[-1].counter
+ 1) * sizeof (dtv_t
));
243 /* Re-initialize the TLS. */
244 _dl_allocate_tls_init (TLS_TPADJ (result
));
250 /* Free stacks until cache size is lower than LIMIT. */
252 free_stacks (size_t limit
)
254 /* We reduce the size of the cache. Remove the last entries until
255 the size is below the limit. */
259 /* Search from the end of the list. */
260 list_for_each_prev_safe (entry
, prev
, &stack_cache
)
262 struct pthread
*curr
;
264 curr
= list_entry (entry
, struct pthread
, list
);
267 /* Unlink the block. */
268 stack_list_del (entry
);
270 /* Account for the freed memory. */
271 stack_cache_actsize
-= curr
->stackblock_size
;
273 /* Free the memory associated with the ELF TLS. */
274 _dl_deallocate_tls (TLS_TPADJ (curr
), false);
276 /* Remove this block. This should never fail. If it does
277 something is really wrong. */
278 if (munmap (curr
->stackblock
, curr
->stackblock_size
) != 0)
281 /* Maybe we have freed enough. */
282 if (stack_cache_actsize
<= limit
)
289 /* Add a stack frame which is not used anymore to the stack. Must be
290 called with the cache lock held. */
292 __attribute ((always_inline
))
293 queue_stack (struct pthread
*stack
)
295 /* We unconditionally add the stack to the list. The memory may
296 still be in use but it will not be reused until the kernel marks
297 the stack as not used anymore. */
298 stack_list_add (&stack
->list
, &stack_cache
);
300 stack_cache_actsize
+= stack
->stackblock_size
;
301 if (__builtin_expect (stack_cache_actsize
> stack_cache_maxsize
, 0))
302 free_stacks (stack_cache_maxsize
);
306 /* This function is called indirectly from the freeres code in libc. */
308 __free_stack_cache (void)
316 change_stack_perm (struct pthread
*pd
317 #ifdef NEED_SEPARATE_REGISTER_STACK
322 #ifdef NEED_SEPARATE_REGISTER_STACK
323 void *stack
= (pd
->stackblock
324 + (((((pd
->stackblock_size
- pd
->guardsize
) / 2)
325 & pagemask
) + pd
->guardsize
) & pagemask
));
326 size_t len
= pd
->stackblock
+ pd
->stackblock_size
- stack
;
327 #elif _STACK_GROWS_DOWN
328 void *stack
= pd
->stackblock
+ pd
->guardsize
;
329 size_t len
= pd
->stackblock_size
- pd
->guardsize
;
330 #elif _STACK_GROWS_UP
331 void *stack
= pd
->stackblock
;
332 size_t len
= (uintptr_t) pd
- pd
->guardsize
- (uintptr_t) pd
->stackblock
;
334 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
336 if (mprotect (stack
, len
, PROT_READ
| PROT_WRITE
| PROT_EXEC
) != 0)
344 allocate_stack (const struct pthread_attr
*attr
, struct pthread
**pdp
,
345 ALLOCATE_STACK_PARMS
)
349 size_t pagesize_m1
= __getpagesize () - 1;
352 assert (attr
!= NULL
);
353 assert (powerof2 (pagesize_m1
+ 1));
354 assert (TCB_ALIGNMENT
>= STACK_ALIGN
);
356 /* Get the stack size from the attribute if it is set. Otherwise we
357 use the default we determined at start time. */
358 size
= attr
->stacksize
?: __default_stacksize
;
360 /* Get memory for the stack. */
361 if (__builtin_expect (attr
->flags
& ATTR_FLAG_STACKADDR
, 0))
365 /* If the user also specified the size of the stack make sure it
367 if (attr
->stacksize
!= 0
368 && attr
->stacksize
< (__static_tls_size
+ MINIMAL_REST_STACK
))
371 /* Adjust stack size for alignment of the TLS block. */
373 adj
= ((uintptr_t) attr
->stackaddr
- TLS_TCB_SIZE
)
374 & __static_tls_align_m1
;
375 assert (size
> adj
+ TLS_TCB_SIZE
);
377 adj
= ((uintptr_t) attr
->stackaddr
- __static_tls_size
)
378 & __static_tls_align_m1
;
382 /* The user provided some memory. Let's hope it matches the
383 size... We do not allocate guard pages if the user provided
384 the stack. It is the user's responsibility to do this if it
387 pd
= (struct pthread
*) ((uintptr_t) attr
->stackaddr
388 - TLS_TCB_SIZE
- adj
);
390 pd
= (struct pthread
*) (((uintptr_t) attr
->stackaddr
391 - __static_tls_size
- adj
)
395 /* The user provided stack memory needs to be cleared. */
396 memset (pd
, '\0', sizeof (struct pthread
));
398 /* The first TSD block is included in the TCB. */
399 pd
->specific
[0] = pd
->specific_1stblock
;
401 /* Remember the stack-related values. */
402 pd
->stackblock
= (char *) attr
->stackaddr
- size
;
403 pd
->stackblock_size
= size
;
405 /* This is a user-provided stack. It will not be queued in the
406 stack cache nor will the memory (except the TLS memory) be freed. */
407 pd
->user_stack
= true;
409 /* This is at least the second thread. */
410 pd
->header
.multiple_threads
= 1;
411 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
412 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
415 #ifndef __ASSUME_PRIVATE_FUTEX
416 /* The thread must know when private futexes are supported. */
417 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
418 header
.private_futex
);
421 #ifdef NEED_DL_SYSINFO
422 /* Copy the sysinfo value from the parent. */
423 THREAD_SYSINFO(pd
) = THREAD_SELF_SYSINFO
;
426 /* The process ID is also the same as that of the caller. */
427 pd
->pid
= THREAD_GETMEM (THREAD_SELF
, pid
);
429 /* Allocate the DTV for this thread. */
430 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
432 /* Something went wrong. */
433 assert (errno
== ENOMEM
);
438 /* Prepare to modify global data. */
439 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
441 /* And add to the list of stacks in use. */
442 list_add (&pd
->list
, &__stack_user
);
444 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
448 /* Allocate some anonymous memory. If possible use the cache. */
452 const int prot
= (PROT_READ
| PROT_WRITE
453 | ((GL(dl_stack_flags
) & PF_X
) ? PROT_EXEC
: 0));
455 #if COLORING_INCREMENT != 0
456 /* Add one more page for stack coloring. Don't do it for stacks
457 with 16 times pagesize or larger. This might just cause
458 unnecessary misalignment. */
459 if (size
<= 16 * pagesize_m1
)
460 size
+= pagesize_m1
+ 1;
463 /* Adjust the stack size for alignment. */
464 size
&= ~__static_tls_align_m1
;
467 /* Make sure the size of the stack is enough for the guard and
468 eventually the thread descriptor. */
469 guardsize
= (attr
->guardsize
+ pagesize_m1
) & ~pagesize_m1
;
470 if (__builtin_expect (size
< ((guardsize
+ __static_tls_size
471 + MINIMAL_REST_STACK
+ pagesize_m1
)
474 /* The stack is too small (or the guard too large). */
477 /* Try to get a stack from the cache. */
479 pd
= get_cached_stack (&size
, &mem
);
482 /* To avoid aliasing effects on a larger scale than pages we
483 adjust the allocated stack size if necessary. This way
484 allocations directly following each other will not have
485 aliasing problems. */
486 #if MULTI_PAGE_ALIASING != 0
487 if ((size
% MULTI_PAGE_ALIASING
) == 0)
488 size
+= pagesize_m1
+ 1;
491 mem
= mmap (NULL
, size
, prot
,
492 MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_STACK
, -1, 0);
494 if (__builtin_expect (mem
== MAP_FAILED
, 0))
497 __set_errno (EAGAIN
);
502 /* SIZE is guaranteed to be greater than zero.
503 So we can never get a null pointer back from mmap. */
504 assert (mem
!= NULL
);
506 #if COLORING_INCREMENT != 0
507 /* Atomically increment NCREATED. */
508 unsigned int ncreated
= atomic_increment_val (&nptl_ncreated
);
510 /* We chose the offset for coloring by incrementing it for
511 every new thread by a fixed amount. The offset used
512 module the page size. Even if coloring would be better
513 relative to higher alignment values it makes no sense to
514 do it since the mmap() interface does not allow us to
515 specify any alignment for the returned memory block. */
516 size_t coloring
= (ncreated
* COLORING_INCREMENT
) & pagesize_m1
;
518 /* Make sure the coloring offsets does not disturb the alignment
519 of the TCB and static TLS block. */
520 if (__builtin_expect ((coloring
& __static_tls_align_m1
) != 0, 0))
521 coloring
= (((coloring
+ __static_tls_align_m1
)
522 & ~(__static_tls_align_m1
))
525 /* Unless specified we do not make any adjustments. */
529 /* Place the thread descriptor at the end of the stack. */
531 pd
= (struct pthread
*) ((char *) mem
+ size
- coloring
) - 1;
533 pd
= (struct pthread
*) ((((uintptr_t) mem
+ size
- coloring
535 & ~__static_tls_align_m1
)
539 /* Remember the stack-related values. */
540 pd
->stackblock
= mem
;
541 pd
->stackblock_size
= size
;
543 /* We allocated the first block thread-specific data array.
544 This address will not change for the lifetime of this
546 pd
->specific
[0] = pd
->specific_1stblock
;
548 /* This is at least the second thread. */
549 pd
->header
.multiple_threads
= 1;
550 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
551 __pthread_multiple_threads
= *__libc_multiple_threads_ptr
= 1;
554 #ifndef __ASSUME_PRIVATE_FUTEX
555 /* The thread must know when private futexes are supported. */
556 pd
->header
.private_futex
= THREAD_GETMEM (THREAD_SELF
,
557 header
.private_futex
);
560 #ifdef NEED_DL_SYSINFO
561 /* Copy the sysinfo value from the parent. */
562 THREAD_SYSINFO(pd
) = THREAD_SELF_SYSINFO
;
565 /* The process ID is also the same as that of the caller. */
566 pd
->pid
= THREAD_GETMEM (THREAD_SELF
, pid
);
568 /* Allocate the DTV for this thread. */
569 if (_dl_allocate_tls (TLS_TPADJ (pd
)) == NULL
)
571 /* Something went wrong. */
572 assert (errno
== ENOMEM
);
574 /* Free the stack memory we just allocated. */
575 (void) munmap (mem
, size
);
581 /* Prepare to modify global data. */
582 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
584 /* And add to the list of stacks in use. */
585 stack_list_add (&pd
->list
, &stack_used
);
587 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
590 /* There might have been a race. Another thread might have
591 caused the stacks to get exec permission while this new
592 stack was prepared. Detect if this was possible and
593 change the permission if necessary. */
594 if (__builtin_expect ((GL(dl_stack_flags
) & PF_X
) != 0
595 && (prot
& PROT_EXEC
) == 0, 0))
597 int err
= change_stack_perm (pd
598 #ifdef NEED_SEPARATE_REGISTER_STACK
604 /* Free the stack memory we just allocated. */
605 (void) munmap (mem
, size
);
612 /* Note that all of the stack and the thread descriptor is
613 zeroed. This means we do not have to initialize fields
614 with initial value zero. This is specifically true for
615 the 'tid' field which is always set back to zero once the
616 stack is not used anymore and for the 'guardsize' field
617 which will be read next. */
620 /* Create or resize the guard area if necessary. */
621 if (__builtin_expect (guardsize
> pd
->guardsize
, 0))
623 #ifdef NEED_SEPARATE_REGISTER_STACK
624 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
625 #elif _STACK_GROWS_DOWN
627 # elif _STACK_GROWS_UP
628 char *guard
= (char *) (((uintptr_t) pd
- guardsize
) & ~pagesize_m1
);
630 if (mprotect (guard
, guardsize
, PROT_NONE
) != 0)
636 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
638 /* Remove the thread from the list. */
639 stack_list_del (&pd
->list
);
641 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
643 /* Get rid of the TLS block we allocated. */
644 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
646 /* Free the stack memory regardless of whether the size
647 of the cache is over the limit or not. If this piece
648 of memory caused problems we better do not use it
649 anymore. Uh, and we ignore possible errors. There
650 is nothing we could do. */
651 (void) munmap (mem
, size
);
656 pd
->guardsize
= guardsize
;
658 else if (__builtin_expect (pd
->guardsize
- guardsize
> size
- reqsize
,
661 /* The old guard area is too large. */
663 #ifdef NEED_SEPARATE_REGISTER_STACK
664 char *guard
= mem
+ (((size
- guardsize
) / 2) & ~pagesize_m1
);
665 char *oldguard
= mem
+ (((size
- pd
->guardsize
) / 2) & ~pagesize_m1
);
668 && mprotect (oldguard
, guard
- oldguard
, prot
) != 0)
671 if (mprotect (guard
+ guardsize
,
672 oldguard
+ pd
->guardsize
- guard
- guardsize
,
675 #elif _STACK_GROWS_DOWN
676 if (mprotect ((char *) mem
+ guardsize
, pd
->guardsize
- guardsize
,
679 #elif _STACK_GROWS_UP
680 if (mprotect ((char *) pd
- pd
->guardsize
,
681 pd
->guardsize
- guardsize
, prot
) != 0)
685 pd
->guardsize
= guardsize
;
687 /* The pthread_getattr_np() calls need to get passed the size
688 requested in the attribute, regardless of how large the
689 actually used guardsize is. */
690 pd
->reported_guardsize
= guardsize
;
693 /* Initialize the lock. We have to do this unconditionally since the
694 stillborn thread could be canceled while the lock is taken. */
695 pd
->lock
= LLL_LOCK_INITIALIZER
;
697 /* The robust mutex lists also need to be initialized
698 unconditionally because the cleanup for the previous stack owner
699 might have happened in the kernel. */
700 pd
->robust_head
.futex_offset
= (offsetof (pthread_mutex_t
, __data
.__lock
)
701 - offsetof (pthread_mutex_t
,
702 __data
.__list
.__next
));
703 pd
->robust_head
.list_op_pending
= NULL
;
704 #ifdef __PTHREAD_MUTEX_HAVE_PREV
705 pd
->robust_prev
= &pd
->robust_head
;
707 pd
->robust_head
.list
= &pd
->robust_head
;
709 /* We place the thread descriptor at the end of the stack. */
713 /* The stack begins before the TCB and the static TLS block. */
714 stacktop
= ((char *) (pd
+ 1) - __static_tls_size
);
716 stacktop
= (char *) (pd
- 1);
719 #ifdef NEED_SEPARATE_REGISTER_STACK
720 *stack
= pd
->stackblock
;
721 *stacksize
= stacktop
- *stack
;
722 #elif _STACK_GROWS_DOWN
724 #elif _STACK_GROWS_UP
725 *stack
= pd
->stackblock
;
735 __deallocate_stack (struct pthread
*pd
)
737 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
739 /* Remove the thread from the list of threads with user defined
741 stack_list_del (&pd
->list
);
743 /* Not much to do. Just free the mmap()ed memory. Note that we do
744 not reset the 'used' flag in the 'tid' field. This is done by
745 the kernel. If no thread has been created yet this field is
747 if (__builtin_expect (! pd
->user_stack
, 1))
748 (void) queue_stack (pd
);
750 /* Free the memory associated with the ELF TLS. */
751 _dl_deallocate_tls (TLS_TPADJ (pd
), false);
753 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
759 __make_stacks_executable (void **stack_endp
)
761 /* First the main thread's stack. */
762 int err
= _dl_make_stack_executable (stack_endp
);
766 #ifdef NEED_SEPARATE_REGISTER_STACK
767 const size_t pagemask
= ~(__getpagesize () - 1);
770 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
773 list_for_each (runp
, &stack_used
)
775 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
776 #ifdef NEED_SEPARATE_REGISTER_STACK
784 /* Also change the permission for the currently unused stacks. This
785 might be wasted time but better spend it here than adding a check
788 list_for_each (runp
, &stack_cache
)
790 err
= change_stack_perm (list_entry (runp
, struct pthread
, list
)
791 #ifdef NEED_SEPARATE_REGISTER_STACK
799 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
805 /* In case of a fork() call the memory allocation in the child will be
806 the same but only one thread is running. All stacks except that of
807 the one running thread are not used anymore. We have to recycle
810 __reclaim_stacks (void)
812 struct pthread
*self
= (struct pthread
*) THREAD_SELF
;
814 /* No locking necessary. The caller is the only stack in use. But
815 we have to be aware that we might have interrupted a list
818 if (in_flight_stack
!= NULL
)
820 bool add_p
= in_flight_stack
& 1;
821 in_flight_stack
= (list_t
*) (in_flight_stack
& ~1l);
825 /* We always add at the beginning of the list. So in this
826 case we only need to check the beginning of these lists. */
827 int check_list (list_t
*l
)
829 if (l
->next
->prev
!= l
)
831 assert (l
->next
->prev
== in_flight_stack
);
833 in_flight_stack
->next
= l
->next
;
834 in_flight_stack
->prev
= l
;
835 l
->next
= in_flight_stack
;
843 if (check_list (&stack_used
) == 0)
844 (void) check_list (&stack_cache
);
848 /* We can simply always replay the delete operation. */
849 in_flight_stack
->next
->prev
= in_flight_stack
->prev
;
850 in_flight_stack
->prev
->next
= in_flight_stack
->next
;
853 in_flight_stack
= NULL
;
856 /* Mark all stacks except the still running one as free. */
858 list_for_each (runp
, &stack_used
)
860 struct pthread
*curp
= list_entry (runp
, struct pthread
, list
);
863 /* This marks the stack as free. */
866 /* The PID field must be initialized for the new process. */
867 curp
->pid
= self
->pid
;
869 /* Account for the size of the stack. */
870 stack_cache_actsize
+= curp
->stackblock_size
;
872 if (curp
->specific_used
)
874 /* Clear the thread-specific data. */
875 memset (curp
->specific_1stblock
, '\0',
876 sizeof (curp
->specific_1stblock
));
878 curp
->specific_used
= false;
880 for (size_t cnt
= 1; cnt
< PTHREAD_KEY_1STLEVEL_SIZE
; ++cnt
)
881 if (curp
->specific
[cnt
] != NULL
)
883 memset (curp
->specific
[cnt
], '\0',
884 sizeof (curp
->specific_1stblock
));
886 /* We have allocated the block which we do not
887 free here so re-set the bit. */
888 curp
->specific_used
= true;
894 /* Reset the PIDs in any cached stacks. */
895 list_for_each (runp
, &stack_cache
)
897 struct pthread
*curp
= list_entry (runp
, struct pthread
, list
);
898 curp
->pid
= self
->pid
;
901 /* Add the stack of all running threads to the cache. */
902 list_splice (&stack_used
, &stack_cache
);
904 /* Remove the entry for the current thread to from the cache list
905 and add it to the list of running threads. Which of the two
906 lists is decided by the user_stack flag. */
907 stack_list_del (&self
->list
);
909 /* Re-initialize the lists for all the threads. */
910 INIT_LIST_HEAD (&stack_used
);
911 INIT_LIST_HEAD (&__stack_user
);
913 if (__builtin_expect (THREAD_GETMEM (self
, user_stack
), 0))
914 list_add (&self
->list
, &__stack_user
);
916 stack_list_add (&self
->list
, &stack_used
);
918 /* There is one thread running. */
921 /* Initialize the lock. */
922 stack_cache_lock
= LLL_LOCK_INITIALIZER
;
927 # undef __find_thread_by_id
928 /* Find a thread given the thread ID. */
931 __find_thread_by_id (pid_t tid
)
933 struct pthread
*result
= NULL
;
935 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
937 /* Iterate over the list with system-allocated threads first. */
939 list_for_each (runp
, &stack_used
)
941 struct pthread
*curp
;
943 curp
= list_entry (runp
, struct pthread
, list
);
945 if (curp
->tid
== tid
)
952 /* Now the list with threads using user-allocated stacks. */
953 list_for_each (runp
, &__stack_user
)
955 struct pthread
*curp
;
957 curp
= list_entry (runp
, struct pthread
, list
);
959 if (curp
->tid
== tid
)
967 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
976 setxid_signal_thread (struct xid_command
*cmdp
, struct pthread
*t
)
978 if (! IS_DETACHED (t
))
983 ch
= t
->cancelhandling
;
985 /* If the thread is exiting right now, ignore it. */
986 if ((ch
& EXITING_BITMASK
) != 0)
989 while (atomic_compare_and_exchange_bool_acq (&t
->cancelhandling
,
990 ch
| SETXID_BITMASK
, ch
));
994 INTERNAL_SYSCALL_DECL (err
);
996 val
= INTERNAL_SYSCALL (tgkill
, err
, 3, THREAD_GETMEM (THREAD_SELF
, pid
),
1000 val
= INTERNAL_SYSCALL (tgkill
, err
, 3, THREAD_GETMEM (THREAD_SELF
, pid
),
1002 if (INTERNAL_SYSCALL_ERROR_P (val
, err
)
1003 && INTERNAL_SYSCALL_ERRNO (val
, err
) == ENOSYS
)
1005 val
= INTERNAL_SYSCALL (tkill
, err
, 2, t
->tid
, SIGSETXID
);
1008 if (!INTERNAL_SYSCALL_ERROR_P (val
, err
))
1009 atomic_increment (&cmdp
->cntr
);
1015 __nptl_setxid (struct xid_command
*cmdp
)
1018 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1023 struct pthread
*self
= THREAD_SELF
;
1025 /* Iterate over the list with system-allocated threads first. */
1027 list_for_each (runp
, &stack_used
)
1029 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1033 setxid_signal_thread (cmdp
, t
);
1036 /* Now the list with threads using user-allocated stacks. */
1037 list_for_each (runp
, &__stack_user
)
1039 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1043 setxid_signal_thread (cmdp
, t
);
1046 int cur
= cmdp
->cntr
;
1049 lll_futex_wait (&cmdp
->cntr
, cur
, LLL_PRIVATE
);
1053 /* This must be last, otherwise the current thread might not have
1054 permissions to send SIGSETXID syscall to the other threads. */
1055 INTERNAL_SYSCALL_DECL (err
);
1056 result
= INTERNAL_SYSCALL_NCS (cmdp
->syscall_no
, err
, 3,
1057 cmdp
->id
[0], cmdp
->id
[1], cmdp
->id
[2]);
1058 if (INTERNAL_SYSCALL_ERROR_P (result
, err
))
1060 __set_errno (INTERNAL_SYSCALL_ERRNO (result
, err
));
1064 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1068 static inline void __attribute__((always_inline
))
1069 init_one_static_tls (struct pthread
*curp
, struct link_map
*map
)
1071 dtv_t
*dtv
= GET_DTV (TLS_TPADJ (curp
));
1073 void *dest
= (char *) curp
- map
->l_tls_offset
;
1074 # elif TLS_DTV_AT_TP
1075 void *dest
= (char *) curp
+ map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
1077 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1080 /* Fill in the DTV slot so that a later LD/GD access will find it. */
1081 dtv
[map
->l_tls_modid
].pointer
.val
= dest
;
1082 dtv
[map
->l_tls_modid
].pointer
.is_static
= true;
1084 /* Initialize the memory. */
1085 memset (__mempcpy (dest
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
1086 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
1091 __pthread_init_static_tls (struct link_map
*map
)
1093 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1095 /* Iterate over the list with system-allocated threads first. */
1097 list_for_each (runp
, &stack_used
)
1098 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1100 /* Now the list with threads using user-allocated stacks. */
1101 list_for_each (runp
, &__stack_user
)
1102 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1104 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);
1110 __wait_lookup_done (void)
1112 lll_lock (stack_cache_lock
, LLL_PRIVATE
);
1114 struct pthread
*self
= THREAD_SELF
;
1116 /* Iterate over the list with system-allocated threads first. */
1118 list_for_each (runp
, &stack_used
)
1120 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1121 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1124 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1126 /* We have to wait until this thread is done with the global
1127 scope. First tell the thread that we are waiting and
1128 possibly have to be woken. */
1129 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1130 THREAD_GSCOPE_FLAG_WAIT
,
1131 THREAD_GSCOPE_FLAG_USED
))
1135 lll_futex_wait (gscope_flagp
, THREAD_GSCOPE_FLAG_WAIT
, LLL_PRIVATE
);
1136 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1139 /* Now the list with threads using user-allocated stacks. */
1140 list_for_each (runp
, &__stack_user
)
1142 struct pthread
*t
= list_entry (runp
, struct pthread
, list
);
1143 if (t
== self
|| t
->header
.gscope_flag
== THREAD_GSCOPE_FLAG_UNUSED
)
1146 int *const gscope_flagp
= &t
->header
.gscope_flag
;
1148 /* We have to wait until this thread is done with the global
1149 scope. First tell the thread that we are waiting and
1150 possibly have to be woken. */
1151 if (atomic_compare_and_exchange_bool_acq (gscope_flagp
,
1152 THREAD_GSCOPE_FLAG_WAIT
,
1153 THREAD_GSCOPE_FLAG_USED
))
1157 lll_futex_wait (gscope_flagp
, THREAD_GSCOPE_FLAG_WAIT
, LLL_PRIVATE
);
1158 while (*gscope_flagp
== THREAD_GSCOPE_FLAG_WAIT
);
1161 lll_unlock (stack_cache_lock
, LLL_PRIVATE
);