Update.
[glibc.git] / nptl / allocatestack.c
blob242da0a5a1f489ff0b2b333d3eb287d5592b0073
1 /* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18 02111-1307 USA. */
20 #include <assert.h>
21 #include <errno.h>
22 #include <signal.h>
23 #include <stdint.h>
24 #include <string.h>
25 #include <unistd.h>
26 #include <sys/mman.h>
27 #include <sys/param.h>
28 #include <dl-sysdep.h>
29 #include <tls.h>
30 #include <lowlevellock.h>
33 #ifndef NEED_SEPARATE_REGISTER_STACK
35 /* Most architectures have exactly one stack pointer. Some have more. */
36 # define STACK_VARIABLES void *stackaddr
38 /* How to pass the values to the 'create_thread' function. */
39 # define STACK_VARIABLES_ARGS stackaddr
41 /* How to declare function which gets there parameters. */
42 # define STACK_VARIABLES_PARMS void *stackaddr
44 /* How to declare allocate_stack. */
45 # define ALLOCATE_STACK_PARMS void **stack
47 /* This is how the function is called. We do it this way to allow
48 other variants of the function to have more parameters. */
49 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
51 #else
53 /* We need two stacks. The kernel will place them but we have to tell
54 the kernel about the size of the reserved address space. */
55 # define STACK_VARIABLES void *stackaddr; size_t stacksize
57 /* How to pass the values to the 'create_thread' function. */
58 # define STACK_VARIABLES_ARGS stackaddr, stacksize
60 /* How to declare function which gets there parameters. */
61 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
63 /* How to declare allocate_stack. */
64 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
66 /* This is how the function is called. We do it this way to allow
67 other variants of the function to have more parameters. */
68 # define ALLOCATE_STACK(attr, pd) \
69 allocate_stack (attr, pd, &stackaddr, &stacksize)
71 #endif
74 /* Default alignment of stack. */
75 #ifndef STACK_ALIGN
76 # define STACK_ALIGN __alignof__ (long double)
77 #endif
79 /* Default value for minimal stack size after allocating thread
80 descriptor and guard. */
81 #ifndef MINIMAL_REST_STACK
82 # define MINIMAL_REST_STACK 4096
83 #endif
86 /* Let the architecture add some flags to the mmap() call used to
87 allocate stacks. */
88 #ifndef ARCH_MAP_FLAGS
89 # define ARCH_MAP_FLAGS 0
90 #endif
92 /* This yields the pointer that TLS support code calls the thread pointer. */
93 #if TLS_TCB_AT_TP
94 # define TLS_TPADJ(pd) (pd)
95 #elif TLS_DTV_AT_TP
96 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
97 #endif
99 /* Cache handling for not-yet free stacks. */
101 /* Maximum size in kB of cache. */
102 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
103 static size_t stack_cache_actsize;
105 /* Mutex protecting this variable. */
106 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
108 /* List of queued stack frames. */
109 static LIST_HEAD (stack_cache);
111 /* List of the stacks in use. */
112 static LIST_HEAD (stack_used);
114 /* List of the threads with user provided stacks in use. No need to
115 initialize this, since it's done in __pthread_initialize_minimal. */
116 list_t __stack_user __attribute__ ((nocommon));
117 hidden_data_def (__stack_user)
119 #if COLORING_INCREMENT != 0
120 /* Number of threads created. */
121 static unsigned int nptl_ncreated;
122 #endif
125 /* Check whether the stack is still used or not. */
126 #define FREE_P(descr) ((descr)->tid <= 0)
129 /* We create a double linked list of all cache entries. Double linked
130 because this allows removing entries from the end. */
133 /* Get a stack frame from the cache. We have to match by size since
134 some blocks might be too small or far too large. */
135 static struct pthread *
136 get_cached_stack (size_t *sizep, void **memp)
138 size_t size = *sizep;
139 struct pthread *result = NULL;
140 list_t *entry;
142 lll_lock (stack_cache_lock);
144 /* Search the cache for a matching entry. We search for the
145 smallest stack which has at least the required size. Note that
146 in normal situations the size of all allocated stacks is the
147 same. As the very least there are only a few different sizes.
148 Therefore this loop will exit early most of the time with an
149 exact match. */
150 list_for_each (entry, &stack_cache)
152 struct pthread *curr;
154 curr = list_entry (entry, struct pthread, list);
155 if (FREE_P (curr) && curr->stackblock_size >= size)
157 if (curr->stackblock_size == size)
159 result = curr;
160 break;
163 if (result == NULL
164 || result->stackblock_size > curr->stackblock_size)
165 result = curr;
169 if (__builtin_expect (result == NULL, 0)
170 /* Make sure the size difference is not too excessive. In that
171 case we do not use the block. */
172 || __builtin_expect (result->stackblock_size > 4 * size, 0))
174 /* Release the lock. */
175 lll_unlock (stack_cache_lock);
177 return NULL;
180 /* Dequeue the entry. */
181 list_del (&result->list);
183 /* And add to the list of stacks in use. */
184 list_add (&result->list, &stack_used);
186 /* And decrease the cache size. */
187 stack_cache_actsize -= result->stackblock_size;
189 /* Release the lock early. */
190 lll_unlock (stack_cache_lock);
192 /* Report size and location of the stack to the caller. */
193 *sizep = result->stackblock_size;
194 *memp = result->stackblock;
196 /* Cancellation handling is back to the default. */
197 result->cancelhandling = 0;
198 result->cleanup = NULL;
200 /* No pending event. */
201 result->nextevent = NULL;
203 /* Clear the DTV. */
204 dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
205 memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
207 /* Re-initialize the TLS. */
208 _dl_allocate_tls_init (TLS_TPADJ (result));
210 return result;
214 /* Add a stack frame which is not used anymore to the stack. Must be
215 called with the cache lock held. */
216 static inline void
217 __attribute ((always_inline))
218 queue_stack (struct pthread *stack)
220 /* We unconditionally add the stack to the list. The memory may
221 still be in use but it will not be reused until the kernel marks
222 the stack as not used anymore. */
223 list_add (&stack->list, &stack_cache);
225 stack_cache_actsize += stack->stackblock_size;
226 if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
228 /* We reduce the size of the cache. Remove the last entries
229 until the size is below the limit. */
230 list_t *entry;
231 list_t *prev;
233 /* Search from the end of the list. */
234 list_for_each_prev_safe (entry, prev, &stack_cache)
236 struct pthread *curr;
238 curr = list_entry (entry, struct pthread, list);
239 if (FREE_P (curr))
241 /* Unlink the block. */
242 list_del (entry);
244 /* Account for the freed memory. */
245 stack_cache_actsize -= curr->stackblock_size;
247 /* Free the memory associated with the ELF TLS. */
248 _dl_deallocate_tls (TLS_TPADJ (curr), false);
250 /* Remove this block. This should never fail. If it
251 does something is really wrong. */
252 if (munmap (curr->stackblock, curr->stackblock_size) != 0)
253 abort ();
255 /* Maybe we have freed enough. */
256 if (stack_cache_actsize <= stack_cache_maxsize)
257 break;
264 static int
265 internal_function
266 change_stack_perm (struct pthread *pd
267 #ifdef NEED_SEPARATE_REGISTER_STACK
268 , size_t pagemask
269 #endif
272 #ifdef NEED_SEPARATE_REGISTER_STACK
273 void *stack = (pd->stackblock
274 + (((((pd->stackblock_size - pd->guardsize) / 2)
275 & pagemask) + pd->guardsize) & pagemask));
276 size_t len = pd->stackblock + pd->stackblock_size - stack;
277 #else
278 void *stack = pd->stackblock + pd->guardsize;
279 size_t len = pd->stackblock_size - pd->guardsize;
280 #endif
281 if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
282 return errno;
284 return 0;
288 static int
289 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
290 ALLOCATE_STACK_PARMS)
292 struct pthread *pd;
293 size_t size;
294 size_t pagesize_m1 = __getpagesize () - 1;
295 void *stacktop;
297 assert (attr != NULL);
298 assert (powerof2 (pagesize_m1 + 1));
299 assert (TCB_ALIGNMENT >= STACK_ALIGN);
301 /* Get the stack size from the attribute if it is set. Otherwise we
302 use the default we determined at start time. */
303 size = attr->stacksize ?: __default_stacksize;
305 /* Get memory for the stack. */
306 if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
308 uintptr_t adj;
310 /* If the user also specified the size of the stack make sure it
311 is large enough. */
312 if (attr->stacksize != 0
313 && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
314 return EINVAL;
316 /* Adjust stack size for alignment of the TLS block. */
317 #if TLS_TCB_AT_TP
318 adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
319 & __static_tls_align_m1;
320 assert (size > adj + TLS_TCB_SIZE);
321 #elif TLS_DTV_AT_TP
322 adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
323 & __static_tls_align_m1;
324 assert (size > adj);
325 #endif
327 /* The user provided some memory. Let's hope it matches the
328 size... We do not allocate guard pages if the user provided
329 the stack. It is the user's responsibility to do this if it
330 is wanted. */
331 #if TLS_TCB_AT_TP
332 pd = (struct pthread *) ((uintptr_t) attr->stackaddr
333 - TLS_TCB_SIZE - adj);
334 #elif TLS_DTV_AT_TP
335 pd = (struct pthread *) (((uintptr_t) attr->stackaddr
336 - __static_tls_size - adj)
337 - TLS_PRE_TCB_SIZE);
338 #endif
340 /* The user provided stack memory needs to be cleared. */
341 memset (pd, '\0', sizeof (struct pthread));
343 /* The first TSD block is included in the TCB. */
344 pd->specific[0] = pd->specific_1stblock;
346 /* Remember the stack-related values. */
347 pd->stackblock = (char *) attr->stackaddr - size;
348 pd->stackblock_size = size;
350 /* This is a user-provided stack. It will not be queued in the
351 stack cache nor will the memory (except the TLS memory) be freed. */
352 pd->user_stack = true;
354 /* This is at least the second thread. */
355 pd->header.multiple_threads = 1;
356 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
357 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
358 #endif
360 #ifdef NEED_DL_SYSINFO
361 /* Copy the sysinfo value from the parent. */
362 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
363 #endif
365 /* The process ID is also the same as that of the caller. */
366 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
368 /* Allocate the DTV for this thread. */
369 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
371 /* Something went wrong. */
372 assert (errno == ENOMEM);
373 return EAGAIN;
377 /* Prepare to modify global data. */
378 lll_lock (stack_cache_lock);
380 /* And add to the list of stacks in use. */
381 list_add (&pd->list, &__stack_user);
383 lll_unlock (stack_cache_lock);
385 else
387 /* Allocate some anonymous memory. If possible use the cache. */
388 size_t guardsize;
389 size_t reqsize;
390 void *mem;
391 const int prot = (PROT_READ | PROT_WRITE
392 | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
394 #if COLORING_INCREMENT != 0
395 /* Add one more page for stack coloring. Don't do it for stacks
396 with 16 times pagesize or larger. This might just cause
397 unnecessary misalignment. */
398 if (size <= 16 * pagesize_m1)
399 size += pagesize_m1 + 1;
400 #endif
402 /* Adjust the stack size for alignment. */
403 size &= ~__static_tls_align_m1;
404 assert (size != 0);
406 /* Make sure the size of the stack is enough for the guard and
407 eventually the thread descriptor. */
408 guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
409 if (__builtin_expect (size < (guardsize + __static_tls_size
410 + MINIMAL_REST_STACK + pagesize_m1 + 1),
412 /* The stack is too small (or the guard too large). */
413 return EINVAL;
415 /* Try to get a stack from the cache. */
416 reqsize = size;
417 pd = get_cached_stack (&size, &mem);
418 if (pd == NULL)
420 /* To avoid aliasing effects on a larger scale then pages we
421 adjust the allocated stack size if necessary. This way
422 allocations directly following each other will not have
423 aliasing problems. */
424 #if MULTI_PAGE_ALIASING != 0
425 if ((size % MULTI_PAGE_ALIASING) == 0)
426 size += pagesize_m1 + 1;
427 #endif
429 mem = mmap (NULL, size, prot,
430 MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
432 if (__builtin_expect (mem == MAP_FAILED, 0))
434 #ifdef ARCH_RETRY_MMAP
435 mem = ARCH_RETRY_MMAP (size);
436 if (__builtin_expect (mem == MAP_FAILED, 0))
437 #endif
438 return errno;
441 /* SIZE is guaranteed to be greater than zero.
442 So we can never get a null pointer back from mmap. */
443 assert (mem != NULL);
445 #if COLORING_INCREMENT != 0
446 /* Atomically increment NCREATED. */
447 unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
449 /* We chose the offset for coloring by incrementing it for
450 every new thread by a fixed amount. The offset used
451 module the page size. Even if coloring would be better
452 relative to higher alignment values it makes no sense to
453 do it since the mmap() interface does not allow us to
454 specify any alignment for the returned memory block. */
455 size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
457 /* Make sure the coloring offsets does not disturb the alignment
458 of the TCB and static TLS block. */
459 if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
460 coloring = (((coloring + __static_tls_align_m1)
461 & ~(__static_tls_align_m1))
462 & ~pagesize_m1);
463 #else
464 /* Unless specified we do not make any adjustments. */
465 # define coloring 0
466 #endif
468 /* Place the thread descriptor at the end of the stack. */
469 #if TLS_TCB_AT_TP
470 pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
471 #elif TLS_DTV_AT_TP
472 pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
473 - __static_tls_size)
474 & ~__static_tls_align_m1)
475 - TLS_PRE_TCB_SIZE);
476 #endif
478 /* Remember the stack-related values. */
479 pd->stackblock = mem;
480 pd->stackblock_size = size;
482 /* We allocated the first block thread-specific data array.
483 This address will not change for the lifetime of this
484 descriptor. */
485 pd->specific[0] = pd->specific_1stblock;
487 /* This is at least the second thread. */
488 pd->header.multiple_threads = 1;
489 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
490 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
491 #endif
493 #ifdef NEED_DL_SYSINFO
494 /* Copy the sysinfo value from the parent. */
495 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
496 #endif
498 /* The process ID is also the same as that of the caller. */
499 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
501 /* Allocate the DTV for this thread. */
502 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
504 /* Something went wrong. */
505 assert (errno == ENOMEM);
507 /* Free the stack memory we just allocated. */
508 (void) munmap (mem, size);
510 return EAGAIN;
514 /* Prepare to modify global data. */
515 lll_lock (stack_cache_lock);
517 /* And add to the list of stacks in use. */
518 list_add (&pd->list, &stack_used);
520 lll_unlock (stack_cache_lock);
523 /* There might have been a race. Another thread might have
524 caused the stacks to get exec permission while this new
525 stack was prepared. Detect if this was possible and
526 change the permission if necessary. */
527 if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
528 && (prot & PROT_EXEC) == 0, 0))
530 int err = change_stack_perm (pd
531 #ifdef NEED_SEPARATE_REGISTER_STACK
532 , ~pagesize_m1
533 #endif
535 if (err != 0)
537 /* Free the stack memory we just allocated. */
538 (void) munmap (mem, size);
540 return err;
545 /* Note that all of the stack and the thread descriptor is
546 zeroed. This means we do not have to initialize fields
547 with initial value zero. This is specifically true for
548 the 'tid' field which is always set back to zero once the
549 stack is not used anymore and for the 'guardsize' field
550 which will be read next. */
553 /* Create or resize the guard area if necessary. */
554 if (__builtin_expect (guardsize > pd->guardsize, 0))
556 #ifdef NEED_SEPARATE_REGISTER_STACK
557 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
558 #else
559 char *guard = mem;
560 #endif
561 if (mprotect (guard, guardsize, PROT_NONE) != 0)
563 int err;
564 mprot_error:
565 err = errno;
567 lll_lock (stack_cache_lock);
569 /* Remove the thread from the list. */
570 list_del (&pd->list);
572 lll_unlock (stack_cache_lock);
574 /* Get rid of the TLS block we allocated. */
575 _dl_deallocate_tls (TLS_TPADJ (pd), false);
577 /* Free the stack memory regardless of whether the size
578 of the cache is over the limit or not. If this piece
579 of memory caused problems we better do not use it
580 anymore. Uh, and we ignore possible errors. There
581 is nothing we could do. */
582 (void) munmap (mem, size);
584 return err;
587 pd->guardsize = guardsize;
589 else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
592 /* The old guard area is too large. */
594 #ifdef NEED_SEPARATE_REGISTER_STACK
595 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
596 char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
598 if (oldguard < guard
599 && mprotect (oldguard, guard - oldguard, prot) != 0)
600 goto mprot_error;
602 if (mprotect (guard + guardsize,
603 oldguard + pd->guardsize - guard - guardsize,
604 prot) != 0)
605 goto mprot_error;
606 #else
607 if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
608 prot) != 0)
609 goto mprot_error;
610 #endif
612 pd->guardsize = guardsize;
614 /* The pthread_getattr_np() calls need to get passed the size
615 requested in the attribute, regardless of how large the
616 actually used guardsize is. */
617 pd->reported_guardsize = guardsize;
620 /* Initialize the lock. We have to do this unconditionally since the
621 stillborn thread could be canceled while the lock is taken. */
622 pd->lock = LLL_LOCK_INITIALIZER;
624 /* We place the thread descriptor at the end of the stack. */
625 *pdp = pd;
627 #if TLS_TCB_AT_TP
628 /* The stack begins before the TCB and the static TLS block. */
629 stacktop = ((char *) (pd + 1) - __static_tls_size);
630 #elif TLS_DTV_AT_TP
631 stacktop = (char *) (pd - 1);
632 #endif
634 #ifdef NEED_SEPARATE_REGISTER_STACK
635 *stack = pd->stackblock;
636 *stacksize = stacktop - *stack;
637 #else
638 *stack = stacktop;
639 #endif
641 return 0;
645 void
646 internal_function
647 __deallocate_stack (struct pthread *pd)
649 lll_lock (stack_cache_lock);
651 /* Remove the thread from the list of threads with user defined
652 stacks. */
653 list_del (&pd->list);
655 /* Not much to do. Just free the mmap()ed memory. Note that we do
656 not reset the 'used' flag in the 'tid' field. This is done by
657 the kernel. If no thread has been created yet this field is
658 still zero. */
659 if (__builtin_expect (! pd->user_stack, 1))
660 (void) queue_stack (pd);
661 else
662 /* Free the memory associated with the ELF TLS. */
663 _dl_deallocate_tls (TLS_TPADJ (pd), false);
665 lll_unlock (stack_cache_lock);
670 internal_function
671 __make_stacks_executable (void **stack_endp)
673 /* First the main thread's stack. */
674 int err = _dl_make_stack_executable (stack_endp);
675 if (err != 0)
676 return err;
678 #ifdef NEED_SEPARATE_REGISTER_STACK
679 const size_t pagemask = ~(__getpagesize () - 1);
680 #endif
682 lll_lock (stack_cache_lock);
684 list_t *runp;
685 list_for_each (runp, &stack_used)
687 err = change_stack_perm (list_entry (runp, struct pthread, list)
688 #ifdef NEED_SEPARATE_REGISTER_STACK
689 , pagemask
690 #endif
692 if (err != 0)
693 break;
696 /* Also change the permission for the currently unused stacks. This
697 might be wasted time but better spend it here than adding a check
698 in the fast path. */
699 if (err == 0)
700 list_for_each (runp, &stack_cache)
702 err = change_stack_perm (list_entry (runp, struct pthread, list)
703 #ifdef NEED_SEPARATE_REGISTER_STACK
704 , pagemask
705 #endif
707 if (err != 0)
708 break;
711 lll_unlock (stack_cache_lock);
713 return err;
717 /* In case of a fork() call the memory allocation in the child will be
718 the same but only one thread is running. All stacks except that of
719 the one running thread are not used anymore. We have to recycle
720 them. */
721 void
722 __reclaim_stacks (void)
724 struct pthread *self = (struct pthread *) THREAD_SELF;
726 /* No locking necessary. The caller is the only stack in use. */
728 /* Mark all stacks except the still running one as free. */
729 list_t *runp;
730 list_for_each (runp, &stack_used)
732 struct pthread *curp;
734 curp = list_entry (runp, struct pthread, list);
735 if (curp != self)
737 /* This marks the stack as free. */
738 curp->tid = 0;
740 /* The PID field must be initialized for the new process. */
741 curp->pid = self->pid;
743 /* Account for the size of the stack. */
744 stack_cache_actsize += curp->stackblock_size;
748 /* Add the stack of all running threads to the cache. */
749 list_splice (&stack_used, &stack_cache);
751 /* Remove the entry for the current thread to from the cache list
752 and add it to the list of running threads. Which of the two
753 lists is decided by the user_stack flag. */
754 list_del (&self->list);
756 /* Re-initialize the lists for all the threads. */
757 INIT_LIST_HEAD (&stack_used);
758 INIT_LIST_HEAD (&__stack_user);
760 if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
761 list_add (&self->list, &__stack_user);
762 else
763 list_add (&self->list, &stack_used);
765 /* There is one thread running. */
766 __nptl_nthreads = 1;
768 /* Initialize the lock. */
769 stack_cache_lock = LLL_LOCK_INITIALIZER;
773 #if HP_TIMING_AVAIL
774 /* Find a thread given the thread ID. */
775 struct pthread *
776 attribute_hidden
777 __find_thread_by_id (pid_t tid)
779 struct pthread *result = NULL;
781 lll_lock (stack_cache_lock);
783 /* Iterate over the list with system-allocated threads first. */
784 list_t *runp;
785 list_for_each (runp, &stack_used)
787 struct pthread *curp;
789 curp = list_entry (runp, struct pthread, list);
791 if (curp->tid == tid)
793 result = curp;
794 goto out;
798 /* Now the list with threads using user-allocated stacks. */
799 list_for_each (runp, &__stack_user)
801 struct pthread *curp;
803 curp = list_entry (runp, struct pthread, list);
805 if (curp->tid == tid)
807 result = curp;
808 goto out;
812 out:
813 lll_unlock (stack_cache_lock);
815 return result;
817 #endif
819 void
820 attribute_hidden
821 __nptl_setxid (struct xid_command *cmdp)
823 lll_lock (stack_cache_lock);
825 __xidcmd = cmdp;
826 cmdp->cntr = 0;
828 INTERNAL_SYSCALL_DECL (err);
830 struct pthread *self = THREAD_SELF;
832 /* Iterate over the list with system-allocated threads first. */
833 list_t *runp;
834 list_for_each (runp, &stack_used)
836 struct pthread *t = list_entry (runp, struct pthread, list);
837 if (t != self)
839 int val;
840 #if __ASSUME_TGKILL
841 val = INTERNAL_SYSCALL (tgkill, err, 3,
842 THREAD_GETMEM (THREAD_SELF, pid),
843 t->tid, SIGSETXID);
844 #else
845 # ifdef __NR_tgkill
846 val = INTERNAL_SYSCALL (tgkill, err, 3,
847 THREAD_GETMEM (THREAD_SELF, pid),
848 t->tid, SIGSETXID);
849 if (INTERNAL_SYSCALL_ERROR_P (val, err)
850 && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
851 # endif
852 val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
853 #endif
855 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
856 atomic_increment (&cmdp->cntr);
860 /* Now the list with threads using user-allocated stacks. */
861 list_for_each (runp, &__stack_user)
863 struct pthread *t = list_entry (runp, struct pthread, list);
864 if (t != self)
866 int val;
867 #if __ASSUME_TGKILL
868 val = INTERNAL_SYSCALL (tgkill, err, 3,
869 THREAD_GETMEM (THREAD_SELF, pid),
870 t->tid, SIGSETXID);
871 #else
872 # ifdef __NR_tgkill
873 val = INTERNAL_SYSCALL (tgkill, err, 3,
874 THREAD_GETMEM (THREAD_SELF, pid),
875 t->tid, SIGSETXID);
876 if (INTERNAL_SYSCALL_ERROR_P (val, err)
877 && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
878 # endif
879 val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
880 #endif
882 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
883 atomic_increment (&cmdp->cntr);
887 int cur = cmdp->cntr;
888 while (cur != 0)
890 lll_futex_wait (&cmdp->cntr, cur);
891 cur = cmdp->cntr;
894 lll_unlock (stack_cache_lock);
897 static inline void __attribute__((always_inline))
898 init_one_static_tls (struct pthread *curp, struct link_map *map)
900 dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
901 # if TLS_TCB_AT_TP
902 void *dest = (char *) curp - map->l_tls_offset;
903 # elif TLS_DTV_AT_TP
904 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
905 # else
906 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
907 # endif
909 /* Fill in the DTV slot so that a later LD/GD access will find it. */
910 dtv[map->l_tls_modid].pointer = dest;
912 /* Initialize the memory. */
913 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
914 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
917 void
918 attribute_hidden
919 __pthread_init_static_tls (struct link_map *map)
921 lll_lock (stack_cache_lock);
923 /* Iterate over the list with system-allocated threads first. */
924 list_t *runp;
925 list_for_each (runp, &stack_used)
926 init_one_static_tls (list_entry (runp, struct pthread, list), map);
928 /* Now the list with threads using user-allocated stacks. */
929 list_for_each (runp, &__stack_user)
930 init_one_static_tls (list_entry (runp, struct pthread, list), map);
932 lll_unlock (stack_cache_lock);