nptl/allocatestack.c

   1 /* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31
  32
  33 #ifndef NEED_SEPARATE_REGISTER_STACK
  34
  35 /* Most architectures have exactly one stack pointer.  Some have more.  */
  36 # define STACK_VARIABLES void *stackaddr = NULL
  37
  38 /* How to pass the values to the 'create_thread' function.  */
  39 # define STACK_VARIABLES_ARGS stackaddr
  40
  41 /* How to declare function which gets there parameters.  */
  42 # define STACK_VARIABLES_PARMS void *stackaddr
  43
  44 /* How to declare allocate_stack.  */
  45 # define ALLOCATE_STACK_PARMS void **stack
  46
  47 /* This is how the function is called.  We do it this way to allow
  48    other variants of the function to have more parameters.  */
  49 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  50
  51 #else
  52
  53 /* We need two stacks.  The kernel will place them but we have to tell
  54    the kernel about the size of the reserved address space.  */
  55 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
  56
  57 /* How to pass the values to the 'create_thread' function.  */
  58 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  59
  60 /* How to declare function which gets there parameters.  */
  61 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  62
  63 /* How to declare allocate_stack.  */
  64 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  65
  66 /* This is how the function is called.  We do it this way to allow
  67    other variants of the function to have more parameters.  */
  68 # define ALLOCATE_STACK(attr, pd) \
  69   allocate_stack (attr, pd, &stackaddr, &stacksize)
  70
  71 #endif
  72
  73
  74 /* Default alignment of stack.  */
  75 #ifndef STACK_ALIGN
  76 # define STACK_ALIGN __alignof__ (long double)
  77 #endif
  78
  79 /* Default value for minimal stack size after allocating thread
  80    descriptor and guard.  */
  81 #ifndef MINIMAL_REST_STACK
  82 # define MINIMAL_REST_STACK     4096
  83 #endif
  84
  85
  86 /* Let the architecture add some flags to the mmap() call used to
  87    allocate stacks.  */
  88 #ifndef ARCH_MAP_FLAGS
  89 # define ARCH_MAP_FLAGS 0
  90 #endif
  91
  92 /* This yields the pointer that TLS support code calls the thread pointer.  */
  93 #if TLS_TCB_AT_TP
  94 # define TLS_TPADJ(pd) (pd)
  95 #elif TLS_DTV_AT_TP
  96 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  97 #endif
  98
  99 /* Cache handling for not-yet free stacks.  */
 100
 101 /* Maximum size in kB of cache.  */
 102 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 103 static size_t stack_cache_actsize;
 104
 105 /* Mutex protecting this variable.  */
 106 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
 107
 108 /* List of queued stack frames.  */
 109 static LIST_HEAD (stack_cache);
 110
 111 /* List of the stacks in use.  */
 112 static LIST_HEAD (stack_used);
 113
 114 /* List of the threads with user provided stacks in use.  No need to
 115    initialize this, since it's done in __pthread_initialize_minimal.  */
 116 list_t __stack_user __attribute__ ((nocommon));
 117 hidden_data_def (__stack_user)
 118
 119 #if COLORING_INCREMENT != 0
 120 /* Number of threads created.  */
 121 static unsigned int nptl_ncreated;
 122 #endif
 123
 124
 125 /* Check whether the stack is still used or not.  */
 126 #define FREE_P(descr) ((descr)->tid <= 0)
 127
 128
 129 /* We create a double linked list of all cache entries.  Double linked
 130    because this allows removing entries from the end.  */
 131
 132
 133 /* Get a stack frame from the cache.  We have to match by size since
 134    some blocks might be too small or far too large.  */
 135 static struct pthread *
 136 get_cached_stack (size_t *sizep, void **memp)
 137 {
 138   size_t size = *sizep;
 139   struct pthread *result = NULL;
 140   list_t *entry;
 141
 142   lll_lock (stack_cache_lock);
 143
 144   /* Search the cache for a matching entry.  We search for the
 145      smallest stack which has at least the required size.  Note that
 146      in normal situations the size of all allocated stacks is the
 147      same.  As the very least there are only a few different sizes.
 148      Therefore this loop will exit early most of the time with an
 149      exact match.  */
 150   list_for_each (entry, &stack_cache)
 151     {
 152       struct pthread *curr;
 153
 154       curr = list_entry (entry, struct pthread, list);
 155       if (FREE_P (curr) && curr->stackblock_size >= size)
 156         {
 157           if (curr->stackblock_size == size)
 158             {
 159               result = curr;
 160               break;
 161             }
 162
 163           if (result == NULL
 164               || result->stackblock_size > curr->stackblock_size)
 165             result = curr;
 166         }
 167     }
 168
 169   if (__builtin_expect (result == NULL, 0)
 170       /* Make sure the size difference is not too excessive.  In that
 171          case we do not use the block.  */
 172       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 173     {
 174       /* Release the lock.  */
 175       lll_unlock (stack_cache_lock);
 176
 177       return NULL;
 178     }
 179
 180   /* Dequeue the entry.  */
 181   list_del (&result->list);
 182
 183   /* And add to the list of stacks in use.  */
 184   list_add (&result->list, &stack_used);
 185
 186   /* And decrease the cache size.  */
 187   stack_cache_actsize -= result->stackblock_size;
 188
 189   /* Release the lock early.  */
 190   lll_unlock (stack_cache_lock);
 191
 192   /* Report size and location of the stack to the caller.  */
 193   *sizep = result->stackblock_size;
 194   *memp = result->stackblock;
 195
 196   /* Cancellation handling is back to the default.  */
 197   result->cancelhandling = 0;
 198   result->cleanup = NULL;
 199
 200   /* No pending event.  */
 201   result->nextevent = NULL;
 202
 203   /* Clear the DTV.  */
 204   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 205   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 206
 207   /* Re-initialize the TLS.  */
 208   _dl_allocate_tls_init (TLS_TPADJ (result));
 209
 210   return result;
 211 }
 212
 213
 214 /* Free stacks until cache size is lower than LIMIT.  */
 215 static void
 216 free_stacks (size_t limit)
 217 {
 218   /* We reduce the size of the cache.  Remove the last entries until
 219      the size is below the limit.  */
 220   list_t *entry;
 221   list_t *prev;
 222
 223   /* Search from the end of the list.  */
 224   list_for_each_prev_safe (entry, prev, &stack_cache)
 225     {
 226       struct pthread *curr;
 227
 228       curr = list_entry (entry, struct pthread, list);
 229       if (FREE_P (curr))
 230         {
 231           /* Unlink the block.  */
 232           list_del (entry);
 233
 234           /* Account for the freed memory.  */
 235           stack_cache_actsize -= curr->stackblock_size;
 236
 237           /* Free the memory associated with the ELF TLS.  */
 238           _dl_deallocate_tls (TLS_TPADJ (curr), false);
 239
 240           /* Remove this block.  This should never fail.  If it does
 241              something is really wrong.  */
 242           if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 243             abort ();
 244
 245           /* Maybe we have freed enough.  */
 246           if (stack_cache_actsize <= limit)
 247             break;
 248         }
 249     }
 250 }
 251
 252
 253 /* Add a stack frame which is not used anymore to the stack.  Must be
 254    called with the cache lock held.  */
 255 static inline void
 256 __attribute ((always_inline))
 257 queue_stack (struct pthread *stack)
 258 {
 259   /* We unconditionally add the stack to the list.  The memory may
 260      still be in use but it will not be reused until the kernel marks
 261      the stack as not used anymore.  */
 262   list_add (&stack->list, &stack_cache);
 263
 264   stack_cache_actsize += stack->stackblock_size;
 265   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 266     free_stacks (stack_cache_maxsize);
 267 }
 268
 269
 270 /* This function is called indirectly from the freeres code in libc.  */
 271 void
 272 __free_stack_cache (void)
 273 {
 274   free_stacks (0);
 275 }
 276
 277
 278 static int
 279 internal_function
 280 change_stack_perm (struct pthread *pd
 281 #ifdef NEED_SEPARATE_REGISTER_STACK
 282                    , size_t pagemask
 283 #endif
 284                    )
 285 {
 286 #ifdef NEED_SEPARATE_REGISTER_STACK
 287   void *stack = (pd->stackblock
 288                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 289                       & pagemask) + pd->guardsize) & pagemask));
 290   size_t len = pd->stackblock + pd->stackblock_size - stack;
 291 #else
 292   void *stack = pd->stackblock + pd->guardsize;
 293   size_t len = pd->stackblock_size - pd->guardsize;
 294 #endif
 295   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 296     return errno;
 297
 298   return 0;
 299 }
 300
 301
 302 static int
 303 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 304                 ALLOCATE_STACK_PARMS)
 305 {
 306   struct pthread *pd;
 307   size_t size;
 308   size_t pagesize_m1 = __getpagesize () - 1;
 309   void *stacktop;
 310
 311   assert (attr != NULL);
 312   assert (powerof2 (pagesize_m1 + 1));
 313   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 314
 315   /* Get the stack size from the attribute if it is set.  Otherwise we
 316      use the default we determined at start time.  */
 317   size = attr->stacksize ?: __default_stacksize;
 318
 319   /* Get memory for the stack.  */
 320   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 321     {
 322       uintptr_t adj;
 323
 324       /* If the user also specified the size of the stack make sure it
 325          is large enough.  */
 326       if (attr->stacksize != 0
 327           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 328         return EINVAL;
 329
 330       /* Adjust stack size for alignment of the TLS block.  */
 331 #if TLS_TCB_AT_TP
 332       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 333             & __static_tls_align_m1;
 334       assert (size > adj + TLS_TCB_SIZE);
 335 #elif TLS_DTV_AT_TP
 336       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 337             & __static_tls_align_m1;
 338       assert (size > adj);
 339 #endif
 340
 341       /* The user provided some memory.  Let's hope it matches the
 342          size...  We do not allocate guard pages if the user provided
 343          the stack.  It is the user's responsibility to do this if it
 344          is wanted.  */
 345 #if TLS_TCB_AT_TP
 346       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 347                                - TLS_TCB_SIZE - adj);
 348 #elif TLS_DTV_AT_TP
 349       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 350                                 - __static_tls_size - adj)
 351                                - TLS_PRE_TCB_SIZE);
 352 #endif
 353
 354       /* The user provided stack memory needs to be cleared.  */
 355       memset (pd, '\0', sizeof (struct pthread));
 356
 357       /* The first TSD block is included in the TCB.  */
 358       pd->specific[0] = pd->specific_1stblock;
 359
 360       /* Remember the stack-related values.  */
 361       pd->stackblock = (char *) attr->stackaddr - size;
 362       pd->stackblock_size = size;
 363
 364       /* This is a user-provided stack.  It will not be queued in the
 365          stack cache nor will the memory (except the TLS memory) be freed.  */
 366       pd->user_stack = true;
 367
 368       /* This is at least the second thread.  */
 369       pd->header.multiple_threads = 1;
 370 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 371       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 372 #endif
 373
 374 #ifdef NEED_DL_SYSINFO
 375       /* Copy the sysinfo value from the parent.  */
 376       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 377 #endif
 378
 379       /* The process ID is also the same as that of the caller.  */
 380       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 381
 382       /* Allocate the DTV for this thread.  */
 383       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 384         {
 385           /* Something went wrong.  */
 386           assert (errno == ENOMEM);
 387           return EAGAIN;
 388         }
 389
 390
 391       /* Prepare to modify global data.  */
 392       lll_lock (stack_cache_lock);
 393
 394       /* And add to the list of stacks in use.  */
 395       list_add (&pd->list, &__stack_user);
 396
 397       lll_unlock (stack_cache_lock);
 398     }
 399   else
 400     {
 401       /* Allocate some anonymous memory.  If possible use the cache.  */
 402       size_t guardsize;
 403       size_t reqsize;
 404       void *mem;
 405       const int prot = (PROT_READ | PROT_WRITE
 406                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 407
 408 #if COLORING_INCREMENT != 0
 409       /* Add one more page for stack coloring.  Don't do it for stacks
 410          with 16 times pagesize or larger.  This might just cause
 411          unnecessary misalignment.  */
 412       if (size <= 16 * pagesize_m1)
 413         size += pagesize_m1 + 1;
 414 #endif
 415
 416       /* Adjust the stack size for alignment.  */
 417       size &= ~__static_tls_align_m1;
 418       assert (size != 0);
 419
 420       /* Make sure the size of the stack is enough for the guard and
 421          eventually the thread descriptor.  */
 422       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 423       if (__builtin_expect (size < ((guardsize + __static_tls_size
 424                                      + MINIMAL_REST_STACK + pagesize_m1)
 425                                     & ~pagesize_m1),
 426                             0))
 427         /* The stack is too small (or the guard too large).  */
 428         return EINVAL;
 429
 430       /* Try to get a stack from the cache.  */
 431       reqsize = size;
 432       pd = get_cached_stack (&size, &mem);
 433       if (pd == NULL)
 434         {
 435           /* To avoid aliasing effects on a larger scale than pages we
 436              adjust the allocated stack size if necessary.  This way
 437              allocations directly following each other will not have
 438              aliasing problems.  */
 439 #if MULTI_PAGE_ALIASING != 0
 440           if ((size % MULTI_PAGE_ALIASING) == 0)
 441             size += pagesize_m1 + 1;
 442 #endif
 443
 444           mem = mmap (NULL, size, prot,
 445                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 446
 447           if (__builtin_expect (mem == MAP_FAILED, 0))
 448             {
 449 #ifdef ARCH_RETRY_MMAP
 450               mem = ARCH_RETRY_MMAP (size);
 451               if (__builtin_expect (mem == MAP_FAILED, 0))
 452 #endif
 453                 return errno;
 454             }
 455
 456           /* SIZE is guaranteed to be greater than zero.
 457              So we can never get a null pointer back from mmap.  */
 458           assert (mem != NULL);
 459
 460 #if COLORING_INCREMENT != 0
 461           /* Atomically increment NCREATED.  */
 462           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 463
 464           /* We chose the offset for coloring by incrementing it for
 465              every new thread by a fixed amount.  The offset used
 466              module the page size.  Even if coloring would be better
 467              relative to higher alignment values it makes no sense to
 468              do it since the mmap() interface does not allow us to
 469              specify any alignment for the returned memory block.  */
 470           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 471
 472           /* Make sure the coloring offsets does not disturb the alignment
 473              of the TCB and static TLS block.  */
 474           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 475             coloring = (((coloring + __static_tls_align_m1)
 476                          & ~(__static_tls_align_m1))
 477                         & ~pagesize_m1);
 478 #else
 479           /* Unless specified we do not make any adjustments.  */
 480 # define coloring 0
 481 #endif
 482
 483           /* Place the thread descriptor at the end of the stack.  */
 484 #if TLS_TCB_AT_TP
 485           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 486 #elif TLS_DTV_AT_TP
 487           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 488                                     - __static_tls_size)
 489                                     & ~__static_tls_align_m1)
 490                                    - TLS_PRE_TCB_SIZE);
 491 #endif
 492
 493           /* Remember the stack-related values.  */
 494           pd->stackblock = mem;
 495           pd->stackblock_size = size;
 496
 497           /* We allocated the first block thread-specific data array.
 498              This address will not change for the lifetime of this
 499              descriptor.  */
 500           pd->specific[0] = pd->specific_1stblock;
 501
 502           /* This is at least the second thread.  */
 503           pd->header.multiple_threads = 1;
 504 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 505           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 506 #endif
 507
 508 #ifdef NEED_DL_SYSINFO
 509           /* Copy the sysinfo value from the parent.  */
 510           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 511 #endif
 512
 513           /* The process ID is also the same as that of the caller.  */
 514           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 515
 516           /* Allocate the DTV for this thread.  */
 517           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 518             {
 519               /* Something went wrong.  */
 520               assert (errno == ENOMEM);
 521
 522               /* Free the stack memory we just allocated.  */
 523               (void) munmap (mem, size);
 524
 525               return EAGAIN;
 526             }
 527
 528
 529           /* Prepare to modify global data.  */
 530           lll_lock (stack_cache_lock);
 531
 532           /* And add to the list of stacks in use.  */
 533           list_add (&pd->list, &stack_used);
 534
 535           lll_unlock (stack_cache_lock);
 536
 537
 538           /* There might have been a race.  Another thread might have
 539              caused the stacks to get exec permission while this new
 540              stack was prepared.  Detect if this was possible and
 541              change the permission if necessary.  */
 542           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 543                                 && (prot & PROT_EXEC) == 0, 0))
 544             {
 545               int err = change_stack_perm (pd
 546 #ifdef NEED_SEPARATE_REGISTER_STACK
 547                                            , ~pagesize_m1
 548 #endif
 549                                            );
 550               if (err != 0)
 551                 {
 552                   /* Free the stack memory we just allocated.  */
 553                   (void) munmap (mem, size);
 554
 555                   return err;
 556                 }
 557             }
 558
 559
 560           /* Note that all of the stack and the thread descriptor is
 561              zeroed.  This means we do not have to initialize fields
 562              with initial value zero.  This is specifically true for
 563              the 'tid' field which is always set back to zero once the
 564              stack is not used anymore and for the 'guardsize' field
 565              which will be read next.  */
 566         }
 567
 568       /* Create or resize the guard area if necessary.  */
 569       if (__builtin_expect (guardsize > pd->guardsize, 0))
 570         {
 571 #ifdef NEED_SEPARATE_REGISTER_STACK
 572           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 573 #else
 574           char *guard = mem;
 575 #endif
 576           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 577             {
 578               int err;
 579             mprot_error:
 580               err = errno;
 581
 582               lll_lock (stack_cache_lock);
 583
 584               /* Remove the thread from the list.  */
 585               list_del (&pd->list);
 586
 587               lll_unlock (stack_cache_lock);
 588
 589               /* Get rid of the TLS block we allocated.  */
 590               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 591
 592               /* Free the stack memory regardless of whether the size
 593                  of the cache is over the limit or not.  If this piece
 594                  of memory caused problems we better do not use it
 595                  anymore.  Uh, and we ignore possible errors.  There
 596                  is nothing we could do.  */
 597               (void) munmap (mem, size);
 598
 599               return err;
 600             }
 601
 602           pd->guardsize = guardsize;
 603         }
 604       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 605                                  0))
 606         {
 607           /* The old guard area is too large.  */
 608
 609 #ifdef NEED_SEPARATE_REGISTER_STACK
 610           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 611           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 612
 613           if (oldguard < guard
 614               && mprotect (oldguard, guard - oldguard, prot) != 0)
 615             goto mprot_error;
 616
 617           if (mprotect (guard + guardsize,
 618                         oldguard + pd->guardsize - guard - guardsize,
 619                         prot) != 0)
 620             goto mprot_error;
 621 #else
 622           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 623                         prot) != 0)
 624             goto mprot_error;
 625 #endif
 626
 627           pd->guardsize = guardsize;
 628         }
 629       /* The pthread_getattr_np() calls need to get passed the size
 630          requested in the attribute, regardless of how large the
 631          actually used guardsize is.  */
 632       pd->reported_guardsize = guardsize;
 633     }
 634
 635   /* Initialize the lock.  We have to do this unconditionally since the
 636      stillborn thread could be canceled while the lock is taken.  */
 637   pd->lock = LLL_LOCK_INITIALIZER;
 638
 639   /* The robust mutex lists also need to be initialized
 640      unconditionally because the cleanup for the previous stack owner
 641      might have happened in the kernel.  */
 642   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
 643                                   - offsetof (pthread_mutex_t,
 644                                               __data.__list.__next));
 645   pd->robust_head.list_op_pending = NULL;
 646 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 647   pd->robust_prev = &pd->robust_head;
 648 #endif
 649   pd->robust_head.list = &pd->robust_head;
 650
 651   /* We place the thread descriptor at the end of the stack.  */
 652   *pdp = pd;
 653
 654 #if TLS_TCB_AT_TP
 655   /* The stack begins before the TCB and the static TLS block.  */
 656   stacktop = ((char *) (pd + 1) - __static_tls_size);
 657 #elif TLS_DTV_AT_TP
 658   stacktop = (char *) (pd - 1);
 659 #endif
 660
 661 #ifdef NEED_SEPARATE_REGISTER_STACK
 662   *stack = pd->stackblock;
 663   *stacksize = stacktop - *stack;
 664 #else
 665   *stack = stacktop;
 666 #endif
 667
 668   return 0;
 669 }
 670
 671
 672 void
 673 internal_function
 674 __deallocate_stack (struct pthread *pd)
 675 {
 676   lll_lock (stack_cache_lock);
 677
 678   /* Remove the thread from the list of threads with user defined
 679      stacks.  */
 680   list_del (&pd->list);
 681
 682   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 683      not reset the 'used' flag in the 'tid' field.  This is done by
 684      the kernel.  If no thread has been created yet this field is
 685      still zero.  */
 686   if (__builtin_expect (! pd->user_stack, 1))
 687     (void) queue_stack (pd);
 688   else
 689     /* Free the memory associated with the ELF TLS.  */
 690     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 691
 692   lll_unlock (stack_cache_lock);
 693 }
 694
 695
 696 int
 697 internal_function
 698 __make_stacks_executable (void **stack_endp)
 699 {
 700   /* First the main thread's stack.  */
 701   int err = _dl_make_stack_executable (stack_endp);
 702   if (err != 0)
 703     return err;
 704
 705 #ifdef NEED_SEPARATE_REGISTER_STACK
 706   const size_t pagemask = ~(__getpagesize () - 1);
 707 #endif
 708
 709   lll_lock (stack_cache_lock);
 710
 711   list_t *runp;
 712   list_for_each (runp, &stack_used)
 713     {
 714       err = change_stack_perm (list_entry (runp, struct pthread, list)
 715 #ifdef NEED_SEPARATE_REGISTER_STACK
 716                                , pagemask
 717 #endif
 718                                );
 719       if (err != 0)
 720         break;
 721     }
 722
 723   /* Also change the permission for the currently unused stacks.  This
 724      might be wasted time but better spend it here than adding a check
 725      in the fast path.  */
 726   if (err == 0)
 727     list_for_each (runp, &stack_cache)
 728       {
 729         err = change_stack_perm (list_entry (runp, struct pthread, list)
 730 #ifdef NEED_SEPARATE_REGISTER_STACK
 731                                  , pagemask
 732 #endif
 733                                  );
 734         if (err != 0)
 735           break;
 736       }
 737
 738   lll_unlock (stack_cache_lock);
 739
 740   return err;
 741 }
 742
 743
 744 /* In case of a fork() call the memory allocation in the child will be
 745    the same but only one thread is running.  All stacks except that of
 746    the one running thread are not used anymore.  We have to recycle
 747    them.  */
 748 void
 749 __reclaim_stacks (void)
 750 {
 751   struct pthread *self = (struct pthread *) THREAD_SELF;
 752
 753   /* No locking necessary.  The caller is the only stack in use.  */
 754
 755   /* Mark all stacks except the still running one as free.  */
 756   list_t *runp;
 757   list_for_each (runp, &stack_used)
 758     {
 759       struct pthread *curp = list_entry (runp, struct pthread, list);
 760       if (curp != self)
 761         {
 762           /* This marks the stack as free.  */
 763           curp->tid = 0;
 764
 765           /* The PID field must be initialized for the new process.  */
 766           curp->pid = self->pid;
 767
 768           /* Account for the size of the stack.  */
 769           stack_cache_actsize += curp->stackblock_size;
 770         }
 771     }
 772
 773   /* Reset the PIDs in any cached stacks.  */
 774   list_for_each (runp, &stack_cache)
 775     {
 776       struct pthread *curp = list_entry (runp, struct pthread, list);
 777       curp->pid = self->pid;
 778     }
 779
 780   /* Add the stack of all running threads to the cache.  */
 781   list_splice (&stack_used, &stack_cache);
 782
 783   /* Remove the entry for the current thread to from the cache list
 784      and add it to the list of running threads.  Which of the two
 785      lists is decided by the user_stack flag.  */
 786   list_del (&self->list);
 787
 788   /* Re-initialize the lists for all the threads.  */
 789   INIT_LIST_HEAD (&stack_used);
 790   INIT_LIST_HEAD (&__stack_user);
 791
 792   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 793     list_add (&self->list, &__stack_user);
 794   else
 795     list_add (&self->list, &stack_used);
 796
 797   /* There is one thread running.  */
 798   __nptl_nthreads = 1;
 799
 800   /* Initialize the lock.  */
 801   stack_cache_lock = LLL_LOCK_INITIALIZER;
 802 }
 803
 804
 805 #if HP_TIMING_AVAIL
 806 # undef __find_thread_by_id
 807 /* Find a thread given the thread ID.  */
 808 attribute_hidden
 809 struct pthread *
 810 __find_thread_by_id (pid_t tid)
 811 {
 812   struct pthread *result = NULL;
 813
 814   lll_lock (stack_cache_lock);
 815
 816   /* Iterate over the list with system-allocated threads first.  */
 817   list_t *runp;
 818   list_for_each (runp, &stack_used)
 819     {
 820       struct pthread *curp;
 821
 822       curp = list_entry (runp, struct pthread, list);
 823
 824       if (curp->tid == tid)
 825         {
 826           result = curp;
 827           goto out;
 828         }
 829     }
 830
 831   /* Now the list with threads using user-allocated stacks.  */
 832   list_for_each (runp, &__stack_user)
 833     {
 834       struct pthread *curp;
 835
 836       curp = list_entry (runp, struct pthread, list);
 837
 838       if (curp->tid == tid)
 839         {
 840           result = curp;
 841           goto out;
 842         }
 843     }
 844
 845  out:
 846   lll_unlock (stack_cache_lock);
 847
 848   return result;
 849 }
 850 #endif
 851
 852
 853 static void
 854 internal_function
 855 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
 856 {
 857   if (! IS_DETACHED (t))
 858     {
 859       int ch;
 860       do
 861         {
 862           ch = t->cancelhandling;
 863
 864           /* If the thread is exiting right now, ignore it.  */
 865           if ((ch & EXITING_BITMASK) != 0)
 866             return;
 867         }
 868       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
 869                                                    ch | SETXID_BITMASK, ch));
 870     }
 871
 872   int val;
 873   INTERNAL_SYSCALL_DECL (err);
 874 #if __ASSUME_TGKILL
 875   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 876                           t->tid, SIGSETXID);
 877 #else
 878 # ifdef __NR_tgkill
 879   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 880                           t->tid, SIGSETXID);
 881   if (INTERNAL_SYSCALL_ERROR_P (val, err)
 882       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 883 # endif
 884     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 885 #endif
 886
 887   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 888     atomic_increment (&cmdp->cntr);
 889 }
 890
 891
 892 int
 893 attribute_hidden
 894 __nptl_setxid (struct xid_command *cmdp)
 895 {
 896   int result;
 897   lll_lock (stack_cache_lock);
 898
 899   __xidcmd = cmdp;
 900   cmdp->cntr = 0;
 901
 902   struct pthread *self = THREAD_SELF;
 903
 904   /* Iterate over the list with system-allocated threads first.  */
 905   list_t *runp;
 906   list_for_each (runp, &stack_used)
 907     {
 908       struct pthread *t = list_entry (runp, struct pthread, list);
 909       if (t == self)
 910         continue;
 911
 912       setxid_signal_thread (cmdp, t);
 913     }
 914
 915   /* Now the list with threads using user-allocated stacks.  */
 916   list_for_each (runp, &__stack_user)
 917     {
 918       struct pthread *t = list_entry (runp, struct pthread, list);
 919       if (t == self)
 920         continue;
 921
 922       setxid_signal_thread (cmdp, t);
 923     }
 924
 925   int cur = cmdp->cntr;
 926   while (cur != 0)
 927     {
 928       lll_futex_wait (&cmdp->cntr, cur);
 929       cur = cmdp->cntr;
 930     }
 931
 932   /* This must be last, otherwise the current thread might not have
 933      permissions to send SIGSETXID syscall to the other threads.  */
 934   INTERNAL_SYSCALL_DECL (err);
 935   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 936                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 937   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 938     {
 939       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 940       result = -1;
 941     }
 942
 943   lll_unlock (stack_cache_lock);
 944   return result;
 945 }
 946
 947 static inline void __attribute__((always_inline))
 948 init_one_static_tls (struct pthread *curp, struct link_map *map)
 949 {
 950   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 951 # if TLS_TCB_AT_TP
 952   void *dest = (char *) curp - map->l_tls_offset;
 953 # elif TLS_DTV_AT_TP
 954   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 955 # else
 956 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 957 # endif
 958
 959   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 960   dtv[map->l_tls_modid].pointer.val = dest;
 961   dtv[map->l_tls_modid].pointer.is_static = true;
 962
 963   /* Initialize the memory.  */
 964   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 965           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 966 }
 967
 968 void
 969 attribute_hidden
 970 __pthread_init_static_tls (struct link_map *map)
 971 {
 972   lll_lock (stack_cache_lock);
 973
 974   /* Iterate over the list with system-allocated threads first.  */
 975   list_t *runp;
 976   list_for_each (runp, &stack_used)
 977     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 978
 979   /* Now the list with threads using user-allocated stacks.  */
 980   list_for_each (runp, &__stack_user)
 981     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 982
 983   lll_unlock (stack_cache_lock);
 984 }