nptl/allocatestack.c

   1 /* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <stdint.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25 #include <sys/mman.h>
  26 #include <sys/param.h>
  27 #include <dl-sysdep.h>
  28 #include <tls.h>
  29
  30
  31
  32 #ifndef NEED_SEPARATE_REGISTER_STACK
  33
  34 /* Most architectures have exactly one stack pointer.  Some have more.  */
  35 #define STACK_VARIABLES void *stackaddr
  36
  37 /* How to pass the values to the 'create_thread' function.  */
  38 #define STACK_VARIABLES_ARGS stackaddr
  39
  40 /* How to declare function which gets there parameters.  */
  41 #define STACK_VARIABLES_PARMS void *stackaddr
  42
  43 /* How to declare allocate_stack.  */
  44 #define ALLOCATE_STACK_PARMS void **stack
  45
  46 /* This is how the function is called.  We do it this way to allow
  47    other variants of the function to have more parameters.  */
  48 #define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  49
  50 #else
  51
  52 #define STACK_VARIABLES void *stackaddr; size_t stacksize
  53 #define STACK_VARIABLES_ARGS stackaddr, stacksize
  54 #define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  55 #define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  56 #define ALLOCATE_STACK(attr, pd) \
  57   allocate_stack (attr, pd, &stackaddr, &stacksize)
  58
  59 #endif
  60
  61
  62 /* Default alignment of stack.  */
  63 #ifndef STACK_ALIGN
  64 # define STACK_ALIGN __alignof__ (long double)
  65 #endif
  66
  67 /* Default value for minimal stack size after allocating thread
  68    descriptor and guard.  */
  69 #ifndef MINIMAL_REST_STACK
  70 # define MINIMAL_REST_STACK     4096
  71 #endif
  72
  73
  74 /* Let the architecture add some flags to the mmap() call used to
  75    allocate stacks.  */
  76 #ifndef ARCH_MAP_FLAGS
  77 # define ARCH_MAP_FLAGS 0
  78 #endif
  79
  80 /* This yields the pointer that TLS support code calls the thread pointer.  */
  81 #if TLS_TCB_AT_TP
  82 # define TLS_TPADJ(pd) (pd)
  83 #elif TLS_DTV_AT_TP
  84 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  85 #endif
  86
  87 /* Cache handling for not-yet free stacks.  */
  88
  89 /* Maximum size in kB of cache.  */
  90 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
  91 static size_t stack_cache_actsize;
  92
  93 /* Mutex protecting this variable.  */
  94 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
  95
  96 /* List of queued stack frames.  */
  97 static LIST_HEAD (stack_cache);
  98
  99 /* List of the stacks in use.  */
 100 static LIST_HEAD (stack_used);
 101
 102 /* List of the threads with user provided stacks in use.  No need to
 103    initialize this, since it's done in __pthread_initialize_minimal.  */
 104 list_t __stack_user __attribute__ ((nocommon));
 105 hidden_data_def (__stack_user)
 106
 107 #if COLORING_INCREMENT != 0
 108 /* Number of threads created.  */
 109 static unsigned int nptl_ncreated;
 110 #endif
 111
 112
 113 /* Check whether the stack is still used or not.  */
 114 #define FREE_P(descr) ((descr)->tid <= 0)
 115
 116
 117 /* We create a double linked list of all cache entries.  Double linked
 118    because this allows removing entries from the end.  */
 119
 120
 121 /* Get a stack frame from the cache.  We have to match by size since
 122    some blocks might be too small or far too large.  */
 123 static struct pthread *
 124 get_cached_stack (size_t *sizep, void **memp)
 125 {
 126   size_t size = *sizep;
 127   struct pthread *result = NULL;
 128   list_t *entry;
 129
 130   lll_lock (stack_cache_lock);
 131
 132   /* Search the cache for a matching entry.  We search for the
 133      smallest stack which has at least the required size.  Note that
 134      in normal situations the size of all allocated stacks is the
 135      same.  As the very least there are only a few different sizes.
 136      Therefore this loop will exit early most of the time with an
 137      exact match.  */
 138   list_for_each (entry, &stack_cache)
 139     {
 140       struct pthread *curr;
 141
 142       curr = list_entry (entry, struct pthread, list);
 143       if (FREE_P (curr) && curr->stackblock_size >= size)
 144         {
 145           if (curr->stackblock_size == size)
 146             {
 147               result = curr;
 148               break;
 149             }
 150
 151           if (result == NULL
 152               || result->stackblock_size > curr->stackblock_size)
 153             result = curr;
 154         }
 155     }
 156
 157   if (__builtin_expect (result == NULL, 0)
 158       /* Make sure the size difference is not too excessive.  In that
 159          case we do not use the block.  */
 160       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 161     {
 162       /* Release the lock.  */
 163       lll_unlock (stack_cache_lock);
 164
 165       return NULL;
 166     }
 167
 168   /* Dequeue the entry.  */
 169   list_del (&result->list);
 170
 171   /* And add to the list of stacks in use.  */
 172   list_add (&result->list, &stack_used);
 173
 174   /* And decrease the cache size.  */
 175   stack_cache_actsize -= result->stackblock_size;
 176
 177   /* Release the lock early.  */
 178   lll_unlock (stack_cache_lock);
 179
 180   /* Report size and location of the stack to the caller.  */
 181   *sizep = result->stackblock_size;
 182   *memp = result->stackblock;
 183
 184   /* Cancellation handling is back to the default.  */
 185   result->cancelhandling = 0;
 186   result->cleanup = NULL;
 187
 188   /* No pending event.  */
 189   result->nextevent = NULL;
 190
 191   /* Clear the DTV.  */
 192   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 193   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 194
 195   /* Re-initialize the TLS.  */
 196   _dl_allocate_tls_init (TLS_TPADJ (result));
 197
 198   return result;
 199 }
 200
 201
 202 /* Add a stack frame which is not used anymore to the stack.  Must be
 203    called with the cache lock held.  */
 204 static inline void
 205 __attribute ((always_inline))
 206 queue_stack (struct pthread *stack)
 207 {
 208   /* We unconditionally add the stack to the list.  The memory may
 209      still be in use but it will not be reused until the kernel marks
 210      the stack as not used anymore.  */
 211   list_add (&stack->list, &stack_cache);
 212
 213   stack_cache_actsize += stack->stackblock_size;
 214   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 215     {
 216       /* We reduce the size of the cache.  Remove the last entries
 217          until the size is below the limit.  */
 218       list_t *entry;
 219       list_t *prev;
 220
 221       /* Search from the end of the list.  */
 222       list_for_each_prev_safe (entry, prev, &stack_cache)
 223         {
 224           struct pthread *curr;
 225
 226           curr = list_entry (entry, struct pthread, list);
 227           if (FREE_P (curr))
 228             {
 229               /* Unlink the block.  */
 230               list_del (entry);
 231
 232               /* Account for the freed memory.  */
 233               stack_cache_actsize -= curr->stackblock_size;
 234
 235               /* Free the memory associated with the ELF TLS.  */
 236               _dl_deallocate_tls (TLS_TPADJ (curr), false);
 237
 238               /* Remove this block.  This should never fail.  If it
 239                  does something is really wrong.  */
 240               if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 241                 abort ();
 242
 243               /* Maybe we have freed enough.  */
 244               if (stack_cache_actsize <= stack_cache_maxsize)
 245                 break;
 246             }
 247         }
 248     }
 249 }
 250
 251
 252 static int
 253 internal_function
 254 change_stack_perm (struct pthread *pd
 255 #ifdef NEED_SEPARATE_REGISTER_STACK
 256                    , size_t pagemask
 257 #endif
 258                    )
 259 {
 260 #ifdef NEED_SEPARATE_REGISTER_STACK
 261   void *stack = (pd->stackblock
 262                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 263                       & pagemask) + pd->guardsize) & pagemask));
 264   size_t len = pd->stackblock + pd->stackblock_size - stack;
 265 #else
 266   void *stack = pd->stackblock + pd->guardsize;
 267   size_t len = pd->stackblock_size - pd->guardsize;
 268 #endif
 269   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 270     return errno;
 271
 272   return 0;
 273 }
 274
 275
 276 static int
 277 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 278                 ALLOCATE_STACK_PARMS)
 279 {
 280   struct pthread *pd;
 281   size_t size;
 282   size_t pagesize_m1 = __getpagesize () - 1;
 283   void *stacktop;
 284
 285   assert (attr != NULL);
 286   assert (powerof2 (pagesize_m1 + 1));
 287   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 288
 289   /* Get the stack size from the attribute if it is set.  Otherwise we
 290      use the default we determined at start time.  */
 291   size = attr->stacksize ?: __default_stacksize;
 292
 293   /* Get memory for the stack.  */
 294   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 295     {
 296       uintptr_t adj;
 297
 298       /* If the user also specified the size of the stack make sure it
 299          is large enough.  */
 300       if (attr->stacksize != 0
 301           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 302         return EINVAL;
 303
 304       /* Adjust stack size for alignment of the TLS block.  */
 305 #if TLS_TCB_AT_TP
 306       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 307             & __static_tls_align_m1;
 308       assert (size > adj + TLS_TCB_SIZE);
 309 #elif TLS_DTV_AT_TP
 310       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 311             & __static_tls_align_m1;
 312       assert (size > adj);
 313 #endif
 314
 315       /* The user provided some memory.  Let's hope it matches the
 316          size...  We do not allocate guard pages if the user provided
 317          the stack.  It is the user's responsibility to do this if it
 318          is wanted.  */
 319 #if TLS_TCB_AT_TP
 320       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 321                                - TLS_TCB_SIZE - adj);
 322 #elif TLS_DTV_AT_TP
 323       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 324                                 - __static_tls_size - adj)
 325                                - TLS_PRE_TCB_SIZE);
 326 #endif
 327
 328       /* The user provided stack memory needs to be cleared.  */
 329       memset (pd, '\0', sizeof (struct pthread));
 330
 331       /* The first TSD block is included in the TCB.  */
 332       pd->specific[0] = pd->specific_1stblock;
 333
 334 #if defined __ASSUME_CLONE_STOPPED && LLL_LOCK_INITIALIZER != 0
 335       /* Initialize the lock.  */
 336       pd->lock = LLL_LOCK_INITIALIZER;
 337 #endif
 338
 339       /* Remember the stack-related values.  */
 340       pd->stackblock = (char *) attr->stackaddr - size;
 341       pd->stackblock_size = size;
 342
 343       /* This is a user-provided stack.  It will not be queued in the
 344          stack cache nor will the memory (except the TLS memory) be freed.  */
 345       pd->user_stack = true;
 346
 347       /* This is at least the second thread.  */
 348       pd->header.multiple_threads = 1;
 349 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 350       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 351 #endif
 352
 353 #ifdef NEED_DL_SYSINFO
 354       /* Copy the sysinfo value from the parent.  */
 355       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 356 #endif
 357
 358       /* The process ID is also the same as that of the caller.  */
 359       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 360
 361       /* Allocate the DTV for this thread.  */
 362       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 363         /* Something went wrong.  */
 364         return errno;
 365
 366
 367       /* Prepare to modify global data.  */
 368       lll_lock (stack_cache_lock);
 369
 370       /* And add to the list of stacks in use.  */
 371       list_add (&pd->list, &__stack_user);
 372
 373       lll_unlock (stack_cache_lock);
 374     }
 375   else
 376     {
 377       /* Allocate some anonymous memory.  If possible use the cache.  */
 378       size_t guardsize;
 379       size_t reqsize;
 380       void *mem;
 381       const int prot = (PROT_READ | PROT_WRITE
 382                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 383
 384 #if COLORING_INCREMENT != 0
 385       /* Add one more page for stack coloring.  Don't do it for stacks
 386          with 16 times pagesize or larger.  This might just cause
 387          unnecessary misalignment.  */
 388       if (size <= 16 * pagesize_m1)
 389         size += pagesize_m1 + 1;
 390 #endif
 391
 392       /* Adjust the stack size for alignment.  */
 393       size &= ~__static_tls_align_m1;
 394       assert (size != 0);
 395
 396       /* Make sure the size of the stack is enough for the guard and
 397          eventually the thread descriptor.  */
 398       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 399       if (__builtin_expect (size < (guardsize + __static_tls_size
 400                                     + MINIMAL_REST_STACK + pagesize_m1 + 1),
 401                             0))
 402         /* The stack is too small (or the guard too large).  */
 403         return EINVAL;
 404
 405       /* Try to get a stack from the cache.  */
 406       reqsize = size;
 407       pd = get_cached_stack (&size, &mem);
 408       if (pd == NULL)
 409         {
 410           /* To avoid aliasing effects on a larger scale then pages we
 411              adjust the allocated stack size if necessary.  This way
 412              allocations directly following each other will not have
 413              aliasing problems.  */
 414 #if MULTI_PAGE_ALIASING != 0
 415           if ((size % MULTI_PAGE_ALIASING) == 0)
 416             size += pagesize_m1 + 1;
 417 #endif
 418
 419           mem = mmap (NULL, size, prot,
 420                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 421
 422           if (__builtin_expect (mem == MAP_FAILED, 0))
 423             {
 424 #ifdef ARCH_RETRY_MMAP
 425               mem = ARCH_RETRY_MMAP (size);
 426               if (__builtin_expect (mem == MAP_FAILED, 0))
 427 #endif
 428                 return errno;
 429             }
 430
 431           /* SIZE is guaranteed to be greater than zero.
 432              So we can never get a null pointer back from mmap.  */
 433           assert (mem != NULL);
 434
 435 #if COLORING_INCREMENT != 0
 436           /* Atomically increment NCREATED.  */
 437           unsigned int ncreated = (atomic_exchange_and_add (&nptl_ncreated, 1)
 438                                    + 1);
 439
 440           /* We chose the offset for coloring by incrementing it for
 441              every new thread by a fixed amount.  The offset used
 442              module the page size.  Even if coloring would be better
 443              relative to higher alignment values it makes no sense to
 444              do it since the mmap() interface does not allow us to
 445              specify any alignment for the returned memory block.  */
 446           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 447
 448           /* Make sure the coloring offsets does not disturb the alignment
 449              of the TCB and static TLS block.  */
 450           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 451             coloring = (((coloring + __static_tls_align_m1)
 452                          & ~(__static_tls_align_m1))
 453                         & ~pagesize_m1);
 454 #else
 455           /* Unless specified we do not make any adjustments.  */
 456 # define coloring 0
 457 #endif
 458
 459           /* Place the thread descriptor at the end of the stack.  */
 460 #if TLS_TCB_AT_TP
 461           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 462 #elif TLS_DTV_AT_TP
 463           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 464                                     - __static_tls_size)
 465                                     & ~__static_tls_align_m1)
 466                                    - TLS_PRE_TCB_SIZE);
 467 #endif
 468
 469           /* Remember the stack-related values.  */
 470           pd->stackblock = mem;
 471           pd->stackblock_size = size;
 472
 473           /* We allocated the first block thread-specific data array.
 474              This address will not change for the lifetime of this
 475              descriptor.  */
 476           pd->specific[0] = pd->specific_1stblock;
 477
 478 #if defined __ASSUME_CLONE_STOPPED && LLL_LOCK_INITIALIZER != 0
 479           /* Initialize the lock.  */
 480           pd->lock = LLL_LOCK_INITIALIZER;
 481 #endif
 482
 483           /* This is at least the second thread.  */
 484           pd->header.multiple_threads = 1;
 485 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 486           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 487 #endif
 488
 489 #ifdef NEED_DL_SYSINFO
 490           /* Copy the sysinfo value from the parent.  */
 491           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 492 #endif
 493
 494           /* The process ID is also the same as that of the caller.  */
 495           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 496
 497           /* Allocate the DTV for this thread.  */
 498           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 499             {
 500               /* Something went wrong.  */
 501               int err = errno;
 502
 503               /* Free the stack memory we just allocated.  */
 504               (void) munmap (mem, size);
 505
 506               return err;
 507             }
 508
 509
 510           /* Prepare to modify global data.  */
 511           lll_lock (stack_cache_lock);
 512
 513           /* And add to the list of stacks in use.  */
 514           list_add (&pd->list, &stack_used);
 515
 516           lll_unlock (stack_cache_lock);
 517
 518
 519           /* There might have been a race.  Another thread might have
 520              caused the stacks to get exec permission while this new
 521              stack was prepared.  Detect if this was possible and
 522              change the permission if necessary.  */
 523           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 524                                 && (prot & PROT_EXEC) == 0, 0))
 525             {
 526               int err = change_stack_perm (pd
 527 #ifdef NEED_SEPARATE_REGISTER_STACK
 528                                            , ~pagesize_m1
 529 #endif
 530                                            );
 531               if (err != 0)
 532                 {
 533                   /* Free the stack memory we just allocated.  */
 534                   (void) munmap (mem, size);
 535
 536                   return err;
 537                 }
 538             }
 539
 540
 541           /* Note that all of the stack and the thread descriptor is
 542              zeroed.  This means we do not have to initialize fields
 543              with initial value zero.  This is specifically true for
 544              the 'tid' field which is always set back to zero once the
 545              stack is not used anymore and for the 'guardsize' field
 546              which will be read next.  */
 547         }
 548
 549       /* Create or resize the guard area if necessary.  */
 550       if (__builtin_expect (guardsize > pd->guardsize, 0))
 551         {
 552 #ifdef NEED_SEPARATE_REGISTER_STACK
 553           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 554 #else
 555           char *guard = mem;
 556 #endif
 557           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 558             {
 559               int err;
 560             mprot_error:
 561               err = errno;
 562
 563               lll_lock (stack_cache_lock);
 564
 565               /* Remove the thread from the list.  */
 566               list_del (&pd->list);
 567
 568               lll_unlock (stack_cache_lock);
 569
 570               /* Get rid of the TLS block we allocated.  */
 571               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 572
 573               /* Free the stack memory regardless of whether the size
 574                  of the cache is over the limit or not.  If this piece
 575                  of memory caused problems we better do not use it
 576                  anymore.  Uh, and we ignore possible errors.  There
 577                  is nothing we could do.  */
 578               (void) munmap (mem, size);
 579
 580               return err;
 581             }
 582
 583           pd->guardsize = guardsize;
 584         }
 585       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 586                                  0))
 587         {
 588           /* The old guard area is too large.  */
 589
 590 #ifdef NEED_SEPARATE_REGISTER_STACK
 591           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 592           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 593
 594           if (oldguard < guard
 595               && mprotect (oldguard, guard - oldguard, prot) != 0)
 596             goto mprot_error;
 597
 598           if (mprotect (guard + guardsize,
 599                         oldguard + pd->guardsize - guard - guardsize,
 600                         prot) != 0)
 601             goto mprot_error;
 602 #else
 603           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 604                         prot) != 0)
 605             goto mprot_error;
 606 #endif
 607
 608           pd->guardsize = guardsize;
 609         }
 610     }
 611
 612 #ifndef __ASSUME_CLONE_STOPPED
 613   /* Initialize the lock.  We have to do this unconditionally if the
 614      CLONE_STOPPED flag is not available since then the stillborn
 615      thread could be canceled while the lock is taken.  */
 616   pd->lock = LLL_LOCK_INITIALIZER;
 617 #endif
 618
 619   /* We place the thread descriptor at the end of the stack.  */
 620   *pdp = pd;
 621
 622 #if TLS_TCB_AT_TP
 623   /* The stack begins before the TCB and the static TLS block.  */
 624   stacktop = ((char *) (pd + 1) - __static_tls_size);
 625 #elif TLS_DTV_AT_TP
 626   stacktop = (char *) (pd - 1);
 627 #endif
 628
 629 #ifdef NEED_SEPARATE_REGISTER_STACK
 630   *stack = pd->stackblock;
 631   *stacksize = stacktop - *stack;
 632 #else
 633   *stack = stacktop;
 634 #endif
 635
 636   return 0;
 637 }
 638
 639
 640 void
 641 internal_function
 642 __deallocate_stack (struct pthread *pd)
 643 {
 644   lll_lock (stack_cache_lock);
 645
 646   /* Remove the thread from the list of threads with user defined
 647      stacks.  */
 648   list_del (&pd->list);
 649
 650   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 651      not reset the 'used' flag in the 'tid' field.  This is done by
 652      the kernel.  If no thread has been created yet this field is
 653      still zero.  */
 654   if (__builtin_expect (! pd->user_stack, 1))
 655     (void) queue_stack (pd);
 656   else
 657     /* Free the memory associated with the ELF TLS.  */
 658     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 659
 660   lll_unlock (stack_cache_lock);
 661 }
 662
 663
 664 int
 665 internal_function
 666 __make_stacks_executable (void)
 667 {
 668 #ifdef NEED_SEPARATE_REGISTER_STACK
 669   const size_t pagemask = ~(__getpagesize () - 1);
 670 #endif
 671
 672   lll_lock (stack_cache_lock);
 673
 674   int err = 0;
 675   list_t *runp;
 676   list_for_each (runp, &stack_used)
 677     {
 678       err = change_stack_perm (list_entry (runp, struct pthread, list)
 679 #ifdef NEED_SEPARATE_REGISTER_STACK
 680                                , pagemask
 681 #endif
 682                                );
 683       if (err != 0)
 684         break;
 685     }
 686
 687   /* Also change the permission for the currently unused stacks.  This
 688      might be wasted time but better spend it here than adding a check
 689      in the fast path.  */
 690   if (err == 0)
 691     list_for_each (runp, &stack_cache)
 692       {
 693         err = change_stack_perm (list_entry (runp, struct pthread, list)
 694 #ifdef NEED_SEPARATE_REGISTER_STACK
 695                                  , pagemask
 696 #endif
 697                                  );
 698         if (err != 0)
 699           break;
 700       }
 701
 702   lll_unlock (stack_cache_lock);
 703
 704   if (err == 0)
 705     err = _dl_make_stack_executable ();
 706
 707   return err;
 708 }
 709
 710
 711 /* In case of a fork() call the memory allocation in the child will be
 712    the same but only one thread is running.  All stacks except that of
 713    the one running thread are not used anymore.  We have to recycle
 714    them.  */
 715 void
 716 __reclaim_stacks (void)
 717 {
 718   struct pthread *self = (struct pthread *) THREAD_SELF;
 719
 720   /* No locking necessary.  The caller is the only stack in use.  */
 721
 722   /* Mark all stacks except the still running one as free.  */
 723   list_t *runp;
 724   list_for_each (runp, &stack_used)
 725     {
 726       struct pthread *curp;
 727
 728       curp = list_entry (runp, struct pthread, list);
 729       if (curp != self)
 730         {
 731           /* This marks the stack as free.  */
 732           curp->tid = 0;
 733
 734           /* The PID field must be initialized for the new process.  */
 735           curp->pid = self->pid;
 736
 737           /* Account for the size of the stack.  */
 738           stack_cache_actsize += curp->stackblock_size;
 739         }
 740     }
 741
 742   /* Add the stack of all running threads to the cache.  */
 743   list_splice (&stack_used, &stack_cache);
 744
 745   /* Remove the entry for the current thread to from the cache list
 746      and add it to the list of running threads.  Which of the two
 747      lists is decided by the user_stack flag.  */
 748   list_del (&self->list);
 749
 750   /* Re-initialize the lists for all the threads.  */
 751   INIT_LIST_HEAD (&stack_used);
 752   INIT_LIST_HEAD (&__stack_user);
 753
 754   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 755     list_add (&self->list, &__stack_user);
 756   else
 757     list_add (&self->list, &stack_used);
 758
 759   /* There is one thread running.  */
 760   __nptl_nthreads = 1;
 761
 762   /* Initialize the lock.  */
 763   stack_cache_lock = LLL_LOCK_INITIALIZER;
 764 }
 765
 766
 767 #if HP_TIMING_AVAIL
 768 /* Find a thread given the thread ID.  */
 769 struct pthread *
 770 attribute_hidden
 771 __find_thread_by_id (pid_t tid)
 772 {
 773   struct pthread *result = NULL;
 774
 775   lll_lock (stack_cache_lock);
 776
 777   /* Iterate over the list with system-allocated threads first.  */
 778   list_t *runp;
 779   list_for_each (runp, &stack_used)
 780     {
 781       struct pthread *curp;
 782
 783       curp = list_entry (runp, struct pthread, list);
 784
 785       if (curp->tid == tid)
 786         {
 787           result = curp;
 788           goto out;
 789         }
 790     }
 791
 792   /* Now the list with threads using user-allocated stacks.  */
 793   list_for_each (runp, &__stack_user)
 794     {
 795       struct pthread *curp;
 796
 797       curp = list_entry (runp, struct pthread, list);
 798
 799       if (curp->tid == tid)
 800         {
 801           result = curp;
 802           goto out;
 803         }
 804     }
 805
 806  out:
 807   lll_unlock (stack_cache_lock);
 808
 809   return result;
 810 }
 811 #endif
 812
 813 static inline void __attribute__((always_inline))
 814 init_one_static_tls (struct pthread *curp, struct link_map *map)
 815 {
 816   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 817 # if TLS_TCB_AT_TP
 818   void *dest = (char *) curp - map->l_tls_offset;
 819 # elif TLS_DTV_AT_TP
 820   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 821 # else
 822 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 823 # endif
 824
 825   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 826   dtv[map->l_tls_modid].pointer = dest;
 827
 828   /* Initialize the memory.  */
 829   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 830           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 831 }
 832
 833 void
 834 attribute_hidden
 835 __pthread_init_static_tls (struct link_map *map)
 836 {
 837   lll_lock (stack_cache_lock);
 838
 839   /* Iterate over the list with system-allocated threads first.  */
 840   list_t *runp;
 841   list_for_each (runp, &stack_used)
 842     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 843
 844   /* Now the list with threads using user-allocated stacks.  */
 845   list_for_each (runp, &__stack_user)
 846     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 847
 848   lll_unlock (stack_cache_lock);
 849 }