nptl/allocatestack.c

   1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31 #include <kernel-features.h>
  32
  33
  34 #ifndef NEED_SEPARATE_REGISTER_STACK
  35
  36 /* Most architectures have exactly one stack pointer.  Some have more.  */
  37 # define STACK_VARIABLES void *stackaddr = NULL
  38
  39 /* How to pass the values to the 'create_thread' function.  */
  40 # define STACK_VARIABLES_ARGS stackaddr
  41
  42 /* How to declare function which gets there parameters.  */
  43 # define STACK_VARIABLES_PARMS void *stackaddr
  44
  45 /* How to declare allocate_stack.  */
  46 # define ALLOCATE_STACK_PARMS void **stack
  47
  48 /* This is how the function is called.  We do it this way to allow
  49    other variants of the function to have more parameters.  */
  50 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  51
  52 #else
  53
  54 /* We need two stacks.  The kernel will place them but we have to tell
  55    the kernel about the size of the reserved address space.  */
  56 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
  57
  58 /* How to pass the values to the 'create_thread' function.  */
  59 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  60
  61 /* How to declare function which gets there parameters.  */
  62 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  63
  64 /* How to declare allocate_stack.  */
  65 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  66
  67 /* This is how the function is called.  We do it this way to allow
  68    other variants of the function to have more parameters.  */
  69 # define ALLOCATE_STACK(attr, pd) \
  70   allocate_stack (attr, pd, &stackaddr, &stacksize)
  71
  72 #endif
  73
  74
  75 /* Default alignment of stack.  */
  76 #ifndef STACK_ALIGN
  77 # define STACK_ALIGN __alignof__ (long double)
  78 #endif
  79
  80 /* Default value for minimal stack size after allocating thread
  81    descriptor and guard.  */
  82 #ifndef MINIMAL_REST_STACK
  83 # define MINIMAL_REST_STACK     4096
  84 #endif
  85
  86
  87 /* Let the architecture add some flags to the mmap() call used to
  88    allocate stacks.  */
  89 #ifndef ARCH_MAP_FLAGS
  90 # define ARCH_MAP_FLAGS 0
  91 #endif
  92
  93 /* This yields the pointer that TLS support code calls the thread pointer.  */
  94 #if TLS_TCB_AT_TP
  95 # define TLS_TPADJ(pd) (pd)
  96 #elif TLS_DTV_AT_TP
  97 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  98 #endif
  99
 100 /* Cache handling for not-yet free stacks.  */
 101
 102 /* Maximum size in kB of cache.  */
 103 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 104 static size_t stack_cache_actsize;
 105
 106 /* Mutex protecting this variable.  */
 107 static int stack_cache_lock = LLL_LOCK_INITIALIZER;
 108
 109 /* List of queued stack frames.  */
 110 static LIST_HEAD (stack_cache);
 111
 112 /* List of the stacks in use.  */
 113 static LIST_HEAD (stack_used);
 114
 115 /* List of the threads with user provided stacks in use.  No need to
 116    initialize this, since it's done in __pthread_initialize_minimal.  */
 117 list_t __stack_user __attribute__ ((nocommon));
 118 hidden_data_def (__stack_user)
 119
 120 #if COLORING_INCREMENT != 0
 121 /* Number of threads created.  */
 122 static unsigned int nptl_ncreated;
 123 #endif
 124
 125
 126 /* Check whether the stack is still used or not.  */
 127 #define FREE_P(descr) ((descr)->tid <= 0)
 128
 129
 130 /* We create a double linked list of all cache entries.  Double linked
 131    because this allows removing entries from the end.  */
 132
 133
 134 /* Get a stack frame from the cache.  We have to match by size since
 135    some blocks might be too small or far too large.  */
 136 static struct pthread *
 137 get_cached_stack (size_t *sizep, void **memp)
 138 {
 139   size_t size = *sizep;
 140   struct pthread *result = NULL;
 141   list_t *entry;
 142
 143   lll_lock (stack_cache_lock, LLL_PRIVATE);
 144
 145   /* Search the cache for a matching entry.  We search for the
 146      smallest stack which has at least the required size.  Note that
 147      in normal situations the size of all allocated stacks is the
 148      same.  As the very least there are only a few different sizes.
 149      Therefore this loop will exit early most of the time with an
 150      exact match.  */
 151   list_for_each (entry, &stack_cache)
 152     {
 153       struct pthread *curr;
 154
 155       curr = list_entry (entry, struct pthread, list);
 156       if (FREE_P (curr) && curr->stackblock_size >= size)
 157         {
 158           if (curr->stackblock_size == size)
 159             {
 160               result = curr;
 161               break;
 162             }
 163
 164           if (result == NULL
 165               || result->stackblock_size > curr->stackblock_size)
 166             result = curr;
 167         }
 168     }
 169
 170   if (__builtin_expect (result == NULL, 0)
 171       /* Make sure the size difference is not too excessive.  In that
 172          case we do not use the block.  */
 173       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 174     {
 175       /* Release the lock.  */
 176       lll_unlock (stack_cache_lock, LLL_PRIVATE);
 177
 178       return NULL;
 179     }
 180
 181   /* Dequeue the entry.  */
 182   list_del (&result->list);
 183
 184   /* And add to the list of stacks in use.  */
 185   list_add (&result->list, &stack_used);
 186
 187   /* And decrease the cache size.  */
 188   stack_cache_actsize -= result->stackblock_size;
 189
 190   /* Release the lock early.  */
 191   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 192
 193   /* Report size and location of the stack to the caller.  */
 194   *sizep = result->stackblock_size;
 195   *memp = result->stackblock;
 196
 197   /* Cancellation handling is back to the default.  */
 198   result->cancelhandling = 0;
 199   result->cleanup = NULL;
 200
 201   /* No pending event.  */
 202   result->nextevent = NULL;
 203
 204   /* Clear the DTV.  */
 205   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 206   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 207
 208   /* Re-initialize the TLS.  */
 209   _dl_allocate_tls_init (TLS_TPADJ (result));
 210
 211   return result;
 212 }
 213
 214
 215 /* Free stacks until cache size is lower than LIMIT.  */
 216 static void
 217 free_stacks (size_t limit)
 218 {
 219   /* We reduce the size of the cache.  Remove the last entries until
 220      the size is below the limit.  */
 221   list_t *entry;
 222   list_t *prev;
 223
 224   /* Search from the end of the list.  */
 225   list_for_each_prev_safe (entry, prev, &stack_cache)
 226     {
 227       struct pthread *curr;
 228
 229       curr = list_entry (entry, struct pthread, list);
 230       if (FREE_P (curr))
 231         {
 232           /* Unlink the block.  */
 233           list_del (entry);
 234
 235           /* Account for the freed memory.  */
 236           stack_cache_actsize -= curr->stackblock_size;
 237
 238           /* Free the memory associated with the ELF TLS.  */
 239           _dl_deallocate_tls (TLS_TPADJ (curr), false);
 240
 241           /* Remove this block.  This should never fail.  If it does
 242              something is really wrong.  */
 243           if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 244             abort ();
 245
 246           /* Maybe we have freed enough.  */
 247           if (stack_cache_actsize <= limit)
 248             break;
 249         }
 250     }
 251 }
 252
 253
 254 /* Add a stack frame which is not used anymore to the stack.  Must be
 255    called with the cache lock held.  */
 256 static inline void
 257 __attribute ((always_inline))
 258 queue_stack (struct pthread *stack)
 259 {
 260   /* We unconditionally add the stack to the list.  The memory may
 261      still be in use but it will not be reused until the kernel marks
 262      the stack as not used anymore.  */
 263   list_add (&stack->list, &stack_cache);
 264
 265   stack_cache_actsize += stack->stackblock_size;
 266   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 267     free_stacks (stack_cache_maxsize);
 268 }
 269
 270
 271 /* This function is called indirectly from the freeres code in libc.  */
 272 void
 273 __free_stack_cache (void)
 274 {
 275   free_stacks (0);
 276 }
 277
 278
 279 static int
 280 internal_function
 281 change_stack_perm (struct pthread *pd
 282 #ifdef NEED_SEPARATE_REGISTER_STACK
 283                    , size_t pagemask
 284 #endif
 285                    )
 286 {
 287 #ifdef NEED_SEPARATE_REGISTER_STACK
 288   void *stack = (pd->stackblock
 289                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 290                       & pagemask) + pd->guardsize) & pagemask));
 291   size_t len = pd->stackblock + pd->stackblock_size - stack;
 292 #elif _STACK_GROWS_DOWN
 293   void *stack = pd->stackblock + pd->guardsize;
 294   size_t len = pd->stackblock_size - pd->guardsize;
 295 #elif _STACK_GROWS_UP
 296   void *stack = pd->stackblock;
 297   size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
 298 #else
 299 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
 300 #endif
 301   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 302     return errno;
 303
 304   return 0;
 305 }
 306
 307
 308 static int
 309 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 310                 ALLOCATE_STACK_PARMS)
 311 {
 312   struct pthread *pd;
 313   size_t size;
 314   size_t pagesize_m1 = __getpagesize () - 1;
 315   void *stacktop;
 316
 317   assert (attr != NULL);
 318   assert (powerof2 (pagesize_m1 + 1));
 319   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 320
 321   /* Get the stack size from the attribute if it is set.  Otherwise we
 322      use the default we determined at start time.  */
 323   size = attr->stacksize ?: __default_stacksize;
 324
 325   /* Get memory for the stack.  */
 326   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 327     {
 328       uintptr_t adj;
 329
 330       /* If the user also specified the size of the stack make sure it
 331          is large enough.  */
 332       if (attr->stacksize != 0
 333           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 334         return EINVAL;
 335
 336       /* Adjust stack size for alignment of the TLS block.  */
 337 #if TLS_TCB_AT_TP
 338       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 339             & __static_tls_align_m1;
 340       assert (size > adj + TLS_TCB_SIZE);
 341 #elif TLS_DTV_AT_TP
 342       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 343             & __static_tls_align_m1;
 344       assert (size > adj);
 345 #endif
 346
 347       /* The user provided some memory.  Let's hope it matches the
 348          size...  We do not allocate guard pages if the user provided
 349          the stack.  It is the user's responsibility to do this if it
 350          is wanted.  */
 351 #if TLS_TCB_AT_TP
 352       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 353                                - TLS_TCB_SIZE - adj);
 354 #elif TLS_DTV_AT_TP
 355       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 356                                 - __static_tls_size - adj)
 357                                - TLS_PRE_TCB_SIZE);
 358 #endif
 359
 360       /* The user provided stack memory needs to be cleared.  */
 361       memset (pd, '\0', sizeof (struct pthread));
 362
 363       /* The first TSD block is included in the TCB.  */
 364       pd->specific[0] = pd->specific_1stblock;
 365
 366       /* Remember the stack-related values.  */
 367       pd->stackblock = (char *) attr->stackaddr - size;
 368       pd->stackblock_size = size;
 369
 370       /* This is a user-provided stack.  It will not be queued in the
 371          stack cache nor will the memory (except the TLS memory) be freed.  */
 372       pd->user_stack = true;
 373
 374       /* This is at least the second thread.  */
 375       pd->header.multiple_threads = 1;
 376 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 377       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 378 #endif
 379
 380 #ifndef __ASSUME_PRIVATE_FUTEX
 381       /* The thread must know when private futexes are supported.  */
 382       pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 383                                                 header.private_futex);
 384 #endif
 385
 386 #ifdef NEED_DL_SYSINFO
 387       /* Copy the sysinfo value from the parent.  */
 388       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 389 #endif
 390
 391       /* The process ID is also the same as that of the caller.  */
 392       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 393
 394       /* Allocate the DTV for this thread.  */
 395       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 396         {
 397           /* Something went wrong.  */
 398           assert (errno == ENOMEM);
 399           return EAGAIN;
 400         }
 401
 402
 403       /* Prepare to modify global data.  */
 404       lll_lock (stack_cache_lock, LLL_PRIVATE);
 405
 406       /* And add to the list of stacks in use.  */
 407       list_add (&pd->list, &__stack_user);
 408
 409       lll_unlock (stack_cache_lock, LLL_PRIVATE);
 410     }
 411   else
 412     {
 413       /* Allocate some anonymous memory.  If possible use the cache.  */
 414       size_t guardsize;
 415       size_t reqsize;
 416       void *mem;
 417       const int prot = (PROT_READ | PROT_WRITE
 418                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 419
 420 #if COLORING_INCREMENT != 0
 421       /* Add one more page for stack coloring.  Don't do it for stacks
 422          with 16 times pagesize or larger.  This might just cause
 423          unnecessary misalignment.  */
 424       if (size <= 16 * pagesize_m1)
 425         size += pagesize_m1 + 1;
 426 #endif
 427
 428       /* Adjust the stack size for alignment.  */
 429       size &= ~__static_tls_align_m1;
 430       assert (size != 0);
 431
 432       /* Make sure the size of the stack is enough for the guard and
 433          eventually the thread descriptor.  */
 434       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 435       if (__builtin_expect (size < ((guardsize + __static_tls_size
 436                                      + MINIMAL_REST_STACK + pagesize_m1)
 437                                     & ~pagesize_m1),
 438                             0))
 439         /* The stack is too small (or the guard too large).  */
 440         return EINVAL;
 441
 442       /* Try to get a stack from the cache.  */
 443       reqsize = size;
 444       pd = get_cached_stack (&size, &mem);
 445       if (pd == NULL)
 446         {
 447           /* To avoid aliasing effects on a larger scale than pages we
 448              adjust the allocated stack size if necessary.  This way
 449              allocations directly following each other will not have
 450              aliasing problems.  */
 451 #if MULTI_PAGE_ALIASING != 0
 452           if ((size % MULTI_PAGE_ALIASING) == 0)
 453             size += pagesize_m1 + 1;
 454 #endif
 455
 456           mem = mmap (NULL, size, prot,
 457                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 458
 459           if (__builtin_expect (mem == MAP_FAILED, 0))
 460             {
 461 #ifdef ARCH_RETRY_MMAP
 462               mem = ARCH_RETRY_MMAP (size);
 463               if (__builtin_expect (mem == MAP_FAILED, 0))
 464 #endif
 465                 return errno;
 466             }
 467
 468           /* SIZE is guaranteed to be greater than zero.
 469              So we can never get a null pointer back from mmap.  */
 470           assert (mem != NULL);
 471
 472 #if COLORING_INCREMENT != 0
 473           /* Atomically increment NCREATED.  */
 474           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 475
 476           /* We chose the offset for coloring by incrementing it for
 477              every new thread by a fixed amount.  The offset used
 478              module the page size.  Even if coloring would be better
 479              relative to higher alignment values it makes no sense to
 480              do it since the mmap() interface does not allow us to
 481              specify any alignment for the returned memory block.  */
 482           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 483
 484           /* Make sure the coloring offsets does not disturb the alignment
 485              of the TCB and static TLS block.  */
 486           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 487             coloring = (((coloring + __static_tls_align_m1)
 488                          & ~(__static_tls_align_m1))
 489                         & ~pagesize_m1);
 490 #else
 491           /* Unless specified we do not make any adjustments.  */
 492 # define coloring 0
 493 #endif
 494
 495           /* Place the thread descriptor at the end of the stack.  */
 496 #if TLS_TCB_AT_TP
 497           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 498 #elif TLS_DTV_AT_TP
 499           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 500                                     - __static_tls_size)
 501                                     & ~__static_tls_align_m1)
 502                                    - TLS_PRE_TCB_SIZE);
 503 #endif
 504
 505           /* Remember the stack-related values.  */
 506           pd->stackblock = mem;
 507           pd->stackblock_size = size;
 508
 509           /* We allocated the first block thread-specific data array.
 510              This address will not change for the lifetime of this
 511              descriptor.  */
 512           pd->specific[0] = pd->specific_1stblock;
 513
 514           /* This is at least the second thread.  */
 515           pd->header.multiple_threads = 1;
 516 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 517           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 518 #endif
 519
 520 #ifndef __ASSUME_PRIVATE_FUTEX
 521           /* The thread must know when private futexes are supported.  */
 522           pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 523                                                     header.private_futex);
 524 #endif
 525
 526 #ifdef NEED_DL_SYSINFO
 527           /* Copy the sysinfo value from the parent.  */
 528           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 529 #endif
 530
 531           /* The process ID is also the same as that of the caller.  */
 532           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 533
 534           /* Allocate the DTV for this thread.  */
 535           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 536             {
 537               /* Something went wrong.  */
 538               assert (errno == ENOMEM);
 539
 540               /* Free the stack memory we just allocated.  */
 541               (void) munmap (mem, size);
 542
 543               return EAGAIN;
 544             }
 545
 546
 547           /* Prepare to modify global data.  */
 548           lll_lock (stack_cache_lock, LLL_PRIVATE);
 549
 550           /* And add to the list of stacks in use.  */
 551           list_add (&pd->list, &stack_used);
 552
 553           lll_unlock (stack_cache_lock, LLL_PRIVATE);
 554
 555
 556           /* There might have been a race.  Another thread might have
 557              caused the stacks to get exec permission while this new
 558              stack was prepared.  Detect if this was possible and
 559              change the permission if necessary.  */
 560           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 561                                 && (prot & PROT_EXEC) == 0, 0))
 562             {
 563               int err = change_stack_perm (pd
 564 #ifdef NEED_SEPARATE_REGISTER_STACK
 565                                            , ~pagesize_m1
 566 #endif
 567                                            );
 568               if (err != 0)
 569                 {
 570                   /* Free the stack memory we just allocated.  */
 571                   (void) munmap (mem, size);
 572
 573                   return err;
 574                 }
 575             }
 576
 577
 578           /* Note that all of the stack and the thread descriptor is
 579              zeroed.  This means we do not have to initialize fields
 580              with initial value zero.  This is specifically true for
 581              the 'tid' field which is always set back to zero once the
 582              stack is not used anymore and for the 'guardsize' field
 583              which will be read next.  */
 584         }
 585
 586       /* Create or resize the guard area if necessary.  */
 587       if (__builtin_expect (guardsize > pd->guardsize, 0))
 588         {
 589 #ifdef NEED_SEPARATE_REGISTER_STACK
 590           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 591 #elif _STACK_GROWS_DOWN
 592           char *guard = mem;
 593 # elif _STACK_GROWS_UP
 594           char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
 595 #endif
 596           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 597             {
 598               int err;
 599             mprot_error:
 600               err = errno;
 601
 602               lll_lock (stack_cache_lock, LLL_PRIVATE);
 603
 604               /* Remove the thread from the list.  */
 605               list_del (&pd->list);
 606
 607               lll_unlock (stack_cache_lock, LLL_PRIVATE);
 608
 609               /* Get rid of the TLS block we allocated.  */
 610               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 611
 612               /* Free the stack memory regardless of whether the size
 613                  of the cache is over the limit or not.  If this piece
 614                  of memory caused problems we better do not use it
 615                  anymore.  Uh, and we ignore possible errors.  There
 616                  is nothing we could do.  */
 617               (void) munmap (mem, size);
 618
 619               return err;
 620             }
 621
 622           pd->guardsize = guardsize;
 623         }
 624       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 625                                  0))
 626         {
 627           /* The old guard area is too large.  */
 628
 629 #ifdef NEED_SEPARATE_REGISTER_STACK
 630           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 631           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 632
 633           if (oldguard < guard
 634               && mprotect (oldguard, guard - oldguard, prot) != 0)
 635             goto mprot_error;
 636
 637           if (mprotect (guard + guardsize,
 638                         oldguard + pd->guardsize - guard - guardsize,
 639                         prot) != 0)
 640             goto mprot_error;
 641 #elif _STACK_GROWS_DOWN
 642           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 643                         prot) != 0)
 644             goto mprot_error;
 645 #elif _STACK_GROWS_UP
 646           if (mprotect ((char *) pd - pd->guardsize,
 647                         pd->guardsize - guardsize, prot) != 0)
 648             goto mprot_error;
 649 #endif
 650
 651           pd->guardsize = guardsize;
 652         }
 653       /* The pthread_getattr_np() calls need to get passed the size
 654          requested in the attribute, regardless of how large the
 655          actually used guardsize is.  */
 656       pd->reported_guardsize = guardsize;
 657     }
 658
 659   /* Initialize the lock.  We have to do this unconditionally since the
 660      stillborn thread could be canceled while the lock is taken.  */
 661   pd->lock = LLL_LOCK_INITIALIZER;
 662
 663   /* The robust mutex lists also need to be initialized
 664      unconditionally because the cleanup for the previous stack owner
 665      might have happened in the kernel.  */
 666   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
 667                                   - offsetof (pthread_mutex_t,
 668                                               __data.__list.__next));
 669   pd->robust_head.list_op_pending = NULL;
 670 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 671   pd->robust_prev = &pd->robust_head;
 672 #endif
 673   pd->robust_head.list = &pd->robust_head;
 674
 675   /* We place the thread descriptor at the end of the stack.  */
 676   *pdp = pd;
 677
 678 #if TLS_TCB_AT_TP
 679   /* The stack begins before the TCB and the static TLS block.  */
 680   stacktop = ((char *) (pd + 1) - __static_tls_size);
 681 #elif TLS_DTV_AT_TP
 682   stacktop = (char *) (pd - 1);
 683 #endif
 684
 685 #ifdef NEED_SEPARATE_REGISTER_STACK
 686   *stack = pd->stackblock;
 687   *stacksize = stacktop - *stack;
 688 #elif _STACK_GROWS_DOWN
 689   *stack = stacktop;
 690 #elif _STACK_GROWS_UP
 691   *stack = pd->stackblock;
 692   assert (*stack > 0);
 693 #endif
 694
 695   return 0;
 696 }
 697
 698
 699 void
 700 internal_function
 701 __deallocate_stack (struct pthread *pd)
 702 {
 703   lll_lock (stack_cache_lock, LLL_PRIVATE);
 704
 705   /* Remove the thread from the list of threads with user defined
 706      stacks.  */
 707   list_del (&pd->list);
 708
 709   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 710      not reset the 'used' flag in the 'tid' field.  This is done by
 711      the kernel.  If no thread has been created yet this field is
 712      still zero.  */
 713   if (__builtin_expect (! pd->user_stack, 1))
 714     (void) queue_stack (pd);
 715   else
 716     /* Free the memory associated with the ELF TLS.  */
 717     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 718
 719   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 720 }
 721
 722
 723 int
 724 internal_function
 725 __make_stacks_executable (void **stack_endp)
 726 {
 727   /* First the main thread's stack.  */
 728   int err = _dl_make_stack_executable (stack_endp);
 729   if (err != 0)
 730     return err;
 731
 732 #ifdef NEED_SEPARATE_REGISTER_STACK
 733   const size_t pagemask = ~(__getpagesize () - 1);
 734 #endif
 735
 736   lll_lock (stack_cache_lock, LLL_PRIVATE);
 737
 738   list_t *runp;
 739   list_for_each (runp, &stack_used)
 740     {
 741       err = change_stack_perm (list_entry (runp, struct pthread, list)
 742 #ifdef NEED_SEPARATE_REGISTER_STACK
 743                                , pagemask
 744 #endif
 745                                );
 746       if (err != 0)
 747         break;
 748     }
 749
 750   /* Also change the permission for the currently unused stacks.  This
 751      might be wasted time but better spend it here than adding a check
 752      in the fast path.  */
 753   if (err == 0)
 754     list_for_each (runp, &stack_cache)
 755       {
 756         err = change_stack_perm (list_entry (runp, struct pthread, list)
 757 #ifdef NEED_SEPARATE_REGISTER_STACK
 758                                  , pagemask
 759 #endif
 760                                  );
 761         if (err != 0)
 762           break;
 763       }
 764
 765   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 766
 767   return err;
 768 }
 769
 770
 771 /* In case of a fork() call the memory allocation in the child will be
 772    the same but only one thread is running.  All stacks except that of
 773    the one running thread are not used anymore.  We have to recycle
 774    them.  */
 775 void
 776 __reclaim_stacks (void)
 777 {
 778   struct pthread *self = (struct pthread *) THREAD_SELF;
 779
 780   /* No locking necessary.  The caller is the only stack in use.  */
 781
 782   /* Mark all stacks except the still running one as free.  */
 783   list_t *runp;
 784   list_for_each (runp, &stack_used)
 785     {
 786       struct pthread *curp = list_entry (runp, struct pthread, list);
 787       if (curp != self)
 788         {
 789           /* This marks the stack as free.  */
 790           curp->tid = 0;
 791
 792           /* The PID field must be initialized for the new process.  */
 793           curp->pid = self->pid;
 794
 795           /* Account for the size of the stack.  */
 796           stack_cache_actsize += curp->stackblock_size;
 797         }
 798     }
 799
 800   /* Reset the PIDs in any cached stacks.  */
 801   list_for_each (runp, &stack_cache)
 802     {
 803       struct pthread *curp = list_entry (runp, struct pthread, list);
 804       curp->pid = self->pid;
 805     }
 806
 807   /* Add the stack of all running threads to the cache.  */
 808   list_splice (&stack_used, &stack_cache);
 809
 810   /* Remove the entry for the current thread to from the cache list
 811      and add it to the list of running threads.  Which of the two
 812      lists is decided by the user_stack flag.  */
 813   list_del (&self->list);
 814
 815   /* Re-initialize the lists for all the threads.  */
 816   INIT_LIST_HEAD (&stack_used);
 817   INIT_LIST_HEAD (&__stack_user);
 818
 819   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 820     list_add (&self->list, &__stack_user);
 821   else
 822     list_add (&self->list, &stack_used);
 823
 824   /* There is one thread running.  */
 825   __nptl_nthreads = 1;
 826
 827   /* Initialize the lock.  */
 828   stack_cache_lock = LLL_LOCK_INITIALIZER;
 829 }
 830
 831
 832 #if HP_TIMING_AVAIL
 833 # undef __find_thread_by_id
 834 /* Find a thread given the thread ID.  */
 835 attribute_hidden
 836 struct pthread *
 837 __find_thread_by_id (pid_t tid)
 838 {
 839   struct pthread *result = NULL;
 840
 841   lll_lock (stack_cache_lock, LLL_PRIVATE);
 842
 843   /* Iterate over the list with system-allocated threads first.  */
 844   list_t *runp;
 845   list_for_each (runp, &stack_used)
 846     {
 847       struct pthread *curp;
 848
 849       curp = list_entry (runp, struct pthread, list);
 850
 851       if (curp->tid == tid)
 852         {
 853           result = curp;
 854           goto out;
 855         }
 856     }
 857
 858   /* Now the list with threads using user-allocated stacks.  */
 859   list_for_each (runp, &__stack_user)
 860     {
 861       struct pthread *curp;
 862
 863       curp = list_entry (runp, struct pthread, list);
 864
 865       if (curp->tid == tid)
 866         {
 867           result = curp;
 868           goto out;
 869         }
 870     }
 871
 872  out:
 873   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 874
 875   return result;
 876 }
 877 #endif
 878
 879
 880 static void
 881 internal_function
 882 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
 883 {
 884   if (! IS_DETACHED (t))
 885     {
 886       int ch;
 887       do
 888         {
 889           ch = t->cancelhandling;
 890
 891           /* If the thread is exiting right now, ignore it.  */
 892           if ((ch & EXITING_BITMASK) != 0)
 893             return;
 894         }
 895       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
 896                                                    ch | SETXID_BITMASK, ch));
 897     }
 898
 899   int val;
 900   INTERNAL_SYSCALL_DECL (err);
 901 #if __ASSUME_TGKILL
 902   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 903                           t->tid, SIGSETXID);
 904 #else
 905 # ifdef __NR_tgkill
 906   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 907                           t->tid, SIGSETXID);
 908   if (INTERNAL_SYSCALL_ERROR_P (val, err)
 909       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 910 # endif
 911     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 912 #endif
 913
 914   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 915     atomic_increment (&cmdp->cntr);
 916 }
 917
 918
 919 int
 920 attribute_hidden
 921 __nptl_setxid (struct xid_command *cmdp)
 922 {
 923   int result;
 924   lll_lock (stack_cache_lock, LLL_PRIVATE);
 925
 926   __xidcmd = cmdp;
 927   cmdp->cntr = 0;
 928
 929   struct pthread *self = THREAD_SELF;
 930
 931   /* Iterate over the list with system-allocated threads first.  */
 932   list_t *runp;
 933   list_for_each (runp, &stack_used)
 934     {
 935       struct pthread *t = list_entry (runp, struct pthread, list);
 936       if (t == self)
 937         continue;
 938
 939       setxid_signal_thread (cmdp, t);
 940     }
 941
 942   /* Now the list with threads using user-allocated stacks.  */
 943   list_for_each (runp, &__stack_user)
 944     {
 945       struct pthread *t = list_entry (runp, struct pthread, list);
 946       if (t == self)
 947         continue;
 948
 949       setxid_signal_thread (cmdp, t);
 950     }
 951
 952   int cur = cmdp->cntr;
 953   while (cur != 0)
 954     {
 955       lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
 956       cur = cmdp->cntr;
 957     }
 958
 959   /* This must be last, otherwise the current thread might not have
 960      permissions to send SIGSETXID syscall to the other threads.  */
 961   INTERNAL_SYSCALL_DECL (err);
 962   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 963                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 964   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 965     {
 966       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 967       result = -1;
 968     }
 969
 970   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 971   return result;
 972 }
 973
 974 static inline void __attribute__((always_inline))
 975 init_one_static_tls (struct pthread *curp, struct link_map *map)
 976 {
 977   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 978 # if TLS_TCB_AT_TP
 979   void *dest = (char *) curp - map->l_tls_offset;
 980 # elif TLS_DTV_AT_TP
 981   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 982 # else
 983 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 984 # endif
 985
 986   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 987   dtv[map->l_tls_modid].pointer.val = dest;
 988   dtv[map->l_tls_modid].pointer.is_static = true;
 989
 990   /* Initialize the memory.  */
 991   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 992           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 993 }
 994
 995 void
 996 attribute_hidden
 997 __pthread_init_static_tls (struct link_map *map)
 998 {
 999   lll_lock (stack_cache_lock, LLL_PRIVATE);
1000
1001   /* Iterate over the list with system-allocated threads first.  */
1002   list_t *runp;
1003   list_for_each (runp, &stack_used)
1004     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1005
1006   /* Now the list with threads using user-allocated stacks.  */
1007   list_for_each (runp, &__stack_user)
1008     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1009
1010   lll_unlock (stack_cache_lock, LLL_PRIVATE);
1011 }
1012
1013
1014 void
1015 attribute_hidden
1016 __wait_lookup_done (void)
1017 {
1018   lll_lock (stack_cache_lock, LLL_PRIVATE);
1019
1020   struct pthread *self = THREAD_SELF;
1021
1022   /* Iterate over the list with system-allocated threads first.  */
1023   list_t *runp;
1024   list_for_each (runp, &stack_used)
1025     {
1026       struct pthread *t = list_entry (runp, struct pthread, list);
1027       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1028         continue;
1029
1030       int *const gscope_flagp = &t->header.gscope_flag;
1031
1032       /* We have to wait until this thread is done with the global
1033          scope.  First tell the thread that we are waiting and
1034          possibly have to be woken.  */
1035       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1036                                                 THREAD_GSCOPE_FLAG_WAIT,
1037                                                 THREAD_GSCOPE_FLAG_USED))
1038         continue;
1039
1040       do
1041         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1042       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1043     }
1044
1045   /* Now the list with threads using user-allocated stacks.  */
1046   list_for_each (runp, &__stack_user)
1047     {
1048       struct pthread *t = list_entry (runp, struct pthread, list);
1049       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1050         continue;
1051
1052       int *const gscope_flagp = &t->header.gscope_flag;
1053
1054       /* We have to wait until this thread is done with the global
1055          scope.  First tell the thread that we are waiting and
1056          possibly have to be woken.  */
1057       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1058                                                 THREAD_GSCOPE_FLAG_WAIT,
1059                                                 THREAD_GSCOPE_FLAG_USED))
1060         continue;
1061
1062       do
1063         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1064       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1065     }
1066
1067   lll_unlock (stack_cache_lock, LLL_PRIVATE);
1068 }