nptl/allocatestack.c

   1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31 #include <kernel-features.h>
  32
  33
  34 #ifndef NEED_SEPARATE_REGISTER_STACK
  35
  36 /* Most architectures have exactly one stack pointer.  Some have more.  */
  37 # define STACK_VARIABLES void *stackaddr = NULL
  38
  39 /* How to pass the values to the 'create_thread' function.  */
  40 # define STACK_VARIABLES_ARGS stackaddr
  41
  42 /* How to declare function which gets there parameters.  */
  43 # define STACK_VARIABLES_PARMS void *stackaddr
  44
  45 /* How to declare allocate_stack.  */
  46 # define ALLOCATE_STACK_PARMS void **stack
  47
  48 /* This is how the function is called.  We do it this way to allow
  49    other variants of the function to have more parameters.  */
  50 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  51
  52 #else
  53
  54 /* We need two stacks.  The kernel will place them but we have to tell
  55    the kernel about the size of the reserved address space.  */
  56 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
  57
  58 /* How to pass the values to the 'create_thread' function.  */
  59 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  60
  61 /* How to declare function which gets there parameters.  */
  62 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  63
  64 /* How to declare allocate_stack.  */
  65 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  66
  67 /* This is how the function is called.  We do it this way to allow
  68    other variants of the function to have more parameters.  */
  69 # define ALLOCATE_STACK(attr, pd) \
  70   allocate_stack (attr, pd, &stackaddr, &stacksize)
  71
  72 #endif
  73
  74
  75 /* Default alignment of stack.  */
  76 #ifndef STACK_ALIGN
  77 # define STACK_ALIGN __alignof__ (long double)
  78 #endif
  79
  80 /* Default value for minimal stack size after allocating thread
  81    descriptor and guard.  */
  82 #ifndef MINIMAL_REST_STACK
  83 # define MINIMAL_REST_STACK     4096
  84 #endif
  85
  86
  87 /* Let the architecture add some flags to the mmap() call used to
  88    allocate stacks.  */
  89 #ifndef ARCH_MAP_FLAGS
  90 # define ARCH_MAP_FLAGS 0
  91 #endif
  92
  93 /* This yields the pointer that TLS support code calls the thread pointer.  */
  94 #if TLS_TCB_AT_TP
  95 # define TLS_TPADJ(pd) (pd)
  96 #elif TLS_DTV_AT_TP
  97 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  98 #endif
  99
 100 /* Cache handling for not-yet free stacks.  */
 101
 102 /* Maximum size in kB of cache.  */
 103 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 104 static size_t stack_cache_actsize;
 105
 106 /* Mutex protecting this variable.  */
 107 static int stack_cache_lock = LLL_LOCK_INITIALIZER;
 108
 109 /* List of queued stack frames.  */
 110 static LIST_HEAD (stack_cache);
 111
 112 /* List of the stacks in use.  */
 113 static LIST_HEAD (stack_used);
 114
 115 /* List of the threads with user provided stacks in use.  No need to
 116    initialize this, since it's done in __pthread_initialize_minimal.  */
 117 list_t __stack_user __attribute__ ((nocommon));
 118 hidden_data_def (__stack_user)
 119
 120 #if COLORING_INCREMENT != 0
 121 /* Number of threads created.  */
 122 static unsigned int nptl_ncreated;
 123 #endif
 124
 125
 126 /* Check whether the stack is still used or not.  */
 127 #define FREE_P(descr) ((descr)->tid <= 0)
 128
 129
 130 /* We create a double linked list of all cache entries.  Double linked
 131    because this allows removing entries from the end.  */
 132
 133
 134 /* Get a stack frame from the cache.  We have to match by size since
 135    some blocks might be too small or far too large.  */
 136 static struct pthread *
 137 get_cached_stack (size_t *sizep, void **memp)
 138 {
 139   size_t size = *sizep;
 140   struct pthread *result = NULL;
 141   list_t *entry;
 142
 143   lll_lock (stack_cache_lock, LLL_PRIVATE);
 144
 145   /* Search the cache for a matching entry.  We search for the
 146      smallest stack which has at least the required size.  Note that
 147      in normal situations the size of all allocated stacks is the
 148      same.  As the very least there are only a few different sizes.
 149      Therefore this loop will exit early most of the time with an
 150      exact match.  */
 151   list_for_each (entry, &stack_cache)
 152     {
 153       struct pthread *curr;
 154
 155       curr = list_entry (entry, struct pthread, list);
 156       if (FREE_P (curr) && curr->stackblock_size >= size)
 157         {
 158           if (curr->stackblock_size == size)
 159             {
 160               result = curr;
 161               break;
 162             }
 163
 164           if (result == NULL
 165               || result->stackblock_size > curr->stackblock_size)
 166             result = curr;
 167         }
 168     }
 169
 170   if (__builtin_expect (result == NULL, 0)
 171       /* Make sure the size difference is not too excessive.  In that
 172          case we do not use the block.  */
 173       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 174     {
 175       /* Release the lock.  */
 176       lll_unlock (stack_cache_lock, LLL_PRIVATE);
 177
 178       return NULL;
 179     }
 180
 181   /* Dequeue the entry.  */
 182   list_del (&result->list);
 183
 184   /* And add to the list of stacks in use.  */
 185   list_add (&result->list, &stack_used);
 186
 187   /* And decrease the cache size.  */
 188   stack_cache_actsize -= result->stackblock_size;
 189
 190   /* Release the lock early.  */
 191   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 192
 193   /* Report size and location of the stack to the caller.  */
 194   *sizep = result->stackblock_size;
 195   *memp = result->stackblock;
 196
 197   /* Cancellation handling is back to the default.  */
 198   result->cancelhandling = 0;
 199   result->cleanup = NULL;
 200
 201   /* No pending event.  */
 202   result->nextevent = NULL;
 203
 204   /* Clear the DTV.  */
 205   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 206   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 207
 208   /* Re-initialize the TLS.  */
 209   _dl_allocate_tls_init (TLS_TPADJ (result));
 210
 211   return result;
 212 }
 213
 214
 215 /* Free stacks until cache size is lower than LIMIT.  */
 216 static void
 217 free_stacks (size_t limit)
 218 {
 219   /* We reduce the size of the cache.  Remove the last entries until
 220      the size is below the limit.  */
 221   list_t *entry;
 222   list_t *prev;
 223
 224   /* Search from the end of the list.  */
 225   list_for_each_prev_safe (entry, prev, &stack_cache)
 226     {
 227       struct pthread *curr;
 228
 229       curr = list_entry (entry, struct pthread, list);
 230       if (FREE_P (curr))
 231         {
 232           /* Unlink the block.  */
 233           list_del (entry);
 234
 235           /* Account for the freed memory.  */
 236           stack_cache_actsize -= curr->stackblock_size;
 237
 238           /* Free the memory associated with the ELF TLS.  */
 239           _dl_deallocate_tls (TLS_TPADJ (curr), false);
 240
 241           /* Remove this block.  This should never fail.  If it does
 242              something is really wrong.  */
 243           if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 244             abort ();
 245
 246           /* Maybe we have freed enough.  */
 247           if (stack_cache_actsize <= limit)
 248             break;
 249         }
 250     }
 251 }
 252
 253
 254 /* Add a stack frame which is not used anymore to the stack.  Must be
 255    called with the cache lock held.  */
 256 static inline void
 257 __attribute ((always_inline))
 258 queue_stack (struct pthread *stack)
 259 {
 260   /* We unconditionally add the stack to the list.  The memory may
 261      still be in use but it will not be reused until the kernel marks
 262      the stack as not used anymore.  */
 263   list_add (&stack->list, &stack_cache);
 264
 265   stack_cache_actsize += stack->stackblock_size;
 266   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 267     free_stacks (stack_cache_maxsize);
 268 }
 269
 270
 271 /* This function is called indirectly from the freeres code in libc.  */
 272 void
 273 __free_stack_cache (void)
 274 {
 275   free_stacks (0);
 276 }
 277
 278
 279 static int
 280 internal_function
 281 change_stack_perm (struct pthread *pd
 282 #ifdef NEED_SEPARATE_REGISTER_STACK
 283                    , size_t pagemask
 284 #endif
 285                    )
 286 {
 287 #ifdef NEED_SEPARATE_REGISTER_STACK
 288   void *stack = (pd->stackblock
 289                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 290                       & pagemask) + pd->guardsize) & pagemask));
 291   size_t len = pd->stackblock + pd->stackblock_size - stack;
 292 #elif _STACK_GROWS_DOWN
 293   void *stack = pd->stackblock + pd->guardsize;
 294   size_t len = pd->stackblock_size - pd->guardsize;
 295 #elif _STACK_GROWS_UP
 296   void *stack = pd->stackblock;
 297   size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
 298 #else
 299 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
 300 #endif
 301   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 302     return errno;
 303
 304   return 0;
 305 }
 306
 307
 308 static int
 309 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 310                 ALLOCATE_STACK_PARMS)
 311 {
 312   struct pthread *pd;
 313   size_t size;
 314   size_t pagesize_m1 = __getpagesize () - 1;
 315   void *stacktop;
 316
 317   assert (attr != NULL);
 318   assert (powerof2 (pagesize_m1 + 1));
 319   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 320
 321   /* Get the stack size from the attribute if it is set.  Otherwise we
 322      use the default we determined at start time.  */
 323   size = attr->stacksize ?: __default_stacksize;
 324
 325   /* Get memory for the stack.  */
 326   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 327     {
 328       uintptr_t adj;
 329
 330       /* If the user also specified the size of the stack make sure it
 331          is large enough.  */
 332       if (attr->stacksize != 0
 333           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 334         return EINVAL;
 335
 336       /* Adjust stack size for alignment of the TLS block.  */
 337 #if TLS_TCB_AT_TP
 338       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 339             & __static_tls_align_m1;
 340       assert (size > adj + TLS_TCB_SIZE);
 341 #elif TLS_DTV_AT_TP
 342       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 343             & __static_tls_align_m1;
 344       assert (size > adj);
 345 #endif
 346
 347       /* The user provided some memory.  Let's hope it matches the
 348          size...  We do not allocate guard pages if the user provided
 349          the stack.  It is the user's responsibility to do this if it
 350          is wanted.  */
 351 #if TLS_TCB_AT_TP
 352       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 353                                - TLS_TCB_SIZE - adj);
 354 #elif TLS_DTV_AT_TP
 355       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 356                                 - __static_tls_size - adj)
 357                                - TLS_PRE_TCB_SIZE);
 358 #endif
 359
 360       /* The user provided stack memory needs to be cleared.  */
 361       memset (pd, '\0', sizeof (struct pthread));
 362
 363       /* The first TSD block is included in the TCB.  */
 364       pd->specific[0] = pd->specific_1stblock;
 365
 366       /* Remember the stack-related values.  */
 367       pd->stackblock = (char *) attr->stackaddr - size;
 368       pd->stackblock_size = size;
 369
 370       /* This is a user-provided stack.  It will not be queued in the
 371          stack cache nor will the memory (except the TLS memory) be freed.  */
 372       pd->user_stack = true;
 373
 374       /* This is at least the second thread.  */
 375       pd->header.multiple_threads = 1;
 376 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 377       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 378 #endif
 379
 380 #ifndef __ASSUME_PRIVATE_FUTEX
 381       /* The thread must know when private futexes are supported.  */
 382       pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 383                                                 header.private_futex);
 384 #endif
 385
 386 #ifdef NEED_DL_SYSINFO
 387       /* Copy the sysinfo value from the parent.  */
 388       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 389 #endif
 390
 391       /* The process ID is also the same as that of the caller.  */
 392       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 393
 394       /* Allocate the DTV for this thread.  */
 395       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 396         {
 397           /* Something went wrong.  */
 398           assert (errno == ENOMEM);
 399           return EAGAIN;
 400         }
 401
 402
 403       /* Prepare to modify global data.  */
 404       lll_lock (stack_cache_lock, LLL_PRIVATE);
 405
 406       /* And add to the list of stacks in use.  */
 407       list_add (&pd->list, &__stack_user);
 408
 409       lll_unlock (stack_cache_lock, LLL_PRIVATE);
 410     }
 411   else
 412     {
 413       /* Allocate some anonymous memory.  If possible use the cache.  */
 414       size_t guardsize;
 415       size_t reqsize;
 416       void *mem;
 417       const int prot = (PROT_READ | PROT_WRITE
 418                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 419
 420 #if COLORING_INCREMENT != 0
 421       /* Add one more page for stack coloring.  Don't do it for stacks
 422          with 16 times pagesize or larger.  This might just cause
 423          unnecessary misalignment.  */
 424       if (size <= 16 * pagesize_m1)
 425         size += pagesize_m1 + 1;
 426 #endif
 427
 428       /* Adjust the stack size for alignment.  */
 429       size &= ~__static_tls_align_m1;
 430       assert (size != 0);
 431
 432       /* Make sure the size of the stack is enough for the guard and
 433          eventually the thread descriptor.  */
 434       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 435       if (__builtin_expect (size < ((guardsize + __static_tls_size
 436                                      + MINIMAL_REST_STACK + pagesize_m1)
 437                                     & ~pagesize_m1),
 438                             0))
 439         /* The stack is too small (or the guard too large).  */
 440         return EINVAL;
 441
 442       /* Try to get a stack from the cache.  */
 443       reqsize = size;
 444       pd = get_cached_stack (&size, &mem);
 445       if (pd == NULL)
 446         {
 447           /* To avoid aliasing effects on a larger scale than pages we
 448              adjust the allocated stack size if necessary.  This way
 449              allocations directly following each other will not have
 450              aliasing problems.  */
 451 #if MULTI_PAGE_ALIASING != 0
 452           if ((size % MULTI_PAGE_ALIASING) == 0)
 453             size += pagesize_m1 + 1;
 454 #endif
 455
 456           mem = mmap (NULL, size, prot,
 457                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 458
 459           if (__builtin_expect (mem == MAP_FAILED, 0))
 460             {
 461 #ifdef ARCH_RETRY_MMAP
 462               mem = ARCH_RETRY_MMAP (size);
 463               if (__builtin_expect (mem == MAP_FAILED, 0))
 464 #endif
 465                 return errno;
 466             }
 467
 468           /* SIZE is guaranteed to be greater than zero.
 469              So we can never get a null pointer back from mmap.  */
 470           assert (mem != NULL);
 471
 472 #if COLORING_INCREMENT != 0
 473           /* Atomically increment NCREATED.  */
 474           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 475
 476           /* We chose the offset for coloring by incrementing it for
 477              every new thread by a fixed amount.  The offset used
 478              module the page size.  Even if coloring would be better
 479              relative to higher alignment values it makes no sense to
 480              do it since the mmap() interface does not allow us to
 481              specify any alignment for the returned memory block.  */
 482           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 483
 484           /* Make sure the coloring offsets does not disturb the alignment
 485              of the TCB and static TLS block.  */
 486           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 487             coloring = (((coloring + __static_tls_align_m1)
 488                          & ~(__static_tls_align_m1))
 489                         & ~pagesize_m1);
 490 #else
 491           /* Unless specified we do not make any adjustments.  */
 492 # define coloring 0
 493 #endif
 494
 495           /* Place the thread descriptor at the end of the stack.  */
 496 #if TLS_TCB_AT_TP
 497           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 498 #elif TLS_DTV_AT_TP
 499           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 500                                     - __static_tls_size)
 501                                     & ~__static_tls_align_m1)
 502                                    - TLS_PRE_TCB_SIZE);
 503 #endif
 504
 505           /* Remember the stack-related values.  */
 506           pd->stackblock = mem;
 507           pd->stackblock_size = size;
 508
 509           /* We allocated the first block thread-specific data array.
 510              This address will not change for the lifetime of this
 511              descriptor.  */
 512           pd->specific[0] = pd->specific_1stblock;
 513
 514           /* This is at least the second thread.  */
 515           pd->header.multiple_threads = 1;
 516 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 517           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 518 #endif
 519
 520 #ifndef __ASSUME_PRIVATE_FUTEX
 521           /* The thread must know when private futexes are supported.  */
 522           pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 523                                                     header.private_futex);
 524 #endif
 525
 526 #ifdef NEED_DL_SYSINFO
 527           /* Copy the sysinfo value from the parent.  */
 528           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 529 #endif
 530
 531           /* The process ID is also the same as that of the caller.  */
 532           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 533
 534           /* Allocate the DTV for this thread.  */
 535           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 536             {
 537               /* Something went wrong.  */
 538               assert (errno == ENOMEM);
 539
 540               /* Free the stack memory we just allocated.  */
 541               (void) munmap (mem, size);
 542
 543               return EAGAIN;
 544             }
 545
 546
 547           /* Prepare to modify global data.  */
 548           lll_lock (stack_cache_lock, LLL_PRIVATE);
 549
 550           /* And add to the list of stacks in use.  */
 551           list_add (&pd->list, &stack_used);
 552
 553           lll_unlock (stack_cache_lock, LLL_PRIVATE);
 554
 555
 556           /* There might have been a race.  Another thread might have
 557              caused the stacks to get exec permission while this new
 558              stack was prepared.  Detect if this was possible and
 559              change the permission if necessary.  */
 560           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 561                                 && (prot & PROT_EXEC) == 0, 0))
 562             {
 563               int err = change_stack_perm (pd
 564 #ifdef NEED_SEPARATE_REGISTER_STACK
 565                                            , ~pagesize_m1
 566 #endif
 567                                            );
 568               if (err != 0)
 569                 {
 570                   /* Free the stack memory we just allocated.  */
 571                   (void) munmap (mem, size);
 572
 573                   return err;
 574                 }
 575             }
 576
 577
 578           /* Note that all of the stack and the thread descriptor is
 579              zeroed.  This means we do not have to initialize fields
 580              with initial value zero.  This is specifically true for
 581              the 'tid' field which is always set back to zero once the
 582              stack is not used anymore and for the 'guardsize' field
 583              which will be read next.  */
 584         }
 585
 586       /* Create or resize the guard area if necessary.  */
 587       if (__builtin_expect (guardsize > pd->guardsize, 0))
 588         {
 589 #ifdef NEED_SEPARATE_REGISTER_STACK
 590           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 591 #elif _STACK_GROWS_DOWN
 592           char *guard = mem;
 593 # elif _STACK_GROWS_UP
 594           char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
 595 #endif
 596           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 597             {
 598               int err;
 599             mprot_error:
 600               err = errno;
 601
 602               lll_lock (stack_cache_lock, LLL_PRIVATE);
 603
 604               /* Remove the thread from the list.  */
 605               list_del (&pd->list);
 606
 607               lll_unlock (stack_cache_lock, LLL_PRIVATE);
 608
 609               /* Get rid of the TLS block we allocated.  */
 610               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 611
 612               /* Free the stack memory regardless of whether the size
 613                  of the cache is over the limit or not.  If this piece
 614                  of memory caused problems we better do not use it
 615                  anymore.  Uh, and we ignore possible errors.  There
 616                  is nothing we could do.  */
 617               (void) munmap (mem, size);
 618
 619               return err;
 620             }
 621
 622           pd->guardsize = guardsize;
 623         }
 624       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 625                                  0))
 626         {
 627           /* The old guard area is too large.  */
 628
 629 #ifdef NEED_SEPARATE_REGISTER_STACK
 630           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 631           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 632
 633           if (oldguard < guard
 634               && mprotect (oldguard, guard - oldguard, prot) != 0)
 635             goto mprot_error;
 636
 637           if (mprotect (guard + guardsize,
 638                         oldguard + pd->guardsize - guard - guardsize,
 639                         prot) != 0)
 640             goto mprot_error;
 641 #elif _STACK_GROWS_DOWN
 642           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 643                         prot) != 0)
 644             goto mprot_error;
 645 #elif _STACK_GROWS_UP
 646           if (mprotect ((char *) pd - pd->guardsize,
 647                         pd->guardsize - guardsize, prot) != 0)
 648             goto mprot_error;
 649 #endif
 650
 651           pd->guardsize = guardsize;
 652         }
 653       /* The pthread_getattr_np() calls need to get passed the size
 654          requested in the attribute, regardless of how large the
 655          actually used guardsize is.  */
 656       pd->reported_guardsize = guardsize;
 657     }
 658
 659   /* Initialize the lock.  We have to do this unconditionally since the
 660      stillborn thread could be canceled while the lock is taken.  */
 661   pd->lock = LLL_LOCK_INITIALIZER;
 662
 663   /* The robust mutex lists also need to be initialized
 664      unconditionally because the cleanup for the previous stack owner
 665      might have happened in the kernel.  */
 666   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
 667                                   - offsetof (pthread_mutex_t,
 668                                               __data.__list.__next));
 669   pd->robust_head.list_op_pending = NULL;
 670 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 671   pd->robust_prev = &pd->robust_head;
 672 #endif
 673   pd->robust_head.list = &pd->robust_head;
 674
 675   /* We place the thread descriptor at the end of the stack.  */
 676   *pdp = pd;
 677
 678 #if TLS_TCB_AT_TP
 679   /* The stack begins before the TCB and the static TLS block.  */
 680   stacktop = ((char *) (pd + 1) - __static_tls_size);
 681 #elif TLS_DTV_AT_TP
 682   stacktop = (char *) (pd - 1);
 683 #endif
 684
 685 #ifdef NEED_SEPARATE_REGISTER_STACK
 686   *stack = pd->stackblock;
 687   *stacksize = stacktop - *stack;
 688 #elif _STACK_GROWS_DOWN
 689   *stack = stacktop;
 690 #elif _STACK_GROWS_UP
 691   *stack = pd->stackblock;
 692   assert (*stack > 0);
 693 #endif
 694
 695   return 0;
 696 }
 697
 698
 699 void
 700 internal_function
 701 __deallocate_stack (struct pthread *pd)
 702 {
 703   lll_lock (stack_cache_lock, LLL_PRIVATE);
 704
 705   /* Remove the thread from the list of threads with user defined
 706      stacks.  */
 707   list_del (&pd->list);
 708
 709   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 710      not reset the 'used' flag in the 'tid' field.  This is done by
 711      the kernel.  If no thread has been created yet this field is
 712      still zero.  */
 713   if (__builtin_expect (! pd->user_stack, 1))
 714     (void) queue_stack (pd);
 715   else
 716     /* Free the memory associated with the ELF TLS.  */
 717     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 718
 719   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 720 }
 721
 722
 723 int
 724 internal_function
 725 __make_stacks_executable (void **stack_endp)
 726 {
 727   /* First the main thread's stack.  */
 728   int err = _dl_make_stack_executable (stack_endp);
 729   if (err != 0)
 730     return err;
 731
 732 #ifdef NEED_SEPARATE_REGISTER_STACK
 733   const size_t pagemask = ~(__getpagesize () - 1);
 734 #endif
 735
 736   lll_lock (stack_cache_lock, LLL_PRIVATE);
 737
 738   list_t *runp;
 739   list_for_each (runp, &stack_used)
 740     {
 741       err = change_stack_perm (list_entry (runp, struct pthread, list)
 742 #ifdef NEED_SEPARATE_REGISTER_STACK
 743                                , pagemask
 744 #endif
 745                                );
 746       if (err != 0)
 747         break;
 748     }
 749
 750   /* Also change the permission for the currently unused stacks.  This
 751      might be wasted time but better spend it here than adding a check
 752      in the fast path.  */
 753   if (err == 0)
 754     list_for_each (runp, &stack_cache)
 755       {
 756         err = change_stack_perm (list_entry (runp, struct pthread, list)
 757 #ifdef NEED_SEPARATE_REGISTER_STACK
 758                                  , pagemask
 759 #endif
 760                                  );
 761         if (err != 0)
 762           break;
 763       }
 764
 765   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 766
 767   return err;
 768 }
 769
 770
 771 /* In case of a fork() call the memory allocation in the child will be
 772    the same but only one thread is running.  All stacks except that of
 773    the one running thread are not used anymore.  We have to recycle
 774    them.  */
 775 void
 776 __reclaim_stacks (void)
 777 {
 778   struct pthread *self = (struct pthread *) THREAD_SELF;
 779
 780   /* No locking necessary.  The caller is the only stack in use.  */
 781
 782   /* Mark all stacks except the still running one as free.  */
 783   list_t *runp;
 784   list_for_each (runp, &stack_used)
 785     {
 786       struct pthread *curp = list_entry (runp, struct pthread, list);
 787       if (curp != self)
 788         {
 789           /* This marks the stack as free.  */
 790           curp->tid = 0;
 791
 792           /* The PID field must be initialized for the new process.  */
 793           curp->pid = self->pid;
 794
 795           /* Account for the size of the stack.  */
 796           stack_cache_actsize += curp->stackblock_size;
 797
 798           if (curp->specific_used)
 799             {
 800               /* Clear the thread-specific data.  */
 801               memset (curp->specific_1stblock, '\0',
 802                       sizeof (curp->specific_1stblock));
 803
 804               curp->specific_used = false;
 805
 806               for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
 807                 if (curp->specific[cnt] != NULL)
 808                   {
 809                     memset (curp->specific[cnt], '\0',
 810                             sizeof (curp->specific_1stblock));
 811
 812                     /* We have allocated the block which we do not
 813                        free here so re-set the bit.  */
 814                     curp->specific_used = true;
 815                   }
 816             }
 817         }
 818     }
 819
 820   /* Reset the PIDs in any cached stacks.  */
 821   list_for_each (runp, &stack_cache)
 822     {
 823       struct pthread *curp = list_entry (runp, struct pthread, list);
 824       curp->pid = self->pid;
 825     }
 826
 827   /* Add the stack of all running threads to the cache.  */
 828   list_splice (&stack_used, &stack_cache);
 829
 830   /* Remove the entry for the current thread to from the cache list
 831      and add it to the list of running threads.  Which of the two
 832      lists is decided by the user_stack flag.  */
 833   list_del (&self->list);
 834
 835   /* Re-initialize the lists for all the threads.  */
 836   INIT_LIST_HEAD (&stack_used);
 837   INIT_LIST_HEAD (&__stack_user);
 838
 839   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 840     list_add (&self->list, &__stack_user);
 841   else
 842     list_add (&self->list, &stack_used);
 843
 844   /* There is one thread running.  */
 845   __nptl_nthreads = 1;
 846
 847   /* Initialize the lock.  */
 848   stack_cache_lock = LLL_LOCK_INITIALIZER;
 849 }
 850
 851
 852 #if HP_TIMING_AVAIL
 853 # undef __find_thread_by_id
 854 /* Find a thread given the thread ID.  */
 855 attribute_hidden
 856 struct pthread *
 857 __find_thread_by_id (pid_t tid)
 858 {
 859   struct pthread *result = NULL;
 860
 861   lll_lock (stack_cache_lock, LLL_PRIVATE);
 862
 863   /* Iterate over the list with system-allocated threads first.  */
 864   list_t *runp;
 865   list_for_each (runp, &stack_used)
 866     {
 867       struct pthread *curp;
 868
 869       curp = list_entry (runp, struct pthread, list);
 870
 871       if (curp->tid == tid)
 872         {
 873           result = curp;
 874           goto out;
 875         }
 876     }
 877
 878   /* Now the list with threads using user-allocated stacks.  */
 879   list_for_each (runp, &__stack_user)
 880     {
 881       struct pthread *curp;
 882
 883       curp = list_entry (runp, struct pthread, list);
 884
 885       if (curp->tid == tid)
 886         {
 887           result = curp;
 888           goto out;
 889         }
 890     }
 891
 892  out:
 893   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 894
 895   return result;
 896 }
 897 #endif
 898
 899
 900 static void
 901 internal_function
 902 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
 903 {
 904   if (! IS_DETACHED (t))
 905     {
 906       int ch;
 907       do
 908         {
 909           ch = t->cancelhandling;
 910
 911           /* If the thread is exiting right now, ignore it.  */
 912           if ((ch & EXITING_BITMASK) != 0)
 913             return;
 914         }
 915       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
 916                                                    ch | SETXID_BITMASK, ch));
 917     }
 918
 919   int val;
 920   INTERNAL_SYSCALL_DECL (err);
 921 #if __ASSUME_TGKILL
 922   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 923                           t->tid, SIGSETXID);
 924 #else
 925 # ifdef __NR_tgkill
 926   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 927                           t->tid, SIGSETXID);
 928   if (INTERNAL_SYSCALL_ERROR_P (val, err)
 929       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 930 # endif
 931     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 932 #endif
 933
 934   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 935     atomic_increment (&cmdp->cntr);
 936 }
 937
 938
 939 int
 940 attribute_hidden
 941 __nptl_setxid (struct xid_command *cmdp)
 942 {
 943   int result;
 944   lll_lock (stack_cache_lock, LLL_PRIVATE);
 945
 946   __xidcmd = cmdp;
 947   cmdp->cntr = 0;
 948
 949   struct pthread *self = THREAD_SELF;
 950
 951   /* Iterate over the list with system-allocated threads first.  */
 952   list_t *runp;
 953   list_for_each (runp, &stack_used)
 954     {
 955       struct pthread *t = list_entry (runp, struct pthread, list);
 956       if (t == self)
 957         continue;
 958
 959       setxid_signal_thread (cmdp, t);
 960     }
 961
 962   /* Now the list with threads using user-allocated stacks.  */
 963   list_for_each (runp, &__stack_user)
 964     {
 965       struct pthread *t = list_entry (runp, struct pthread, list);
 966       if (t == self)
 967         continue;
 968
 969       setxid_signal_thread (cmdp, t);
 970     }
 971
 972   int cur = cmdp->cntr;
 973   while (cur != 0)
 974     {
 975       lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
 976       cur = cmdp->cntr;
 977     }
 978
 979   /* This must be last, otherwise the current thread might not have
 980      permissions to send SIGSETXID syscall to the other threads.  */
 981   INTERNAL_SYSCALL_DECL (err);
 982   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 983                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 984   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 985     {
 986       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 987       result = -1;
 988     }
 989
 990   lll_unlock (stack_cache_lock, LLL_PRIVATE);
 991   return result;
 992 }
 993
 994 static inline void __attribute__((always_inline))
 995 init_one_static_tls (struct pthread *curp, struct link_map *map)
 996 {
 997   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 998 # if TLS_TCB_AT_TP
 999   void *dest = (char *) curp - map->l_tls_offset;
1000 # elif TLS_DTV_AT_TP
1001   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
1002 # else
1003 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1004 # endif
1005
1006   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
1007   dtv[map->l_tls_modid].pointer.val = dest;
1008   dtv[map->l_tls_modid].pointer.is_static = true;
1009
1010   /* Initialize the memory.  */
1011   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
1012           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
1013 }
1014
1015 void
1016 attribute_hidden
1017 __pthread_init_static_tls (struct link_map *map)
1018 {
1019   lll_lock (stack_cache_lock, LLL_PRIVATE);
1020
1021   /* Iterate over the list with system-allocated threads first.  */
1022   list_t *runp;
1023   list_for_each (runp, &stack_used)
1024     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1025
1026   /* Now the list with threads using user-allocated stacks.  */
1027   list_for_each (runp, &__stack_user)
1028     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1029
1030   lll_unlock (stack_cache_lock, LLL_PRIVATE);
1031 }
1032
1033
1034 void
1035 attribute_hidden
1036 __wait_lookup_done (void)
1037 {
1038   lll_lock (stack_cache_lock, LLL_PRIVATE);
1039
1040   struct pthread *self = THREAD_SELF;
1041
1042   /* Iterate over the list with system-allocated threads first.  */
1043   list_t *runp;
1044   list_for_each (runp, &stack_used)
1045     {
1046       struct pthread *t = list_entry (runp, struct pthread, list);
1047       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1048         continue;
1049
1050       int *const gscope_flagp = &t->header.gscope_flag;
1051
1052       /* We have to wait until this thread is done with the global
1053          scope.  First tell the thread that we are waiting and
1054          possibly have to be woken.  */
1055       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1056                                                 THREAD_GSCOPE_FLAG_WAIT,
1057                                                 THREAD_GSCOPE_FLAG_USED))
1058         continue;
1059
1060       do
1061         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1062       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1063     }
1064
1065   /* Now the list with threads using user-allocated stacks.  */
1066   list_for_each (runp, &__stack_user)
1067     {
1068       struct pthread *t = list_entry (runp, struct pthread, list);
1069       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1070         continue;
1071
1072       int *const gscope_flagp = &t->header.gscope_flag;
1073
1074       /* We have to wait until this thread is done with the global
1075          scope.  First tell the thread that we are waiting and
1076          possibly have to be woken.  */
1077       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1078                                                 THREAD_GSCOPE_FLAG_WAIT,
1079                                                 THREAD_GSCOPE_FLAG_USED))
1080         continue;
1081
1082       do
1083         lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1084       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1085     }
1086
1087   lll_unlock (stack_cache_lock, LLL_PRIVATE);
1088 }