nptl/allocatestack.c

   1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31
  32
  33 #ifndef NEED_SEPARATE_REGISTER_STACK
  34
  35 /* Most architectures have exactly one stack pointer.  Some have more.  */
  36 # define STACK_VARIABLES void *stackaddr = NULL
  37
  38 /* How to pass the values to the 'create_thread' function.  */
  39 # define STACK_VARIABLES_ARGS stackaddr
  40
  41 /* How to declare function which gets there parameters.  */
  42 # define STACK_VARIABLES_PARMS void *stackaddr
  43
  44 /* How to declare allocate_stack.  */
  45 # define ALLOCATE_STACK_PARMS void **stack
  46
  47 /* This is how the function is called.  We do it this way to allow
  48    other variants of the function to have more parameters.  */
  49 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  50
  51 #else
  52
  53 /* We need two stacks.  The kernel will place them but we have to tell
  54    the kernel about the size of the reserved address space.  */
  55 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
  56
  57 /* How to pass the values to the 'create_thread' function.  */
  58 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  59
  60 /* How to declare function which gets there parameters.  */
  61 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  62
  63 /* How to declare allocate_stack.  */
  64 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  65
  66 /* This is how the function is called.  We do it this way to allow
  67    other variants of the function to have more parameters.  */
  68 # define ALLOCATE_STACK(attr, pd) \
  69   allocate_stack (attr, pd, &stackaddr, &stacksize)
  70
  71 #endif
  72
  73
  74 /* Default alignment of stack.  */
  75 #ifndef STACK_ALIGN
  76 # define STACK_ALIGN __alignof__ (long double)
  77 #endif
  78
  79 /* Default value for minimal stack size after allocating thread
  80    descriptor and guard.  */
  81 #ifndef MINIMAL_REST_STACK
  82 # define MINIMAL_REST_STACK     4096
  83 #endif
  84
  85
  86 /* Let the architecture add some flags to the mmap() call used to
  87    allocate stacks.  */
  88 #ifndef ARCH_MAP_FLAGS
  89 # define ARCH_MAP_FLAGS 0
  90 #endif
  91
  92 /* This yields the pointer that TLS support code calls the thread pointer.  */
  93 #if TLS_TCB_AT_TP
  94 # define TLS_TPADJ(pd) (pd)
  95 #elif TLS_DTV_AT_TP
  96 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  97 #endif
  98
  99 /* Cache handling for not-yet free stacks.  */
 100
 101 /* Maximum size in kB of cache.  */
 102 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 103 static size_t stack_cache_actsize;
 104
 105 /* Mutex protecting this variable.  */
 106 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
 107
 108 /* List of queued stack frames.  */
 109 static LIST_HEAD (stack_cache);
 110
 111 /* List of the stacks in use.  */
 112 static LIST_HEAD (stack_used);
 113
 114 /* List of the threads with user provided stacks in use.  No need to
 115    initialize this, since it's done in __pthread_initialize_minimal.  */
 116 list_t __stack_user __attribute__ ((nocommon));
 117 hidden_data_def (__stack_user)
 118
 119 #if COLORING_INCREMENT != 0
 120 /* Number of threads created.  */
 121 static unsigned int nptl_ncreated;
 122 #endif
 123
 124
 125 /* Check whether the stack is still used or not.  */
 126 #define FREE_P(descr) ((descr)->tid <= 0)
 127
 128
 129 /* We create a double linked list of all cache entries.  Double linked
 130    because this allows removing entries from the end.  */
 131
 132
 133 /* Get a stack frame from the cache.  We have to match by size since
 134    some blocks might be too small or far too large.  */
 135 static struct pthread *
 136 get_cached_stack (size_t *sizep, void **memp)
 137 {
 138   size_t size = *sizep;
 139   struct pthread *result = NULL;
 140   list_t *entry;
 141
 142   lll_lock (stack_cache_lock);
 143
 144   /* Search the cache for a matching entry.  We search for the
 145      smallest stack which has at least the required size.  Note that
 146      in normal situations the size of all allocated stacks is the
 147      same.  As the very least there are only a few different sizes.
 148      Therefore this loop will exit early most of the time with an
 149      exact match.  */
 150   list_for_each (entry, &stack_cache)
 151     {
 152       struct pthread *curr;
 153
 154       curr = list_entry (entry, struct pthread, list);
 155       if (FREE_P (curr) && curr->stackblock_size >= size)
 156         {
 157           if (curr->stackblock_size == size)
 158             {
 159               result = curr;
 160               break;
 161             }
 162
 163           if (result == NULL
 164               || result->stackblock_size > curr->stackblock_size)
 165             result = curr;
 166         }
 167     }
 168
 169   if (__builtin_expect (result == NULL, 0)
 170       /* Make sure the size difference is not too excessive.  In that
 171          case we do not use the block.  */
 172       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 173     {
 174       /* Release the lock.  */
 175       lll_unlock (stack_cache_lock);
 176
 177       return NULL;
 178     }
 179
 180   /* Dequeue the entry.  */
 181   list_del (&result->list);
 182
 183   /* And add to the list of stacks in use.  */
 184   list_add (&result->list, &stack_used);
 185
 186   /* And decrease the cache size.  */
 187   stack_cache_actsize -= result->stackblock_size;
 188
 189   /* Release the lock early.  */
 190   lll_unlock (stack_cache_lock);
 191
 192   /* Report size and location of the stack to the caller.  */
 193   *sizep = result->stackblock_size;
 194   *memp = result->stackblock;
 195
 196   /* Cancellation handling is back to the default.  */
 197   result->cancelhandling = 0;
 198   result->cleanup = NULL;
 199
 200   /* No pending event.  */
 201   result->nextevent = NULL;
 202
 203   /* Clear the DTV.  */
 204   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 205   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 206
 207   /* Re-initialize the TLS.  */
 208   _dl_allocate_tls_init (TLS_TPADJ (result));
 209
 210   return result;
 211 }
 212
 213
 214 /* Free stacks until cache size is lower than LIMIT.  */
 215 static void
 216 free_stacks (size_t limit)
 217 {
 218   /* We reduce the size of the cache.  Remove the last entries until
 219      the size is below the limit.  */
 220   list_t *entry;
 221   list_t *prev;
 222
 223   /* Search from the end of the list.  */
 224   list_for_each_prev_safe (entry, prev, &stack_cache)
 225     {
 226       struct pthread *curr;
 227
 228       curr = list_entry (entry, struct pthread, list);
 229       if (FREE_P (curr))
 230         {
 231           /* Unlink the block.  */
 232           list_del (entry);
 233
 234           /* Account for the freed memory.  */
 235           stack_cache_actsize -= curr->stackblock_size;
 236
 237           /* Free the memory associated with the ELF TLS.  */
 238           _dl_deallocate_tls (TLS_TPADJ (curr), false);
 239
 240           /* Remove this block.  This should never fail.  If it does
 241              something is really wrong.  */
 242           if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 243             abort ();
 244
 245           /* Maybe we have freed enough.  */
 246           if (stack_cache_actsize <= limit)
 247             break;
 248         }
 249     }
 250 }
 251
 252
 253 /* Add a stack frame which is not used anymore to the stack.  Must be
 254    called with the cache lock held.  */
 255 static inline void
 256 __attribute ((always_inline))
 257 queue_stack (struct pthread *stack)
 258 {
 259   /* We unconditionally add the stack to the list.  The memory may
 260      still be in use but it will not be reused until the kernel marks
 261      the stack as not used anymore.  */
 262   list_add (&stack->list, &stack_cache);
 263
 264   stack_cache_actsize += stack->stackblock_size;
 265   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 266     free_stacks (stack_cache_maxsize);
 267 }
 268
 269
 270 /* This function is called indirectly from the freeres code in libc.  */
 271 void
 272 __free_stack_cache (void)
 273 {
 274   free_stacks (0);
 275 }
 276
 277
 278 static int
 279 internal_function
 280 change_stack_perm (struct pthread *pd
 281 #ifdef NEED_SEPARATE_REGISTER_STACK
 282                    , size_t pagemask
 283 #endif
 284                    )
 285 {
 286 #ifdef NEED_SEPARATE_REGISTER_STACK
 287   void *stack = (pd->stackblock
 288                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 289                       & pagemask) + pd->guardsize) & pagemask));
 290   size_t len = pd->stackblock + pd->stackblock_size - stack;
 291 #elif _STACK_GROWS_DOWN
 292   void *stack = pd->stackblock + pd->guardsize;
 293   size_t len = pd->stackblock_size - pd->guardsize;
 294 #elif _STACK_GROWS_UP
 295   void *stack = pd->stackblock;
 296   size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
 297 #else
 298 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
 299 #endif
 300   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 301     return errno;
 302
 303   return 0;
 304 }
 305
 306
 307 static int
 308 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 309                 ALLOCATE_STACK_PARMS)
 310 {
 311   struct pthread *pd;
 312   size_t size;
 313   size_t pagesize_m1 = __getpagesize () - 1;
 314   void *stacktop;
 315
 316   assert (attr != NULL);
 317   assert (powerof2 (pagesize_m1 + 1));
 318   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 319
 320   /* Get the stack size from the attribute if it is set.  Otherwise we
 321      use the default we determined at start time.  */
 322   size = attr->stacksize ?: __default_stacksize;
 323
 324   /* Get memory for the stack.  */
 325   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 326     {
 327       uintptr_t adj;
 328
 329       /* If the user also specified the size of the stack make sure it
 330          is large enough.  */
 331       if (attr->stacksize != 0
 332           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 333         return EINVAL;
 334
 335       /* Adjust stack size for alignment of the TLS block.  */
 336 #if TLS_TCB_AT_TP
 337       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 338             & __static_tls_align_m1;
 339       assert (size > adj + TLS_TCB_SIZE);
 340 #elif TLS_DTV_AT_TP
 341       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 342             & __static_tls_align_m1;
 343       assert (size > adj);
 344 #endif
 345
 346       /* The user provided some memory.  Let's hope it matches the
 347          size...  We do not allocate guard pages if the user provided
 348          the stack.  It is the user's responsibility to do this if it
 349          is wanted.  */
 350 #if TLS_TCB_AT_TP
 351       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 352                                - TLS_TCB_SIZE - adj);
 353 #elif TLS_DTV_AT_TP
 354       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 355                                 - __static_tls_size - adj)
 356                                - TLS_PRE_TCB_SIZE);
 357 #endif
 358
 359       /* The user provided stack memory needs to be cleared.  */
 360       memset (pd, '\0', sizeof (struct pthread));
 361
 362       /* The first TSD block is included in the TCB.  */
 363       pd->specific[0] = pd->specific_1stblock;
 364
 365       /* Remember the stack-related values.  */
 366       pd->stackblock = (char *) attr->stackaddr - size;
 367       pd->stackblock_size = size;
 368
 369       /* This is a user-provided stack.  It will not be queued in the
 370          stack cache nor will the memory (except the TLS memory) be freed.  */
 371       pd->user_stack = true;
 372
 373       /* This is at least the second thread.  */
 374       pd->header.multiple_threads = 1;
 375 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 376       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 377 #endif
 378
 379 #ifndef __ASSUME_PRIVATE_FUTEX
 380       /* The thread must know when private futexes are supported.  */
 381       pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 382                                                 header.private_futex);
 383 #endif
 384
 385 #ifdef NEED_DL_SYSINFO
 386       /* Copy the sysinfo value from the parent.  */
 387       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 388 #endif
 389
 390       /* The process ID is also the same as that of the caller.  */
 391       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 392
 393       /* Allocate the DTV for this thread.  */
 394       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 395         {
 396           /* Something went wrong.  */
 397           assert (errno == ENOMEM);
 398           return EAGAIN;
 399         }
 400
 401
 402       /* Prepare to modify global data.  */
 403       lll_lock (stack_cache_lock);
 404
 405       /* And add to the list of stacks in use.  */
 406       list_add (&pd->list, &__stack_user);
 407
 408       lll_unlock (stack_cache_lock);
 409     }
 410   else
 411     {
 412       /* Allocate some anonymous memory.  If possible use the cache.  */
 413       size_t guardsize;
 414       size_t reqsize;
 415       void *mem;
 416       const int prot = (PROT_READ | PROT_WRITE
 417                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 418
 419 #if COLORING_INCREMENT != 0
 420       /* Add one more page for stack coloring.  Don't do it for stacks
 421          with 16 times pagesize or larger.  This might just cause
 422          unnecessary misalignment.  */
 423       if (size <= 16 * pagesize_m1)
 424         size += pagesize_m1 + 1;
 425 #endif
 426
 427       /* Adjust the stack size for alignment.  */
 428       size &= ~__static_tls_align_m1;
 429       assert (size != 0);
 430
 431       /* Make sure the size of the stack is enough for the guard and
 432          eventually the thread descriptor.  */
 433       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 434       if (__builtin_expect (size < ((guardsize + __static_tls_size
 435                                      + MINIMAL_REST_STACK + pagesize_m1)
 436                                     & ~pagesize_m1),
 437                             0))
 438         /* The stack is too small (or the guard too large).  */
 439         return EINVAL;
 440
 441       /* Try to get a stack from the cache.  */
 442       reqsize = size;
 443       pd = get_cached_stack (&size, &mem);
 444       if (pd == NULL)
 445         {
 446           /* To avoid aliasing effects on a larger scale than pages we
 447              adjust the allocated stack size if necessary.  This way
 448              allocations directly following each other will not have
 449              aliasing problems.  */
 450 #if MULTI_PAGE_ALIASING != 0
 451           if ((size % MULTI_PAGE_ALIASING) == 0)
 452             size += pagesize_m1 + 1;
 453 #endif
 454
 455           mem = mmap (NULL, size, prot,
 456                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 457
 458           if (__builtin_expect (mem == MAP_FAILED, 0))
 459             {
 460 #ifdef ARCH_RETRY_MMAP
 461               mem = ARCH_RETRY_MMAP (size);
 462               if (__builtin_expect (mem == MAP_FAILED, 0))
 463 #endif
 464                 return errno;
 465             }
 466
 467           /* SIZE is guaranteed to be greater than zero.
 468              So we can never get a null pointer back from mmap.  */
 469           assert (mem != NULL);
 470
 471 #if COLORING_INCREMENT != 0
 472           /* Atomically increment NCREATED.  */
 473           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 474
 475           /* We chose the offset for coloring by incrementing it for
 476              every new thread by a fixed amount.  The offset used
 477              module the page size.  Even if coloring would be better
 478              relative to higher alignment values it makes no sense to
 479              do it since the mmap() interface does not allow us to
 480              specify any alignment for the returned memory block.  */
 481           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 482
 483           /* Make sure the coloring offsets does not disturb the alignment
 484              of the TCB and static TLS block.  */
 485           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 486             coloring = (((coloring + __static_tls_align_m1)
 487                          & ~(__static_tls_align_m1))
 488                         & ~pagesize_m1);
 489 #else
 490           /* Unless specified we do not make any adjustments.  */
 491 # define coloring 0
 492 #endif
 493
 494           /* Place the thread descriptor at the end of the stack.  */
 495 #if TLS_TCB_AT_TP
 496           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 497 #elif TLS_DTV_AT_TP
 498           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 499                                     - __static_tls_size)
 500                                     & ~__static_tls_align_m1)
 501                                    - TLS_PRE_TCB_SIZE);
 502 #endif
 503
 504           /* Remember the stack-related values.  */
 505           pd->stackblock = mem;
 506           pd->stackblock_size = size;
 507
 508           /* We allocated the first block thread-specific data array.
 509              This address will not change for the lifetime of this
 510              descriptor.  */
 511           pd->specific[0] = pd->specific_1stblock;
 512
 513           /* This is at least the second thread.  */
 514           pd->header.multiple_threads = 1;
 515 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 516           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 517 #endif
 518
 519 #ifndef __ASSUME_PRIVATE_FUTEX
 520           /* The thread must know when private futexes are supported.  */
 521           pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
 522                                                     header.private_futex);
 523 #endif
 524
 525 #ifdef NEED_DL_SYSINFO
 526           /* Copy the sysinfo value from the parent.  */
 527           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 528 #endif
 529
 530           /* The process ID is also the same as that of the caller.  */
 531           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 532
 533           /* Allocate the DTV for this thread.  */
 534           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 535             {
 536               /* Something went wrong.  */
 537               assert (errno == ENOMEM);
 538
 539               /* Free the stack memory we just allocated.  */
 540               (void) munmap (mem, size);
 541
 542               return EAGAIN;
 543             }
 544
 545
 546           /* Prepare to modify global data.  */
 547           lll_lock (stack_cache_lock);
 548
 549           /* And add to the list of stacks in use.  */
 550           list_add (&pd->list, &stack_used);
 551
 552           lll_unlock (stack_cache_lock);
 553
 554
 555           /* There might have been a race.  Another thread might have
 556              caused the stacks to get exec permission while this new
 557              stack was prepared.  Detect if this was possible and
 558              change the permission if necessary.  */
 559           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 560                                 && (prot & PROT_EXEC) == 0, 0))
 561             {
 562               int err = change_stack_perm (pd
 563 #ifdef NEED_SEPARATE_REGISTER_STACK
 564                                            , ~pagesize_m1
 565 #endif
 566                                            );
 567               if (err != 0)
 568                 {
 569                   /* Free the stack memory we just allocated.  */
 570                   (void) munmap (mem, size);
 571
 572                   return err;
 573                 }
 574             }
 575
 576
 577           /* Note that all of the stack and the thread descriptor is
 578              zeroed.  This means we do not have to initialize fields
 579              with initial value zero.  This is specifically true for
 580              the 'tid' field which is always set back to zero once the
 581              stack is not used anymore and for the 'guardsize' field
 582              which will be read next.  */
 583         }
 584
 585       /* Create or resize the guard area if necessary.  */
 586       if (__builtin_expect (guardsize > pd->guardsize, 0))
 587         {
 588 #ifdef NEED_SEPARATE_REGISTER_STACK
 589           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 590 #elif _STACK_GROWS_DOWN
 591           char *guard = mem;
 592 # elif _STACK_GROWS_UP
 593           char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
 594 #endif
 595           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 596             {
 597               int err;
 598             mprot_error:
 599               err = errno;
 600
 601               lll_lock (stack_cache_lock);
 602
 603               /* Remove the thread from the list.  */
 604               list_del (&pd->list);
 605
 606               lll_unlock (stack_cache_lock);
 607
 608               /* Get rid of the TLS block we allocated.  */
 609               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 610
 611               /* Free the stack memory regardless of whether the size
 612                  of the cache is over the limit or not.  If this piece
 613                  of memory caused problems we better do not use it
 614                  anymore.  Uh, and we ignore possible errors.  There
 615                  is nothing we could do.  */
 616               (void) munmap (mem, size);
 617
 618               return err;
 619             }
 620
 621           pd->guardsize = guardsize;
 622         }
 623       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 624                                  0))
 625         {
 626           /* The old guard area is too large.  */
 627
 628 #ifdef NEED_SEPARATE_REGISTER_STACK
 629           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 630           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 631
 632           if (oldguard < guard
 633               && mprotect (oldguard, guard - oldguard, prot) != 0)
 634             goto mprot_error;
 635
 636           if (mprotect (guard + guardsize,
 637                         oldguard + pd->guardsize - guard - guardsize,
 638                         prot) != 0)
 639             goto mprot_error;
 640 #elif _STACK_GROWS_DOWN
 641           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 642                         prot) != 0)
 643             goto mprot_error;
 644 #elif _STACK_GROWS_UP
 645           if (mprotect ((char *) pd - pd->guardsize,
 646                         pd->guardsize - guardsize, prot) != 0)
 647             goto mprot_error;
 648 #endif
 649
 650           pd->guardsize = guardsize;
 651         }
 652       /* The pthread_getattr_np() calls need to get passed the size
 653          requested in the attribute, regardless of how large the
 654          actually used guardsize is.  */
 655       pd->reported_guardsize = guardsize;
 656     }
 657
 658   /* Initialize the lock.  We have to do this unconditionally since the
 659      stillborn thread could be canceled while the lock is taken.  */
 660   pd->lock = LLL_LOCK_INITIALIZER;
 661
 662   /* The robust mutex lists also need to be initialized
 663      unconditionally because the cleanup for the previous stack owner
 664      might have happened in the kernel.  */
 665   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
 666                                   - offsetof (pthread_mutex_t,
 667                                               __data.__list.__next));
 668   pd->robust_head.list_op_pending = NULL;
 669 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 670   pd->robust_prev = &pd->robust_head;
 671 #endif
 672   pd->robust_head.list = &pd->robust_head;
 673
 674   /* We place the thread descriptor at the end of the stack.  */
 675   *pdp = pd;
 676
 677 #if TLS_TCB_AT_TP
 678   /* The stack begins before the TCB and the static TLS block.  */
 679   stacktop = ((char *) (pd + 1) - __static_tls_size);
 680 #elif TLS_DTV_AT_TP
 681   stacktop = (char *) (pd - 1);
 682 #endif
 683
 684 #ifdef NEED_SEPARATE_REGISTER_STACK
 685   *stack = pd->stackblock;
 686   *stacksize = stacktop - *stack;
 687 #elif _STACK_GROWS_DOWN
 688   *stack = stacktop;
 689 #elif _STACK_GROWS_UP
 690   *stack = pd->stackblock;
 691   assert (*stack > 0);
 692 #endif
 693
 694   return 0;
 695 }
 696
 697
 698 void
 699 internal_function
 700 __deallocate_stack (struct pthread *pd)
 701 {
 702   lll_lock (stack_cache_lock);
 703
 704   /* Remove the thread from the list of threads with user defined
 705      stacks.  */
 706   list_del (&pd->list);
 707
 708   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 709      not reset the 'used' flag in the 'tid' field.  This is done by
 710      the kernel.  If no thread has been created yet this field is
 711      still zero.  */
 712   if (__builtin_expect (! pd->user_stack, 1))
 713     (void) queue_stack (pd);
 714   else
 715     /* Free the memory associated with the ELF TLS.  */
 716     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 717
 718   lll_unlock (stack_cache_lock);
 719 }
 720
 721
 722 int
 723 internal_function
 724 __make_stacks_executable (void **stack_endp)
 725 {
 726   /* First the main thread's stack.  */
 727   int err = _dl_make_stack_executable (stack_endp);
 728   if (err != 0)
 729     return err;
 730
 731 #ifdef NEED_SEPARATE_REGISTER_STACK
 732   const size_t pagemask = ~(__getpagesize () - 1);
 733 #endif
 734
 735   lll_lock (stack_cache_lock);
 736
 737   list_t *runp;
 738   list_for_each (runp, &stack_used)
 739     {
 740       err = change_stack_perm (list_entry (runp, struct pthread, list)
 741 #ifdef NEED_SEPARATE_REGISTER_STACK
 742                                , pagemask
 743 #endif
 744                                );
 745       if (err != 0)
 746         break;
 747     }
 748
 749   /* Also change the permission for the currently unused stacks.  This
 750      might be wasted time but better spend it here than adding a check
 751      in the fast path.  */
 752   if (err == 0)
 753     list_for_each (runp, &stack_cache)
 754       {
 755         err = change_stack_perm (list_entry (runp, struct pthread, list)
 756 #ifdef NEED_SEPARATE_REGISTER_STACK
 757                                  , pagemask
 758 #endif
 759                                  );
 760         if (err != 0)
 761           break;
 762       }
 763
 764   lll_unlock (stack_cache_lock);
 765
 766   return err;
 767 }
 768
 769
 770 /* In case of a fork() call the memory allocation in the child will be
 771    the same but only one thread is running.  All stacks except that of
 772    the one running thread are not used anymore.  We have to recycle
 773    them.  */
 774 void
 775 __reclaim_stacks (void)
 776 {
 777   struct pthread *self = (struct pthread *) THREAD_SELF;
 778
 779   /* No locking necessary.  The caller is the only stack in use.  */
 780
 781   /* Mark all stacks except the still running one as free.  */
 782   list_t *runp;
 783   list_for_each (runp, &stack_used)
 784     {
 785       struct pthread *curp = list_entry (runp, struct pthread, list);
 786       if (curp != self)
 787         {
 788           /* This marks the stack as free.  */
 789           curp->tid = 0;
 790
 791           /* The PID field must be initialized for the new process.  */
 792           curp->pid = self->pid;
 793
 794           /* Account for the size of the stack.  */
 795           stack_cache_actsize += curp->stackblock_size;
 796         }
 797     }
 798
 799   /* Reset the PIDs in any cached stacks.  */
 800   list_for_each (runp, &stack_cache)
 801     {
 802       struct pthread *curp = list_entry (runp, struct pthread, list);
 803       curp->pid = self->pid;
 804     }
 805
 806   /* Add the stack of all running threads to the cache.  */
 807   list_splice (&stack_used, &stack_cache);
 808
 809   /* Remove the entry for the current thread to from the cache list
 810      and add it to the list of running threads.  Which of the two
 811      lists is decided by the user_stack flag.  */
 812   list_del (&self->list);
 813
 814   /* Re-initialize the lists for all the threads.  */
 815   INIT_LIST_HEAD (&stack_used);
 816   INIT_LIST_HEAD (&__stack_user);
 817
 818   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 819     list_add (&self->list, &__stack_user);
 820   else
 821     list_add (&self->list, &stack_used);
 822
 823   /* There is one thread running.  */
 824   __nptl_nthreads = 1;
 825
 826   /* Initialize the lock.  */
 827   stack_cache_lock = LLL_LOCK_INITIALIZER;
 828 }
 829
 830
 831 #if HP_TIMING_AVAIL
 832 # undef __find_thread_by_id
 833 /* Find a thread given the thread ID.  */
 834 attribute_hidden
 835 struct pthread *
 836 __find_thread_by_id (pid_t tid)
 837 {
 838   struct pthread *result = NULL;
 839
 840   lll_lock (stack_cache_lock);
 841
 842   /* Iterate over the list with system-allocated threads first.  */
 843   list_t *runp;
 844   list_for_each (runp, &stack_used)
 845     {
 846       struct pthread *curp;
 847
 848       curp = list_entry (runp, struct pthread, list);
 849
 850       if (curp->tid == tid)
 851         {
 852           result = curp;
 853           goto out;
 854         }
 855     }
 856
 857   /* Now the list with threads using user-allocated stacks.  */
 858   list_for_each (runp, &__stack_user)
 859     {
 860       struct pthread *curp;
 861
 862       curp = list_entry (runp, struct pthread, list);
 863
 864       if (curp->tid == tid)
 865         {
 866           result = curp;
 867           goto out;
 868         }
 869     }
 870
 871  out:
 872   lll_unlock (stack_cache_lock);
 873
 874   return result;
 875 }
 876 #endif
 877
 878
 879 static void
 880 internal_function
 881 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
 882 {
 883   if (! IS_DETACHED (t))
 884     {
 885       int ch;
 886       do
 887         {
 888           ch = t->cancelhandling;
 889
 890           /* If the thread is exiting right now, ignore it.  */
 891           if ((ch & EXITING_BITMASK) != 0)
 892             return;
 893         }
 894       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
 895                                                    ch | SETXID_BITMASK, ch));
 896     }
 897
 898   int val;
 899   INTERNAL_SYSCALL_DECL (err);
 900 #if __ASSUME_TGKILL
 901   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 902                           t->tid, SIGSETXID);
 903 #else
 904 # ifdef __NR_tgkill
 905   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 906                           t->tid, SIGSETXID);
 907   if (INTERNAL_SYSCALL_ERROR_P (val, err)
 908       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 909 # endif
 910     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 911 #endif
 912
 913   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 914     atomic_increment (&cmdp->cntr);
 915 }
 916
 917
 918 int
 919 attribute_hidden
 920 __nptl_setxid (struct xid_command *cmdp)
 921 {
 922   int result;
 923   lll_lock (stack_cache_lock);
 924
 925   __xidcmd = cmdp;
 926   cmdp->cntr = 0;
 927
 928   struct pthread *self = THREAD_SELF;
 929
 930   /* Iterate over the list with system-allocated threads first.  */
 931   list_t *runp;
 932   list_for_each (runp, &stack_used)
 933     {
 934       struct pthread *t = list_entry (runp, struct pthread, list);
 935       if (t == self)
 936         continue;
 937
 938       setxid_signal_thread (cmdp, t);
 939     }
 940
 941   /* Now the list with threads using user-allocated stacks.  */
 942   list_for_each (runp, &__stack_user)
 943     {
 944       struct pthread *t = list_entry (runp, struct pthread, list);
 945       if (t == self)
 946         continue;
 947
 948       setxid_signal_thread (cmdp, t);
 949     }
 950
 951   int cur = cmdp->cntr;
 952   while (cur != 0)
 953     {
 954       lll_private_futex_wait (&cmdp->cntr, cur);
 955       cur = cmdp->cntr;
 956     }
 957
 958   /* This must be last, otherwise the current thread might not have
 959      permissions to send SIGSETXID syscall to the other threads.  */
 960   INTERNAL_SYSCALL_DECL (err);
 961   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 962                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 963   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 964     {
 965       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 966       result = -1;
 967     }
 968
 969   lll_unlock (stack_cache_lock);
 970   return result;
 971 }
 972
 973 static inline void __attribute__((always_inline))
 974 init_one_static_tls (struct pthread *curp, struct link_map *map)
 975 {
 976   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 977 # if TLS_TCB_AT_TP
 978   void *dest = (char *) curp - map->l_tls_offset;
 979 # elif TLS_DTV_AT_TP
 980   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 981 # else
 982 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 983 # endif
 984
 985   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 986   dtv[map->l_tls_modid].pointer.val = dest;
 987   dtv[map->l_tls_modid].pointer.is_static = true;
 988
 989   /* Initialize the memory.  */
 990   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 991           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 992 }
 993
 994 void
 995 attribute_hidden
 996 __pthread_init_static_tls (struct link_map *map)
 997 {
 998   lll_lock (stack_cache_lock);
 999
1000   /* Iterate over the list with system-allocated threads first.  */
1001   list_t *runp;
1002   list_for_each (runp, &stack_used)
1003     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1004
1005   /* Now the list with threads using user-allocated stacks.  */
1006   list_for_each (runp, &__stack_user)
1007     init_one_static_tls (list_entry (runp, struct pthread, list), map);
1008
1009   lll_unlock (stack_cache_lock);
1010 }
1011
1012
1013 void
1014 attribute_hidden
1015 __wait_lookup_done (void)
1016 {
1017   lll_lock (stack_cache_lock);
1018
1019   struct pthread *self = THREAD_SELF;
1020
1021   /* Iterate over the list with system-allocated threads first.  */
1022   list_t *runp;
1023   list_for_each (runp, &stack_used)
1024     {
1025       struct pthread *t = list_entry (runp, struct pthread, list);
1026       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1027         continue;
1028
1029       int *const gscope_flagp = &t->header.gscope_flag;
1030
1031       /* We have to wait until this thread is done with the global
1032          scope.  First tell the thread that we are waiting and
1033          possibly have to be woken.  */
1034       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1035                                                 THREAD_GSCOPE_FLAG_WAIT,
1036                                                 THREAD_GSCOPE_FLAG_USED))
1037         continue;
1038
1039       do
1040         lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
1041       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1042     }
1043
1044   /* Now the list with threads using user-allocated stacks.  */
1045   list_for_each (runp, &__stack_user)
1046     {
1047       struct pthread *t = list_entry (runp, struct pthread, list);
1048       if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1049         continue;
1050
1051       int *const gscope_flagp = &t->header.gscope_flag;
1052
1053       /* We have to wait until this thread is done with the global
1054          scope.  First tell the thread that we are waiting and
1055          possibly have to be woken.  */
1056       if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1057                                                 THREAD_GSCOPE_FLAG_WAIT,
1058                                                 THREAD_GSCOPE_FLAG_USED))
1059         continue;
1060
1061       do
1062         lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
1063       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1064     }
1065
1066   lll_unlock (stack_cache_lock);
1067 }