nptl/allocatestack.c

   1 /* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #include <assert.h>
  21 #include <errno.h>
  22 #include <signal.h>
  23 #include <stdint.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include <sys/param.h>
  28 #include <dl-sysdep.h>
  29 #include <tls.h>
  30 #include <lowlevellock.h>
  31
  32
  33 #ifndef NEED_SEPARATE_REGISTER_STACK
  34
  35 /* Most architectures have exactly one stack pointer.  Some have more.  */
  36 # define STACK_VARIABLES void *stackaddr = NULL
  37
  38 /* How to pass the values to the 'create_thread' function.  */
  39 # define STACK_VARIABLES_ARGS stackaddr
  40
  41 /* How to declare function which gets there parameters.  */
  42 # define STACK_VARIABLES_PARMS void *stackaddr
  43
  44 /* How to declare allocate_stack.  */
  45 # define ALLOCATE_STACK_PARMS void **stack
  46
  47 /* This is how the function is called.  We do it this way to allow
  48    other variants of the function to have more parameters.  */
  49 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
  50
  51 #else
  52
  53 /* We need two stacks.  The kernel will place them but we have to tell
  54    the kernel about the size of the reserved address space.  */
  55 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
  56
  57 /* How to pass the values to the 'create_thread' function.  */
  58 # define STACK_VARIABLES_ARGS stackaddr, stacksize
  59
  60 /* How to declare function which gets there parameters.  */
  61 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
  62
  63 /* How to declare allocate_stack.  */
  64 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
  65
  66 /* This is how the function is called.  We do it this way to allow
  67    other variants of the function to have more parameters.  */
  68 # define ALLOCATE_STACK(attr, pd) \
  69   allocate_stack (attr, pd, &stackaddr, &stacksize)
  70
  71 #endif
  72
  73
  74 /* Default alignment of stack.  */
  75 #ifndef STACK_ALIGN
  76 # define STACK_ALIGN __alignof__ (long double)
  77 #endif
  78
  79 /* Default value for minimal stack size after allocating thread
  80    descriptor and guard.  */
  81 #ifndef MINIMAL_REST_STACK
  82 # define MINIMAL_REST_STACK     4096
  83 #endif
  84
  85
  86 /* Let the architecture add some flags to the mmap() call used to
  87    allocate stacks.  */
  88 #ifndef ARCH_MAP_FLAGS
  89 # define ARCH_MAP_FLAGS 0
  90 #endif
  91
  92 /* This yields the pointer that TLS support code calls the thread pointer.  */
  93 #if TLS_TCB_AT_TP
  94 # define TLS_TPADJ(pd) (pd)
  95 #elif TLS_DTV_AT_TP
  96 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
  97 #endif
  98
  99 /* Cache handling for not-yet free stacks.  */
 100
 101 /* Maximum size in kB of cache.  */
 102 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default.  */
 103 static size_t stack_cache_actsize;
 104
 105 /* Mutex protecting this variable.  */
 106 static lll_lock_t stack_cache_lock = LLL_LOCK_INITIALIZER;
 107
 108 /* List of queued stack frames.  */
 109 static LIST_HEAD (stack_cache);
 110
 111 /* List of the stacks in use.  */
 112 static LIST_HEAD (stack_used);
 113
 114 /* List of the threads with user provided stacks in use.  No need to
 115    initialize this, since it's done in __pthread_initialize_minimal.  */
 116 list_t __stack_user __attribute__ ((nocommon));
 117 hidden_data_def (__stack_user)
 118
 119 #if COLORING_INCREMENT != 0
 120 /* Number of threads created.  */
 121 static unsigned int nptl_ncreated;
 122 #endif
 123
 124
 125 /* Check whether the stack is still used or not.  */
 126 #define FREE_P(descr) ((descr)->tid <= 0)
 127
 128
 129 /* We create a double linked list of all cache entries.  Double linked
 130    because this allows removing entries from the end.  */
 131
 132
 133 /* Get a stack frame from the cache.  We have to match by size since
 134    some blocks might be too small or far too large.  */
 135 static struct pthread *
 136 get_cached_stack (size_t *sizep, void **memp)
 137 {
 138   size_t size = *sizep;
 139   struct pthread *result = NULL;
 140   list_t *entry;
 141
 142   lll_lock (stack_cache_lock);
 143
 144   /* Search the cache for a matching entry.  We search for the
 145      smallest stack which has at least the required size.  Note that
 146      in normal situations the size of all allocated stacks is the
 147      same.  As the very least there are only a few different sizes.
 148      Therefore this loop will exit early most of the time with an
 149      exact match.  */
 150   list_for_each (entry, &stack_cache)
 151     {
 152       struct pthread *curr;
 153
 154       curr = list_entry (entry, struct pthread, list);
 155       if (FREE_P (curr) && curr->stackblock_size >= size)
 156         {
 157           if (curr->stackblock_size == size)
 158             {
 159               result = curr;
 160               break;
 161             }
 162
 163           if (result == NULL
 164               || result->stackblock_size > curr->stackblock_size)
 165             result = curr;
 166         }
 167     }
 168
 169   if (__builtin_expect (result == NULL, 0)
 170       /* Make sure the size difference is not too excessive.  In that
 171          case we do not use the block.  */
 172       || __builtin_expect (result->stackblock_size > 4 * size, 0))
 173     {
 174       /* Release the lock.  */
 175       lll_unlock (stack_cache_lock);
 176
 177       return NULL;
 178     }
 179
 180   /* Dequeue the entry.  */
 181   list_del (&result->list);
 182
 183   /* And add to the list of stacks in use.  */
 184   list_add (&result->list, &stack_used);
 185
 186   /* And decrease the cache size.  */
 187   stack_cache_actsize -= result->stackblock_size;
 188
 189   /* Release the lock early.  */
 190   lll_unlock (stack_cache_lock);
 191
 192   /* Report size and location of the stack to the caller.  */
 193   *sizep = result->stackblock_size;
 194   *memp = result->stackblock;
 195
 196   /* Cancellation handling is back to the default.  */
 197   result->cancelhandling = 0;
 198   result->cleanup = NULL;
 199
 200   /* No pending event.  */
 201   result->nextevent = NULL;
 202
 203   /* Clear the DTV.  */
 204   dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
 205   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 206
 207   /* Re-initialize the TLS.  */
 208   _dl_allocate_tls_init (TLS_TPADJ (result));
 209
 210   return result;
 211 }
 212
 213
 214 /* Add a stack frame which is not used anymore to the stack.  Must be
 215    called with the cache lock held.  */
 216 static inline void
 217 __attribute ((always_inline))
 218 queue_stack (struct pthread *stack)
 219 {
 220   /* We unconditionally add the stack to the list.  The memory may
 221      still be in use but it will not be reused until the kernel marks
 222      the stack as not used anymore.  */
 223   list_add (&stack->list, &stack_cache);
 224
 225   stack_cache_actsize += stack->stackblock_size;
 226   if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
 227     {
 228       /* We reduce the size of the cache.  Remove the last entries
 229          until the size is below the limit.  */
 230       list_t *entry;
 231       list_t *prev;
 232
 233       /* Search from the end of the list.  */
 234       list_for_each_prev_safe (entry, prev, &stack_cache)
 235         {
 236           struct pthread *curr;
 237
 238           curr = list_entry (entry, struct pthread, list);
 239           if (FREE_P (curr))
 240             {
 241               /* Unlink the block.  */
 242               list_del (entry);
 243
 244               /* Account for the freed memory.  */
 245               stack_cache_actsize -= curr->stackblock_size;
 246
 247               /* Free the memory associated with the ELF TLS.  */
 248               _dl_deallocate_tls (TLS_TPADJ (curr), false);
 249
 250               /* Remove this block.  This should never fail.  If it
 251                  does something is really wrong.  */
 252               if (munmap (curr->stackblock, curr->stackblock_size) != 0)
 253                 abort ();
 254
 255               /* Maybe we have freed enough.  */
 256               if (stack_cache_actsize <= stack_cache_maxsize)
 257                 break;
 258             }
 259         }
 260     }
 261 }
 262
 263
 264 static int
 265 internal_function
 266 change_stack_perm (struct pthread *pd
 267 #ifdef NEED_SEPARATE_REGISTER_STACK
 268                    , size_t pagemask
 269 #endif
 270                    )
 271 {
 272 #ifdef NEED_SEPARATE_REGISTER_STACK
 273   void *stack = (pd->stackblock
 274                  + (((((pd->stackblock_size - pd->guardsize) / 2)
 275                       & pagemask) + pd->guardsize) & pagemask));
 276   size_t len = pd->stackblock + pd->stackblock_size - stack;
 277 #else
 278   void *stack = pd->stackblock + pd->guardsize;
 279   size_t len = pd->stackblock_size - pd->guardsize;
 280 #endif
 281   if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
 282     return errno;
 283
 284   return 0;
 285 }
 286
 287
 288 static int
 289 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 290                 ALLOCATE_STACK_PARMS)
 291 {
 292   struct pthread *pd;
 293   size_t size;
 294   size_t pagesize_m1 = __getpagesize () - 1;
 295   void *stacktop;
 296
 297   assert (attr != NULL);
 298   assert (powerof2 (pagesize_m1 + 1));
 299   assert (TCB_ALIGNMENT >= STACK_ALIGN);
 300
 301   /* Get the stack size from the attribute if it is set.  Otherwise we
 302      use the default we determined at start time.  */
 303   size = attr->stacksize ?: __default_stacksize;
 304
 305   /* Get memory for the stack.  */
 306   if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
 307     {
 308       uintptr_t adj;
 309
 310       /* If the user also specified the size of the stack make sure it
 311          is large enough.  */
 312       if (attr->stacksize != 0
 313           && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
 314         return EINVAL;
 315
 316       /* Adjust stack size for alignment of the TLS block.  */
 317 #if TLS_TCB_AT_TP
 318       adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
 319             & __static_tls_align_m1;
 320       assert (size > adj + TLS_TCB_SIZE);
 321 #elif TLS_DTV_AT_TP
 322       adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
 323             & __static_tls_align_m1;
 324       assert (size > adj);
 325 #endif
 326
 327       /* The user provided some memory.  Let's hope it matches the
 328          size...  We do not allocate guard pages if the user provided
 329          the stack.  It is the user's responsibility to do this if it
 330          is wanted.  */
 331 #if TLS_TCB_AT_TP
 332       pd = (struct pthread *) ((uintptr_t) attr->stackaddr
 333                                - TLS_TCB_SIZE - adj);
 334 #elif TLS_DTV_AT_TP
 335       pd = (struct pthread *) (((uintptr_t) attr->stackaddr
 336                                 - __static_tls_size - adj)
 337                                - TLS_PRE_TCB_SIZE);
 338 #endif
 339
 340       /* The user provided stack memory needs to be cleared.  */
 341       memset (pd, '\0', sizeof (struct pthread));
 342
 343       /* The first TSD block is included in the TCB.  */
 344       pd->specific[0] = pd->specific_1stblock;
 345
 346       /* Remember the stack-related values.  */
 347       pd->stackblock = (char *) attr->stackaddr - size;
 348       pd->stackblock_size = size;
 349
 350       /* This is a user-provided stack.  It will not be queued in the
 351          stack cache nor will the memory (except the TLS memory) be freed.  */
 352       pd->user_stack = true;
 353
 354       /* This is at least the second thread.  */
 355       pd->header.multiple_threads = 1;
 356 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 357       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 358 #endif
 359
 360 #ifdef NEED_DL_SYSINFO
 361       /* Copy the sysinfo value from the parent.  */
 362       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 363 #endif
 364
 365       /* The process ID is also the same as that of the caller.  */
 366       pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 367
 368       /* Allocate the DTV for this thread.  */
 369       if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 370         {
 371           /* Something went wrong.  */
 372           assert (errno == ENOMEM);
 373           return EAGAIN;
 374         }
 375
 376
 377       /* Prepare to modify global data.  */
 378       lll_lock (stack_cache_lock);
 379
 380       /* And add to the list of stacks in use.  */
 381       list_add (&pd->list, &__stack_user);
 382
 383       lll_unlock (stack_cache_lock);
 384     }
 385   else
 386     {
 387       /* Allocate some anonymous memory.  If possible use the cache.  */
 388       size_t guardsize;
 389       size_t reqsize;
 390       void *mem;
 391       const int prot = (PROT_READ | PROT_WRITE
 392                         | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
 393
 394 #if COLORING_INCREMENT != 0
 395       /* Add one more page for stack coloring.  Don't do it for stacks
 396          with 16 times pagesize or larger.  This might just cause
 397          unnecessary misalignment.  */
 398       if (size <= 16 * pagesize_m1)
 399         size += pagesize_m1 + 1;
 400 #endif
 401
 402       /* Adjust the stack size for alignment.  */
 403       size &= ~__static_tls_align_m1;
 404       assert (size != 0);
 405
 406       /* Make sure the size of the stack is enough for the guard and
 407          eventually the thread descriptor.  */
 408       guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
 409       if (__builtin_expect (size < ((guardsize + __static_tls_size
 410                                      + MINIMAL_REST_STACK + pagesize_m1)
 411                                     & ~pagesize_m1),
 412                             0))
 413         /* The stack is too small (or the guard too large).  */
 414         return EINVAL;
 415
 416       /* Try to get a stack from the cache.  */
 417       reqsize = size;
 418       pd = get_cached_stack (&size, &mem);
 419       if (pd == NULL)
 420         {
 421           /* To avoid aliasing effects on a larger scale than pages we
 422              adjust the allocated stack size if necessary.  This way
 423              allocations directly following each other will not have
 424              aliasing problems.  */
 425 #if MULTI_PAGE_ALIASING != 0
 426           if ((size % MULTI_PAGE_ALIASING) == 0)
 427             size += pagesize_m1 + 1;
 428 #endif
 429
 430           mem = mmap (NULL, size, prot,
 431                       MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
 432
 433           if (__builtin_expect (mem == MAP_FAILED, 0))
 434             {
 435 #ifdef ARCH_RETRY_MMAP
 436               mem = ARCH_RETRY_MMAP (size);
 437               if (__builtin_expect (mem == MAP_FAILED, 0))
 438 #endif
 439                 return errno;
 440             }
 441
 442           /* SIZE is guaranteed to be greater than zero.
 443              So we can never get a null pointer back from mmap.  */
 444           assert (mem != NULL);
 445
 446 #if COLORING_INCREMENT != 0
 447           /* Atomically increment NCREATED.  */
 448           unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
 449
 450           /* We chose the offset for coloring by incrementing it for
 451              every new thread by a fixed amount.  The offset used
 452              module the page size.  Even if coloring would be better
 453              relative to higher alignment values it makes no sense to
 454              do it since the mmap() interface does not allow us to
 455              specify any alignment for the returned memory block.  */
 456           size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
 457
 458           /* Make sure the coloring offsets does not disturb the alignment
 459              of the TCB and static TLS block.  */
 460           if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
 461             coloring = (((coloring + __static_tls_align_m1)
 462                          & ~(__static_tls_align_m1))
 463                         & ~pagesize_m1);
 464 #else
 465           /* Unless specified we do not make any adjustments.  */
 466 # define coloring 0
 467 #endif
 468
 469           /* Place the thread descriptor at the end of the stack.  */
 470 #if TLS_TCB_AT_TP
 471           pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
 472 #elif TLS_DTV_AT_TP
 473           pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
 474                                     - __static_tls_size)
 475                                     & ~__static_tls_align_m1)
 476                                    - TLS_PRE_TCB_SIZE);
 477 #endif
 478
 479           /* Remember the stack-related values.  */
 480           pd->stackblock = mem;
 481           pd->stackblock_size = size;
 482
 483           /* We allocated the first block thread-specific data array.
 484              This address will not change for the lifetime of this
 485              descriptor.  */
 486           pd->specific[0] = pd->specific_1stblock;
 487
 488           /* This is at least the second thread.  */
 489           pd->header.multiple_threads = 1;
 490 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
 491           __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 492 #endif
 493
 494 #ifdef NEED_DL_SYSINFO
 495           /* Copy the sysinfo value from the parent.  */
 496           THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
 497 #endif
 498
 499           /* The process ID is also the same as that of the caller.  */
 500           pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
 501
 502           /* Allocate the DTV for this thread.  */
 503           if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
 504             {
 505               /* Something went wrong.  */
 506               assert (errno == ENOMEM);
 507
 508               /* Free the stack memory we just allocated.  */
 509               (void) munmap (mem, size);
 510
 511               return EAGAIN;
 512             }
 513
 514
 515           /* Prepare to modify global data.  */
 516           lll_lock (stack_cache_lock);
 517
 518           /* And add to the list of stacks in use.  */
 519           list_add (&pd->list, &stack_used);
 520
 521           lll_unlock (stack_cache_lock);
 522
 523
 524           /* There might have been a race.  Another thread might have
 525              caused the stacks to get exec permission while this new
 526              stack was prepared.  Detect if this was possible and
 527              change the permission if necessary.  */
 528           if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
 529                                 && (prot & PROT_EXEC) == 0, 0))
 530             {
 531               int err = change_stack_perm (pd
 532 #ifdef NEED_SEPARATE_REGISTER_STACK
 533                                            , ~pagesize_m1
 534 #endif
 535                                            );
 536               if (err != 0)
 537                 {
 538                   /* Free the stack memory we just allocated.  */
 539                   (void) munmap (mem, size);
 540
 541                   return err;
 542                 }
 543             }
 544
 545
 546           /* Note that all of the stack and the thread descriptor is
 547              zeroed.  This means we do not have to initialize fields
 548              with initial value zero.  This is specifically true for
 549              the 'tid' field which is always set back to zero once the
 550              stack is not used anymore and for the 'guardsize' field
 551              which will be read next.  */
 552         }
 553
 554       /* Create or resize the guard area if necessary.  */
 555       if (__builtin_expect (guardsize > pd->guardsize, 0))
 556         {
 557 #ifdef NEED_SEPARATE_REGISTER_STACK
 558           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 559 #else
 560           char *guard = mem;
 561 #endif
 562           if (mprotect (guard, guardsize, PROT_NONE) != 0)
 563             {
 564               int err;
 565             mprot_error:
 566               err = errno;
 567
 568               lll_lock (stack_cache_lock);
 569
 570               /* Remove the thread from the list.  */
 571               list_del (&pd->list);
 572
 573               lll_unlock (stack_cache_lock);
 574
 575               /* Get rid of the TLS block we allocated.  */
 576               _dl_deallocate_tls (TLS_TPADJ (pd), false);
 577
 578               /* Free the stack memory regardless of whether the size
 579                  of the cache is over the limit or not.  If this piece
 580                  of memory caused problems we better do not use it
 581                  anymore.  Uh, and we ignore possible errors.  There
 582                  is nothing we could do.  */
 583               (void) munmap (mem, size);
 584
 585               return err;
 586             }
 587
 588           pd->guardsize = guardsize;
 589         }
 590       else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
 591                                  0))
 592         {
 593           /* The old guard area is too large.  */
 594
 595 #ifdef NEED_SEPARATE_REGISTER_STACK
 596           char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
 597           char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
 598
 599           if (oldguard < guard
 600               && mprotect (oldguard, guard - oldguard, prot) != 0)
 601             goto mprot_error;
 602
 603           if (mprotect (guard + guardsize,
 604                         oldguard + pd->guardsize - guard - guardsize,
 605                         prot) != 0)
 606             goto mprot_error;
 607 #else
 608           if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
 609                         prot) != 0)
 610             goto mprot_error;
 611 #endif
 612
 613           pd->guardsize = guardsize;
 614         }
 615       /* The pthread_getattr_np() calls need to get passed the size
 616          requested in the attribute, regardless of how large the
 617          actually used guardsize is.  */
 618       pd->reported_guardsize = guardsize;
 619     }
 620
 621   /* Initialize the lock.  We have to do this unconditionally since the
 622      stillborn thread could be canceled while the lock is taken.  */
 623   pd->lock = LLL_LOCK_INITIALIZER;
 624
 625   /* The robust mutex lists also need to be initialized
 626      unconditionally because the cleanup for the previous stack owner
 627      might have happened in the kernel.  */
 628   pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
 629                                   - offsetof (pthread_mutex_t,
 630                                               __data.__list.__next));
 631   pd->robust_head.list_op_pending = NULL;
 632 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 633   pd->robust_prev = &pd->robust_head;
 634 #endif
 635   pd->robust_head.list = &pd->robust_head;
 636
 637   /* We place the thread descriptor at the end of the stack.  */
 638   *pdp = pd;
 639
 640 #if TLS_TCB_AT_TP
 641   /* The stack begins before the TCB and the static TLS block.  */
 642   stacktop = ((char *) (pd + 1) - __static_tls_size);
 643 #elif TLS_DTV_AT_TP
 644   stacktop = (char *) (pd - 1);
 645 #endif
 646
 647 #ifdef NEED_SEPARATE_REGISTER_STACK
 648   *stack = pd->stackblock;
 649   *stacksize = stacktop - *stack;
 650 #else
 651   *stack = stacktop;
 652 #endif
 653
 654   return 0;
 655 }
 656
 657
 658 void
 659 internal_function
 660 __deallocate_stack (struct pthread *pd)
 661 {
 662   lll_lock (stack_cache_lock);
 663
 664   /* Remove the thread from the list of threads with user defined
 665      stacks.  */
 666   list_del (&pd->list);
 667
 668   /* Not much to do.  Just free the mmap()ed memory.  Note that we do
 669      not reset the 'used' flag in the 'tid' field.  This is done by
 670      the kernel.  If no thread has been created yet this field is
 671      still zero.  */
 672   if (__builtin_expect (! pd->user_stack, 1))
 673     (void) queue_stack (pd);
 674   else
 675     /* Free the memory associated with the ELF TLS.  */
 676     _dl_deallocate_tls (TLS_TPADJ (pd), false);
 677
 678   lll_unlock (stack_cache_lock);
 679 }
 680
 681
 682 int
 683 internal_function
 684 __make_stacks_executable (void **stack_endp)
 685 {
 686   /* First the main thread's stack.  */
 687   int err = _dl_make_stack_executable (stack_endp);
 688   if (err != 0)
 689     return err;
 690
 691 #ifdef NEED_SEPARATE_REGISTER_STACK
 692   const size_t pagemask = ~(__getpagesize () - 1);
 693 #endif
 694
 695   lll_lock (stack_cache_lock);
 696
 697   list_t *runp;
 698   list_for_each (runp, &stack_used)
 699     {
 700       err = change_stack_perm (list_entry (runp, struct pthread, list)
 701 #ifdef NEED_SEPARATE_REGISTER_STACK
 702                                , pagemask
 703 #endif
 704                                );
 705       if (err != 0)
 706         break;
 707     }
 708
 709   /* Also change the permission for the currently unused stacks.  This
 710      might be wasted time but better spend it here than adding a check
 711      in the fast path.  */
 712   if (err == 0)
 713     list_for_each (runp, &stack_cache)
 714       {
 715         err = change_stack_perm (list_entry (runp, struct pthread, list)
 716 #ifdef NEED_SEPARATE_REGISTER_STACK
 717                                  , pagemask
 718 #endif
 719                                  );
 720         if (err != 0)
 721           break;
 722       }
 723
 724   lll_unlock (stack_cache_lock);
 725
 726   return err;
 727 }
 728
 729
 730 /* In case of a fork() call the memory allocation in the child will be
 731    the same but only one thread is running.  All stacks except that of
 732    the one running thread are not used anymore.  We have to recycle
 733    them.  */
 734 void
 735 __reclaim_stacks (void)
 736 {
 737   struct pthread *self = (struct pthread *) THREAD_SELF;
 738
 739   /* No locking necessary.  The caller is the only stack in use.  */
 740
 741   /* Mark all stacks except the still running one as free.  */
 742   list_t *runp;
 743   list_for_each (runp, &stack_used)
 744     {
 745       struct pthread *curp = list_entry (runp, struct pthread, list);
 746       if (curp != self)
 747         {
 748           /* This marks the stack as free.  */
 749           curp->tid = 0;
 750
 751           /* The PID field must be initialized for the new process.  */
 752           curp->pid = self->pid;
 753
 754           /* Account for the size of the stack.  */
 755           stack_cache_actsize += curp->stackblock_size;
 756         }
 757     }
 758
 759   /* Reset the PIDs in any cached stacks.  */
 760   list_for_each (runp, &stack_cache)
 761     {
 762       struct pthread *curp = list_entry (runp, struct pthread, list);
 763       curp->pid = self->pid;
 764     }
 765
 766   /* Add the stack of all running threads to the cache.  */
 767   list_splice (&stack_used, &stack_cache);
 768
 769   /* Remove the entry for the current thread to from the cache list
 770      and add it to the list of running threads.  Which of the two
 771      lists is decided by the user_stack flag.  */
 772   list_del (&self->list);
 773
 774   /* Re-initialize the lists for all the threads.  */
 775   INIT_LIST_HEAD (&stack_used);
 776   INIT_LIST_HEAD (&__stack_user);
 777
 778   if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
 779     list_add (&self->list, &__stack_user);
 780   else
 781     list_add (&self->list, &stack_used);
 782
 783   /* There is one thread running.  */
 784   __nptl_nthreads = 1;
 785
 786   /* Initialize the lock.  */
 787   stack_cache_lock = LLL_LOCK_INITIALIZER;
 788 }
 789
 790
 791 #if HP_TIMING_AVAIL
 792 # undef __find_thread_by_id
 793 /* Find a thread given the thread ID.  */
 794 attribute_hidden
 795 struct pthread *
 796 __find_thread_by_id (pid_t tid)
 797 {
 798   struct pthread *result = NULL;
 799
 800   lll_lock (stack_cache_lock);
 801
 802   /* Iterate over the list with system-allocated threads first.  */
 803   list_t *runp;
 804   list_for_each (runp, &stack_used)
 805     {
 806       struct pthread *curp;
 807
 808       curp = list_entry (runp, struct pthread, list);
 809
 810       if (curp->tid == tid)
 811         {
 812           result = curp;
 813           goto out;
 814         }
 815     }
 816
 817   /* Now the list with threads using user-allocated stacks.  */
 818   list_for_each (runp, &__stack_user)
 819     {
 820       struct pthread *curp;
 821
 822       curp = list_entry (runp, struct pthread, list);
 823
 824       if (curp->tid == tid)
 825         {
 826           result = curp;
 827           goto out;
 828         }
 829     }
 830
 831  out:
 832   lll_unlock (stack_cache_lock);
 833
 834   return result;
 835 }
 836 #endif
 837
 838
 839 static void
 840 internal_function
 841 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
 842 {
 843   if (! IS_DETACHED (t))
 844     {
 845       int ch;
 846       do
 847         {
 848           ch = t->cancelhandling;
 849
 850           /* If the thread is exiting right now, ignore it.  */
 851           if ((ch & EXITING_BITMASK) != 0)
 852             return;
 853         }
 854       while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
 855                                                    ch | SETXID_BITMASK, ch));
 856     }
 857
 858   int val;
 859   INTERNAL_SYSCALL_DECL (err);
 860 #if __ASSUME_TGKILL
 861   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 862                           t->tid, SIGSETXID);
 863 #else
 864 # ifdef __NR_tgkill
 865   val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
 866                           t->tid, SIGSETXID);
 867   if (INTERNAL_SYSCALL_ERROR_P (val, err)
 868       && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
 869 # endif
 870     val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
 871 #endif
 872
 873   if (!INTERNAL_SYSCALL_ERROR_P (val, err))
 874     atomic_increment (&cmdp->cntr);
 875 }
 876
 877
 878 int
 879 attribute_hidden
 880 __nptl_setxid (struct xid_command *cmdp)
 881 {
 882   int result;
 883   lll_lock (stack_cache_lock);
 884
 885   __xidcmd = cmdp;
 886   cmdp->cntr = 0;
 887
 888   struct pthread *self = THREAD_SELF;
 889
 890   /* Iterate over the list with system-allocated threads first.  */
 891   list_t *runp;
 892   list_for_each (runp, &stack_used)
 893     {
 894       struct pthread *t = list_entry (runp, struct pthread, list);
 895       if (t == self)
 896         continue;
 897
 898       setxid_signal_thread (cmdp, t);
 899     }
 900
 901   /* Now the list with threads using user-allocated stacks.  */
 902   list_for_each (runp, &__stack_user)
 903     {
 904       struct pthread *t = list_entry (runp, struct pthread, list);
 905       if (t == self)
 906         continue;
 907
 908       setxid_signal_thread (cmdp, t);
 909     }
 910
 911   int cur = cmdp->cntr;
 912   while (cur != 0)
 913     {
 914       lll_futex_wait (&cmdp->cntr, cur);
 915       cur = cmdp->cntr;
 916     }
 917
 918   /* This must be last, otherwise the current thread might not have
 919      permissions to send SIGSETXID syscall to the other threads.  */
 920   INTERNAL_SYSCALL_DECL (err);
 921   result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
 922                                  cmdp->id[0], cmdp->id[1], cmdp->id[2]);
 923   if (INTERNAL_SYSCALL_ERROR_P (result, err))
 924     {
 925       __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
 926       result = -1;
 927     }
 928
 929   lll_unlock (stack_cache_lock);
 930   return result;
 931 }
 932
 933 static inline void __attribute__((always_inline))
 934 init_one_static_tls (struct pthread *curp, struct link_map *map)
 935 {
 936   dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
 937 # if TLS_TCB_AT_TP
 938   void *dest = (char *) curp - map->l_tls_offset;
 939 # elif TLS_DTV_AT_TP
 940   void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
 941 # else
 942 #  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
 943 # endif
 944
 945   /* Fill in the DTV slot so that a later LD/GD access will find it.  */
 946   dtv[map->l_tls_modid].pointer.val = dest;
 947   dtv[map->l_tls_modid].pointer.is_static = true;
 948
 949   /* Initialize the memory.  */
 950   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 951           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 952 }
 953
 954 void
 955 attribute_hidden
 956 __pthread_init_static_tls (struct link_map *map)
 957 {
 958   lll_lock (stack_cache_lock);
 959
 960   /* Iterate over the list with system-allocated threads first.  */
 961   list_t *runp;
 962   list_for_each (runp, &stack_used)
 963     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 964
 965   /* Now the list with threads using user-allocated stacks.  */
 966   list_for_each (runp, &__stack_user)
 967     init_one_static_tls (list_entry (runp, struct pthread, list), map);
 968
 969   lll_unlock (stack_cache_lock);
 970 }