Add tst-wcstod-round
[glibc.git] / elf / dl-tls.c
blob17567ad1b3e22460880be34d3fc59e9ef3bf2ebb
1 /* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2016 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
19 #include <assert.h>
20 #include <errno.h>
21 #include <libintl.h>
22 #include <signal.h>
23 #include <stdlib.h>
24 #include <unistd.h>
25 #include <sys/param.h>
26 #include <atomic.h>
28 #include <tls.h>
29 #include <dl-tls.h>
30 #include <ldsodefs.h>
32 /* Amount of excess space to allocate in the static TLS area
33 to allow dynamic loading of modules defining IE-model TLS data. */
34 #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
37 /* Out-of-memory handler. */
38 static void
39 __attribute__ ((__noreturn__))
40 oom (void)
42 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
46 size_t
47 internal_function
48 _dl_next_tls_modid (void)
50 size_t result;
52 if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
54 size_t disp = 0;
55 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
57 /* Note that this branch will never be executed during program
58 start since there are no gaps at that time. Therefore it
59 does not matter that the dl_tls_dtv_slotinfo is not allocated
60 yet when the function is called for the first times.
62 NB: the offset +1 is due to the fact that DTV[0] is used
63 for something else. */
64 result = GL(dl_tls_static_nelem) + 1;
65 if (result <= GL(dl_tls_max_dtv_idx))
68 while (result - disp < runp->len)
70 if (runp->slotinfo[result - disp].map == NULL)
71 break;
73 ++result;
74 assert (result <= GL(dl_tls_max_dtv_idx) + 1);
77 if (result - disp < runp->len)
78 break;
80 disp += runp->len;
82 while ((runp = runp->next) != NULL);
84 if (result > GL(dl_tls_max_dtv_idx))
86 /* The new index must indeed be exactly one higher than the
87 previous high. */
88 assert (result == GL(dl_tls_max_dtv_idx) + 1);
89 /* There is no gap anymore. */
90 GL(dl_tls_dtv_gaps) = false;
92 goto nogaps;
95 else
97 /* No gaps, allocate a new entry. */
98 nogaps:
100 result = ++GL(dl_tls_max_dtv_idx);
103 return result;
107 size_t
108 internal_function
109 _dl_count_modids (void)
111 /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
112 we fail to load a module and unload it leaving a gap. If we don't
113 have gaps then the number of modids is the current maximum so
114 return that. */
115 if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
116 return GL(dl_tls_max_dtv_idx);
118 /* We have gaps and are forced to count the non-NULL entries. */
119 size_t n = 0;
120 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
121 while (runp != NULL)
123 for (size_t i = 0; i < runp->len; ++i)
124 if (runp->slotinfo[i].map != NULL)
125 ++n;
127 runp = runp->next;
130 return n;
134 #ifdef SHARED
135 void
136 internal_function
137 _dl_determine_tlsoffset (void)
139 size_t max_align = TLS_TCB_ALIGN;
140 size_t freetop = 0;
141 size_t freebottom = 0;
143 /* The first element of the dtv slot info list is allocated. */
144 assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
145 /* There is at this point only one element in the
146 dl_tls_dtv_slotinfo_list list. */
147 assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
149 struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
151 /* Determining the offset of the various parts of the static TLS
152 block has several dependencies. In addition we have to work
153 around bugs in some toolchains.
155 Each TLS block from the objects available at link time has a size
156 and an alignment requirement. The GNU ld computes the alignment
157 requirements for the data at the positions *in the file*, though.
158 I.e, it is not simply possible to allocate a block with the size
159 of the TLS program header entry. The data is layed out assuming
160 that the first byte of the TLS block fulfills
162 p_vaddr mod p_align == &TLS_BLOCK mod p_align
164 This means we have to add artificial padding at the beginning of
165 the TLS block. These bytes are never used for the TLS data in
166 this module but the first byte allocated must be aligned
167 according to mod p_align == 0 so that the first byte of the TLS
168 block is aligned according to p_vaddr mod p_align. This is ugly
169 and the linker can help by computing the offsets in the TLS block
170 assuming the first byte of the TLS block is aligned according to
171 p_align.
173 The extra space which might be allocated before the first byte of
174 the TLS block need not go unused. The code below tries to use
175 that memory for the next TLS block. This can work if the total
176 memory requirement for the next TLS block is smaller than the
177 gap. */
179 #if TLS_TCB_AT_TP
180 /* We simply start with zero. */
181 size_t offset = 0;
183 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
185 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
187 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
188 & (slotinfo[cnt].map->l_tls_align - 1));
189 size_t off;
190 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
192 if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
194 off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
195 - firstbyte, slotinfo[cnt].map->l_tls_align)
196 + firstbyte;
197 if (off <= freebottom)
199 freetop = off;
201 /* XXX For some architectures we perhaps should store the
202 negative offset. */
203 slotinfo[cnt].map->l_tls_offset = off;
204 continue;
208 off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
209 slotinfo[cnt].map->l_tls_align) + firstbyte;
210 if (off > offset + slotinfo[cnt].map->l_tls_blocksize
211 + (freebottom - freetop))
213 freetop = offset;
214 freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
216 offset = off;
218 /* XXX For some architectures we perhaps should store the
219 negative offset. */
220 slotinfo[cnt].map->l_tls_offset = off;
223 GL(dl_tls_static_used) = offset;
224 GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
225 + TLS_TCB_SIZE);
226 #elif TLS_DTV_AT_TP
227 /* The TLS blocks start right after the TCB. */
228 size_t offset = TLS_TCB_SIZE;
230 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
232 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
234 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
235 & (slotinfo[cnt].map->l_tls_align - 1));
236 size_t off;
237 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
239 if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
241 off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
242 if (off - freebottom < firstbyte)
243 off += slotinfo[cnt].map->l_tls_align;
244 if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
246 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
247 freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
248 - firstbyte);
249 continue;
253 off = roundup (offset, slotinfo[cnt].map->l_tls_align);
254 if (off - offset < firstbyte)
255 off += slotinfo[cnt].map->l_tls_align;
257 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
258 if (off - firstbyte - offset > freetop - freebottom)
260 freebottom = offset;
261 freetop = off - firstbyte;
264 offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
267 GL(dl_tls_static_used) = offset;
268 GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
269 TLS_TCB_ALIGN);
270 #else
271 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
272 #endif
274 /* The alignment requirement for the static TLS block. */
275 GL(dl_tls_static_align) = max_align;
279 /* This is called only when the data structure setup was skipped at startup,
280 when there was no need for it then. Now we have dynamically loaded
281 something needing TLS, or libpthread needs it. */
283 internal_function
284 _dl_tls_setup (void)
286 assert (GL(dl_tls_dtv_slotinfo_list) == NULL);
287 assert (GL(dl_tls_max_dtv_idx) == 0);
289 const size_t nelem = 2 + TLS_SLOTINFO_SURPLUS;
291 GL(dl_tls_dtv_slotinfo_list)
292 = calloc (1, (sizeof (struct dtv_slotinfo_list)
293 + nelem * sizeof (struct dtv_slotinfo)));
294 if (GL(dl_tls_dtv_slotinfo_list) == NULL)
295 return -1;
297 GL(dl_tls_dtv_slotinfo_list)->len = nelem;
299 /* Number of elements in the static TLS block. It can't be zero
300 because of various assumptions. The one element is null. */
301 GL(dl_tls_static_nelem) = GL(dl_tls_max_dtv_idx) = 1;
303 /* This initializes more variables for us. */
304 _dl_determine_tlsoffset ();
306 return 0;
308 rtld_hidden_def (_dl_tls_setup)
309 #endif
311 static void *
312 internal_function
313 allocate_dtv (void *result)
315 dtv_t *dtv;
316 size_t dtv_length;
318 /* We allocate a few more elements in the dtv than are needed for the
319 initial set of modules. This should avoid in most cases expansions
320 of the dtv. */
321 dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
322 dtv = calloc (dtv_length + 2, sizeof (dtv_t));
323 if (dtv != NULL)
325 /* This is the initial length of the dtv. */
326 dtv[0].counter = dtv_length;
328 /* The rest of the dtv (including the generation counter) is
329 Initialize with zero to indicate nothing there. */
331 /* Add the dtv to the thread data structures. */
332 INSTALL_DTV (result, dtv);
334 else
335 result = NULL;
337 return result;
341 /* Get size and alignment requirements of the static TLS block. */
342 void
343 internal_function
344 _dl_get_tls_static_info (size_t *sizep, size_t *alignp)
346 *sizep = GL(dl_tls_static_size);
347 *alignp = GL(dl_tls_static_align);
350 /* Derive the location of the pointer to the start of the original
351 allocation (before alignment) from the pointer to the TCB. */
352 static inline void **
353 tcb_to_pointer_to_free_location (void *tcb)
355 #if TLS_TCB_AT_TP
356 /* The TCB follows the TLS blocks, and the pointer to the front
357 follows the TCB. */
358 void **original_pointer_location = tcb + TLS_TCB_SIZE;
359 #elif TLS_DTV_AT_TP
360 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
361 before that. */
362 void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *);
363 #endif
364 return original_pointer_location;
367 void *
368 internal_function
369 _dl_allocate_tls_storage (void)
371 void *result;
372 size_t size = GL(dl_tls_static_size);
374 #if TLS_DTV_AT_TP
375 /* Memory layout is:
376 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
377 ^ This should be returned. */
378 size += TLS_PRE_TCB_SIZE;
379 #endif
381 /* Perform the allocation. Reserve space for the required alignment
382 and the pointer to the original allocation. */
383 size_t alignment = GL(dl_tls_static_align);
384 void *allocated = malloc (size + alignment + sizeof (void *));
385 if (__glibc_unlikely (allocated == NULL))
386 return NULL;
388 /* Perform alignment and allocate the DTV. */
389 #if TLS_TCB_AT_TP
390 /* The TCB follows the TLS blocks, which determine the alignment.
391 (TCB alignment requirements have been taken into account when
392 calculating GL(dl_tls_static_align).) */
393 void *aligned = (void *) roundup ((uintptr_t) allocated, alignment);
394 result = aligned + size - TLS_TCB_SIZE;
396 /* Clear the TCB data structure. We can't ask the caller (i.e.
397 libpthread) to do it, because we will initialize the DTV et al. */
398 memset (result, '\0', TLS_TCB_SIZE);
399 #elif TLS_DTV_AT_TP
400 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
401 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
402 TLS block alignment, and not just the TLS blocks after it. This
403 can leave an unused alignment gap between the TCB and the TLS
404 blocks. */
405 result = (void *) roundup
406 (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated,
407 alignment);
409 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
410 it. We can't ask the caller (i.e. libpthread) to do it, because
411 we will initialize the DTV et al. */
412 memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
413 #endif
415 /* Record the value of the original pointer for later
416 deallocation. */
417 *tcb_to_pointer_to_free_location (result) = allocated;
419 result = allocate_dtv (result);
420 if (result == NULL)
421 free (allocated);
422 return result;
426 #ifndef SHARED
427 extern dtv_t _dl_static_dtv[];
428 # define _dl_initial_dtv (&_dl_static_dtv[1])
429 #endif
431 static dtv_t *
432 _dl_resize_dtv (dtv_t *dtv)
434 /* Resize the dtv. */
435 dtv_t *newp;
436 /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
437 other threads concurrently. */
438 size_t newsize
439 = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS;
440 size_t oldsize = dtv[-1].counter;
442 if (dtv == GL(dl_initial_dtv))
444 /* This is the initial dtv that was either statically allocated in
445 __libc_setup_tls or allocated during rtld startup using the
446 dl-minimal.c malloc instead of the real malloc. We can't free
447 it, we have to abandon the old storage. */
449 newp = malloc ((2 + newsize) * sizeof (dtv_t));
450 if (newp == NULL)
451 oom ();
452 memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
454 else
456 newp = realloc (&dtv[-1],
457 (2 + newsize) * sizeof (dtv_t));
458 if (newp == NULL)
459 oom ();
462 newp[0].counter = newsize;
464 /* Clear the newly allocated part. */
465 memset (newp + 2 + oldsize, '\0',
466 (newsize - oldsize) * sizeof (dtv_t));
468 /* Return the generation counter. */
469 return &newp[1];
473 void *
474 internal_function
475 _dl_allocate_tls_init (void *result)
477 if (result == NULL)
478 /* The memory allocation failed. */
479 return NULL;
481 dtv_t *dtv = GET_DTV (result);
482 struct dtv_slotinfo_list *listp;
483 size_t total = 0;
484 size_t maxgen = 0;
486 /* Check if the current dtv is big enough. */
487 if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
489 /* Resize the dtv. */
490 dtv = _dl_resize_dtv (dtv);
492 /* Install this new dtv in the thread data structures. */
493 INSTALL_DTV (result, &dtv[-1]);
496 /* We have to prepare the dtv for all currently loaded modules using
497 TLS. For those which are dynamically loaded we add the values
498 indicating deferred allocation. */
499 listp = GL(dl_tls_dtv_slotinfo_list);
500 while (1)
502 size_t cnt;
504 for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
506 struct link_map *map;
507 void *dest;
509 /* Check for the total number of used slots. */
510 if (total + cnt > GL(dl_tls_max_dtv_idx))
511 break;
513 map = listp->slotinfo[cnt].map;
514 if (map == NULL)
515 /* Unused entry. */
516 continue;
518 /* Keep track of the maximum generation number. This might
519 not be the generation counter. */
520 assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
521 maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
523 dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
524 dtv[map->l_tls_modid].pointer.to_free = NULL;
526 if (map->l_tls_offset == NO_TLS_OFFSET
527 || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
528 continue;
530 assert (map->l_tls_modid == total + cnt);
531 assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
532 #if TLS_TCB_AT_TP
533 assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
534 dest = (char *) result - map->l_tls_offset;
535 #elif TLS_DTV_AT_TP
536 dest = (char *) result + map->l_tls_offset;
537 #else
538 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
539 #endif
541 /* Copy the initialization image and clear the BSS part. */
542 memset (__mempcpy (dest, map->l_tls_initimage,
543 map->l_tls_initimage_size), '\0',
544 map->l_tls_blocksize - map->l_tls_initimage_size);
547 total += cnt;
548 if (total >= GL(dl_tls_max_dtv_idx))
549 break;
551 listp = listp->next;
552 assert (listp != NULL);
555 /* The DTV version is up-to-date now. */
556 dtv[0].counter = maxgen;
558 return result;
560 rtld_hidden_def (_dl_allocate_tls_init)
562 void *
563 internal_function
564 _dl_allocate_tls (void *mem)
566 return _dl_allocate_tls_init (mem == NULL
567 ? _dl_allocate_tls_storage ()
568 : allocate_dtv (mem));
570 rtld_hidden_def (_dl_allocate_tls)
573 void
574 internal_function
575 _dl_deallocate_tls (void *tcb, bool dealloc_tcb)
577 dtv_t *dtv = GET_DTV (tcb);
579 /* We need to free the memory allocated for non-static TLS. */
580 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
581 free (dtv[1 + cnt].pointer.to_free);
583 /* The array starts with dtv[-1]. */
584 if (dtv != GL(dl_initial_dtv))
585 free (dtv - 1);
587 if (dealloc_tcb)
588 free (*tcb_to_pointer_to_free_location (tcb));
590 rtld_hidden_def (_dl_deallocate_tls)
593 #ifdef SHARED
594 /* The __tls_get_addr function has two basic forms which differ in the
595 arguments. The IA-64 form takes two parameters, the module ID and
596 offset. The form used, among others, on IA-32 takes a reference to
597 a special structure which contain the same information. The second
598 form seems to be more often used (in the moment) so we default to
599 it. Users of the IA-64 form have to provide adequate definitions
600 of the following macros. */
601 # ifndef GET_ADDR_ARGS
602 # define GET_ADDR_ARGS tls_index *ti
603 # define GET_ADDR_PARAM ti
604 # endif
605 # ifndef GET_ADDR_MODULE
606 # define GET_ADDR_MODULE ti->ti_module
607 # endif
608 # ifndef GET_ADDR_OFFSET
609 # define GET_ADDR_OFFSET ti->ti_offset
610 # endif
612 /* Allocate one DTV entry. */
613 static struct dtv_pointer
614 allocate_dtv_entry (size_t alignment, size_t size)
616 if (powerof2 (alignment) && alignment <= _Alignof (max_align_t))
618 /* The alignment is supported by malloc. */
619 void *ptr = malloc (size);
620 return (struct dtv_pointer) { ptr, ptr };
623 /* Emulate memalign to by manually aligning a pointer returned by
624 malloc. First compute the size with an overflow check. */
625 size_t alloc_size = size + alignment;
626 if (alloc_size < size)
627 return (struct dtv_pointer) {};
629 /* Perform the allocation. This is the pointer we need to free
630 later. */
631 void *start = malloc (alloc_size);
632 if (start == NULL)
633 return (struct dtv_pointer) {};
635 /* Find the aligned position within the larger allocation. */
636 void *aligned = (void *) roundup ((uintptr_t) start, alignment);
638 return (struct dtv_pointer) { .val = aligned, .to_free = start };
641 static struct dtv_pointer
642 allocate_and_init (struct link_map *map)
644 struct dtv_pointer result = allocate_dtv_entry
645 (map->l_tls_align, map->l_tls_blocksize);
646 if (result.val == NULL)
647 oom ();
649 /* Initialize the memory. */
650 memset (__mempcpy (result.val, map->l_tls_initimage,
651 map->l_tls_initimage_size),
652 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
654 return result;
658 struct link_map *
659 _dl_update_slotinfo (unsigned long int req_modid)
661 struct link_map *the_map = NULL;
662 dtv_t *dtv = THREAD_DTV ();
664 /* The global dl_tls_dtv_slotinfo array contains for each module
665 index the generation counter current when the entry was created.
666 This array never shrinks so that all module indices which were
667 valid at some time can be used to access it. Before the first
668 use of a new module index in this function the array was extended
669 appropriately. Access also does not have to be guarded against
670 modifications of the array. It is assumed that pointer-size
671 values can be read atomically even in SMP environments. It is
672 possible that other threads at the same time dynamically load
673 code and therefore add to the slotinfo list. This is a problem
674 since we must not pick up any information about incomplete work.
675 The solution to this is to ignore all dtv slots which were
676 created after the one we are currently interested. We know that
677 dynamic loading for this module is completed and this is the last
678 load operation we know finished. */
679 unsigned long int idx = req_modid;
680 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
682 while (idx >= listp->len)
684 idx -= listp->len;
685 listp = listp->next;
688 if (dtv[0].counter < listp->slotinfo[idx].gen)
690 /* The generation counter for the slot is higher than what the
691 current dtv implements. We have to update the whole dtv but
692 only those entries with a generation counter <= the one for
693 the entry we need. */
694 size_t new_gen = listp->slotinfo[idx].gen;
695 size_t total = 0;
697 /* We have to look through the entire dtv slotinfo list. */
698 listp = GL(dl_tls_dtv_slotinfo_list);
701 for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
703 size_t gen = listp->slotinfo[cnt].gen;
705 if (gen > new_gen)
706 /* This is a slot for a generation younger than the
707 one we are handling now. It might be incompletely
708 set up so ignore it. */
709 continue;
711 /* If the entry is older than the current dtv layout we
712 know we don't have to handle it. */
713 if (gen <= dtv[0].counter)
714 continue;
716 /* If there is no map this means the entry is empty. */
717 struct link_map *map = listp->slotinfo[cnt].map;
718 if (map == NULL)
720 if (dtv[-1].counter >= total + cnt)
722 /* If this modid was used at some point the memory
723 might still be allocated. */
724 free (dtv[total + cnt].pointer.to_free);
725 dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
726 dtv[total + cnt].pointer.to_free = NULL;
729 continue;
732 /* Check whether the current dtv array is large enough. */
733 size_t modid = map->l_tls_modid;
734 assert (total + cnt == modid);
735 if (dtv[-1].counter < modid)
737 /* Resize the dtv. */
738 dtv = _dl_resize_dtv (dtv);
740 assert (modid <= dtv[-1].counter);
742 /* Install this new dtv in the thread data
743 structures. */
744 INSTALL_NEW_DTV (dtv);
747 /* If there is currently memory allocate for this
748 dtv entry free it. */
749 /* XXX Ideally we will at some point create a memory
750 pool. */
751 free (dtv[modid].pointer.to_free);
752 dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
753 dtv[modid].pointer.to_free = NULL;
755 if (modid == req_modid)
756 the_map = map;
759 total += listp->len;
761 while ((listp = listp->next) != NULL);
763 /* This will be the new maximum generation counter. */
764 dtv[0].counter = new_gen;
767 return the_map;
771 static void *
772 __attribute_noinline__
773 tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
775 /* The allocation was deferred. Do it now. */
776 if (the_map == NULL)
778 /* Find the link map for this module. */
779 size_t idx = GET_ADDR_MODULE;
780 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
782 while (idx >= listp->len)
784 idx -= listp->len;
785 listp = listp->next;
788 the_map = listp->slotinfo[idx].map;
791 /* Make sure that, if a dlopen running in parallel forces the
792 variable into static storage, we'll wait until the address in the
793 static TLS block is set up, and use that. If we're undecided
794 yet, make sure we make the decision holding the lock as well. */
795 if (__glibc_unlikely (the_map->l_tls_offset
796 != FORCED_DYNAMIC_TLS_OFFSET))
798 __rtld_lock_lock_recursive (GL(dl_load_lock));
799 if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
801 the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
802 __rtld_lock_unlock_recursive (GL(dl_load_lock));
804 else if (__glibc_likely (the_map->l_tls_offset
805 != FORCED_DYNAMIC_TLS_OFFSET))
807 #if TLS_TCB_AT_TP
808 void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
809 #elif TLS_DTV_AT_TP
810 void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
811 #else
812 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
813 #endif
814 __rtld_lock_unlock_recursive (GL(dl_load_lock));
816 dtv[GET_ADDR_MODULE].pointer.to_free = NULL;
817 dtv[GET_ADDR_MODULE].pointer.val = p;
819 return (char *) p + GET_ADDR_OFFSET;
821 else
822 __rtld_lock_unlock_recursive (GL(dl_load_lock));
824 struct dtv_pointer result = allocate_and_init (the_map);
825 dtv[GET_ADDR_MODULE].pointer = result;
826 assert (result.to_free != NULL);
828 return (char *) result.val + GET_ADDR_OFFSET;
832 static struct link_map *
833 __attribute_noinline__
834 update_get_addr (GET_ADDR_ARGS)
836 struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
837 dtv_t *dtv = THREAD_DTV ();
839 void *p = dtv[GET_ADDR_MODULE].pointer.val;
841 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
842 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
844 return (void *) p + GET_ADDR_OFFSET;
847 /* For all machines that have a non-macro version of __tls_get_addr, we
848 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
849 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
850 in ld.so for __tls_get_addr. */
852 #ifndef __tls_get_addr
853 extern void * __tls_get_addr (GET_ADDR_ARGS);
854 rtld_hidden_proto (__tls_get_addr)
855 rtld_hidden_def (__tls_get_addr)
856 #endif
858 /* The generic dynamic and local dynamic model cannot be used in
859 statically linked applications. */
860 void *
861 __tls_get_addr (GET_ADDR_ARGS)
863 dtv_t *dtv = THREAD_DTV ();
865 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
866 return update_get_addr (GET_ADDR_PARAM);
868 void *p = dtv[GET_ADDR_MODULE].pointer.val;
870 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
871 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
873 return (char *) p + GET_ADDR_OFFSET;
875 #endif
878 /* Look up the module's TLS block as for __tls_get_addr,
879 but never touch anything. Return null if it's not allocated yet. */
880 void *
881 _dl_tls_get_addr_soft (struct link_map *l)
883 if (__glibc_unlikely (l->l_tls_modid == 0))
884 /* This module has no TLS segment. */
885 return NULL;
887 dtv_t *dtv = THREAD_DTV ();
888 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
890 /* This thread's DTV is not completely current,
891 but it might already cover this module. */
893 if (l->l_tls_modid >= dtv[-1].counter)
894 /* Nope. */
895 return NULL;
897 size_t idx = l->l_tls_modid;
898 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
899 while (idx >= listp->len)
901 idx -= listp->len;
902 listp = listp->next;
905 /* We've reached the slot for this module.
906 If its generation counter is higher than the DTV's,
907 this thread does not know about this module yet. */
908 if (dtv[0].counter < listp->slotinfo[idx].gen)
909 return NULL;
912 void *data = dtv[l->l_tls_modid].pointer.val;
913 if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
914 /* The DTV is current, but this thread has not yet needed
915 to allocate this module's segment. */
916 data = NULL;
918 return data;
922 void
923 _dl_add_to_slotinfo (struct link_map *l)
925 /* Now that we know the object is loaded successfully add
926 modules containing TLS data to the dtv info table. We
927 might have to increase its size. */
928 struct dtv_slotinfo_list *listp;
929 struct dtv_slotinfo_list *prevp;
930 size_t idx = l->l_tls_modid;
932 /* Find the place in the dtv slotinfo list. */
933 listp = GL(dl_tls_dtv_slotinfo_list);
934 prevp = NULL; /* Needed to shut up gcc. */
937 /* Does it fit in the array of this list element? */
938 if (idx < listp->len)
939 break;
940 idx -= listp->len;
941 prevp = listp;
942 listp = listp->next;
944 while (listp != NULL);
946 if (listp == NULL)
948 /* When we come here it means we have to add a new element
949 to the slotinfo list. And the new module must be in
950 the first slot. */
951 assert (idx == 0);
953 listp = prevp->next = (struct dtv_slotinfo_list *)
954 malloc (sizeof (struct dtv_slotinfo_list)
955 + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
956 if (listp == NULL)
958 /* We ran out of memory. We will simply fail this
959 call but don't undo anything we did so far. The
960 application will crash or be terminated anyway very
961 soon. */
963 /* We have to do this since some entries in the dtv
964 slotinfo array might already point to this
965 generation. */
966 ++GL(dl_tls_generation);
968 _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
969 cannot create TLS data structures"));
972 listp->len = TLS_SLOTINFO_SURPLUS;
973 listp->next = NULL;
974 memset (listp->slotinfo, '\0',
975 TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
978 /* Add the information into the slotinfo data structure. */
979 listp->slotinfo[idx].map = l;
980 listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;