1 /* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2017 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
25 #include <sys/param.h>
32 /* Amount of excess space to allocate in the static TLS area
33 to allow dynamic loading of modules defining IE-model TLS data. */
34 #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
37 /* Out-of-memory handler. */
39 __attribute__ ((__noreturn__
))
42 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
48 _dl_next_tls_modid (void)
52 if (__builtin_expect (GL(dl_tls_dtv_gaps
), false))
55 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
57 /* Note that this branch will never be executed during program
58 start since there are no gaps at that time. Therefore it
59 does not matter that the dl_tls_dtv_slotinfo is not allocated
60 yet when the function is called for the first times.
62 NB: the offset +1 is due to the fact that DTV[0] is used
63 for something else. */
64 result
= GL(dl_tls_static_nelem
) + 1;
65 if (result
<= GL(dl_tls_max_dtv_idx
))
68 while (result
- disp
< runp
->len
)
70 if (runp
->slotinfo
[result
- disp
].map
== NULL
)
74 assert (result
<= GL(dl_tls_max_dtv_idx
) + 1);
77 if (result
- disp
< runp
->len
)
82 while ((runp
= runp
->next
) != NULL
);
84 if (result
> GL(dl_tls_max_dtv_idx
))
86 /* The new index must indeed be exactly one higher than the
88 assert (result
== GL(dl_tls_max_dtv_idx
) + 1);
89 /* There is no gap anymore. */
90 GL(dl_tls_dtv_gaps
) = false;
97 /* No gaps, allocate a new entry. */
100 result
= ++GL(dl_tls_max_dtv_idx
);
109 _dl_count_modids (void)
111 /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
112 we fail to load a module and unload it leaving a gap. If we don't
113 have gaps then the number of modids is the current maximum so
115 if (__glibc_likely (!GL(dl_tls_dtv_gaps
)))
116 return GL(dl_tls_max_dtv_idx
);
118 /* We have gaps and are forced to count the non-NULL entries. */
120 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
123 for (size_t i
= 0; i
< runp
->len
; ++i
)
124 if (runp
->slotinfo
[i
].map
!= NULL
)
137 _dl_determine_tlsoffset (void)
139 size_t max_align
= TLS_TCB_ALIGN
;
141 size_t freebottom
= 0;
143 /* The first element of the dtv slot info list is allocated. */
144 assert (GL(dl_tls_dtv_slotinfo_list
) != NULL
);
145 /* There is at this point only one element in the
146 dl_tls_dtv_slotinfo_list list. */
147 assert (GL(dl_tls_dtv_slotinfo_list
)->next
== NULL
);
149 struct dtv_slotinfo
*slotinfo
= GL(dl_tls_dtv_slotinfo_list
)->slotinfo
;
151 /* Determining the offset of the various parts of the static TLS
152 block has several dependencies. In addition we have to work
153 around bugs in some toolchains.
155 Each TLS block from the objects available at link time has a size
156 and an alignment requirement. The GNU ld computes the alignment
157 requirements for the data at the positions *in the file*, though.
158 I.e, it is not simply possible to allocate a block with the size
159 of the TLS program header entry. The data is layed out assuming
160 that the first byte of the TLS block fulfills
162 p_vaddr mod p_align == &TLS_BLOCK mod p_align
164 This means we have to add artificial padding at the beginning of
165 the TLS block. These bytes are never used for the TLS data in
166 this module but the first byte allocated must be aligned
167 according to mod p_align == 0 so that the first byte of the TLS
168 block is aligned according to p_vaddr mod p_align. This is ugly
169 and the linker can help by computing the offsets in the TLS block
170 assuming the first byte of the TLS block is aligned according to
173 The extra space which might be allocated before the first byte of
174 the TLS block need not go unused. The code below tries to use
175 that memory for the next TLS block. This can work if the total
176 memory requirement for the next TLS block is smaller than the
180 /* We simply start with zero. */
183 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
185 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
187 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
188 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
190 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
192 if (freebottom
- freetop
>= slotinfo
[cnt
].map
->l_tls_blocksize
)
194 off
= roundup (freetop
+ slotinfo
[cnt
].map
->l_tls_blocksize
195 - firstbyte
, slotinfo
[cnt
].map
->l_tls_align
)
197 if (off
<= freebottom
)
201 /* XXX For some architectures we perhaps should store the
203 slotinfo
[cnt
].map
->l_tls_offset
= off
;
208 off
= roundup (offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
,
209 slotinfo
[cnt
].map
->l_tls_align
) + firstbyte
;
210 if (off
> offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
211 + (freebottom
- freetop
))
214 freebottom
= off
- slotinfo
[cnt
].map
->l_tls_blocksize
;
218 /* XXX For some architectures we perhaps should store the
220 slotinfo
[cnt
].map
->l_tls_offset
= off
;
223 GL(dl_tls_static_used
) = offset
;
224 GL(dl_tls_static_size
) = (roundup (offset
+ TLS_STATIC_SURPLUS
, max_align
)
227 /* The TLS blocks start right after the TCB. */
228 size_t offset
= TLS_TCB_SIZE
;
230 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
232 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
234 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
235 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
237 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
239 if (slotinfo
[cnt
].map
->l_tls_blocksize
<= freetop
- freebottom
)
241 off
= roundup (freebottom
, slotinfo
[cnt
].map
->l_tls_align
);
242 if (off
- freebottom
< firstbyte
)
243 off
+= slotinfo
[cnt
].map
->l_tls_align
;
244 if (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
<= freetop
)
246 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
247 freebottom
= (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
253 off
= roundup (offset
, slotinfo
[cnt
].map
->l_tls_align
);
254 if (off
- offset
< firstbyte
)
255 off
+= slotinfo
[cnt
].map
->l_tls_align
;
257 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
258 if (off
- firstbyte
- offset
> freetop
- freebottom
)
261 freetop
= off
- firstbyte
;
264 offset
= off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
;
267 GL(dl_tls_static_used
) = offset
;
268 GL(dl_tls_static_size
) = roundup (offset
+ TLS_STATIC_SURPLUS
,
271 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
274 /* The alignment requirement for the static TLS block. */
275 GL(dl_tls_static_align
) = max_align
;
281 allocate_dtv (void *result
)
286 /* We allocate a few more elements in the dtv than are needed for the
287 initial set of modules. This should avoid in most cases expansions
289 dtv_length
= GL(dl_tls_max_dtv_idx
) + DTV_SURPLUS
;
290 dtv
= calloc (dtv_length
+ 2, sizeof (dtv_t
));
293 /* This is the initial length of the dtv. */
294 dtv
[0].counter
= dtv_length
;
296 /* The rest of the dtv (including the generation counter) is
297 Initialize with zero to indicate nothing there. */
299 /* Add the dtv to the thread data structures. */
300 INSTALL_DTV (result
, dtv
);
309 /* Get size and alignment requirements of the static TLS block. */
312 _dl_get_tls_static_info (size_t *sizep
, size_t *alignp
)
314 *sizep
= GL(dl_tls_static_size
);
315 *alignp
= GL(dl_tls_static_align
);
318 /* Derive the location of the pointer to the start of the original
319 allocation (before alignment) from the pointer to the TCB. */
320 static inline void **
321 tcb_to_pointer_to_free_location (void *tcb
)
324 /* The TCB follows the TLS blocks, and the pointer to the front
326 void **original_pointer_location
= tcb
+ TLS_TCB_SIZE
;
328 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
330 void **original_pointer_location
= tcb
- TLS_PRE_TCB_SIZE
- sizeof (void *);
332 return original_pointer_location
;
337 _dl_allocate_tls_storage (void)
340 size_t size
= GL(dl_tls_static_size
);
344 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
345 ^ This should be returned. */
346 size
+= TLS_PRE_TCB_SIZE
;
349 /* Perform the allocation. Reserve space for the required alignment
350 and the pointer to the original allocation. */
351 size_t alignment
= GL(dl_tls_static_align
);
352 void *allocated
= malloc (size
+ alignment
+ sizeof (void *));
353 if (__glibc_unlikely (allocated
== NULL
))
356 /* Perform alignment and allocate the DTV. */
358 /* The TCB follows the TLS blocks, which determine the alignment.
359 (TCB alignment requirements have been taken into account when
360 calculating GL(dl_tls_static_align).) */
361 void *aligned
= (void *) roundup ((uintptr_t) allocated
, alignment
);
362 result
= aligned
+ size
- TLS_TCB_SIZE
;
364 /* Clear the TCB data structure. We can't ask the caller (i.e.
365 libpthread) to do it, because we will initialize the DTV et al. */
366 memset (result
, '\0', TLS_TCB_SIZE
);
368 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
369 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
370 TLS block alignment, and not just the TLS blocks after it. This
371 can leave an unused alignment gap between the TCB and the TLS
373 result
= (void *) roundup
374 (sizeof (void *) + TLS_PRE_TCB_SIZE
+ (uintptr_t) allocated
,
377 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
378 it. We can't ask the caller (i.e. libpthread) to do it, because
379 we will initialize the DTV et al. */
380 memset (result
- TLS_PRE_TCB_SIZE
, '\0', TLS_PRE_TCB_SIZE
+ TLS_TCB_SIZE
);
383 /* Record the value of the original pointer for later
385 *tcb_to_pointer_to_free_location (result
) = allocated
;
387 result
= allocate_dtv (result
);
395 extern dtv_t _dl_static_dtv
[];
396 # define _dl_initial_dtv (&_dl_static_dtv[1])
400 _dl_resize_dtv (dtv_t
*dtv
)
402 /* Resize the dtv. */
404 /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
405 other threads concurrently. */
407 = atomic_load_acquire (&GL(dl_tls_max_dtv_idx
)) + DTV_SURPLUS
;
408 size_t oldsize
= dtv
[-1].counter
;
410 if (dtv
== GL(dl_initial_dtv
))
412 /* This is the initial dtv that was either statically allocated in
413 __libc_setup_tls or allocated during rtld startup using the
414 dl-minimal.c malloc instead of the real malloc. We can't free
415 it, we have to abandon the old storage. */
417 newp
= malloc ((2 + newsize
) * sizeof (dtv_t
));
420 memcpy (newp
, &dtv
[-1], (2 + oldsize
) * sizeof (dtv_t
));
424 newp
= realloc (&dtv
[-1],
425 (2 + newsize
) * sizeof (dtv_t
));
430 newp
[0].counter
= newsize
;
432 /* Clear the newly allocated part. */
433 memset (newp
+ 2 + oldsize
, '\0',
434 (newsize
- oldsize
) * sizeof (dtv_t
));
436 /* Return the generation counter. */
442 _dl_allocate_tls_init (void *result
)
445 /* The memory allocation failed. */
448 dtv_t
*dtv
= GET_DTV (result
);
449 struct dtv_slotinfo_list
*listp
;
453 /* Check if the current dtv is big enough. */
454 if (dtv
[-1].counter
< GL(dl_tls_max_dtv_idx
))
456 /* Resize the dtv. */
457 dtv
= _dl_resize_dtv (dtv
);
459 /* Install this new dtv in the thread data structures. */
460 INSTALL_DTV (result
, &dtv
[-1]);
463 /* We have to prepare the dtv for all currently loaded modules using
464 TLS. For those which are dynamically loaded we add the values
465 indicating deferred allocation. */
466 listp
= GL(dl_tls_dtv_slotinfo_list
);
471 for (cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
473 struct link_map
*map
;
476 /* Check for the total number of used slots. */
477 if (total
+ cnt
> GL(dl_tls_max_dtv_idx
))
480 map
= listp
->slotinfo
[cnt
].map
;
485 /* Keep track of the maximum generation number. This might
486 not be the generation counter. */
487 assert (listp
->slotinfo
[cnt
].gen
<= GL(dl_tls_generation
));
488 maxgen
= MAX (maxgen
, listp
->slotinfo
[cnt
].gen
);
490 dtv
[map
->l_tls_modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
491 dtv
[map
->l_tls_modid
].pointer
.to_free
= NULL
;
493 if (map
->l_tls_offset
== NO_TLS_OFFSET
494 || map
->l_tls_offset
== FORCED_DYNAMIC_TLS_OFFSET
)
497 assert (map
->l_tls_modid
== total
+ cnt
);
498 assert (map
->l_tls_blocksize
>= map
->l_tls_initimage_size
);
500 assert ((size_t) map
->l_tls_offset
>= map
->l_tls_blocksize
);
501 dest
= (char *) result
- map
->l_tls_offset
;
503 dest
= (char *) result
+ map
->l_tls_offset
;
505 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
508 /* Set up the DTV entry. The simplified __tls_get_addr that
509 some platforms use in static programs requires it. */
510 dtv
[map
->l_tls_modid
].pointer
.val
= dest
;
512 /* Copy the initialization image and clear the BSS part. */
513 memset (__mempcpy (dest
, map
->l_tls_initimage
,
514 map
->l_tls_initimage_size
), '\0',
515 map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
519 if (total
>= GL(dl_tls_max_dtv_idx
))
523 assert (listp
!= NULL
);
526 /* The DTV version is up-to-date now. */
527 dtv
[0].counter
= maxgen
;
531 rtld_hidden_def (_dl_allocate_tls_init
)
534 _dl_allocate_tls (void *mem
)
536 return _dl_allocate_tls_init (mem
== NULL
537 ? _dl_allocate_tls_storage ()
538 : allocate_dtv (mem
));
540 rtld_hidden_def (_dl_allocate_tls
)
544 _dl_deallocate_tls (void *tcb
, bool dealloc_tcb
)
546 dtv_t
*dtv
= GET_DTV (tcb
);
548 /* We need to free the memory allocated for non-static TLS. */
549 for (size_t cnt
= 0; cnt
< dtv
[-1].counter
; ++cnt
)
550 free (dtv
[1 + cnt
].pointer
.to_free
);
552 /* The array starts with dtv[-1]. */
553 if (dtv
!= GL(dl_initial_dtv
))
557 free (*tcb_to_pointer_to_free_location (tcb
));
559 rtld_hidden_def (_dl_deallocate_tls
)
563 /* The __tls_get_addr function has two basic forms which differ in the
564 arguments. The IA-64 form takes two parameters, the module ID and
565 offset. The form used, among others, on IA-32 takes a reference to
566 a special structure which contain the same information. The second
567 form seems to be more often used (in the moment) so we default to
568 it. Users of the IA-64 form have to provide adequate definitions
569 of the following macros. */
570 # ifndef GET_ADDR_ARGS
571 # define GET_ADDR_ARGS tls_index *ti
572 # define GET_ADDR_PARAM ti
574 # ifndef GET_ADDR_MODULE
575 # define GET_ADDR_MODULE ti->ti_module
577 # ifndef GET_ADDR_OFFSET
578 # define GET_ADDR_OFFSET ti->ti_offset
581 /* Allocate one DTV entry. */
582 static struct dtv_pointer
583 allocate_dtv_entry (size_t alignment
, size_t size
)
585 if (powerof2 (alignment
) && alignment
<= _Alignof (max_align_t
))
587 /* The alignment is supported by malloc. */
588 void *ptr
= malloc (size
);
589 return (struct dtv_pointer
) { ptr
, ptr
};
592 /* Emulate memalign to by manually aligning a pointer returned by
593 malloc. First compute the size with an overflow check. */
594 size_t alloc_size
= size
+ alignment
;
595 if (alloc_size
< size
)
596 return (struct dtv_pointer
) {};
598 /* Perform the allocation. This is the pointer we need to free
600 void *start
= malloc (alloc_size
);
602 return (struct dtv_pointer
) {};
604 /* Find the aligned position within the larger allocation. */
605 void *aligned
= (void *) roundup ((uintptr_t) start
, alignment
);
607 return (struct dtv_pointer
) { .val
= aligned
, .to_free
= start
};
610 static struct dtv_pointer
611 allocate_and_init (struct link_map
*map
)
613 struct dtv_pointer result
= allocate_dtv_entry
614 (map
->l_tls_align
, map
->l_tls_blocksize
);
615 if (result
.val
== NULL
)
618 /* Initialize the memory. */
619 memset (__mempcpy (result
.val
, map
->l_tls_initimage
,
620 map
->l_tls_initimage_size
),
621 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
628 _dl_update_slotinfo (unsigned long int req_modid
)
630 struct link_map
*the_map
= NULL
;
631 dtv_t
*dtv
= THREAD_DTV ();
633 /* The global dl_tls_dtv_slotinfo array contains for each module
634 index the generation counter current when the entry was created.
635 This array never shrinks so that all module indices which were
636 valid at some time can be used to access it. Before the first
637 use of a new module index in this function the array was extended
638 appropriately. Access also does not have to be guarded against
639 modifications of the array. It is assumed that pointer-size
640 values can be read atomically even in SMP environments. It is
641 possible that other threads at the same time dynamically load
642 code and therefore add to the slotinfo list. This is a problem
643 since we must not pick up any information about incomplete work.
644 The solution to this is to ignore all dtv slots which were
645 created after the one we are currently interested. We know that
646 dynamic loading for this module is completed and this is the last
647 load operation we know finished. */
648 unsigned long int idx
= req_modid
;
649 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
651 while (idx
>= listp
->len
)
657 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
659 /* The generation counter for the slot is higher than what the
660 current dtv implements. We have to update the whole dtv but
661 only those entries with a generation counter <= the one for
662 the entry we need. */
663 size_t new_gen
= listp
->slotinfo
[idx
].gen
;
666 /* We have to look through the entire dtv slotinfo list. */
667 listp
= GL(dl_tls_dtv_slotinfo_list
);
670 for (size_t cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
672 size_t gen
= listp
->slotinfo
[cnt
].gen
;
675 /* This is a slot for a generation younger than the
676 one we are handling now. It might be incompletely
677 set up so ignore it. */
680 /* If the entry is older than the current dtv layout we
681 know we don't have to handle it. */
682 if (gen
<= dtv
[0].counter
)
685 /* If there is no map this means the entry is empty. */
686 struct link_map
*map
= listp
->slotinfo
[cnt
].map
;
689 if (dtv
[-1].counter
>= total
+ cnt
)
691 /* If this modid was used at some point the memory
692 might still be allocated. */
693 free (dtv
[total
+ cnt
].pointer
.to_free
);
694 dtv
[total
+ cnt
].pointer
.val
= TLS_DTV_UNALLOCATED
;
695 dtv
[total
+ cnt
].pointer
.to_free
= NULL
;
701 /* Check whether the current dtv array is large enough. */
702 size_t modid
= map
->l_tls_modid
;
703 assert (total
+ cnt
== modid
);
704 if (dtv
[-1].counter
< modid
)
706 /* Resize the dtv. */
707 dtv
= _dl_resize_dtv (dtv
);
709 assert (modid
<= dtv
[-1].counter
);
711 /* Install this new dtv in the thread data
713 INSTALL_NEW_DTV (dtv
);
716 /* If there is currently memory allocate for this
717 dtv entry free it. */
718 /* XXX Ideally we will at some point create a memory
720 free (dtv
[modid
].pointer
.to_free
);
721 dtv
[modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
722 dtv
[modid
].pointer
.to_free
= NULL
;
724 if (modid
== req_modid
)
730 while ((listp
= listp
->next
) != NULL
);
732 /* This will be the new maximum generation counter. */
733 dtv
[0].counter
= new_gen
;
741 __attribute_noinline__
742 tls_get_addr_tail (GET_ADDR_ARGS
, dtv_t
*dtv
, struct link_map
*the_map
)
744 /* The allocation was deferred. Do it now. */
747 /* Find the link map for this module. */
748 size_t idx
= GET_ADDR_MODULE
;
749 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
751 while (idx
>= listp
->len
)
757 the_map
= listp
->slotinfo
[idx
].map
;
760 /* Make sure that, if a dlopen running in parallel forces the
761 variable into static storage, we'll wait until the address in the
762 static TLS block is set up, and use that. If we're undecided
763 yet, make sure we make the decision holding the lock as well. */
764 if (__glibc_unlikely (the_map
->l_tls_offset
765 != FORCED_DYNAMIC_TLS_OFFSET
))
767 __rtld_lock_lock_recursive (GL(dl_load_lock
));
768 if (__glibc_likely (the_map
->l_tls_offset
== NO_TLS_OFFSET
))
770 the_map
->l_tls_offset
= FORCED_DYNAMIC_TLS_OFFSET
;
771 __rtld_lock_unlock_recursive (GL(dl_load_lock
));
773 else if (__glibc_likely (the_map
->l_tls_offset
774 != FORCED_DYNAMIC_TLS_OFFSET
))
777 void *p
= (char *) THREAD_SELF
- the_map
->l_tls_offset
;
779 void *p
= (char *) THREAD_SELF
+ the_map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
781 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
783 __rtld_lock_unlock_recursive (GL(dl_load_lock
));
785 dtv
[GET_ADDR_MODULE
].pointer
.to_free
= NULL
;
786 dtv
[GET_ADDR_MODULE
].pointer
.val
= p
;
788 return (char *) p
+ GET_ADDR_OFFSET
;
791 __rtld_lock_unlock_recursive (GL(dl_load_lock
));
793 struct dtv_pointer result
= allocate_and_init (the_map
);
794 dtv
[GET_ADDR_MODULE
].pointer
= result
;
795 assert (result
.to_free
!= NULL
);
797 return (char *) result
.val
+ GET_ADDR_OFFSET
;
801 static struct link_map
*
802 __attribute_noinline__
803 update_get_addr (GET_ADDR_ARGS
)
805 struct link_map
*the_map
= _dl_update_slotinfo (GET_ADDR_MODULE
);
806 dtv_t
*dtv
= THREAD_DTV ();
808 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
810 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
811 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, the_map
);
813 return (void *) p
+ GET_ADDR_OFFSET
;
816 /* For all machines that have a non-macro version of __tls_get_addr, we
817 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
818 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
819 in ld.so for __tls_get_addr. */
821 #ifndef __tls_get_addr
822 extern void * __tls_get_addr (GET_ADDR_ARGS
);
823 rtld_hidden_proto (__tls_get_addr
)
824 rtld_hidden_def (__tls_get_addr
)
827 /* The generic dynamic and local dynamic model cannot be used in
828 statically linked applications. */
830 __tls_get_addr (GET_ADDR_ARGS
)
832 dtv_t
*dtv
= THREAD_DTV ();
834 if (__glibc_unlikely (dtv
[0].counter
!= GL(dl_tls_generation
)))
835 return update_get_addr (GET_ADDR_PARAM
);
837 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
839 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
840 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, NULL
);
842 return (char *) p
+ GET_ADDR_OFFSET
;
847 /* Look up the module's TLS block as for __tls_get_addr,
848 but never touch anything. Return null if it's not allocated yet. */
850 _dl_tls_get_addr_soft (struct link_map
*l
)
852 if (__glibc_unlikely (l
->l_tls_modid
== 0))
853 /* This module has no TLS segment. */
856 dtv_t
*dtv
= THREAD_DTV ();
857 if (__glibc_unlikely (dtv
[0].counter
!= GL(dl_tls_generation
)))
859 /* This thread's DTV is not completely current,
860 but it might already cover this module. */
862 if (l
->l_tls_modid
>= dtv
[-1].counter
)
866 size_t idx
= l
->l_tls_modid
;
867 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
868 while (idx
>= listp
->len
)
874 /* We've reached the slot for this module.
875 If its generation counter is higher than the DTV's,
876 this thread does not know about this module yet. */
877 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
881 void *data
= dtv
[l
->l_tls_modid
].pointer
.val
;
882 if (__glibc_unlikely (data
== TLS_DTV_UNALLOCATED
))
883 /* The DTV is current, but this thread has not yet needed
884 to allocate this module's segment. */
892 _dl_add_to_slotinfo (struct link_map
*l
)
894 /* Now that we know the object is loaded successfully add
895 modules containing TLS data to the dtv info table. We
896 might have to increase its size. */
897 struct dtv_slotinfo_list
*listp
;
898 struct dtv_slotinfo_list
*prevp
;
899 size_t idx
= l
->l_tls_modid
;
901 /* Find the place in the dtv slotinfo list. */
902 listp
= GL(dl_tls_dtv_slotinfo_list
);
903 prevp
= NULL
; /* Needed to shut up gcc. */
906 /* Does it fit in the array of this list element? */
907 if (idx
< listp
->len
)
913 while (listp
!= NULL
);
917 /* When we come here it means we have to add a new element
918 to the slotinfo list. And the new module must be in
922 listp
= prevp
->next
= (struct dtv_slotinfo_list
*)
923 malloc (sizeof (struct dtv_slotinfo_list
)
924 + TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
927 /* We ran out of memory. We will simply fail this
928 call but don't undo anything we did so far. The
929 application will crash or be terminated anyway very
932 /* We have to do this since some entries in the dtv
933 slotinfo array might already point to this
935 ++GL(dl_tls_generation
);
937 _dl_signal_error (ENOMEM
, "dlopen", NULL
, N_("\
938 cannot create TLS data structures"));
941 listp
->len
= TLS_SLOTINFO_SURPLUS
;
943 memset (listp
->slotinfo
, '\0',
944 TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
947 /* Add the information into the slotinfo data structure. */
948 listp
->slotinfo
[idx
].map
= l
;
949 listp
->slotinfo
[idx
].gen
= GL(dl_tls_generation
) + 1;