1 /* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
25 #include <sys/param.h>
36 #define TUNABLE_NAMESPACE rtld
37 #include <dl-tunables.h>
39 /* Surplus static TLS, GLRO(dl_tls_static_surplus), is used for
41 - IE TLS in libc.so for all dlmopen namespaces except in the initial
42 one where libc.so is not loaded dynamically but at startup time,
43 - IE TLS in other libraries which may be dynamically loaded even in the
45 - and optionally for optimizing dynamic TLS access.
47 The maximum number of namespaces is DL_NNS, but to support that many
48 namespaces correctly the static TLS allocation should be significantly
49 increased, which may cause problems with small thread stacks due to the
50 way static TLS is accounted (bug 11787).
52 So there is a rtld.nns tunable limit on the number of supported namespaces
53 that affects the size of the static TLS and by default it's small enough
54 not to cause problems with existing applications. The limit is not
55 enforced or checked: it is the user's responsibility to increase rtld.nns
56 if more dlmopen namespaces are used.
58 Audit modules use their own namespaces, they are not included in rtld.nns,
59 but come on top when computing the number of namespaces. */
61 /* Size of initial-exec TLS in libc.so. This should be the maximum of
62 observed PT_GNU_TLS sizes across all architectures. Some
63 architectures have lower values due to differences in type sizes
64 and link editor capabilities. */
65 #define LIBC_IE_TLS 144
67 /* Size of initial-exec TLS in libraries other than libc.so.
68 This should be large enough to cover runtime libraries of the
69 compiler such as libgomp and libraries in libc other than libc.so. */
70 #define OTHER_IE_TLS 144
72 /* Default number of namespaces. */
75 /* Default for dl_tls_static_optional. */
76 #define OPTIONAL_TLS 512
78 /* Compute the static TLS surplus based on the namespace count and the
79 TLS space that can be used for optimizations. */
81 tls_static_surplus (int nns
, int opt_tls
)
83 return (nns
- 1) * LIBC_IE_TLS
+ nns
* OTHER_IE_TLS
+ opt_tls
;
86 /* This value is chosen so that with default values for the tunables,
87 the computation of dl_tls_static_surplus in
88 _dl_tls_static_surplus_init yields the historic value 1664, for
89 backwards compatibility. */
90 #define LEGACY_TLS (1664 - tls_static_surplus (DEFAULT_NNS, OPTIONAL_TLS))
92 /* Calculate the size of the static TLS surplus, when the given
93 number of audit modules are loaded. Must be called after the
94 number of audit modules is known and before static TLS allocation. */
96 _dl_tls_static_surplus_init (size_t naudit
)
100 nns
= TUNABLE_GET (nns
, size_t, NULL
);
101 opt_tls
= TUNABLE_GET (optional_static_tls
, size_t, NULL
);
104 if (DL_NNS
- nns
< naudit
)
105 _dl_fatal_printf ("Failed loading %lu audit modules, %lu are supported.\n",
106 (unsigned long) naudit
, (unsigned long) (DL_NNS
- nns
));
109 GL(dl_tls_static_optional
) = opt_tls
;
110 assert (LEGACY_TLS
>= 0);
111 GLRO(dl_tls_static_surplus
) = tls_static_surplus (nns
, opt_tls
) + LEGACY_TLS
;
114 /* Out-of-memory handler. */
116 __attribute__ ((__noreturn__
))
119 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
124 _dl_assign_tls_modid (struct link_map
*l
)
128 if (__builtin_expect (GL(dl_tls_dtv_gaps
), false))
131 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
133 /* Note that this branch will never be executed during program
134 start since there are no gaps at that time. Therefore it
135 does not matter that the dl_tls_dtv_slotinfo is not allocated
136 yet when the function is called for the first times.
138 NB: the offset +1 is due to the fact that DTV[0] is used
139 for something else. */
140 result
= GL(dl_tls_static_nelem
) + 1;
141 if (result
<= GL(dl_tls_max_dtv_idx
))
144 while (result
- disp
< runp
->len
)
146 if (runp
->slotinfo
[result
- disp
].map
== NULL
)
150 assert (result
<= GL(dl_tls_max_dtv_idx
) + 1);
153 if (result
- disp
< runp
->len
)
155 /* Mark the entry as used, so any dependency see it. */
156 atomic_store_relaxed (&runp
->slotinfo
[result
- disp
].map
, l
);
157 atomic_store_relaxed (&runp
->slotinfo
[result
- disp
].gen
, 0);
163 while ((runp
= runp
->next
) != NULL
);
165 if (result
> GL(dl_tls_max_dtv_idx
))
167 /* The new index must indeed be exactly one higher than the
169 assert (result
== GL(dl_tls_max_dtv_idx
) + 1);
170 /* There is no gap anymore. */
171 GL(dl_tls_dtv_gaps
) = false;
178 /* No gaps, allocate a new entry. */
181 result
= GL(dl_tls_max_dtv_idx
) + 1;
182 /* Can be read concurrently. */
183 atomic_store_relaxed (&GL(dl_tls_max_dtv_idx
), result
);
186 l
->l_tls_modid
= result
;
191 _dl_count_modids (void)
193 /* The count is the max unless dlclose or failed dlopen created gaps. */
194 if (__glibc_likely (!GL(dl_tls_dtv_gaps
)))
195 return GL(dl_tls_max_dtv_idx
);
197 /* We have gaps and are forced to count the non-NULL entries. */
199 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
202 for (size_t i
= 0; i
< runp
->len
; ++i
)
203 if (runp
->slotinfo
[i
].map
!= NULL
)
215 _dl_determine_tlsoffset (void)
217 size_t max_align
= TCB_ALIGNMENT
;
219 size_t freebottom
= 0;
221 /* The first element of the dtv slot info list is allocated. */
222 assert (GL(dl_tls_dtv_slotinfo_list
) != NULL
);
223 /* There is at this point only one element in the
224 dl_tls_dtv_slotinfo_list list. */
225 assert (GL(dl_tls_dtv_slotinfo_list
)->next
== NULL
);
227 struct dtv_slotinfo
*slotinfo
= GL(dl_tls_dtv_slotinfo_list
)->slotinfo
;
229 /* Determining the offset of the various parts of the static TLS
230 block has several dependencies. In addition we have to work
231 around bugs in some toolchains.
233 Each TLS block from the objects available at link time has a size
234 and an alignment requirement. The GNU ld computes the alignment
235 requirements for the data at the positions *in the file*, though.
236 I.e, it is not simply possible to allocate a block with the size
237 of the TLS program header entry. The data is laid out assuming
238 that the first byte of the TLS block fulfills
240 p_vaddr mod p_align == &TLS_BLOCK mod p_align
242 This means we have to add artificial padding at the beginning of
243 the TLS block. These bytes are never used for the TLS data in
244 this module but the first byte allocated must be aligned
245 according to mod p_align == 0 so that the first byte of the TLS
246 block is aligned according to p_vaddr mod p_align. This is ugly
247 and the linker can help by computing the offsets in the TLS block
248 assuming the first byte of the TLS block is aligned according to
251 The extra space which might be allocated before the first byte of
252 the TLS block need not go unused. The code below tries to use
253 that memory for the next TLS block. This can work if the total
254 memory requirement for the next TLS block is smaller than the
258 /* We simply start with zero. */
261 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
263 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
265 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
266 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
268 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
270 if (freebottom
- freetop
>= slotinfo
[cnt
].map
->l_tls_blocksize
)
272 off
= roundup (freetop
+ slotinfo
[cnt
].map
->l_tls_blocksize
273 - firstbyte
, slotinfo
[cnt
].map
->l_tls_align
)
275 if (off
<= freebottom
)
279 /* XXX For some architectures we perhaps should store the
281 slotinfo
[cnt
].map
->l_tls_offset
= off
;
286 off
= roundup (offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
,
287 slotinfo
[cnt
].map
->l_tls_align
) + firstbyte
;
288 if (off
> offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
289 + (freebottom
- freetop
))
292 freebottom
= off
- slotinfo
[cnt
].map
->l_tls_blocksize
;
296 /* XXX For some architectures we perhaps should store the
298 slotinfo
[cnt
].map
->l_tls_offset
= off
;
301 GL(dl_tls_static_used
) = offset
;
302 GLRO (dl_tls_static_size
) = (roundup (offset
+ GLRO(dl_tls_static_surplus
),
306 /* The TLS blocks start right after the TCB. */
307 size_t offset
= TLS_TCB_SIZE
;
309 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
311 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
313 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
314 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
316 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
318 if (slotinfo
[cnt
].map
->l_tls_blocksize
<= freetop
- freebottom
)
320 off
= roundup (freebottom
, slotinfo
[cnt
].map
->l_tls_align
);
321 if (off
- freebottom
< firstbyte
)
322 off
+= slotinfo
[cnt
].map
->l_tls_align
;
323 if (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
<= freetop
)
325 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
326 freebottom
= (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
332 off
= roundup (offset
, slotinfo
[cnt
].map
->l_tls_align
);
333 if (off
- offset
< firstbyte
)
334 off
+= slotinfo
[cnt
].map
->l_tls_align
;
336 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
337 if (off
- firstbyte
- offset
> freetop
- freebottom
)
340 freetop
= off
- firstbyte
;
343 offset
= off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
;
346 GL(dl_tls_static_used
) = offset
;
347 GLRO (dl_tls_static_size
) = roundup (offset
+ GLRO(dl_tls_static_surplus
),
350 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
353 /* The alignment requirement for the static TLS block. */
354 GLRO (dl_tls_static_align
) = max_align
;
359 allocate_dtv (void *result
)
364 /* Relaxed MO, because the dtv size is later rechecked, not relied on. */
365 size_t max_modid
= atomic_load_relaxed (&GL(dl_tls_max_dtv_idx
));
366 /* We allocate a few more elements in the dtv than are needed for the
367 initial set of modules. This should avoid in most cases expansions
369 dtv_length
= max_modid
+ DTV_SURPLUS
;
370 dtv
= calloc (dtv_length
+ 2, sizeof (dtv_t
));
373 /* This is the initial length of the dtv. */
374 dtv
[0].counter
= dtv_length
;
376 /* The rest of the dtv (including the generation counter) is
377 Initialize with zero to indicate nothing there. */
379 /* Add the dtv to the thread data structures. */
380 INSTALL_DTV (result
, dtv
);
388 /* Get size and alignment requirements of the static TLS block. This
389 function is no longer used by glibc itself, but the GCC sanitizers
390 use it despite the GLIBC_PRIVATE status. */
392 _dl_get_tls_static_info (size_t *sizep
, size_t *alignp
)
394 *sizep
= GLRO (dl_tls_static_size
);
395 *alignp
= GLRO (dl_tls_static_align
);
398 /* Derive the location of the pointer to the start of the original
399 allocation (before alignment) from the pointer to the TCB. */
400 static inline void **
401 tcb_to_pointer_to_free_location (void *tcb
)
404 /* The TCB follows the TLS blocks, and the pointer to the front
406 void **original_pointer_location
= tcb
+ TLS_TCB_SIZE
;
408 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
410 void **original_pointer_location
= tcb
- TLS_PRE_TCB_SIZE
- sizeof (void *);
412 return original_pointer_location
;
416 _dl_allocate_tls_storage (void)
419 size_t size
= GLRO (dl_tls_static_size
);
423 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
424 ^ This should be returned. */
425 size
+= TLS_PRE_TCB_SIZE
;
428 /* Perform the allocation. Reserve space for the required alignment
429 and the pointer to the original allocation. */
430 size_t alignment
= GLRO (dl_tls_static_align
);
431 void *allocated
= malloc (size
+ alignment
+ sizeof (void *));
432 if (__glibc_unlikely (allocated
== NULL
))
435 /* Perform alignment and allocate the DTV. */
437 /* The TCB follows the TLS blocks, which determine the alignment.
438 (TCB alignment requirements have been taken into account when
439 calculating GLRO (dl_tls_static_align).) */
440 void *aligned
= (void *) roundup ((uintptr_t) allocated
, alignment
);
441 result
= aligned
+ size
- TLS_TCB_SIZE
;
443 /* Clear the TCB data structure. We can't ask the caller (i.e.
444 libpthread) to do it, because we will initialize the DTV et al. */
445 memset (result
, '\0', TLS_TCB_SIZE
);
447 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
448 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
449 TLS block alignment, and not just the TLS blocks after it. This
450 can leave an unused alignment gap between the TCB and the TLS
452 result
= (void *) roundup
453 (sizeof (void *) + TLS_PRE_TCB_SIZE
+ (uintptr_t) allocated
,
456 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
457 it. We can't ask the caller (i.e. libpthread) to do it, because
458 we will initialize the DTV et al. */
459 memset (result
- TLS_PRE_TCB_SIZE
, '\0', TLS_PRE_TCB_SIZE
+ TLS_TCB_SIZE
);
462 /* Record the value of the original pointer for later
464 *tcb_to_pointer_to_free_location (result
) = allocated
;
466 result
= allocate_dtv (result
);
474 extern dtv_t _dl_static_dtv
[];
475 # define _dl_initial_dtv (&_dl_static_dtv[1])
479 _dl_resize_dtv (dtv_t
*dtv
, size_t max_modid
)
481 /* Resize the dtv. */
483 size_t newsize
= max_modid
+ DTV_SURPLUS
;
484 size_t oldsize
= dtv
[-1].counter
;
486 if (dtv
== GL(dl_initial_dtv
))
488 /* This is the initial dtv that was either statically allocated in
489 __libc_setup_tls or allocated during rtld startup using the
490 dl-minimal.c malloc instead of the real malloc. We can't free
491 it, we have to abandon the old storage. */
493 newp
= malloc ((2 + newsize
) * sizeof (dtv_t
));
496 memcpy (newp
, &dtv
[-1], (2 + oldsize
) * sizeof (dtv_t
));
500 newp
= realloc (&dtv
[-1],
501 (2 + newsize
) * sizeof (dtv_t
));
506 newp
[0].counter
= newsize
;
508 /* Clear the newly allocated part. */
509 memset (newp
+ 2 + oldsize
, '\0',
510 (newsize
- oldsize
) * sizeof (dtv_t
));
512 /* Return the generation counter. */
517 /* Allocate initial TLS. RESULT should be a non-NULL pointer to storage
518 for the TLS space. The DTV may be resized, and so this function may
519 call malloc to allocate that space. The loader's GL(dl_load_tls_lock)
520 is taken when manipulating global TLS-related data in the loader. */
522 _dl_allocate_tls_init (void *result
, bool init_tls
)
525 /* The memory allocation failed. */
528 dtv_t
*dtv
= GET_DTV (result
);
529 struct dtv_slotinfo_list
*listp
;
533 /* Protects global dynamic TLS related state. */
534 __rtld_lock_lock_recursive (GL(dl_load_tls_lock
));
536 /* Check if the current dtv is big enough. */
537 if (dtv
[-1].counter
< GL(dl_tls_max_dtv_idx
))
539 /* Resize the dtv. */
540 dtv
= _dl_resize_dtv (dtv
, GL(dl_tls_max_dtv_idx
));
542 /* Install this new dtv in the thread data structures. */
543 INSTALL_DTV (result
, &dtv
[-1]);
546 /* We have to prepare the dtv for all currently loaded modules using
547 TLS. For those which are dynamically loaded we add the values
548 indicating deferred allocation. */
549 listp
= GL(dl_tls_dtv_slotinfo_list
);
554 for (cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
556 struct link_map
*map
;
559 /* Check for the total number of used slots. */
560 if (total
+ cnt
> GL(dl_tls_max_dtv_idx
))
563 map
= listp
->slotinfo
[cnt
].map
;
568 /* Keep track of the maximum generation number. This might
569 not be the generation counter. */
570 assert (listp
->slotinfo
[cnt
].gen
<= GL(dl_tls_generation
));
571 maxgen
= MAX (maxgen
, listp
->slotinfo
[cnt
].gen
);
573 dtv
[map
->l_tls_modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
574 dtv
[map
->l_tls_modid
].pointer
.to_free
= NULL
;
576 if (map
->l_tls_offset
== NO_TLS_OFFSET
577 || map
->l_tls_offset
== FORCED_DYNAMIC_TLS_OFFSET
)
580 assert (map
->l_tls_modid
== total
+ cnt
);
581 assert (map
->l_tls_blocksize
>= map
->l_tls_initimage_size
);
583 assert ((size_t) map
->l_tls_offset
>= map
->l_tls_blocksize
);
584 dest
= (char *) result
- map
->l_tls_offset
;
586 dest
= (char *) result
+ map
->l_tls_offset
;
588 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
591 /* Set up the DTV entry. The simplified __tls_get_addr that
592 some platforms use in static programs requires it. */
593 dtv
[map
->l_tls_modid
].pointer
.val
= dest
;
595 /* Copy the initialization image and clear the BSS part. For
596 audit modules or dependencies with initial-exec TLS, we can not
597 set the initial TLS image on default loader initialization
598 because it would already be set by the audit setup. However,
599 subsequent thread creation would need to follow the default
601 if (map
->l_ns
!= LM_ID_BASE
&& !init_tls
)
603 memset (__mempcpy (dest
, map
->l_tls_initimage
,
604 map
->l_tls_initimage_size
), '\0',
605 map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
609 if (total
> GL(dl_tls_max_dtv_idx
))
613 assert (listp
!= NULL
);
615 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
617 /* The DTV version is up-to-date now. */
618 dtv
[0].counter
= maxgen
;
622 rtld_hidden_def (_dl_allocate_tls_init
)
625 _dl_allocate_tls (void *mem
)
627 return _dl_allocate_tls_init (mem
== NULL
628 ? _dl_allocate_tls_storage ()
629 : allocate_dtv (mem
), true);
631 rtld_hidden_def (_dl_allocate_tls
)
635 _dl_deallocate_tls (void *tcb
, bool dealloc_tcb
)
637 dtv_t
*dtv
= GET_DTV (tcb
);
639 /* We need to free the memory allocated for non-static TLS. */
640 for (size_t cnt
= 0; cnt
< dtv
[-1].counter
; ++cnt
)
641 free (dtv
[1 + cnt
].pointer
.to_free
);
643 /* The array starts with dtv[-1]. */
644 if (dtv
!= GL(dl_initial_dtv
))
648 free (*tcb_to_pointer_to_free_location (tcb
));
650 rtld_hidden_def (_dl_deallocate_tls
)
654 /* The __tls_get_addr function has two basic forms which differ in the
655 arguments. The IA-64 form takes two parameters, the module ID and
656 offset. The form used, among others, on IA-32 takes a reference to
657 a special structure which contain the same information. The second
658 form seems to be more often used (in the moment) so we default to
659 it. Users of the IA-64 form have to provide adequate definitions
660 of the following macros. */
661 # ifndef GET_ADDR_ARGS
662 # define GET_ADDR_ARGS tls_index *ti
663 # define GET_ADDR_PARAM ti
665 # ifndef GET_ADDR_MODULE
666 # define GET_ADDR_MODULE ti->ti_module
668 # ifndef GET_ADDR_OFFSET
669 # define GET_ADDR_OFFSET ti->ti_offset
672 /* Allocate one DTV entry. */
673 static struct dtv_pointer
674 allocate_dtv_entry (size_t alignment
, size_t size
)
676 if (powerof2 (alignment
) && alignment
<= _Alignof (max_align_t
))
678 /* The alignment is supported by malloc. */
679 void *ptr
= malloc (size
);
680 return (struct dtv_pointer
) { ptr
, ptr
};
683 /* Emulate memalign to by manually aligning a pointer returned by
684 malloc. First compute the size with an overflow check. */
685 size_t alloc_size
= size
+ alignment
;
686 if (alloc_size
< size
)
687 return (struct dtv_pointer
) {};
689 /* Perform the allocation. This is the pointer we need to free
691 void *start
= malloc (alloc_size
);
693 return (struct dtv_pointer
) {};
695 /* Find the aligned position within the larger allocation. */
696 void *aligned
= (void *) roundup ((uintptr_t) start
, alignment
);
698 return (struct dtv_pointer
) { .val
= aligned
, .to_free
= start
};
701 static struct dtv_pointer
702 allocate_and_init (struct link_map
*map
)
704 struct dtv_pointer result
= allocate_dtv_entry
705 (map
->l_tls_align
, map
->l_tls_blocksize
);
706 if (result
.val
== NULL
)
709 /* Initialize the memory. */
710 memset (__mempcpy (result
.val
, map
->l_tls_initimage
,
711 map
->l_tls_initimage_size
),
712 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
719 _dl_update_slotinfo (unsigned long int req_modid
, size_t new_gen
)
721 struct link_map
*the_map
= NULL
;
722 dtv_t
*dtv
= THREAD_DTV ();
724 /* CONCURRENCY NOTES:
726 The global dl_tls_dtv_slotinfo_list array contains for each module
727 index the generation counter current when that entry was updated.
728 This array never shrinks so that all module indices which were
729 valid at some time can be used to access it. Concurrent loading
730 and unloading of modules can update slotinfo entries or extend
731 the array. The updates happen under the GL(dl_load_tls_lock) and
732 finish with the release store of the generation counter to
733 GL(dl_tls_generation) which is synchronized with the load of
734 new_gen in the caller. So updates up to new_gen are synchronized
735 but updates for later generations may not be.
737 Here we update the thread dtv from old_gen (== dtv[0].counter) to
738 new_gen generation. For this, each dtv[i] entry is either set to
739 an unallocated state (set), or left unmodified (nop). Where (set)
740 may resize the dtv first if modid i >= dtv[-1].counter. The rules
741 for the decision between (set) and (nop) are
743 (1) If slotinfo entry i is concurrently updated then either (set)
744 or (nop) is valid: TLS access cannot use dtv[i] unless it is
745 synchronized with a generation > new_gen.
747 Otherwise, if the generation of slotinfo entry i is gen and the
748 loaded module for this entry is map then
750 (2) If gen <= old_gen then do (nop).
752 (3) If old_gen < gen <= new_gen then
753 (3.1) if map != 0 then (set)
754 (3.2) if map == 0 then either (set) or (nop).
756 Note that (1) cannot be reliably detected, but since both actions
757 are valid it does not have to be. Only (2) and (3.1) cases need
758 to be distinguished for which relaxed mo access of gen and map is
759 enough: their value is synchronized when it matters.
761 Note that a relaxed mo load may give an out-of-thin-air value since
762 it is used in decisions that can affect concurrent stores. But this
763 should only happen if the OOTA value causes UB that justifies the
764 concurrent store of the value. This is not expected to be an issue
766 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
768 if (dtv
[0].counter
< new_gen
)
771 size_t max_modid
= atomic_load_relaxed (&GL(dl_tls_max_dtv_idx
));
772 assert (max_modid
>= req_modid
);
774 /* We have to look through the entire dtv slotinfo list. */
775 listp
= GL(dl_tls_dtv_slotinfo_list
);
778 for (size_t cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
780 size_t modid
= total
+ cnt
;
782 /* Case (1) for all later modids. */
783 if (modid
> max_modid
)
786 size_t gen
= atomic_load_relaxed (&listp
->slotinfo
[cnt
].gen
);
792 /* Case (2) or (1). */
793 if (gen
<= dtv
[0].counter
)
796 /* Case (3) or (1). */
798 /* If there is no map this means the entry is empty. */
800 = atomic_load_relaxed (&listp
->slotinfo
[cnt
].map
);
801 /* Check whether the current dtv array is large enough. */
802 if (dtv
[-1].counter
< modid
)
804 /* Case (3.2) or (1). */
808 /* Resizing the dtv aborts on failure: bug 16134. */
809 dtv
= _dl_resize_dtv (dtv
, max_modid
);
811 assert (modid
<= dtv
[-1].counter
);
813 /* Install this new dtv in the thread data
815 INSTALL_NEW_DTV (dtv
);
818 /* If there is currently memory allocate for this
819 dtv entry free it. Note: this is not AS-safe. */
820 /* XXX Ideally we will at some point create a memory
822 free (dtv
[modid
].pointer
.to_free
);
823 dtv
[modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
824 dtv
[modid
].pointer
.to_free
= NULL
;
826 if (modid
== req_modid
)
831 if (total
> max_modid
)
834 /* Synchronize with _dl_add_to_slotinfo. Ideally this would
835 be consume MO since we only need to order the accesses to
836 the next node after the read of the address and on most
837 hardware (other than alpha) a normal load would do that
838 because of the address dependency. */
839 listp
= atomic_load_acquire (&listp
->next
);
841 while (listp
!= NULL
);
843 /* This will be the new maximum generation counter. */
844 dtv
[0].counter
= new_gen
;
852 __attribute_noinline__
853 tls_get_addr_tail (GET_ADDR_ARGS
, dtv_t
*dtv
, struct link_map
*the_map
)
855 /* The allocation was deferred. Do it now. */
858 /* Find the link map for this module. */
859 size_t idx
= GET_ADDR_MODULE
;
860 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
862 while (idx
>= listp
->len
)
868 the_map
= listp
->slotinfo
[idx
].map
;
871 /* Make sure that, if a dlopen running in parallel forces the
872 variable into static storage, we'll wait until the address in the
873 static TLS block is set up, and use that. If we're undecided
874 yet, make sure we make the decision holding the lock as well. */
875 if (__glibc_unlikely (the_map
->l_tls_offset
876 != FORCED_DYNAMIC_TLS_OFFSET
))
878 __rtld_lock_lock_recursive (GL(dl_load_tls_lock
));
879 if (__glibc_likely (the_map
->l_tls_offset
== NO_TLS_OFFSET
))
881 the_map
->l_tls_offset
= FORCED_DYNAMIC_TLS_OFFSET
;
882 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
884 else if (__glibc_likely (the_map
->l_tls_offset
885 != FORCED_DYNAMIC_TLS_OFFSET
))
888 void *p
= (char *) THREAD_SELF
- the_map
->l_tls_offset
;
890 void *p
= (char *) THREAD_SELF
+ the_map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
892 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
894 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
896 dtv
[GET_ADDR_MODULE
].pointer
.to_free
= NULL
;
897 dtv
[GET_ADDR_MODULE
].pointer
.val
= p
;
899 return (char *) p
+ GET_ADDR_OFFSET
;
902 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
904 struct dtv_pointer result
= allocate_and_init (the_map
);
905 dtv
[GET_ADDR_MODULE
].pointer
= result
;
906 assert (result
.to_free
!= NULL
);
908 return (char *) result
.val
+ GET_ADDR_OFFSET
;
912 static struct link_map
*
913 __attribute_noinline__
914 update_get_addr (GET_ADDR_ARGS
, size_t gen
)
916 struct link_map
*the_map
= _dl_update_slotinfo (GET_ADDR_MODULE
, gen
);
917 dtv_t
*dtv
= THREAD_DTV ();
919 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
921 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
922 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, the_map
);
924 return (void *) p
+ GET_ADDR_OFFSET
;
927 /* For all machines that have a non-macro version of __tls_get_addr, we
928 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
929 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
930 in ld.so for __tls_get_addr. */
932 #ifndef __tls_get_addr
933 extern void * __tls_get_addr (GET_ADDR_ARGS
);
934 rtld_hidden_proto (__tls_get_addr
)
935 rtld_hidden_def (__tls_get_addr
)
938 /* The generic dynamic and local dynamic model cannot be used in
939 statically linked applications. */
941 __tls_get_addr (GET_ADDR_ARGS
)
943 dtv_t
*dtv
= THREAD_DTV ();
945 /* Update is needed if dtv[0].counter < the generation of the accessed
946 module, but the global generation counter is easier to check (which
947 must be synchronized up to the generation of the accessed module by
948 user code doing the TLS access so relaxed mo read is enough). */
949 size_t gen
= atomic_load_relaxed (&GL(dl_tls_generation
));
950 if (__glibc_unlikely (dtv
[0].counter
!= gen
))
952 /* Update DTV up to the global generation, see CONCURRENCY NOTES
953 in _dl_update_slotinfo. */
954 gen
= atomic_load_acquire (&GL(dl_tls_generation
));
955 return update_get_addr (GET_ADDR_PARAM
, gen
);
958 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
960 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
961 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, NULL
);
963 return (char *) p
+ GET_ADDR_OFFSET
;
968 /* Look up the module's TLS block as for __tls_get_addr,
969 but never touch anything. Return null if it's not allocated yet. */
971 _dl_tls_get_addr_soft (struct link_map
*l
)
973 if (__glibc_unlikely (l
->l_tls_modid
== 0))
974 /* This module has no TLS segment. */
977 dtv_t
*dtv
= THREAD_DTV ();
978 /* This may be called without holding the GL(dl_load_tls_lock). Reading
979 arbitrary gen value is fine since this is best effort code. */
980 size_t gen
= atomic_load_relaxed (&GL(dl_tls_generation
));
981 if (__glibc_unlikely (dtv
[0].counter
!= gen
))
983 /* This thread's DTV is not completely current,
984 but it might already cover this module. */
986 if (l
->l_tls_modid
>= dtv
[-1].counter
)
990 size_t idx
= l
->l_tls_modid
;
991 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
992 while (idx
>= listp
->len
)
998 /* We've reached the slot for this module.
999 If its generation counter is higher than the DTV's,
1000 this thread does not know about this module yet. */
1001 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
1005 void *data
= dtv
[l
->l_tls_modid
].pointer
.val
;
1006 if (__glibc_unlikely (data
== TLS_DTV_UNALLOCATED
))
1007 /* The DTV is current, but this thread has not yet needed
1008 to allocate this module's segment. */
1016 _dl_add_to_slotinfo (struct link_map
*l
, bool do_add
)
1018 /* Now that we know the object is loaded successfully add
1019 modules containing TLS data to the dtv info table. We
1020 might have to increase its size. */
1021 struct dtv_slotinfo_list
*listp
;
1022 struct dtv_slotinfo_list
*prevp
;
1023 size_t idx
= l
->l_tls_modid
;
1025 /* Find the place in the dtv slotinfo list. */
1026 listp
= GL(dl_tls_dtv_slotinfo_list
);
1027 prevp
= NULL
; /* Needed to shut up gcc. */
1030 /* Does it fit in the array of this list element? */
1031 if (idx
< listp
->len
)
1035 listp
= listp
->next
;
1037 while (listp
!= NULL
);
1041 /* When we come here it means we have to add a new element
1042 to the slotinfo list. And the new module must be in
1046 listp
= (struct dtv_slotinfo_list
*)
1047 malloc (sizeof (struct dtv_slotinfo_list
)
1048 + TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
1051 /* We ran out of memory while resizing the dtv slotinfo list. */
1052 _dl_signal_error (ENOMEM
, "dlopen", NULL
, N_("\
1053 cannot create TLS data structures"));
1056 listp
->len
= TLS_SLOTINFO_SURPLUS
;
1058 memset (listp
->slotinfo
, '\0',
1059 TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
1060 /* Synchronize with _dl_update_slotinfo. */
1061 atomic_store_release (&prevp
->next
, listp
);
1064 /* Add the information into the slotinfo data structure. */
1067 /* Can be read concurrently. See _dl_update_slotinfo. */
1068 atomic_store_relaxed (&listp
->slotinfo
[idx
].map
, l
);
1069 atomic_store_relaxed (&listp
->slotinfo
[idx
].gen
,
1070 GL(dl_tls_generation
) + 1);
1075 static inline void __attribute__((always_inline
))
1076 init_one_static_tls (struct pthread
*curp
, struct link_map
*map
)
1079 void *dest
= (char *) curp
- map
->l_tls_offset
;
1080 # elif TLS_DTV_AT_TP
1081 void *dest
= (char *) curp
+ map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
1083 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1086 /* Initialize the memory. */
1087 memset (__mempcpy (dest
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
1088 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
1092 _dl_init_static_tls (struct link_map
*map
)
1094 lll_lock (GL (dl_stack_cache_lock
), LLL_PRIVATE
);
1096 /* Iterate over the list with system-allocated threads first. */
1098 list_for_each (runp
, &GL (dl_stack_used
))
1099 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1101 /* Now the list with threads using user-allocated stacks. */
1102 list_for_each (runp
, &GL (dl_stack_user
))
1103 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1105 lll_unlock (GL (dl_stack_cache_lock
), LLL_PRIVATE
);
1107 #endif /* PTHREAD_IN_LIBC */