1 /* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
24 #include <sys/param.h>
26 #include <abort-instr.h>
29 /* We don't need any of this if TLS is not supported. */
33 # include <ldsodefs.h>
35 /* Value used for dtv entries for which the allocation is delayed. */
36 # define TLS_DTV_UNALLOCATED ((void *) -1l)
39 /* Out-of-memory handler. */
42 __attribute__ ((__noreturn__
))
45 static const char msg
[] = "\
46 cannot allocate memory for thread-local data: ABORT\n";
48 __libc_write (STDERR_FILENO
, msg
, sizeof (msg
) - 1);
51 __kill (__getpid (), SIGKILL
);
53 /* Just in case something goes wrong with the kill. */
56 # ifdef ABORT_INSTRUCTION
67 _dl_next_tls_modid (void)
71 if (__builtin_expect (GL(dl_tls_dtv_gaps
), false))
74 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
76 /* Note that this branch will never be executed during program
77 start since there are no gaps at that time. Therefore it
78 does not matter that the dl_tls_dtv_slotinfo is not allocated
79 yet when the function is called for the first times. */
80 result
= GL(dl_tls_static_nelem
) + 1;
81 /* If the following would not be true we mustn't have assumed
83 assert (result
<= GL(dl_tls_max_dtv_idx
));
86 while (result
- disp
< runp
->len
)
88 if (runp
->slotinfo
[result
- disp
].map
== NULL
)
92 assert (result
<= GL(dl_tls_max_dtv_idx
) + 1);
95 if (result
- disp
< runp
->len
)
100 while ((runp
= runp
->next
) != NULL
);
102 if (result
>= GL(dl_tls_max_dtv_idx
))
104 /* The new index must indeed be exactly one higher than the
106 assert (result
== GL(dl_tls_max_dtv_idx
));
108 /* There is no gap anymore. */
109 GL(dl_tls_dtv_gaps
) = false;
116 /* No gaps, allocate a new entry. */
118 result
= ++GL(dl_tls_max_dtv_idx
);
127 _dl_determine_tlsoffset (void)
129 struct dtv_slotinfo
*slotinfo
;
130 size_t max_align
= __alignof__ (void *);
134 /* The first element of the dtv slot info list is allocated. */
135 assert (GL(dl_tls_dtv_slotinfo_list
) != NULL
);
136 /* There is at this point only one element in the
137 dl_tls_dtv_slotinfo_list list. */
138 assert (GL(dl_tls_dtv_slotinfo_list
)->next
== NULL
);
141 /* We simply start with zero. */
144 slotinfo
= GL(dl_tls_dtv_slotinfo_list
)->slotinfo
;
145 for (cnt
= 1; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
147 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
149 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
151 /* Compute the offset of the next TLS block. */
152 offset
= roundup (offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
,
153 slotinfo
[cnt
].map
->l_tls_align
);
155 /* XXX For some architectures we perhaps should store the
157 slotinfo
[cnt
].map
->l_tls_offset
= offset
;
160 /* The thread descriptor (pointed to by the thread pointer) has its
161 own alignment requirement. Adjust the static TLS size
162 and TLS offsets appropriately. */
163 // XXX How to deal with this. We cannot simply add zero bytes
164 // XXX after the first (closest to the TCB) TLS block since this
165 // XXX would invalidate the offsets the linker creates for the LE
168 GL(dl_tls_static_size
) = offset
+ TLS_TCB_SIZE
;
170 /* The TLS blocks start right after the TCB. */
171 offset
= TLS_TCB_SIZE
;
173 /* The first block starts right after the TCB. */
174 slotinfo
= GL(dl_tls_dtv_slotinfo_list
)->slotinfo
;
175 if (slotinfo
[1].map
!= NULL
)
179 offset
= roundup (offset
, slotinfo
[1].map
->l_tls_align
);
180 slotinfo
[1].map
->l_tls_offset
= offset
;
181 max_align
= slotinfo
[1].map
->l_tls_align
;
182 prev_size
= slotinfo
[1].map
->l_tls_blocksize
;
184 for (cnt
= 2; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
186 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
188 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
190 /* Compute the offset of the next TLS block. */
191 offset
= roundup (offset
+ prev_size
,
192 slotinfo
[cnt
].map
->l_tls_align
);
194 /* XXX For some architectures we perhaps should store the
196 slotinfo
[cnt
].map
->l_tls_offset
= offset
;
198 prev_size
= slotinfo
[cnt
].map
->l_tls_blocksize
;
204 GL(dl_tls_static_size
) = offset
;
206 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
209 /* The alignment requirement for the static TLS block. */
210 GL(dl_tls_static_align
) = MAX (TLS_TCB_ALIGN
, max_align
);
216 _dl_allocate_tls (void)
222 /* Allocate a correctly aligned chunk of memory. */
224 assert (GL(dl_tls_static_align
) <= GL(dl_pagesize
));
226 # define _dl_zerofd (-1)
228 # define _dl_zerofd GL(dl_zerofd)
229 if ((dl_zerofd
) == -1)
230 GL(dl_zerofd
) = _dl_sysdep_open_zero_fill ();
233 result
= __mmap (0, GL(dl_tls_static_size
), PROT_READ
|PROT_WRITE
,
234 MAP_ANON
|MAP_PRIVATE
, _dl_zerofd
, 0);
236 /* We allocate a few more elements in the dtv than are needed for the
237 initial set of modules. This should avoid in most cases expansions
239 dtv_length
= GL(dl_tls_max_dtv_idx
) + DTV_SURPLUS
;
240 dtv
= (dtv_t
*) malloc ((dtv_length
+ 2) * sizeof (dtv_t
));
241 if (result
!= MAP_FAILED
&& dtv
!= NULL
)
243 struct dtv_slotinfo_list
*listp
;
244 bool first_block
= true;
248 /* The TCB follows the TLS blocks. */
249 result
= (char *) result
+ GL(dl_tls_static_size
) - TLS_TCB_SIZE
;
252 /* This is the initial length of the dtv. */
253 dtv
[0].counter
= dtv_length
;
254 /* Fill in the generation number. */
255 dtv
[1].counter
= GL(dl_tls_generation
) = 0;
256 /* Initialize all of the rest of the dtv with zero to indicate
258 memset (dtv
+ 2, '\0', dtv_length
* sizeof (dtv_t
));
260 /* We have to look prepare the dtv for all currently loaded
261 modules using TLS. For those which are dynamically loaded we
262 add the values indicating deferred allocation. */
263 listp
= GL(dl_tls_dtv_slotinfo_list
);
268 for (cnt
= first_block
? 1 : 0; cnt
< listp
->len
; ++cnt
)
270 struct link_map
*map
;
273 /* Check for the total number of used slots. */
274 if (total
+ cnt
>= GL(dl_tls_max_dtv_idx
))
277 map
= listp
->slotinfo
[cnt
].map
;
282 if (map
->l_type
== lt_loaded
)
284 /* For dynamically loaded modules we simply store
285 the value indicating deferred allocation. */
286 dtv
[1 + map
->l_tls_modid
].pointer
= TLS_DTV_UNALLOCATED
;
290 assert (map
->l_tls_modid
== cnt
);
291 assert (map
->l_tls_blocksize
>= map
->l_tls_initimage_size
);
293 assert (map
->l_tls_offset
>= map
->l_tls_blocksize
);
294 dest
= (char *) result
- map
->l_tls_offset
;
296 dest
= (char *) result
+ map
->l_tls_offset
;
298 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
301 /* We don't have to clear the BSS part of the TLS block
302 since mmap is used to allocate the memory which
303 guarantees it is initialized to zero. */
304 dtv
[1 + cnt
].pointer
= memcpy (dest
, map
->l_tls_initimage
,
305 map
->l_tls_initimage_size
);
309 if (total
>= GL(dl_tls_max_dtv_idx
))
313 assert (listp
!= NULL
);
316 /* Add the dtv to the thread data structures. */
317 INSTALL_DTV (result
, dtv
);
319 else if (result
!= NULL
)
327 INTDEF(_dl_allocate_tls
)
332 _dl_deallocate_tls (void *tcb
)
334 dtv_t
*dtv
= GET_DTV (tcb
);
336 /* The array starts with dtv[-1]. */
339 munmap (tcb
, GL(dl_tls_static_size
));
345 /* The __tls_get_addr function has two basic forms which differ in the
346 arguments. The IA-64 form takes two parameters, the module ID and
347 offset. The form used, among others, on IA-32 takes a reference to
348 a special structure which contain the same information. The second
349 form seems to be more often used (in the moment) so we default to
350 it. Users of the IA-64 form have to provide adequate definitions
351 of the following macros. */
352 # ifndef GET_ADDR_ARGS
353 # define GET_ADDR_ARGS tls_index *ti
355 # ifndef GET_ADDR_MODULE
356 # define GET_ADDR_MODULE ti->ti_module
358 # ifndef GET_ADDR_OFFSET
359 # define GET_ADDR_OFFSET ti->ti_offset
361 /* Systems which do not have tls_index also probably have to define
362 DONT_USE_TLS_INDEX. */
364 # ifndef __TLS_GET_ADDR
365 # define __TLS_GET_ADDR __tls_get_addr
369 /* Return the symbol address given the map of the module it is in and
370 the symbol record. This is used in dl-sym.c. */
373 _dl_tls_symaddr (struct link_map
*map
, const ElfW(Sym
) *ref
)
375 # ifndef DONT_USE_TLS_INDEX
378 .ti_module
= map
->l_tls_modid
,
379 .ti_offset
= ref
->st_value
382 return __TLS_GET_ADDR (&tmp
);
384 return __TLS_GET_ADDR (map
->l_tls_modid
, ref
->st_value
);
390 allocate_and_init (struct link_map
*map
)
394 newp
= __libc_memalign (map
->l_tls_align
, map
->l_tls_blocksize
);
398 /* Initialize the memory. */
399 memset (__mempcpy (newp
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
400 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
406 /* The generic dynamic and local dynamic model cannot be used in
407 statically linked applications. */
409 __tls_get_addr (GET_ADDR_ARGS
)
411 dtv_t
*dtv
= THREAD_DTV ();
412 struct link_map
*the_map
= NULL
;
415 if (__builtin_expect (dtv
[0].counter
!= GL(dl_tls_generation
), 0))
417 struct dtv_slotinfo_list
*listp
;
420 /* The global dl_tls_dtv_slotinfo array contains for each module
421 index the generation counter current when the entry was
422 created. This array never shrinks so that all module indices
423 which were valid at some time can be used to access it.
424 Before the first use of a new module index in this function
425 the array was extended appropriately. Access also does not
426 have to be guarded against modifications of the array. It is
427 assumed that pointer-size values can be read atomically even
428 in SMP environments. It is possible that other threads at
429 the same time dynamically load code and therefore add to the
430 slotinfo list. This is a problem since we must not pick up
431 any information about incomplete work. The solution to this
432 is to ignore all dtv slots which were created after the one
433 we are currently interested. We know that dynamic loading
434 for this module is completed and this is the last load
435 operation we know finished. */
436 idx
= GET_ADDR_MODULE
;
437 listp
= GL(dl_tls_dtv_slotinfo_list
);
438 while (idx
>= listp
->len
)
444 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
446 /* The generation counter for the slot is higher than what
447 the current dtv implements. We have to update the whole
448 dtv but only those entries with a generation counter <=
449 the one for the entry we need. */
450 size_t new_gen
= listp
->slotinfo
[idx
].gen
;
453 /* We have to look through the entire dtv slotinfo list. */
454 listp
= GL(dl_tls_dtv_slotinfo_list
);
459 for (cnt
= total
= 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
461 size_t gen
= listp
->slotinfo
[cnt
].gen
;
462 struct link_map
*map
;
466 /* This is a slot for a generation younger than
467 the one we are handling now. It might be
468 incompletely set up so ignore it. */
471 /* If the entry is older than the current dtv layout
472 we know we don't have to handle it. */
473 if (gen
<= dtv
[0].counter
)
476 /* If there is no map this means the entry is empty. */
477 map
= listp
->slotinfo
[cnt
].map
;
480 /* If this modid was used at some point the memory
481 might still be allocated. */
482 if (dtv
[total
+ cnt
].pointer
!= TLS_DTV_UNALLOCATED
)
483 free (dtv
[total
+ cnt
].pointer
);
488 /* Check whether the current dtv array is large enough. */
489 modid
= map
->l_tls_modid
;
490 assert (total
+ cnt
== modid
);
491 if (dtv
[-1].counter
< modid
)
493 /* Reallocate the dtv. */
495 size_t newsize
= GL(dl_tls_max_dtv_idx
) + DTV_SURPLUS
;
496 size_t oldsize
= dtv
[-1].counter
;
498 assert (map
->l_tls_modid
<= newsize
);
500 newp
= (dtv_t
*) realloc (&dtv
[-1],
506 newp
[0].counter
= newsize
;
508 /* Clear the newly allocate part. */
509 memset (newp
+ 2 + oldsize
, '\0',
510 (newsize
- oldsize
) * sizeof (dtv_t
));
512 /* Point dtv to the generation counter. */
515 /* Install this new dtv in the thread data
517 INSTALL_NEW_DTV (dtv
);
520 /* If there is currently memory allocate for this
521 dtv entry free it. */
522 /* XXX Ideally we will at some point create a memory
524 if (dtv
[modid
].pointer
!= TLS_DTV_UNALLOCATED
)
525 /* Note that free is called for NULL is well. We
526 deallocate even if it is this dtv entry we are
527 supposed to load. The reason is that we call
528 memalign and not malloc. */
529 free (dtv
[modid
].pointer
);
531 /* This module is loaded dynamically- We defer
532 memory allocation. */
533 dtv
[modid
].pointer
= TLS_DTV_UNALLOCATED
;
535 if (modid
== GET_ADDR_MODULE
)
541 while ((listp
= listp
->next
) != NULL
);
543 /* This will be the new maximum generation counter. */
544 dtv
[0].counter
= new_gen
;
548 p
= dtv
[GET_ADDR_MODULE
].pointer
;
550 if (__builtin_expect (p
== TLS_DTV_UNALLOCATED
, 0))
552 /* The allocation was deferred. Do it now. */
555 /* Find the link map for this module. */
556 size_t idx
= GET_ADDR_MODULE
;
557 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
559 while (idx
>= listp
->len
)
565 the_map
= listp
->slotinfo
[idx
].map
;
568 p
= dtv
[GET_ADDR_MODULE
].pointer
= allocate_and_init (the_map
);
571 return (char *) p
+ GET_ADDR_OFFSET
;