20 #include "pthread_impl.h"
24 static void error(const char *, ...);
28 #define MAXP2(a,b) (-(-(a)&-(b)))
29 #define ALIGN(x,y) ((x)+(y)-1 & -(y))
41 struct td_index
*next
;
48 struct dso
*next
, *prev
;
67 struct dso
**deps
, *needed_by
;
68 char *rpath_orig
, *rpath
;
70 size_t tls_len
, tls_size
, tls_align
, tls_id
, tls_offset
;
71 size_t relro_start
, relro_end
;
73 unsigned char *new_tls
;
74 volatile int new_dtv_idx
, new_tls_idx
;
75 struct td_index
*td_index
;
76 struct dso
*fini_next
;
86 int __init_tp(void *);
87 void __init_libc(char **, char *);
89 const char *__libc_get_version(void);
91 static struct builtin_tls
{
96 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
98 #define ADDEND_LIMIT 4096
99 static size_t *saved_addends
, *apply_addends_to
;
101 static struct dso ldso
;
102 static struct dso
*head
, *tail
, *fini_head
;
103 static char *env_path
, *sys_path
;
104 static unsigned long long gencnt
;
107 static int ldso_fail
;
109 static jmp_buf *rtld_fail
;
110 static pthread_rwlock_t lock
;
111 static struct debug debug
;
112 static size_t tls_cnt
, tls_offset
, tls_align
= MIN_TLS_ALIGN
;
113 static size_t static_tls_cnt
;
114 static pthread_mutex_t init_fini_lock
= { ._m_type
= PTHREAD_MUTEX_RECURSIVE
};
116 struct debug
*_dl_debug_addr
= &debug
;
118 static int dl_strcmp(const char *l
, const char *r
)
120 for (; *l
==*r
&& *l
; l
++, r
++);
121 return *(unsigned char *)l
- *(unsigned char *)r
;
123 #define strcmp(l,r) dl_strcmp(l,r)
125 static void decode_vec(size_t *v
, size_t *a
, size_t cnt
)
128 for (i
=0; i
<cnt
; i
++) a
[i
] = 0;
129 for (; v
[0]; v
+=2) if (v
[0]-1<cnt
-1) {
135 static int search_vec(size_t *v
, size_t *r
, size_t key
)
137 for (; v
[0]!=key
; v
+=2)
143 static uint32_t sysv_hash(const char *s0
)
145 const unsigned char *s
= (void *)s0
;
151 return h
& 0xfffffff;
154 static uint32_t gnu_hash(const char *s0
)
156 const unsigned char *s
= (void *)s0
;
157 uint_fast32_t h
= 5381;
163 static Sym
*sysv_lookup(const char *s
, uint32_t h
, struct dso
*dso
)
166 Sym
*syms
= dso
->syms
;
167 uint32_t *hashtab
= dso
->hashtab
;
168 char *strings
= dso
->strings
;
169 for (i
=hashtab
[2+h
%hashtab
[0]]; i
; i
=hashtab
[2+hashtab
[0]+i
]) {
170 if ((!dso
->versym
|| dso
->versym
[i
] >= 0)
171 && (!strcmp(s
, strings
+syms
[i
].st_name
)))
177 static Sym
*gnu_lookup(const char *s
, uint32_t h1
, struct dso
*dso
)
179 Sym
*syms
= dso
->syms
;
180 char *strings
= dso
->strings
;
181 uint32_t *hashtab
= dso
->ghashtab
;
182 uint32_t nbuckets
= hashtab
[0];
183 uint32_t *buckets
= hashtab
+ 4 + hashtab
[2]*(sizeof(size_t)/4);
186 uint32_t i
= buckets
[h1
% nbuckets
];
190 hashval
= buckets
+ nbuckets
+ (i
- hashtab
[1]);
192 for (h1
|= 1; ; i
++) {
194 if ((!dso
->versym
|| dso
->versym
[i
] >= 0)
195 && (h1
== (h2
|1)) && !strcmp(s
, strings
+ syms
[i
].st_name
))
203 #define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON | 1<<STT_TLS)
204 #define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
206 #ifndef ARCH_SYM_REJECT_UND
207 #define ARCH_SYM_REJECT_UND(s) 0
210 static struct symdef
find_sym(struct dso
*dso
, const char *s
, int need_def
)
212 uint32_t h
= 0, gh
= 0;
213 struct symdef def
= {0};
214 for (; dso
; dso
=dso
->next
) {
216 if (!dso
->global
) continue;
218 if (!gh
) gh
= gnu_hash(s
);
219 sym
= gnu_lookup(s
, gh
, dso
);
221 if (!h
) h
= sysv_hash(s
);
222 sym
= sysv_lookup(s
, h
, dso
);
226 if (need_def
|| (sym
->st_info
&0xf) == STT_TLS
227 || ARCH_SYM_REJECT_UND(sym
))
230 if ((sym
->st_info
&0xf) != STT_TLS
)
232 if (!(1<<(sym
->st_info
&0xf) & OK_TYPES
)) continue;
233 if (!(1<<(sym
->st_info
>>4) & OK_BINDS
)) continue;
235 if (def
.sym
&& sym
->st_info
>>4 == STB_WEAK
) continue;
238 if (sym
->st_info
>>4 == STB_GLOBAL
) break;
243 __attribute__((__visibility__("hidden")))
244 ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic();
246 static void do_relocs(struct dso
*dso
, size_t *rel
, size_t rel_size
, size_t stride
)
248 unsigned char *base
= dso
->base
;
249 Sym
*syms
= dso
->syms
;
250 char *strings
= dso
->strings
;
261 int skip_relative
= 0, reuse_addends
= 0, save_slot
= 0;
264 /* Only ldso's REL table needs addend saving/reuse. */
265 if (rel
== apply_addends_to
)
270 for (; rel_size
; rel
+=stride
, rel_size
-=stride
*sizeof(size_t)) {
271 if (skip_relative
&& IS_RELATIVE(rel
[1])) continue;
272 type
= R_TYPE(rel
[1]);
273 if (type
== REL_NONE
) continue;
274 sym_index
= R_SYM(rel
[1]);
275 reloc_addr
= (void *)(base
+ rel
[0]);
277 sym
= syms
+ sym_index
;
278 name
= strings
+ sym
->st_name
;
279 ctx
= type
==REL_COPY
? head
->next
: head
;
280 def
= find_sym(ctx
, name
, type
==REL_PLT
);
281 if (!def
.sym
&& (sym
->st_shndx
!= SHN_UNDEF
282 || sym
->st_info
>>4 != STB_WEAK
)) {
283 error("Error relocating %s: %s: symbol not found",
285 if (runtime
) longjmp(*rtld_fail
, 1);
296 } else if (type
==REL_GOT
|| type
==REL_PLT
|| type
==REL_COPY
) {
298 } else if (reuse_addends
) {
299 /* Save original addend in stage 2 where the dso
300 * chain consists of just ldso; otherwise read back
301 * saved addend since the inline one was clobbered. */
303 saved_addends
[save_slot
] = *reloc_addr
;
304 addend
= saved_addends
[save_slot
++];
306 addend
= *reloc_addr
;
309 sym_val
= def
.sym
? (size_t)def
.dso
->base
+def
.sym
->st_value
: 0;
310 tls_val
= def
.sym
? def
.sym
->st_value
: 0;
316 addend
-= (size_t)reloc_addr
;
320 *reloc_addr
= sym_val
+ addend
;
323 *reloc_addr
= (size_t)base
+ addend
;
326 if (sym
) *reloc_addr
= sym_val
+ addend
;
327 else *reloc_addr
= (size_t)base
+ addend
;
330 memcpy(reloc_addr
, (void *)sym_val
, sym
->st_size
);
333 *(uint32_t *)reloc_addr
= sym_val
+ addend
334 - (size_t)reloc_addr
;
337 *reloc_addr
= def
.dso
->tls_id
;
340 *reloc_addr
= tls_val
+ addend
- DTP_OFFSET
;
344 *reloc_addr
= tls_val
+ def
.dso
->tls_offset
+ TPOFF_K
+ addend
;
348 *reloc_addr
= tls_val
- def
.dso
->tls_offset
+ addend
;
351 *reloc_addr
= def
.dso
->tls_offset
- tls_val
+ addend
;
355 if (stride
<3) addend
= reloc_addr
[1];
356 if (runtime
&& def
.dso
->tls_id
>= static_tls_cnt
) {
357 struct td_index
*new = malloc(sizeof *new);
360 "Error relocating %s: cannot allocate TLSDESC for %s",
361 dso
->name
, sym
? name
: "(local)" );
362 longjmp(*rtld_fail
, 1);
364 new->next
= dso
->td_index
;
366 new->args
[0] = def
.dso
->tls_id
;
367 new->args
[1] = tls_val
+ addend
;
368 reloc_addr
[0] = (size_t)__tlsdesc_dynamic
;
369 reloc_addr
[1] = (size_t)new;
371 reloc_addr
[0] = (size_t)__tlsdesc_static
;
373 reloc_addr
[1] = tls_val
+ def
.dso
->tls_offset
376 reloc_addr
[1] = tls_val
- def
.dso
->tls_offset
382 error("Error relocating %s: unsupported relocation type %d",
384 if (runtime
) longjmp(*rtld_fail
, 1);
390 /* A huge hack: to make up for the wastefulness of shared libraries
391 * needing at least a page of dirty memory even if they have no global
392 * data, we reclaim the gaps at the beginning and end of writable maps
393 * and "donate" them to the heap by setting up minimal malloc
394 * structures and then freeing them. */
396 static void reclaim(struct dso
*dso
, size_t start
, size_t end
)
399 if (start
>= dso
->relro_start
&& start
< dso
->relro_end
) start
= dso
->relro_end
;
400 if (end
>= dso
->relro_start
&& end
< dso
->relro_end
) end
= dso
->relro_start
;
401 start
= start
+ 6*sizeof(size_t)-1 & -4*sizeof(size_t);
402 end
= (end
& -4*sizeof(size_t)) - 2*sizeof(size_t);
403 if (start
>end
|| end
-start
< 4*sizeof(size_t)) return;
404 a
= (size_t *)(dso
->base
+ start
);
405 z
= (size_t *)(dso
->base
+ end
);
407 a
[-1] = z
[0] = end
-start
+ 2*sizeof(size_t) | 1;
412 static void reclaim_gaps(struct dso
*dso
)
414 Phdr
*ph
= dso
->phdr
;
415 size_t phcnt
= dso
->phnum
;
417 for (; phcnt
--; ph
=(void *)((char *)ph
+dso
->phentsize
)) {
418 if (ph
->p_type
!=PT_LOAD
) continue;
419 if ((ph
->p_flags
&(PF_R
|PF_W
))!=(PF_R
|PF_W
)) continue;
420 reclaim(dso
, ph
->p_vaddr
& -PAGE_SIZE
, ph
->p_vaddr
);
421 reclaim(dso
, ph
->p_vaddr
+ph
->p_memsz
,
422 ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
);
426 static void *mmap_fixed(void *p
, size_t n
, int prot
, int flags
, int fd
, off_t off
)
428 char *q
= mmap(p
, n
, prot
, flags
, fd
, off
);
429 if (q
!= MAP_FAILED
|| errno
!= EINVAL
) return q
;
430 /* Fallbacks for MAP_FIXED failure on NOMMU kernels. */
431 if (flags
& MAP_ANONYMOUS
) {
436 if (lseek(fd
, off
, SEEK_SET
) < 0) return MAP_FAILED
;
437 for (q
=p
; n
; q
+=r
, off
+=r
, n
-=r
) {
439 if (r
< 0 && errno
!= EINTR
) return MAP_FAILED
;
448 static void *map_library(int fd
, struct dso
*dso
)
450 Ehdr buf
[(896+sizeof(Ehdr
))/sizeof(Ehdr
)];
451 void *allocated_buf
=0;
453 size_t addr_min
=SIZE_MAX
, addr_max
=0, map_len
;
454 size_t this_min
, this_max
;
459 unsigned char *map
=MAP_FAILED
, *base
;
464 ssize_t l
= read(fd
, buf
, sizeof buf
);
467 if (l
<sizeof *eh
|| (eh
->e_type
!= ET_DYN
&& eh
->e_type
!= ET_EXEC
))
469 phsize
= eh
->e_phentsize
* eh
->e_phnum
;
470 if (phsize
> sizeof buf
- sizeof *eh
) {
471 allocated_buf
= malloc(phsize
);
472 if (!allocated_buf
) return 0;
473 l
= pread(fd
, allocated_buf
, phsize
, eh
->e_phoff
);
474 if (l
< 0) goto error
;
475 if (l
!= phsize
) goto noexec
;
476 ph
= ph0
= allocated_buf
;
477 } else if (eh
->e_phoff
+ phsize
> l
) {
478 l
= pread(fd
, buf
+1, phsize
, eh
->e_phoff
);
479 if (l
< 0) goto error
;
480 if (l
!= phsize
) goto noexec
;
481 ph
= ph0
= (void *)(buf
+ 1);
483 ph
= ph0
= (void *)((char *)buf
+ eh
->e_phoff
);
485 for (i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
486 if (ph
->p_type
== PT_DYNAMIC
) {
488 } else if (ph
->p_type
== PT_TLS
) {
489 tls_image
= ph
->p_vaddr
;
490 dso
->tls_align
= ph
->p_align
;
491 dso
->tls_len
= ph
->p_filesz
;
492 dso
->tls_size
= ph
->p_memsz
;
493 } else if (ph
->p_type
== PT_GNU_RELRO
) {
494 dso
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
495 dso
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
497 if (ph
->p_type
!= PT_LOAD
) continue;
498 if (ph
->p_vaddr
< addr_min
) {
499 addr_min
= ph
->p_vaddr
;
500 off_start
= ph
->p_offset
;
501 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
502 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
503 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
505 if (ph
->p_vaddr
+ph
->p_memsz
> addr_max
) {
506 addr_max
= ph
->p_vaddr
+ph
->p_memsz
;
509 if (!dyn
) goto noexec
;
510 addr_max
+= PAGE_SIZE
-1;
511 addr_max
&= -PAGE_SIZE
;
512 addr_min
&= -PAGE_SIZE
;
513 off_start
&= -PAGE_SIZE
;
514 map_len
= addr_max
- addr_min
+ off_start
;
515 /* The first time, we map too much, possibly even more than
516 * the length of the file. This is okay because we will not
517 * use the invalid part; we just need to reserve the right
518 * amount of virtual address space to map over later. */
519 map
= mmap((void *)addr_min
, map_len
, prot
, MAP_PRIVATE
, fd
, off_start
);
520 if (map
==MAP_FAILED
) goto error
;
521 /* If the loaded file is not relocatable and the requested address is
522 * not available, then the load operation must fail. */
523 if (eh
->e_type
!= ET_DYN
&& addr_min
&& map
!=(void *)addr_min
) {
527 base
= map
- addr_min
;
530 for (ph
=ph0
, i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
531 if (ph
->p_type
!= PT_LOAD
) continue;
532 /* Check if the programs headers are in this load segment, and
533 * if so, record the address for use by dl_iterate_phdr. */
534 if (!dso
->phdr
&& eh
->e_phoff
>= ph
->p_offset
535 && eh
->e_phoff
+phsize
<= ph
->p_offset
+ph
->p_filesz
) {
536 dso
->phdr
= (void *)(base
+ ph
->p_vaddr
537 + (eh
->e_phoff
-ph
->p_offset
));
538 dso
->phnum
= eh
->e_phnum
;
539 dso
->phentsize
= eh
->e_phentsize
;
541 /* Reuse the existing mapping for the lowest-address LOAD */
542 if ((ph
->p_vaddr
& -PAGE_SIZE
) == addr_min
) continue;
543 this_min
= ph
->p_vaddr
& -PAGE_SIZE
;
544 this_max
= ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
;
545 off_start
= ph
->p_offset
& -PAGE_SIZE
;
546 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
547 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
548 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
549 if (mmap_fixed(base
+this_min
, this_max
-this_min
, prot
, MAP_PRIVATE
|MAP_FIXED
, fd
, off_start
) == MAP_FAILED
)
551 if (ph
->p_memsz
> ph
->p_filesz
) {
552 size_t brk
= (size_t)base
+ph
->p_vaddr
+ph
->p_filesz
;
553 size_t pgbrk
= brk
+PAGE_SIZE
-1 & -PAGE_SIZE
;
554 memset((void *)brk
, 0, pgbrk
-brk
& PAGE_SIZE
-1);
555 if (pgbrk
-(size_t)base
< this_max
&& mmap_fixed((void *)pgbrk
, (size_t)base
+this_max
-pgbrk
, prot
, MAP_PRIVATE
|MAP_FIXED
|MAP_ANONYMOUS
, -1, 0) == MAP_FAILED
)
559 for (i
=0; ((size_t *)(base
+dyn
))[i
]; i
+=2)
560 if (((size_t *)(base
+dyn
))[i
]==DT_TEXTREL
) {
561 if (mprotect(map
, map_len
, PROT_READ
|PROT_WRITE
|PROT_EXEC
)
567 dso
->map_len
= map_len
;
569 dso
->dynv
= (void *)(base
+dyn
);
570 if (dso
->tls_size
) dso
->tls_image
= (void *)(base
+tls_image
);
571 if (!runtime
) reclaim_gaps(dso
);
577 if (map
!=MAP_FAILED
) munmap(map
, map_len
);
582 static int path_open(const char *name
, const char *s
, char *buf
, size_t buf_size
)
587 s
+= strspn(s
, ":\n");
588 l
= strcspn(s
, ":\n");
589 if (l
-1 >= INT_MAX
) return -1;
590 if (snprintf(buf
, buf_size
, "%.*s/%s", (int)l
, s
, name
) < buf_size
) {
591 if ((fd
= open(buf
, O_RDONLY
|O_CLOEXEC
))>=0) return fd
;
599 /* Any negative value but -1 will inhibit
600 * futher path search. */
608 static int fixup_rpath(struct dso
*p
, char *buf
, size_t buf_size
)
611 const char *s
, *t
, *origin
;
613 if (p
->rpath
|| !p
->rpath_orig
) return 0;
614 if (!strchr(p
->rpath_orig
, '$')) {
615 p
->rpath
= p
->rpath_orig
;
620 while ((t
=strchr(s
, '$'))) {
621 if (strncmp(t
, "$ORIGIN", 7) && strncmp(t
, "${ORIGIN}", 9))
626 if (n
> SSIZE_MAX
/PATH_MAX
) return 0;
628 if (p
->kernel_mapped
) {
629 /* $ORIGIN searches cannot be performed for the main program
630 * when it is suid/sgid/AT_SECURE. This is because the
631 * pathname is under the control of the caller of execve.
632 * For libraries, however, $ORIGIN can be processed safely
633 * since the library's pathname came from a trusted source
634 * (either system paths or a call to dlopen). */
637 l
= readlink("/proc/self/exe", buf
, buf_size
);
638 if (l
== -1) switch (errno
) {
653 t
= strrchr(origin
, '/');
654 l
= t
? t
-origin
: 0;
655 p
->rpath
= malloc(strlen(p
->rpath_orig
) + n
*l
+ 1);
656 if (!p
->rpath
) return -1;
660 while ((t
=strchr(s
, '$'))) {
663 memcpy(d
, origin
, l
);
665 /* It was determined previously that the '$' is followed
666 * either by "ORIGIN" or "{ORIGIN}". */
667 s
= t
+ 7 + 2*(t
[1]=='{');
673 static void decode_dyn(struct dso
*p
)
676 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
677 p
->syms
= (void *)(p
->base
+ dyn
[DT_SYMTAB
]);
678 p
->strings
= (void *)(p
->base
+ dyn
[DT_STRTAB
]);
679 if (dyn
[0]&(1<<DT_HASH
))
680 p
->hashtab
= (void *)(p
->base
+ dyn
[DT_HASH
]);
681 if (dyn
[0]&(1<<DT_RPATH
))
682 p
->rpath_orig
= (void *)(p
->strings
+ dyn
[DT_RPATH
]);
683 if (dyn
[0]&(1<<DT_RUNPATH
))
684 p
->rpath_orig
= (void *)(p
->strings
+ dyn
[DT_RUNPATH
]);
685 if (search_vec(p
->dynv
, dyn
, DT_GNU_HASH
))
686 p
->ghashtab
= (void *)(p
->base
+ *dyn
);
687 if (search_vec(p
->dynv
, dyn
, DT_VERSYM
))
688 p
->versym
= (void *)(p
->base
+ *dyn
);
691 static struct dso
*load_library(const char *name
, struct dso
*needed_by
)
693 char buf
[2*NAME_MAX
+2];
694 const char *pathname
;
696 struct dso
*p
, temp_dso
= {0};
708 /* Catch and block attempts to reload the implementation itself */
709 if (name
[0]=='l' && name
[1]=='i' && name
[2]=='b') {
710 static const char *rp
, reserved
[] =
711 "c\0pthread\0rt\0m\0dl\0util\0xnet\0";
712 char *z
= strchr(name
, '.');
715 for (rp
=reserved
; *rp
&& strncmp(name
+3, rp
, l
-3); rp
+=strlen(rp
)+1);
718 /* Track which names have been resolved
719 * and only report each one once. */
720 static unsigned reported
;
721 unsigned mask
= 1U<<(rp
-reserved
);
722 if (!(reported
& mask
)) {
724 dprintf(1, "\t%s => %s (%p)\n",
733 if (!strcmp(name
, ldso
.name
)) is_self
= 1;
738 tail
= ldso
.next
? ldso
.next
: &ldso
;
742 if (strchr(name
, '/')) {
744 fd
= open(name
, O_RDONLY
|O_CLOEXEC
);
746 /* Search for the name to see if it's already loaded */
747 for (p
=head
->next
; p
; p
=p
->next
) {
748 if (p
->shortname
&& !strcmp(p
->shortname
, name
)) {
753 if (strlen(name
) > NAME_MAX
) return 0;
755 if (env_path
) fd
= path_open(name
, env_path
, buf
, sizeof buf
);
756 for (p
=needed_by
; fd
== -1 && p
; p
=p
->needed_by
) {
757 if (fixup_rpath(p
, buf
, sizeof buf
) < 0)
758 fd
= -2; /* Inhibit further search. */
760 fd
= path_open(name
, p
->rpath
, buf
, sizeof buf
);
766 if (ldso
.name
[0]=='/') {
768 for (s
=t
=z
=ldso
.name
; *s
; s
++)
769 if (*s
=='/') z
=t
, t
=s
;
770 prefix_len
= z
-ldso
.name
;
771 if (prefix_len
< PATH_MAX
)
778 char etc_ldso_path
[prefix_len
+ 1
779 + sizeof "/etc/ld-musl-" LDSO_ARCH
".path"];
780 snprintf(etc_ldso_path
, sizeof etc_ldso_path
,
781 "%.*s/etc/ld-musl-" LDSO_ARCH
".path",
782 (int)prefix_len
, prefix
);
783 FILE *f
= fopen(etc_ldso_path
, "rbe");
785 if (getdelim(&sys_path
, (size_t[1]){0}, 0, f
) <= 0) {
790 } else if (errno
!= ENOENT
) {
794 if (!sys_path
) sys_path
= "/lib:/usr/local/lib:/usr/lib";
795 fd
= path_open(name
, sys_path
, buf
, sizeof buf
);
799 if (fd
< 0) return 0;
800 if (fstat(fd
, &st
) < 0) {
804 for (p
=head
->next
; p
; p
=p
->next
) {
805 if (p
->dev
== st
.st_dev
&& p
->ino
== st
.st_ino
) {
806 /* If this library was previously loaded with a
807 * pathname but a search found the same inode,
808 * setup its shortname so it can be found by name. */
809 if (!p
->shortname
&& pathname
!= name
)
810 p
->shortname
= strrchr(p
->name
, '/')+1;
816 map
= noload
? 0 : map_library(fd
, &temp_dso
);
820 /* Allocate storage for the new DSO. When there is TLS, this
821 * storage must include a reservation for all pre-existing
822 * threads to obtain copies of both the new TLS, and an
823 * extended DTV capable of storing an additional slot for
824 * the newly-loaded DSO. */
825 alloc_size
= sizeof *p
+ strlen(pathname
) + 1;
826 if (runtime
&& temp_dso
.tls_image
) {
827 size_t per_th
= temp_dso
.tls_size
+ temp_dso
.tls_align
828 + sizeof(void *) * (tls_cnt
+3);
829 n_th
= libc
.threads_minus_1
+ 1;
830 if (n_th
> SSIZE_MAX
/ per_th
) alloc_size
= SIZE_MAX
;
831 else alloc_size
+= n_th
* per_th
;
833 p
= calloc(1, alloc_size
);
835 munmap(map
, temp_dso
.map_len
);
838 memcpy(p
, &temp_dso
, sizeof temp_dso
);
843 p
->needed_by
= needed_by
;
845 strcpy(p
->name
, pathname
);
846 /* Add a shortname only if name arg was not an explicit pathname. */
847 if (pathname
!= name
) p
->shortname
= strrchr(p
->name
, '/')+1;
849 p
->tls_id
= ++tls_cnt
;
850 tls_align
= MAXP2(tls_align
, p
->tls_align
);
852 p
->tls_offset
= tls_offset
+ ( (tls_align
-1) &
853 -(tls_offset
+ (uintptr_t)p
->tls_image
) );
854 tls_offset
+= p
->tls_size
;
856 tls_offset
+= p
->tls_size
+ p
->tls_align
- 1;
857 tls_offset
-= (tls_offset
+ (uintptr_t)p
->tls_image
)
859 p
->tls_offset
= tls_offset
;
861 p
->new_dtv
= (void *)(-sizeof(size_t) &
862 (uintptr_t)(p
->name
+strlen(p
->name
)+sizeof(size_t)));
863 p
->new_tls
= (void *)(p
->new_dtv
+ n_th
*(tls_cnt
+1));
870 if (ldd_mode
) dprintf(1, "\t%s => %s (%p)\n", name
, pathname
, p
->base
);
875 static void load_deps(struct dso
*p
)
878 struct dso
***deps
= &p
->deps
, **tmp
, *dep
;
879 for (; p
; p
=p
->next
) {
880 for (i
=0; p
->dynv
[i
]; i
+=2) {
881 if (p
->dynv
[i
] != DT_NEEDED
) continue;
882 dep
= load_library(p
->strings
+ p
->dynv
[i
+1], p
);
884 error("Error loading shared library %s: %m (needed by %s)",
885 p
->strings
+ p
->dynv
[i
+1], p
->name
);
886 if (runtime
) longjmp(*rtld_fail
, 1);
890 tmp
= realloc(*deps
, sizeof(*tmp
)*(ndeps
+2));
891 if (!tmp
) longjmp(*rtld_fail
, 1);
900 static void load_preload(char *s
)
905 for ( ; *s
&& (isspace(*s
) || *s
==':'); s
++);
906 for (z
=s
; *z
&& !isspace(*z
) && *z
!=':'; z
++);
914 static void make_global(struct dso
*p
)
916 for (; p
; p
=p
->next
) p
->global
= 1;
919 static void do_mips_relocs(struct dso
*p
, size_t *got
)
922 unsigned char *base
= p
->base
;
923 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_LOCAL_GOTNO
);
927 while (i
--) *got
++ += (size_t)base
;
929 j
=0; search_vec(p
->dynv
, &j
, DT_MIPS_GOTSYM
);
930 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_SYMTABNO
);
931 Sym
*sym
= p
->syms
+ j
;
932 rel
[0] = (unsigned char *)got
- base
;
933 for (i
-=j
; i
; i
--, sym
++, rel
[0]+=sizeof(size_t)) {
934 rel
[1] = sym
-p
->syms
<< 8 | R_MIPS_JUMP_SLOT
;
935 do_relocs(p
, rel
, sizeof rel
, 2);
939 static void reloc_all(struct dso
*p
)
942 for (; p
; p
=p
->next
) {
943 if (p
->relocated
) continue;
944 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
945 if (NEED_MIPS_GOT_RELOCS
)
946 do_mips_relocs(p
, (void *)(p
->base
+dyn
[DT_PLTGOT
]));
947 do_relocs(p
, (void *)(p
->base
+dyn
[DT_JMPREL
]), dyn
[DT_PLTRELSZ
],
948 2+(dyn
[DT_PLTREL
]==DT_RELA
));
949 do_relocs(p
, (void *)(p
->base
+dyn
[DT_REL
]), dyn
[DT_RELSZ
], 2);
950 do_relocs(p
, (void *)(p
->base
+dyn
[DT_RELA
]), dyn
[DT_RELASZ
], 3);
952 if (head
!= &ldso
&& p
->relro_start
!= p
->relro_end
&&
953 mprotect(p
->base
+p
->relro_start
, p
->relro_end
-p
->relro_start
, PROT_READ
)
954 && errno
!= ENOSYS
) {
955 error("Error relocating %s: RELRO protection failed: %m",
957 if (runtime
) longjmp(*rtld_fail
, 1);
964 static void kernel_mapped_dso(struct dso
*p
)
966 size_t min_addr
= -1, max_addr
= 0, cnt
;
968 for (cnt
= p
->phnum
; cnt
--; ph
= (void *)((char *)ph
+ p
->phentsize
)) {
969 if (ph
->p_type
== PT_DYNAMIC
) {
970 p
->dynv
= (void *)(p
->base
+ ph
->p_vaddr
);
971 } else if (ph
->p_type
== PT_GNU_RELRO
) {
972 p
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
973 p
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
975 if (ph
->p_type
!= PT_LOAD
) continue;
976 if (ph
->p_vaddr
< min_addr
)
977 min_addr
= ph
->p_vaddr
;
978 if (ph
->p_vaddr
+ph
->p_memsz
> max_addr
)
979 max_addr
= ph
->p_vaddr
+ph
->p_memsz
;
981 min_addr
&= -PAGE_SIZE
;
982 max_addr
= (max_addr
+ PAGE_SIZE
-1) & -PAGE_SIZE
;
983 p
->map
= p
->base
+ min_addr
;
984 p
->map_len
= max_addr
- min_addr
;
985 p
->kernel_mapped
= 1;
988 static void do_fini()
992 for (p
=fini_head
; p
; p
=p
->fini_next
) {
993 if (!p
->constructed
) continue;
994 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
995 if (dyn
[0] & (1<<DT_FINI_ARRAY
)) {
996 size_t n
= dyn
[DT_FINI_ARRAYSZ
]/sizeof(size_t);
997 size_t *fn
= (size_t *)(p
->base
+ dyn
[DT_FINI_ARRAY
])+n
;
998 while (n
--) ((void (*)(void))*--fn
)();
1000 #ifndef NO_LEGACY_INITFINI
1001 if ((dyn
[0] & (1<<DT_FINI
)) && dyn
[DT_FINI
])
1002 ((void (*)(void))(p
->base
+ dyn
[DT_FINI
]))();
1007 static void do_init_fini(struct dso
*p
)
1009 size_t dyn
[DYN_CNT
];
1010 int need_locking
= libc
.threads_minus_1
;
1011 /* Allow recursive calls that arise when a library calls
1012 * dlopen from one of its constructors, but block any
1013 * other threads until all ctors have finished. */
1014 if (need_locking
) pthread_mutex_lock(&init_fini_lock
);
1015 for (; p
; p
=p
->prev
) {
1016 if (p
->constructed
) continue;
1018 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1019 if (dyn
[0] & ((1<<DT_FINI
) | (1<<DT_FINI_ARRAY
))) {
1020 p
->fini_next
= fini_head
;
1023 #ifndef NO_LEGACY_INITFINI
1024 if ((dyn
[0] & (1<<DT_INIT
)) && dyn
[DT_INIT
])
1025 ((void (*)(void))(p
->base
+ dyn
[DT_INIT
]))();
1027 if (dyn
[0] & (1<<DT_INIT_ARRAY
)) {
1028 size_t n
= dyn
[DT_INIT_ARRAYSZ
]/sizeof(size_t);
1029 size_t *fn
= (void *)(p
->base
+ dyn
[DT_INIT_ARRAY
]);
1030 while (n
--) ((void (*)(void))*fn
++)();
1032 if (!need_locking
&& libc
.threads_minus_1
) {
1034 pthread_mutex_lock(&init_fini_lock
);
1037 if (need_locking
) pthread_mutex_unlock(&init_fini_lock
);
1040 static void dl_debug_state(void)
1044 weak_alias(dl_debug_state
, _dl_debug_state
);
1048 pthread_t self
= __pthread_self();
1050 for (p
=head
; p
; p
=p
->next
) {
1051 if (!p
->tls_id
|| !self
->dtv
[p
->tls_id
]) continue;
1052 memcpy(self
->dtv
[p
->tls_id
], p
->tls_image
, p
->tls_len
);
1053 memset((char *)self
->dtv
[p
->tls_id
]+p
->tls_len
, 0,
1054 p
->tls_size
- p
->tls_len
);
1055 if (p
->tls_id
== (size_t)self
->dtv
[0]) break;
1059 void *__copy_tls(unsigned char *mem
)
1066 dtv
= (void **)(mem
+ libc
.tls_size
) - (tls_cnt
+ 1);
1068 mem
+= -((uintptr_t)mem
+ sizeof(struct pthread
)) & (tls_align
-1);
1069 td
= (pthread_t
)mem
;
1070 mem
+= sizeof(struct pthread
);
1072 for (p
=head
; p
; p
=p
->next
) {
1073 if (!p
->tls_id
) continue;
1074 dtv
[p
->tls_id
] = mem
+ p
->tls_offset
;
1075 memcpy(dtv
[p
->tls_id
], p
->tls_image
, p
->tls_len
);
1080 mem
+= libc
.tls_size
- sizeof(struct pthread
);
1081 mem
-= (uintptr_t)mem
& (tls_align
-1);
1082 td
= (pthread_t
)mem
;
1084 for (p
=head
; p
; p
=p
->next
) {
1085 if (!p
->tls_id
) continue;
1086 dtv
[p
->tls_id
] = mem
- p
->tls_offset
;
1087 memcpy(dtv
[p
->tls_id
], p
->tls_image
, p
->tls_len
);
1090 dtv
[0] = (void *)tls_cnt
;
1091 td
->dtv
= td
->dtv_copy
= dtv
;
1095 __attribute__((__visibility__("hidden")))
1096 void *__tls_get_new(size_t *v
)
1098 pthread_t self
= __pthread_self();
1100 /* Block signals to make accessing new TLS async-signal-safe */
1102 __block_all_sigs(&set
);
1103 if (v
[0]<=(size_t)self
->dtv
[0]) {
1104 __restore_sigs(&set
);
1105 return (char *)self
->dtv
[v
[0]]+v
[1]+DTP_OFFSET
;
1108 /* This is safe without any locks held because, if the caller
1109 * is able to request the Nth entry of the DTV, the DSO list
1110 * must be valid at least that far out and it was synchronized
1111 * at program startup or by an already-completed call to dlopen. */
1113 for (p
=head
; p
->tls_id
!= v
[0]; p
=p
->next
);
1115 /* Get new DTV space from new DSO if needed */
1116 if (v
[0] > (size_t)self
->dtv
[0]) {
1117 void **newdtv
= p
->new_dtv
+
1118 (v
[0]+1)*sizeof(void *)*a_fetch_add(&p
->new_dtv_idx
,1);
1119 memcpy(newdtv
, self
->dtv
,
1120 ((size_t)self
->dtv
[0]+1) * sizeof(void *));
1121 newdtv
[0] = (void *)v
[0];
1122 self
->dtv
= self
->dtv_copy
= newdtv
;
1125 /* Get new TLS memory from all new DSOs up to the requested one */
1127 for (p
=head
; ; p
=p
->next
) {
1128 if (!p
->tls_id
|| self
->dtv
[p
->tls_id
]) continue;
1129 mem
= p
->new_tls
+ (p
->tls_size
+ p
->tls_align
)
1130 * a_fetch_add(&p
->new_tls_idx
,1);
1131 mem
+= ((uintptr_t)p
->tls_image
- (uintptr_t)mem
)
1133 self
->dtv
[p
->tls_id
] = mem
;
1134 memcpy(mem
, p
->tls_image
, p
->tls_len
);
1135 if (p
->tls_id
== v
[0]) break;
1137 __restore_sigs(&set
);
1138 return mem
+ v
[1] + DTP_OFFSET
;
1141 static void update_tls_size()
1143 libc
.tls_size
= ALIGN(
1144 (1+tls_cnt
) * sizeof(void *) +
1146 sizeof(struct pthread
) +
1151 /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
1152 * following stage 2 and stage 3 functions via primitive symbolic lookup
1153 * since it does not have access to their addresses to begin with. */
1155 /* Stage 2 of the dynamic linker is called after relative relocations
1156 * have been processed. It can make function calls to static functions
1157 * and access string literals and static data, but cannot use extern
1158 * symbols. Its job is to perform symbolic relocations on the dynamic
1159 * linker itself, but some of the relocations performed may need to be
1160 * replaced later due to copy relocations in the main program. */
1162 void __dls2(unsigned char *base
, size_t *sp
)
1164 Ehdr
*ehdr
= (void *)base
;
1166 ldso
.name
= ldso
.shortname
= "libc.so";
1168 ldso
.phnum
= ehdr
->e_phnum
;
1169 ldso
.phdr
= (void *)(base
+ ehdr
->e_phoff
);
1170 ldso
.phentsize
= ehdr
->e_phentsize
;
1171 kernel_mapped_dso(&ldso
);
1174 /* Prepare storage for to save clobbered REL addends so they
1175 * can be reused in stage 3. There should be very few. If
1176 * something goes wrong and there are a huge number, abort
1177 * instead of risking stack overflow. */
1178 size_t dyn
[DYN_CNT
];
1179 decode_vec(ldso
.dynv
, dyn
, DYN_CNT
);
1180 size_t *rel
= (void *)(base
+dyn
[DT_REL
]);
1181 size_t rel_size
= dyn
[DT_RELSZ
];
1182 size_t symbolic_rel_cnt
= 0;
1183 apply_addends_to
= rel
;
1184 for (; rel_size
; rel
+=2, rel_size
-=2*sizeof(size_t))
1185 if (!IS_RELATIVE(rel
[1])) symbolic_rel_cnt
++;
1186 if (symbolic_rel_cnt
>= ADDEND_LIMIT
) a_crash();
1187 size_t addends
[symbolic_rel_cnt
+1];
1188 saved_addends
= addends
;
1195 /* Call dynamic linker stage-3, __dls3, looking it up
1196 * symbolically as a barrier against moving the address
1197 * load across the above relocation processing. */
1198 struct symdef dls3_def
= find_sym(&ldso
, "__dls3", 0);
1199 ((stage3_func
)(ldso
.base
+dls3_def
.sym
->st_value
))(sp
);
1202 /* Stage 3 of the dynamic linker is called with the dynamic linker/libc
1203 * fully functional. Its job is to load (if not already loaded) and
1204 * process dependencies and relocations for the main application and
1205 * transfer control to its entry point. */
1207 _Noreturn
void __dls3(size_t *sp
)
1209 static struct dso app
, vdso
;
1210 size_t aux
[AUX_CNT
], *auxv
;
1212 char *env_preload
=0;
1215 char **argv
= (void *)(sp
+1);
1216 char **argv_orig
= argv
;
1217 char **envp
= argv
+argc
+1;
1219 /* Find aux vector just past environ[] and use it to initialize
1220 * global data that may be needed before we can make syscalls. */
1222 for (i
=argc
+1; argv
[i
]; i
++);
1223 libc
.auxv
= auxv
= (void *)(argv
+i
+1);
1224 decode_vec(auxv
, aux
, AUX_CNT
);
1225 __hwcap
= aux
[AT_HWCAP
];
1226 libc
.page_size
= aux
[AT_PAGESZ
];
1227 libc
.secure
= ((aux
[0]&0x7800)!=0x7800 || aux
[AT_UID
]!=aux
[AT_EUID
]
1228 || aux
[AT_GID
]!=aux
[AT_EGID
] || aux
[AT_SECURE
]);
1230 /* Setup early thread pointer in builtin_tls for ldso/libc itself to
1231 * use during dynamic linking. If possible it will also serve as the
1232 * thread pointer at runtime. */
1233 libc
.tls_size
= sizeof builtin_tls
;
1234 if (__init_tp(__copy_tls((void *)builtin_tls
)) < 0) {
1238 /* Only trust user/env if kernel says we're not suid/sgid */
1240 env_path
= getenv("LD_LIBRARY_PATH");
1241 env_preload
= getenv("LD_PRELOAD");
1244 /* If the main program was already loaded by the kernel,
1245 * AT_PHDR will point to some location other than the dynamic
1246 * linker's program headers. */
1247 if (aux
[AT_PHDR
] != (size_t)ldso
.phdr
) {
1248 size_t interp_off
= 0;
1249 size_t tls_image
= 0;
1250 /* Find load address of the main program, via AT_PHDR vs PT_PHDR. */
1251 Phdr
*phdr
= app
.phdr
= (void *)aux
[AT_PHDR
];
1252 app
.phnum
= aux
[AT_PHNUM
];
1253 app
.phentsize
= aux
[AT_PHENT
];
1254 for (i
=aux
[AT_PHNUM
]; i
; i
--, phdr
=(void *)((char *)phdr
+ aux
[AT_PHENT
])) {
1255 if (phdr
->p_type
== PT_PHDR
)
1256 app
.base
= (void *)(aux
[AT_PHDR
] - phdr
->p_vaddr
);
1257 else if (phdr
->p_type
== PT_INTERP
)
1258 interp_off
= (size_t)phdr
->p_vaddr
;
1259 else if (phdr
->p_type
== PT_TLS
) {
1260 tls_image
= phdr
->p_vaddr
;
1261 app
.tls_len
= phdr
->p_filesz
;
1262 app
.tls_size
= phdr
->p_memsz
;
1263 app
.tls_align
= phdr
->p_align
;
1266 if (app
.tls_size
) app
.tls_image
= (char *)app
.base
+ tls_image
;
1267 if (interp_off
) ldso
.name
= (char *)app
.base
+ interp_off
;
1268 if ((aux
[0] & (1UL<<AT_EXECFN
))
1269 && strncmp((char *)aux
[AT_EXECFN
], "/proc/", 6))
1270 app
.name
= (char *)aux
[AT_EXECFN
];
1273 kernel_mapped_dso(&app
);
1276 char *ldname
= argv
[0];
1277 size_t l
= strlen(ldname
);
1278 if (l
>= 3 && !strcmp(ldname
+l
-3, "ldd")) ldd_mode
= 1;
1280 while (argv
[0] && argv
[0][0]=='-' && argv
[0][1]=='-') {
1281 char *opt
= argv
[0]+2;
1282 *argv
++ = (void *)-1;
1285 } else if (!memcmp(opt
, "list", 5)) {
1287 } else if (!memcmp(opt
, "library-path", 12)) {
1288 if (opt
[12]=='=') env_path
= opt
+13;
1289 else if (opt
[12]) *argv
= 0;
1290 else if (*argv
) env_path
= *argv
++;
1291 } else if (!memcmp(opt
, "preload", 7)) {
1292 if (opt
[7]=='=') env_preload
= opt
+8;
1293 else if (opt
[7]) *argv
= 0;
1294 else if (*argv
) env_preload
= *argv
++;
1299 argv
[-1] = (void *)(argc
- (argv
-argv_orig
));
1301 dprintf(2, "musl libc\n"
1303 "Dynamic Program Loader\n"
1304 "Usage: %s [options] [--] pathname%s\n",
1305 __libc_get_version(), ldname
,
1306 ldd_mode
? "" : " [args]");
1309 fd
= open(argv
[0], O_RDONLY
);
1311 dprintf(2, "%s: cannot load %s: %s\n", ldname
, argv
[0], strerror(errno
));
1315 Ehdr
*ehdr
= (void *)map_library(fd
, &app
);
1317 dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname
, argv
[0]);
1324 aux
[AT_ENTRY
] = (size_t)app
.base
+ ehdr
->e_entry
;
1325 /* Find the name that would have been used for the dynamic
1326 * linker had ldd not taken its place. */
1328 for (i
=0; i
<app
.phnum
; i
++) {
1329 if (app
.phdr
[i
].p_type
== PT_INTERP
)
1330 ldso
.name
= (void *)(app
.base
1331 + app
.phdr
[i
].p_vaddr
);
1333 dprintf(1, "\t%s (%p)\n", ldso
.name
, ldso
.base
);
1337 app
.tls_id
= tls_cnt
= 1;
1340 tls_offset
= app
.tls_size
1341 + ( -((uintptr_t)app
.tls_image
+ app
.tls_size
)
1342 & (app
.tls_align
-1) );
1344 tls_offset
= app
.tls_offset
= app
.tls_size
1345 + ( -((uintptr_t)app
.tls_image
+ app
.tls_size
)
1346 & (app
.tls_align
-1) );
1348 tls_align
= MAXP2(tls_align
, app
.tls_align
);
1353 /* Attach to vdso, if provided by the kernel */
1354 if (search_vec(auxv
, &vdso_base
, AT_SYSINFO_EHDR
)) {
1355 Ehdr
*ehdr
= (void *)vdso_base
;
1356 Phdr
*phdr
= vdso
.phdr
= (void *)(vdso_base
+ ehdr
->e_phoff
);
1357 vdso
.phnum
= ehdr
->e_phnum
;
1358 vdso
.phentsize
= ehdr
->e_phentsize
;
1359 for (i
=ehdr
->e_phnum
; i
; i
--, phdr
=(void *)((char *)phdr
+ ehdr
->e_phentsize
)) {
1360 if (phdr
->p_type
== PT_DYNAMIC
)
1361 vdso
.dynv
= (void *)(vdso_base
+ phdr
->p_offset
);
1362 if (phdr
->p_type
== PT_LOAD
)
1363 vdso
.base
= (void *)(vdso_base
- phdr
->p_vaddr
+ phdr
->p_offset
);
1366 vdso
.shortname
= "linux-gate.so.1";
1374 /* Initial dso chain consists only of the app. */
1377 /* Donate unused parts of app and library mapping to malloc */
1379 reclaim_gaps(&ldso
);
1381 /* Load preload/needed libraries, add their symbols to the global
1382 * namespace, and perform all remaining relocations. */
1383 if (env_preload
) load_preload(env_preload
);
1387 #ifndef DYNAMIC_IS_RO
1388 for (i
=0; app
.dynv
[i
]; i
+=2)
1389 if (app
.dynv
[i
]==DT_DEBUG
)
1390 app
.dynv
[i
+1] = (size_t)&debug
;
1393 /* The main program must be relocated LAST since it may contin
1394 * copy relocations which depend on libraries' relocations. */
1395 reloc_all(app
.next
);
1399 if (libc
.tls_size
> sizeof builtin_tls
|| tls_align
> MIN_TLS_ALIGN
) {
1400 void *initial_tls
= calloc(libc
.tls_size
, 1);
1402 dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
1403 argv
[0], libc
.tls_size
);
1406 if (__init_tp(__copy_tls(initial_tls
)) < 0) {
1410 size_t tmp_tls_size
= libc
.tls_size
;
1411 pthread_t self
= __pthread_self();
1412 /* Temporarily set the tls size to the full size of
1413 * builtin_tls so that __copy_tls will use the same layout
1414 * as it did for before. Then check, just to be safe. */
1415 libc
.tls_size
= sizeof builtin_tls
;
1416 if (__copy_tls((void*)builtin_tls
) != self
) a_crash();
1417 libc
.tls_size
= tmp_tls_size
;
1419 static_tls_cnt
= tls_cnt
;
1421 if (ldso_fail
) _exit(127);
1422 if (ldd_mode
) _exit(0);
1424 /* Switch to runtime mode: any further failures in the dynamic
1425 * linker are a reportable failure rather than a fatal startup
1430 debug
.bp
= dl_debug_state
;
1432 debug
.base
= ldso
.base
;
1436 __init_libc(envp
, argv
[0]);
1441 CRTJMP((void *)aux
[AT_ENTRY
], argv
-1);
1445 void *dlopen(const char *file
, int mode
)
1447 struct dso
*volatile p
, *orig_tail
, *next
;
1448 size_t orig_tls_cnt
, orig_tls_offset
, orig_tls_align
;
1453 if (!file
) return head
;
1455 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE
, &cs
);
1456 pthread_rwlock_wrlock(&lock
);
1460 orig_tls_cnt
= tls_cnt
;
1461 orig_tls_offset
= tls_offset
;
1462 orig_tls_align
= tls_align
;
1464 noload
= mode
& RTLD_NOLOAD
;
1467 if (setjmp(*rtld_fail
)) {
1468 /* Clean up anything new that was (partially) loaded */
1469 if (p
&& p
->deps
) for (i
=0; p
->deps
[i
]; i
++)
1470 if (p
->deps
[i
]->global
< 0)
1471 p
->deps
[i
]->global
= 0;
1472 for (p
=orig_tail
->next
; p
; p
=next
) {
1474 munmap(p
->map
, p
->map_len
);
1475 while (p
->td_index
) {
1476 void *tmp
= p
->td_index
->next
;
1480 if (p
->rpath
!= p
->rpath_orig
)
1485 tls_cnt
= orig_tls_cnt
;
1486 tls_offset
= orig_tls_offset
;
1487 tls_align
= orig_tls_align
;
1492 } else p
= load_library(file
, head
);
1496 "Library %s is not already loaded" :
1497 "Error loading shared library %s: %m",
1502 /* First load handling */
1505 if (p
->deps
) for (i
=0; p
->deps
[i
]; i
++)
1506 if (!p
->deps
[i
]->global
)
1507 p
->deps
[i
]->global
= -1;
1508 if (!p
->global
) p
->global
= -1;
1510 if (p
->deps
) for (i
=0; p
->deps
[i
]; i
++)
1511 if (p
->deps
[i
]->global
< 0)
1512 p
->deps
[i
]->global
= 0;
1513 if (p
->global
< 0) p
->global
= 0;
1516 if (mode
& RTLD_GLOBAL
) {
1517 if (p
->deps
) for (i
=0; p
->deps
[i
]; i
++)
1518 p
->deps
[i
]->global
= 1;
1528 pthread_rwlock_unlock(&lock
);
1529 if (p
) do_init_fini(orig_tail
);
1530 pthread_setcancelstate(cs
, 0);
1534 static int invalid_dso_handle(void *h
)
1537 for (p
=head
; p
; p
=p
->next
) if (h
==p
) return 0;
1538 error("Invalid library handle %p", (void *)h
);
1542 void *__tls_get_addr(size_t *);
1544 static void *do_dlsym(struct dso
*p
, const char *s
, void *ra
)
1547 uint32_t h
= 0, gh
= 0;
1549 if (p
== head
|| p
== RTLD_DEFAULT
|| p
== RTLD_NEXT
) {
1550 if (p
== RTLD_DEFAULT
) {
1552 } else if (p
== RTLD_NEXT
) {
1553 for (p
=head
; p
&& (unsigned char *)ra
-p
->map
>p
->map_len
; p
=p
->next
);
1557 struct symdef def
= find_sym(p
, s
, 0);
1558 if (!def
.sym
) goto failed
;
1559 if ((def
.sym
->st_info
&0xf) == STT_TLS
)
1560 return __tls_get_addr((size_t []){def
.dso
->tls_id
, def
.sym
->st_value
});
1561 return def
.dso
->base
+ def
.sym
->st_value
;
1563 if (invalid_dso_handle(p
))
1567 sym
= gnu_lookup(s
, gh
, p
);
1570 sym
= sysv_lookup(s
, h
, p
);
1572 if (sym
&& (sym
->st_info
&0xf) == STT_TLS
)
1573 return __tls_get_addr((size_t []){p
->tls_id
, sym
->st_value
});
1574 if (sym
&& sym
->st_value
&& (1<<(sym
->st_info
&0xf) & OK_TYPES
))
1575 return p
->base
+ sym
->st_value
;
1576 if (p
->deps
) for (i
=0; p
->deps
[i
]; i
++) {
1577 if (p
->deps
[i
]->ghashtab
) {
1578 if (!gh
) gh
= gnu_hash(s
);
1579 sym
= gnu_lookup(s
, gh
, p
->deps
[i
]);
1581 if (!h
) h
= sysv_hash(s
);
1582 sym
= sysv_lookup(s
, h
, p
->deps
[i
]);
1584 if (sym
&& (sym
->st_info
&0xf) == STT_TLS
)
1585 return __tls_get_addr((size_t []){p
->deps
[i
]->tls_id
, sym
->st_value
});
1586 if (sym
&& sym
->st_value
&& (1<<(sym
->st_info
&0xf) & OK_TYPES
))
1587 return p
->deps
[i
]->base
+ sym
->st_value
;
1590 error("Symbol not found: %s", s
);
1594 int __dladdr(const void *addr
, Dl_info
*info
)
1604 pthread_rwlock_rdlock(&lock
);
1605 for (p
=head
; p
&& (unsigned char *)addr
-p
->map
>p
->map_len
; p
=p
->next
);
1606 pthread_rwlock_unlock(&lock
);
1611 strings
= p
->strings
;
1613 nsym
= p
->hashtab
[1];
1617 buckets
= p
->ghashtab
+ 4 + (p
->ghashtab
[2]*sizeof(size_t)/4);
1618 sym
+= p
->ghashtab
[1];
1619 for (i
= nsym
= 0; i
< p
->ghashtab
[0]; i
++) {
1620 if (buckets
[i
] > nsym
)
1624 nsym
-= p
->ghashtab
[1];
1625 hashval
= buckets
+ p
->ghashtab
[0] + nsym
;
1627 while (!(*hashval
++ & 1));
1631 for (; nsym
; nsym
--, sym
++) {
1633 && (1<<(sym
->st_info
&0xf) & OK_TYPES
)
1634 && (1<<(sym
->st_info
>>4) & OK_BINDS
)) {
1635 void *symaddr
= p
->base
+ sym
->st_value
;
1636 if (symaddr
> addr
|| symaddr
< best
)
1639 bestname
= strings
+ sym
->st_name
;
1640 if (addr
== symaddr
)
1645 if (!best
) return 0;
1647 info
->dli_fname
= p
->name
;
1648 info
->dli_fbase
= p
->base
;
1649 info
->dli_sname
= bestname
;
1650 info
->dli_saddr
= best
;
1655 __attribute__((__visibility__("hidden")))
1656 void *__dlsym(void *restrict p
, const char *restrict s
, void *restrict ra
)
1659 pthread_rwlock_rdlock(&lock
);
1660 res
= do_dlsym(p
, s
, ra
);
1661 pthread_rwlock_unlock(&lock
);
1665 int dl_iterate_phdr(int(*callback
)(struct dl_phdr_info
*info
, size_t size
, void *data
), void *data
)
1667 struct dso
*current
;
1668 struct dl_phdr_info info
;
1670 for(current
= head
; current
;) {
1671 info
.dlpi_addr
= (uintptr_t)current
->base
;
1672 info
.dlpi_name
= current
->name
;
1673 info
.dlpi_phdr
= current
->phdr
;
1674 info
.dlpi_phnum
= current
->phnum
;
1675 info
.dlpi_adds
= gencnt
;
1677 info
.dlpi_tls_modid
= current
->tls_id
;
1678 info
.dlpi_tls_data
= current
->tls_image
;
1680 ret
= (callback
)(&info
, sizeof (info
), data
);
1682 if (ret
!= 0) break;
1684 pthread_rwlock_rdlock(&lock
);
1685 current
= current
->next
;
1686 pthread_rwlock_unlock(&lock
);
1691 static int invalid_dso_handle(void *h
)
1693 error("Invalid library handle %p", (void *)h
);
1696 void *dlopen(const char *file
, int mode
)
1698 error("Dynamic loading not supported");
1701 void *__dlsym(void *restrict p
, const char *restrict s
, void *restrict ra
)
1703 error("Symbol not found: %s", s
);
1706 int __dladdr (const void *addr
, Dl_info
*info
)
1712 int __dlinfo(void *dso
, int req
, void *res
)
1714 if (invalid_dso_handle(dso
)) return -1;
1715 if (req
!= RTLD_DI_LINKMAP
) {
1716 error("Unsupported request %d", req
);
1719 *(struct link_map
**)res
= dso
;
1725 pthread_t self
= __pthread_self();
1726 if (!self
->dlerror_flag
) return 0;
1727 self
->dlerror_flag
= 0;
1728 char *s
= self
->dlerror_buf
;
1729 if (s
== (void *)-1)
1730 return "Dynamic linker failed to allocate memory for error message";
1735 int dlclose(void *p
)
1737 return invalid_dso_handle(p
);
1740 void __dl_thread_cleanup(void)
1742 pthread_t self
= __pthread_self();
1743 if (self
->dlerror_buf
!= (void *)-1)
1744 free(self
->dlerror_buf
);
1747 static void error(const char *fmt
, ...)
1753 vdprintf(2, fmt
, ap
);
1760 pthread_t self
= __pthread_self();
1761 if (self
->dlerror_buf
!= (void *)-1)
1762 free(self
->dlerror_buf
);
1763 size_t len
= vsnprintf(0, 0, fmt
, ap
);
1765 char *buf
= malloc(len
+1);
1768 vsnprintf(buf
, len
+1, fmt
, ap
);
1773 self
->dlerror_buf
= buf
;
1774 self
->dlerror_flag
= 1;