20 #include "pthread_impl.h"
24 static void error(const char *, ...);
26 #define MAXP2(a,b) (-(-(a)&-(b)))
27 #define ALIGN(x,y) ((x)+(y)-1 & -(y))
39 struct td_index
*next
;
44 struct fdpic_loadmap
*loadmap
;
50 struct dso
*next
, *prev
;
60 struct dso
*syms_next
, *lazy_next
;
61 size_t *lazy
, lazy_cnt
;
69 struct dso
**deps
, *needed_by
;
70 char *rpath_orig
, *rpath
;
71 struct tls_module tls
;
73 size_t relro_start
, relro_end
;
75 unsigned char *new_tls
;
76 volatile int new_dtv_idx
, new_tls_idx
;
77 struct td_index
*td_index
;
78 struct dso
*fini_next
;
83 struct fdpic_loadmap
*loadmap
;
98 int __init_tp(void *);
99 void __init_libc(char **, char *);
100 void *__copy_tls(unsigned char *);
102 hidden
const char *__libc_get_version(void);
104 static struct builtin_tls
{
109 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
111 #define ADDEND_LIMIT 4096
112 static size_t *saved_addends
, *apply_addends_to
;
114 static struct dso ldso
;
115 static struct dso
*head
, *tail
, *fini_head
, *syms_tail
, *lazy_head
;
116 static char *env_path
, *sys_path
;
117 static unsigned long long gencnt
;
120 static int ldso_fail
;
122 static jmp_buf *rtld_fail
;
123 static pthread_rwlock_t lock
;
124 static struct debug debug
;
125 static struct tls_module
*tls_tail
;
126 static size_t tls_cnt
, tls_offset
, tls_align
= MIN_TLS_ALIGN
;
127 static size_t static_tls_cnt
;
128 static pthread_mutex_t init_fini_lock
= { ._m_type
= PTHREAD_MUTEX_RECURSIVE
};
129 static struct fdpic_loadmap
*app_loadmap
;
130 static struct fdpic_dummy_loadmap app_dummy_loadmap
;
131 static struct dso
*const nodeps_dummy
;
133 struct debug
*_dl_debug_addr
= &debug
;
135 extern hidden
int __malloc_replaced
;
137 hidden
void (*const __init_array_start
)(void)=0, (*const __fini_array_start
)(void)=0;
139 extern hidden
void (*const __init_array_end
)(void), (*const __fini_array_end
)(void);
141 weak_alias(__init_array_start
, __init_array_end
);
142 weak_alias(__fini_array_start
, __fini_array_end
);
144 static int dl_strcmp(const char *l
, const char *r
)
146 for (; *l
==*r
&& *l
; l
++, r
++);
147 return *(unsigned char *)l
- *(unsigned char *)r
;
149 #define strcmp(l,r) dl_strcmp(l,r)
151 /* Compute load address for a virtual address in a given dso. */
153 static void *laddr(const struct dso
*p
, size_t v
)
156 if (!p
->loadmap
) return p
->base
+ v
;
157 for (j
=0; v
-p
->loadmap
->segs
[j
].p_vaddr
>= p
->loadmap
->segs
[j
].p_memsz
; j
++);
158 return (void *)(v
- p
->loadmap
->segs
[j
].p_vaddr
+ p
->loadmap
->segs
[j
].addr
);
160 static void *laddr_pg(const struct dso
*p
, size_t v
)
163 size_t pgsz
= PAGE_SIZE
;
164 if (!p
->loadmap
) return p
->base
+ v
;
166 size_t a
= p
->loadmap
->segs
[j
].p_vaddr
;
167 size_t b
= a
+ p
->loadmap
->segs
[j
].p_memsz
;
173 return (void *)(v
- p
->loadmap
->segs
[j
].p_vaddr
+ p
->loadmap
->segs
[j
].addr
);
175 #define fpaddr(p, v) ((void (*)())&(struct funcdesc){ \
176 laddr(p, v), (p)->got })
178 #define laddr(p, v) (void *)((p)->base + (v))
179 #define laddr_pg(p, v) laddr(p, v)
180 #define fpaddr(p, v) ((void (*)())laddr(p, v))
183 static void decode_vec(size_t *v
, size_t *a
, size_t cnt
)
186 for (i
=0; i
<cnt
; i
++) a
[i
] = 0;
187 for (; v
[0]; v
+=2) if (v
[0]-1<cnt
-1) {
193 static int search_vec(size_t *v
, size_t *r
, size_t key
)
195 for (; v
[0]!=key
; v
+=2)
201 static uint32_t sysv_hash(const char *s0
)
203 const unsigned char *s
= (void *)s0
;
209 return h
& 0xfffffff;
212 static uint32_t gnu_hash(const char *s0
)
214 const unsigned char *s
= (void *)s0
;
215 uint_fast32_t h
= 5381;
221 static Sym
*sysv_lookup(const char *s
, uint32_t h
, struct dso
*dso
)
224 Sym
*syms
= dso
->syms
;
225 Elf_Symndx
*hashtab
= dso
->hashtab
;
226 char *strings
= dso
->strings
;
227 for (i
=hashtab
[2+h
%hashtab
[0]]; i
; i
=hashtab
[2+hashtab
[0]+i
]) {
228 if ((!dso
->versym
|| dso
->versym
[i
] >= 0)
229 && (!strcmp(s
, strings
+syms
[i
].st_name
)))
235 static Sym
*gnu_lookup(uint32_t h1
, uint32_t *hashtab
, struct dso
*dso
, const char *s
)
237 uint32_t nbuckets
= hashtab
[0];
238 uint32_t *buckets
= hashtab
+ 4 + hashtab
[2]*(sizeof(size_t)/4);
239 uint32_t i
= buckets
[h1
% nbuckets
];
243 uint32_t *hashval
= buckets
+ nbuckets
+ (i
- hashtab
[1]);
245 for (h1
|= 1; ; i
++) {
246 uint32_t h2
= *hashval
++;
247 if ((h1
== (h2
|1)) && (!dso
->versym
|| dso
->versym
[i
] >= 0)
248 && !strcmp(s
, dso
->strings
+ dso
->syms
[i
].st_name
))
256 static Sym
*gnu_lookup_filtered(uint32_t h1
, uint32_t *hashtab
, struct dso
*dso
, const char *s
, uint32_t fofs
, size_t fmask
)
258 const size_t *bloomwords
= (const void *)(hashtab
+4);
259 size_t f
= bloomwords
[fofs
& (hashtab
[2]-1)];
260 if (!(f
& fmask
)) return 0;
262 f
>>= (h1
>> hashtab
[3]) % (8 * sizeof f
);
263 if (!(f
& 1)) return 0;
265 return gnu_lookup(h1
, hashtab
, dso
, s
);
268 #define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON | 1<<STT_TLS)
269 #define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
271 #ifndef ARCH_SYM_REJECT_UND
272 #define ARCH_SYM_REJECT_UND(s) 0
275 static struct symdef
find_sym(struct dso
*dso
, const char *s
, int need_def
)
277 uint32_t h
= 0, gh
= gnu_hash(s
), gho
= gh
/ (8*sizeof(size_t)), *ght
;
278 size_t ghm
= 1ul << gh
% (8*sizeof(size_t));
279 struct symdef def
= {0};
280 for (; dso
; dso
=dso
->syms_next
) {
282 if ((ght
= dso
->ghashtab
)) {
283 sym
= gnu_lookup_filtered(gh
, ght
, dso
, s
, gho
, ghm
);
285 if (!h
) h
= sysv_hash(s
);
286 sym
= sysv_lookup(s
, h
, dso
);
290 if (need_def
|| (sym
->st_info
&0xf) == STT_TLS
291 || ARCH_SYM_REJECT_UND(sym
))
294 if ((sym
->st_info
&0xf) != STT_TLS
)
296 if (!(1<<(sym
->st_info
&0xf) & OK_TYPES
)) continue;
297 if (!(1<<(sym
->st_info
>>4) & OK_BINDS
)) continue;
305 hidden
ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic();
307 static void do_relocs(struct dso
*dso
, size_t *rel
, size_t rel_size
, size_t stride
)
309 unsigned char *base
= dso
->base
;
310 Sym
*syms
= dso
->syms
;
311 char *strings
= dso
->strings
;
322 int skip_relative
= 0, reuse_addends
= 0, save_slot
= 0;
325 /* Only ldso's REL table needs addend saving/reuse. */
326 if (rel
== apply_addends_to
)
331 for (; rel_size
; rel
+=stride
, rel_size
-=stride
*sizeof(size_t)) {
332 if (skip_relative
&& IS_RELATIVE(rel
[1], dso
->syms
)) continue;
333 type
= R_TYPE(rel
[1]);
334 if (type
== REL_NONE
) continue;
335 reloc_addr
= laddr(dso
, rel
[0]);
339 } else if (type
==REL_GOT
|| type
==REL_PLT
|| type
==REL_COPY
) {
341 } else if (reuse_addends
) {
342 /* Save original addend in stage 2 where the dso
343 * chain consists of just ldso; otherwise read back
344 * saved addend since the inline one was clobbered. */
346 saved_addends
[save_slot
] = *reloc_addr
;
347 addend
= saved_addends
[save_slot
++];
349 addend
= *reloc_addr
;
352 sym_index
= R_SYM(rel
[1]);
354 sym
= syms
+ sym_index
;
355 name
= strings
+ sym
->st_name
;
356 ctx
= type
==REL_COPY
? head
->syms_next
: head
;
357 def
= (sym
->st_info
&0xf) == STT_SECTION
358 ? (struct symdef
){ .dso
= dso
, .sym
= sym
}
359 : find_sym(ctx
, name
, type
==REL_PLT
);
360 if (!def
.sym
&& (sym
->st_shndx
!= SHN_UNDEF
361 || sym
->st_info
>>4 != STB_WEAK
)) {
362 if (dso
->lazy
&& (type
==REL_PLT
|| type
==REL_GOT
)) {
363 dso
->lazy
[3*dso
->lazy_cnt
+0] = rel
[0];
364 dso
->lazy
[3*dso
->lazy_cnt
+1] = rel
[1];
365 dso
->lazy
[3*dso
->lazy_cnt
+2] = addend
;
369 error("Error relocating %s: %s: symbol not found",
371 if (runtime
) longjmp(*rtld_fail
, 1);
380 sym_val
= def
.sym
? (size_t)laddr(def
.dso
, def
.sym
->st_value
) : 0;
381 tls_val
= def
.sym
? def
.sym
->st_value
: 0;
383 if ((type
== REL_TPOFF
|| type
== REL_TPOFF_NEG
)
384 && runtime
&& def
.dso
->tls_id
> static_tls_cnt
) {
385 error("Error relocating %s: %s: initial-exec TLS "
386 "resolves to dynamic definition in %s",
387 dso
->name
, name
, def
.dso
->name
);
388 longjmp(*rtld_fail
, 1);
395 addend
-= (size_t)reloc_addr
;
399 *reloc_addr
= sym_val
+ addend
;
402 *reloc_addr
= (size_t)base
+ addend
;
405 if (sym
) *reloc_addr
= sym_val
+ addend
;
406 else *reloc_addr
= (size_t)base
+ addend
;
409 memcpy(reloc_addr
, (void *)sym_val
, sym
->st_size
);
412 *(uint32_t *)reloc_addr
= sym_val
+ addend
413 - (size_t)reloc_addr
;
416 *reloc_addr
= def
.sym
? (size_t)(def
.dso
->funcdescs
417 + (def
.sym
- def
.dso
->syms
)) : 0;
419 case REL_FUNCDESC_VAL
:
420 if ((sym
->st_info
&0xf) == STT_SECTION
) *reloc_addr
+= sym_val
;
421 else *reloc_addr
= sym_val
;
422 reloc_addr
[1] = def
.sym
? (size_t)def
.dso
->got
: 0;
425 *reloc_addr
= def
.dso
->tls_id
;
428 *reloc_addr
= tls_val
+ addend
- DTP_OFFSET
;
432 *reloc_addr
= tls_val
+ def
.dso
->tls
.offset
+ TPOFF_K
+ addend
;
436 *reloc_addr
= tls_val
- def
.dso
->tls
.offset
+ addend
;
439 *reloc_addr
= def
.dso
->tls
.offset
- tls_val
+ addend
;
443 if (stride
<3) addend
= reloc_addr
[1];
444 if (runtime
&& def
.dso
->tls_id
> static_tls_cnt
) {
445 struct td_index
*new = malloc(sizeof *new);
448 "Error relocating %s: cannot allocate TLSDESC for %s",
449 dso
->name
, sym
? name
: "(local)" );
450 longjmp(*rtld_fail
, 1);
452 new->next
= dso
->td_index
;
454 new->args
[0] = def
.dso
->tls_id
;
455 new->args
[1] = tls_val
+ addend
;
456 reloc_addr
[0] = (size_t)__tlsdesc_dynamic
;
457 reloc_addr
[1] = (size_t)new;
459 reloc_addr
[0] = (size_t)__tlsdesc_static
;
461 reloc_addr
[1] = tls_val
+ def
.dso
->tls
.offset
464 reloc_addr
[1] = tls_val
- def
.dso
->tls
.offset
470 error("Error relocating %s: unsupported relocation type %d",
472 if (runtime
) longjmp(*rtld_fail
, 1);
478 static void redo_lazy_relocs()
480 struct dso
*p
= lazy_head
, *next
;
484 size_t size
= p
->lazy_cnt
*3*sizeof(size_t);
486 do_relocs(p
, p
->lazy
, size
, 3);
488 p
->lazy_next
= lazy_head
;
498 /* A huge hack: to make up for the wastefulness of shared libraries
499 * needing at least a page of dirty memory even if they have no global
500 * data, we reclaim the gaps at the beginning and end of writable maps
501 * and "donate" them to the heap. */
503 static void reclaim(struct dso
*dso
, size_t start
, size_t end
)
505 void __malloc_donate(char *, char *);
506 if (start
>= dso
->relro_start
&& start
< dso
->relro_end
) start
= dso
->relro_end
;
507 if (end
>= dso
->relro_start
&& end
< dso
->relro_end
) end
= dso
->relro_start
;
508 if (start
>= end
) return;
509 char *base
= laddr_pg(dso
, start
);
510 __malloc_donate(base
, base
+(end
-start
));
513 static void reclaim_gaps(struct dso
*dso
)
515 Phdr
*ph
= dso
->phdr
;
516 size_t phcnt
= dso
->phnum
;
518 for (; phcnt
--; ph
=(void *)((char *)ph
+dso
->phentsize
)) {
519 if (ph
->p_type
!=PT_LOAD
) continue;
520 if ((ph
->p_flags
&(PF_R
|PF_W
))!=(PF_R
|PF_W
)) continue;
521 reclaim(dso
, ph
->p_vaddr
& -PAGE_SIZE
, ph
->p_vaddr
);
522 reclaim(dso
, ph
->p_vaddr
+ph
->p_memsz
,
523 ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
);
527 static void *mmap_fixed(void *p
, size_t n
, int prot
, int flags
, int fd
, off_t off
)
529 static int no_map_fixed
;
532 q
= mmap(p
, n
, prot
, flags
|MAP_FIXED
, fd
, off
);
533 if (!DL_NOMMU_SUPPORT
|| q
!= MAP_FAILED
|| errno
!= EINVAL
)
537 /* Fallbacks for MAP_FIXED failure on NOMMU kernels. */
538 if (flags
& MAP_ANONYMOUS
) {
543 if (lseek(fd
, off
, SEEK_SET
) < 0) return MAP_FAILED
;
544 for (q
=p
; n
; q
+=r
, off
+=r
, n
-=r
) {
546 if (r
< 0 && errno
!= EINTR
) return MAP_FAILED
;
555 static void unmap_library(struct dso
*dso
)
559 for (i
=0; i
<dso
->loadmap
->nsegs
; i
++) {
560 if (!dso
->loadmap
->segs
[i
].p_memsz
)
562 munmap((void *)dso
->loadmap
->segs
[i
].addr
,
563 dso
->loadmap
->segs
[i
].p_memsz
);
566 } else if (dso
->map
&& dso
->map_len
) {
567 munmap(dso
->map
, dso
->map_len
);
571 static void *map_library(int fd
, struct dso
*dso
)
573 Ehdr buf
[(896+sizeof(Ehdr
))/sizeof(Ehdr
)];
574 void *allocated_buf
=0;
576 size_t addr_min
=SIZE_MAX
, addr_max
=0, map_len
;
577 size_t this_min
, this_max
;
583 unsigned char *map
=MAP_FAILED
, *base
;
588 ssize_t l
= read(fd
, buf
, sizeof buf
);
591 if (l
<sizeof *eh
|| (eh
->e_type
!= ET_DYN
&& eh
->e_type
!= ET_EXEC
))
593 phsize
= eh
->e_phentsize
* eh
->e_phnum
;
594 if (phsize
> sizeof buf
- sizeof *eh
) {
595 allocated_buf
= malloc(phsize
);
596 if (!allocated_buf
) return 0;
597 l
= pread(fd
, allocated_buf
, phsize
, eh
->e_phoff
);
598 if (l
< 0) goto error
;
599 if (l
!= phsize
) goto noexec
;
600 ph
= ph0
= allocated_buf
;
601 } else if (eh
->e_phoff
+ phsize
> l
) {
602 l
= pread(fd
, buf
+1, phsize
, eh
->e_phoff
);
603 if (l
< 0) goto error
;
604 if (l
!= phsize
) goto noexec
;
605 ph
= ph0
= (void *)(buf
+ 1);
607 ph
= ph0
= (void *)((char *)buf
+ eh
->e_phoff
);
609 for (i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
610 if (ph
->p_type
== PT_DYNAMIC
) {
612 } else if (ph
->p_type
== PT_TLS
) {
613 tls_image
= ph
->p_vaddr
;
614 dso
->tls
.align
= ph
->p_align
;
615 dso
->tls
.len
= ph
->p_filesz
;
616 dso
->tls
.size
= ph
->p_memsz
;
617 } else if (ph
->p_type
== PT_GNU_RELRO
) {
618 dso
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
619 dso
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
621 if (ph
->p_type
!= PT_LOAD
) continue;
623 if (ph
->p_vaddr
< addr_min
) {
624 addr_min
= ph
->p_vaddr
;
625 off_start
= ph
->p_offset
;
626 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
627 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
628 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
630 if (ph
->p_vaddr
+ph
->p_memsz
> addr_max
) {
631 addr_max
= ph
->p_vaddr
+ph
->p_memsz
;
634 if (!dyn
) goto noexec
;
635 if (DL_FDPIC
&& !(eh
->e_flags
& FDPIC_CONSTDISP_FLAG
)) {
636 dso
->loadmap
= calloc(1, sizeof *dso
->loadmap
637 + nsegs
* sizeof *dso
->loadmap
->segs
);
638 if (!dso
->loadmap
) goto error
;
639 dso
->loadmap
->nsegs
= nsegs
;
640 for (ph
=ph0
, i
=0; i
<nsegs
; ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
641 if (ph
->p_type
!= PT_LOAD
) continue;
642 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
643 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
644 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
645 map
= mmap(0, ph
->p_memsz
+ (ph
->p_vaddr
& PAGE_SIZE
-1),
647 fd
, ph
->p_offset
& -PAGE_SIZE
);
648 if (map
== MAP_FAILED
) {
652 dso
->loadmap
->segs
[i
].addr
= (size_t)map
+
653 (ph
->p_vaddr
& PAGE_SIZE
-1);
654 dso
->loadmap
->segs
[i
].p_vaddr
= ph
->p_vaddr
;
655 dso
->loadmap
->segs
[i
].p_memsz
= ph
->p_memsz
;
657 if (prot
& PROT_WRITE
) {
658 size_t brk
= (ph
->p_vaddr
& PAGE_SIZE
-1)
660 size_t pgbrk
= brk
+ PAGE_SIZE
-1 & -PAGE_SIZE
;
661 size_t pgend
= brk
+ ph
->p_memsz
- ph
->p_filesz
662 + PAGE_SIZE
-1 & -PAGE_SIZE
;
663 if (pgend
> pgbrk
&& mmap_fixed(map
+pgbrk
,
665 MAP_PRIVATE
|MAP_FIXED
|MAP_ANONYMOUS
,
666 -1, off_start
) == MAP_FAILED
)
668 memset(map
+ brk
, 0, pgbrk
-brk
);
671 map
= (void *)dso
->loadmap
->segs
[0].addr
;
675 addr_max
+= PAGE_SIZE
-1;
676 addr_max
&= -PAGE_SIZE
;
677 addr_min
&= -PAGE_SIZE
;
678 off_start
&= -PAGE_SIZE
;
679 map_len
= addr_max
- addr_min
+ off_start
;
680 /* The first time, we map too much, possibly even more than
681 * the length of the file. This is okay because we will not
682 * use the invalid part; we just need to reserve the right
683 * amount of virtual address space to map over later. */
684 map
= DL_NOMMU_SUPPORT
685 ? mmap((void *)addr_min
, map_len
, PROT_READ
|PROT_WRITE
|PROT_EXEC
,
686 MAP_PRIVATE
|MAP_ANONYMOUS
, -1, 0)
687 : mmap((void *)addr_min
, map_len
, prot
,
688 MAP_PRIVATE
, fd
, off_start
);
689 if (map
==MAP_FAILED
) goto error
;
691 dso
->map_len
= map_len
;
692 /* If the loaded file is not relocatable and the requested address is
693 * not available, then the load operation must fail. */
694 if (eh
->e_type
!= ET_DYN
&& addr_min
&& map
!=(void *)addr_min
) {
698 base
= map
- addr_min
;
701 for (ph
=ph0
, i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
702 if (ph
->p_type
!= PT_LOAD
) continue;
703 /* Check if the programs headers are in this load segment, and
704 * if so, record the address for use by dl_iterate_phdr. */
705 if (!dso
->phdr
&& eh
->e_phoff
>= ph
->p_offset
706 && eh
->e_phoff
+phsize
<= ph
->p_offset
+ph
->p_filesz
) {
707 dso
->phdr
= (void *)(base
+ ph
->p_vaddr
708 + (eh
->e_phoff
-ph
->p_offset
));
709 dso
->phnum
= eh
->e_phnum
;
710 dso
->phentsize
= eh
->e_phentsize
;
712 this_min
= ph
->p_vaddr
& -PAGE_SIZE
;
713 this_max
= ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
;
714 off_start
= ph
->p_offset
& -PAGE_SIZE
;
715 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
716 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
717 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
718 /* Reuse the existing mapping for the lowest-address LOAD */
719 if ((ph
->p_vaddr
& -PAGE_SIZE
) != addr_min
|| DL_NOMMU_SUPPORT
)
720 if (mmap_fixed(base
+this_min
, this_max
-this_min
, prot
, MAP_PRIVATE
|MAP_FIXED
, fd
, off_start
) == MAP_FAILED
)
722 if (ph
->p_memsz
> ph
->p_filesz
&& (ph
->p_flags
&PF_W
)) {
723 size_t brk
= (size_t)base
+ph
->p_vaddr
+ph
->p_filesz
;
724 size_t pgbrk
= brk
+PAGE_SIZE
-1 & -PAGE_SIZE
;
725 memset((void *)brk
, 0, pgbrk
-brk
& PAGE_SIZE
-1);
726 if (pgbrk
-(size_t)base
< this_max
&& mmap_fixed((void *)pgbrk
, (size_t)base
+this_max
-pgbrk
, prot
, MAP_PRIVATE
|MAP_FIXED
|MAP_ANONYMOUS
, -1, 0) == MAP_FAILED
)
730 for (i
=0; ((size_t *)(base
+dyn
))[i
]; i
+=2)
731 if (((size_t *)(base
+dyn
))[i
]==DT_TEXTREL
) {
732 if (mprotect(map
, map_len
, PROT_READ
|PROT_WRITE
|PROT_EXEC
)
739 dso
->dynv
= laddr(dso
, dyn
);
740 if (dso
->tls
.size
) dso
->tls
.image
= laddr(dso
, tls_image
);
746 if (map
!=MAP_FAILED
) unmap_library(dso
);
751 static int path_open(const char *name
, const char *s
, char *buf
, size_t buf_size
)
756 s
+= strspn(s
, ":\n");
757 l
= strcspn(s
, ":\n");
758 if (l
-1 >= INT_MAX
) return -1;
759 if (snprintf(buf
, buf_size
, "%.*s/%s", (int)l
, s
, name
) < buf_size
) {
760 if ((fd
= open(buf
, O_RDONLY
|O_CLOEXEC
))>=0) return fd
;
768 /* Any negative value but -1 will inhibit
769 * futher path search. */
777 static int fixup_rpath(struct dso
*p
, char *buf
, size_t buf_size
)
780 const char *s
, *t
, *origin
;
782 if (p
->rpath
|| !p
->rpath_orig
) return 0;
783 if (!strchr(p
->rpath_orig
, '$')) {
784 p
->rpath
= p
->rpath_orig
;
789 while ((t
=strchr(s
, '$'))) {
790 if (strncmp(t
, "$ORIGIN", 7) && strncmp(t
, "${ORIGIN}", 9))
795 if (n
> SSIZE_MAX
/PATH_MAX
) return 0;
797 if (p
->kernel_mapped
) {
798 /* $ORIGIN searches cannot be performed for the main program
799 * when it is suid/sgid/AT_SECURE. This is because the
800 * pathname is under the control of the caller of execve.
801 * For libraries, however, $ORIGIN can be processed safely
802 * since the library's pathname came from a trusted source
803 * (either system paths or a call to dlopen). */
806 l
= readlink("/proc/self/exe", buf
, buf_size
);
807 if (l
== -1) switch (errno
) {
822 t
= strrchr(origin
, '/');
826 /* Normally p->name will always be an absolute or relative
827 * pathname containing at least one '/' character, but in the
828 * case where ldso was invoked as a command to execute a
829 * program in the working directory, app.name may not. Fix. */
833 /* Disallow non-absolute origins for suid/sgid/AT_SECURE. */
834 if (libc
.secure
&& *origin
!= '/')
836 p
->rpath
= malloc(strlen(p
->rpath_orig
) + n
*l
+ 1);
837 if (!p
->rpath
) return -1;
841 while ((t
=strchr(s
, '$'))) {
844 memcpy(d
, origin
, l
);
846 /* It was determined previously that the '$' is followed
847 * either by "ORIGIN" or "{ORIGIN}". */
848 s
= t
+ 7 + 2*(t
[1]=='{');
854 static void decode_dyn(struct dso
*p
)
857 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
858 p
->syms
= laddr(p
, dyn
[DT_SYMTAB
]);
859 p
->strings
= laddr(p
, dyn
[DT_STRTAB
]);
860 if (dyn
[0]&(1<<DT_HASH
))
861 p
->hashtab
= laddr(p
, dyn
[DT_HASH
]);
862 if (dyn
[0]&(1<<DT_RPATH
))
863 p
->rpath_orig
= p
->strings
+ dyn
[DT_RPATH
];
864 if (dyn
[0]&(1<<DT_RUNPATH
))
865 p
->rpath_orig
= p
->strings
+ dyn
[DT_RUNPATH
];
866 if (dyn
[0]&(1<<DT_PLTGOT
))
867 p
->got
= laddr(p
, dyn
[DT_PLTGOT
]);
868 if (search_vec(p
->dynv
, dyn
, DT_GNU_HASH
))
869 p
->ghashtab
= laddr(p
, *dyn
);
870 if (search_vec(p
->dynv
, dyn
, DT_VERSYM
))
871 p
->versym
= laddr(p
, *dyn
);
874 static size_t count_syms(struct dso
*p
)
876 if (p
->hashtab
) return p
->hashtab
[1];
879 uint32_t *buckets
= p
->ghashtab
+ 4 + (p
->ghashtab
[2]*sizeof(size_t)/4);
881 for (i
= nsym
= 0; i
< p
->ghashtab
[0]; i
++) {
882 if (buckets
[i
] > nsym
)
886 hashval
= buckets
+ p
->ghashtab
[0] + (nsym
- p
->ghashtab
[1]);
888 while (!(*hashval
++ & 1));
893 static void *dl_mmap(size_t n
)
896 int prot
= PROT_READ
|PROT_WRITE
, flags
= MAP_ANONYMOUS
|MAP_PRIVATE
;
898 p
= (void *)__syscall(SYS_mmap2
, 0, n
, prot
, flags
, -1, 0);
900 p
= (void *)__syscall(SYS_mmap
, 0, n
, prot
, flags
, -1, 0);
902 return p
== MAP_FAILED
? 0 : p
;
905 static void makefuncdescs(struct dso
*p
)
907 static int self_done
;
908 size_t nsym
= count_syms(p
);
909 size_t i
, size
= nsym
* sizeof(*p
->funcdescs
);
912 p
->funcdescs
= dl_mmap(size
);
915 p
->funcdescs
= malloc(size
);
918 if (!runtime
) a_crash();
919 error("Error allocating function descriptors for %s", p
->name
);
920 longjmp(*rtld_fail
, 1);
922 for (i
=0; i
<nsym
; i
++) {
923 if ((p
->syms
[i
].st_info
&0xf)==STT_FUNC
&& p
->syms
[i
].st_shndx
) {
924 p
->funcdescs
[i
].addr
= laddr(p
, p
->syms
[i
].st_value
);
925 p
->funcdescs
[i
].got
= p
->got
;
927 p
->funcdescs
[i
].addr
= 0;
928 p
->funcdescs
[i
].got
= 0;
933 static struct dso
*load_library(const char *name
, struct dso
*needed_by
)
935 char buf
[2*NAME_MAX
+2];
936 const char *pathname
;
938 struct dso
*p
, temp_dso
= {0};
950 /* Catch and block attempts to reload the implementation itself */
951 if (name
[0]=='l' && name
[1]=='i' && name
[2]=='b') {
952 static const char reserved
[] =
953 "c.pthread.rt.m.dl.util.xnet.";
954 const char *rp
, *next
;
955 for (rp
=reserved
; *rp
; rp
=next
) {
956 next
= strchr(rp
, '.') + 1;
957 if (strncmp(name
+3, rp
, next
-rp
) == 0)
962 /* Track which names have been resolved
963 * and only report each one once. */
964 static unsigned reported
;
965 unsigned mask
= 1U<<(rp
-reserved
);
966 if (!(reported
& mask
)) {
968 dprintf(1, "\t%s => %s (%p)\n",
976 if (!strcmp(name
, ldso
.name
)) is_self
= 1;
985 if (strchr(name
, '/')) {
987 fd
= open(name
, O_RDONLY
|O_CLOEXEC
);
989 /* Search for the name to see if it's already loaded */
990 for (p
=head
->next
; p
; p
=p
->next
) {
991 if (p
->shortname
&& !strcmp(p
->shortname
, name
)) {
995 if (strlen(name
) > NAME_MAX
) return 0;
997 if (env_path
) fd
= path_open(name
, env_path
, buf
, sizeof buf
);
998 for (p
=needed_by
; fd
== -1 && p
; p
=p
->needed_by
) {
999 if (fixup_rpath(p
, buf
, sizeof buf
) < 0)
1000 fd
= -2; /* Inhibit further search. */
1002 fd
= path_open(name
, p
->rpath
, buf
, sizeof buf
);
1008 if (ldso
.name
[0]=='/') {
1010 for (s
=t
=z
=ldso
.name
; *s
; s
++)
1011 if (*s
=='/') z
=t
, t
=s
;
1012 prefix_len
= z
-ldso
.name
;
1013 if (prefix_len
< PATH_MAX
)
1020 char etc_ldso_path
[prefix_len
+ 1
1021 + sizeof "/etc/ld-musl-" LDSO_ARCH
".path"];
1022 snprintf(etc_ldso_path
, sizeof etc_ldso_path
,
1023 "%.*s/etc/ld-musl-" LDSO_ARCH
".path",
1024 (int)prefix_len
, prefix
);
1025 FILE *f
= fopen(etc_ldso_path
, "rbe");
1027 if (getdelim(&sys_path
, (size_t[1]){0}, 0, f
) <= 0) {
1032 } else if (errno
!= ENOENT
) {
1036 if (!sys_path
) sys_path
= "/lib:/usr/local/lib:/usr/lib";
1037 fd
= path_open(name
, sys_path
, buf
, sizeof buf
);
1041 if (fd
< 0) return 0;
1042 if (fstat(fd
, &st
) < 0) {
1046 for (p
=head
->next
; p
; p
=p
->next
) {
1047 if (p
->dev
== st
.st_dev
&& p
->ino
== st
.st_ino
) {
1048 /* If this library was previously loaded with a
1049 * pathname but a search found the same inode,
1050 * setup its shortname so it can be found by name. */
1051 if (!p
->shortname
&& pathname
!= name
)
1052 p
->shortname
= strrchr(p
->name
, '/')+1;
1057 map
= noload
? 0 : map_library(fd
, &temp_dso
);
1061 /* Avoid the danger of getting two versions of libc mapped into the
1062 * same process when an absolute pathname was used. The symbols
1063 * checked are chosen to catch both musl and glibc, and to avoid
1064 * false positives from interposition-hack libraries. */
1065 decode_dyn(&temp_dso
);
1066 if (find_sym(&temp_dso
, "__libc_start_main", 1).sym
&&
1067 find_sym(&temp_dso
, "stdin", 1).sym
) {
1068 unmap_library(&temp_dso
);
1069 return load_library("libc.so", needed_by
);
1071 /* Past this point, if we haven't reached runtime yet, ldso has
1072 * committed either to use the mapped library or to abort execution.
1073 * Unmapping is not possible, so we can safely reclaim gaps. */
1074 if (!runtime
) reclaim_gaps(&temp_dso
);
1076 /* Allocate storage for the new DSO. When there is TLS, this
1077 * storage must include a reservation for all pre-existing
1078 * threads to obtain copies of both the new TLS, and an
1079 * extended DTV capable of storing an additional slot for
1080 * the newly-loaded DSO. */
1081 alloc_size
= sizeof *p
+ strlen(pathname
) + 1;
1082 if (runtime
&& temp_dso
.tls
.image
) {
1083 size_t per_th
= temp_dso
.tls
.size
+ temp_dso
.tls
.align
1084 + sizeof(void *) * (tls_cnt
+3);
1085 n_th
= libc
.threads_minus_1
+ 1;
1086 if (n_th
> SSIZE_MAX
/ per_th
) alloc_size
= SIZE_MAX
;
1087 else alloc_size
+= n_th
* per_th
;
1089 p
= calloc(1, alloc_size
);
1091 unmap_library(&temp_dso
);
1094 memcpy(p
, &temp_dso
, sizeof temp_dso
);
1097 p
->needed_by
= needed_by
;
1099 strcpy(p
->name
, pathname
);
1100 /* Add a shortname only if name arg was not an explicit pathname. */
1101 if (pathname
!= name
) p
->shortname
= strrchr(p
->name
, '/')+1;
1103 p
->tls_id
= ++tls_cnt
;
1104 tls_align
= MAXP2(tls_align
, p
->tls
.align
);
1106 p
->tls
.offset
= tls_offset
+ ( (tls_align
-1) &
1107 -(tls_offset
+ (uintptr_t)p
->tls
.image
) );
1108 tls_offset
+= p
->tls
.size
;
1110 tls_offset
+= p
->tls
.size
+ p
->tls
.align
- 1;
1111 tls_offset
-= (tls_offset
+ (uintptr_t)p
->tls
.image
)
1113 p
->tls
.offset
= tls_offset
;
1115 p
->new_dtv
= (void *)(-sizeof(size_t) &
1116 (uintptr_t)(p
->name
+strlen(p
->name
)+sizeof(size_t)));
1117 p
->new_tls
= (void *)(p
->new_dtv
+ n_th
*(tls_cnt
+1));
1118 if (tls_tail
) tls_tail
->next
= &p
->tls
;
1119 else libc
.tls_head
= &p
->tls
;
1127 if (DL_FDPIC
) makefuncdescs(p
);
1129 if (ldd_mode
) dprintf(1, "\t%s => %s (%p)\n", name
, pathname
, p
->base
);
1134 static void load_deps(struct dso
*p
)
1137 struct dso
***deps
= &p
->deps
, **tmp
, *dep
;
1138 for (; p
; p
=p
->next
) {
1139 for (i
=0; p
->dynv
[i
]; i
+=2) {
1140 if (p
->dynv
[i
] != DT_NEEDED
) continue;
1141 dep
= load_library(p
->strings
+ p
->dynv
[i
+1], p
);
1143 error("Error loading shared library %s: %m (needed by %s)",
1144 p
->strings
+ p
->dynv
[i
+1], p
->name
);
1145 if (runtime
) longjmp(*rtld_fail
, 1);
1149 tmp
= realloc(*deps
, sizeof(*tmp
)*(ndeps
+2));
1150 if (!tmp
) longjmp(*rtld_fail
, 1);
1157 if (!*deps
) *deps
= (struct dso
**)&nodeps_dummy
;
1160 static void load_preload(char *s
)
1164 for (z
=s
; *z
; s
=z
) {
1165 for ( ; *s
&& (isspace(*s
) || *s
==':'); s
++);
1166 for (z
=s
; *z
&& !isspace(*z
) && *z
!=':'; z
++);
1174 static void add_syms(struct dso
*p
)
1176 if (!p
->syms_next
&& syms_tail
!= p
) {
1177 syms_tail
->syms_next
= p
;
1182 static void revert_syms(struct dso
*old_tail
)
1184 struct dso
*p
, *next
;
1185 /* Chop off the tail of the list of dsos that participate in
1186 * the global symbol table, reverting them to RTLD_LOCAL. */
1187 for (p
=old_tail
; p
; p
=next
) {
1188 next
= p
->syms_next
;
1191 syms_tail
= old_tail
;
1194 static void do_mips_relocs(struct dso
*p
, size_t *got
)
1196 size_t i
, j
, rel
[2];
1197 unsigned char *base
= p
->base
;
1198 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_LOCAL_GOTNO
);
1202 while (i
--) *got
++ += (size_t)base
;
1204 j
=0; search_vec(p
->dynv
, &j
, DT_MIPS_GOTSYM
);
1205 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_SYMTABNO
);
1206 Sym
*sym
= p
->syms
+ j
;
1207 rel
[0] = (unsigned char *)got
- base
;
1208 for (i
-=j
; i
; i
--, sym
++, rel
[0]+=sizeof(size_t)) {
1209 rel
[1] = R_INFO(sym
-p
->syms
, R_MIPS_JUMP_SLOT
);
1210 do_relocs(p
, rel
, sizeof rel
, 2);
1214 static void reloc_all(struct dso
*p
)
1216 size_t dyn
[DYN_CNT
];
1217 for (; p
; p
=p
->next
) {
1218 if (p
->relocated
) continue;
1219 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1220 if (NEED_MIPS_GOT_RELOCS
)
1221 do_mips_relocs(p
, laddr(p
, dyn
[DT_PLTGOT
]));
1222 do_relocs(p
, laddr(p
, dyn
[DT_JMPREL
]), dyn
[DT_PLTRELSZ
],
1223 2+(dyn
[DT_PLTREL
]==DT_RELA
));
1224 do_relocs(p
, laddr(p
, dyn
[DT_REL
]), dyn
[DT_RELSZ
], 2);
1225 do_relocs(p
, laddr(p
, dyn
[DT_RELA
]), dyn
[DT_RELASZ
], 3);
1227 if (head
!= &ldso
&& p
->relro_start
!= p
->relro_end
&&
1228 mprotect(laddr(p
, p
->relro_start
), p
->relro_end
-p
->relro_start
, PROT_READ
)
1229 && errno
!= ENOSYS
) {
1230 error("Error relocating %s: RELRO protection failed: %m",
1232 if (runtime
) longjmp(*rtld_fail
, 1);
1239 static void kernel_mapped_dso(struct dso
*p
)
1241 size_t min_addr
= -1, max_addr
= 0, cnt
;
1243 for (cnt
= p
->phnum
; cnt
--; ph
= (void *)((char *)ph
+ p
->phentsize
)) {
1244 if (ph
->p_type
== PT_DYNAMIC
) {
1245 p
->dynv
= laddr(p
, ph
->p_vaddr
);
1246 } else if (ph
->p_type
== PT_GNU_RELRO
) {
1247 p
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
1248 p
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
1250 if (ph
->p_type
!= PT_LOAD
) continue;
1251 if (ph
->p_vaddr
< min_addr
)
1252 min_addr
= ph
->p_vaddr
;
1253 if (ph
->p_vaddr
+ph
->p_memsz
> max_addr
)
1254 max_addr
= ph
->p_vaddr
+ph
->p_memsz
;
1256 min_addr
&= -PAGE_SIZE
;
1257 max_addr
= (max_addr
+ PAGE_SIZE
-1) & -PAGE_SIZE
;
1258 p
->map
= p
->base
+ min_addr
;
1259 p
->map_len
= max_addr
- min_addr
;
1260 p
->kernel_mapped
= 1;
1263 void __libc_exit_fini()
1266 size_t dyn
[DYN_CNT
];
1267 for (p
=fini_head
; p
; p
=p
->fini_next
) {
1268 if (!p
->constructed
) continue;
1269 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1270 if (dyn
[0] & (1<<DT_FINI_ARRAY
)) {
1271 size_t n
= dyn
[DT_FINI_ARRAYSZ
]/sizeof(size_t);
1272 size_t *fn
= (size_t *)laddr(p
, dyn
[DT_FINI_ARRAY
])+n
;
1273 while (n
--) ((void (*)(void))*--fn
)();
1275 #ifndef NO_LEGACY_INITFINI
1276 if ((dyn
[0] & (1<<DT_FINI
)) && dyn
[DT_FINI
])
1277 fpaddr(p
, dyn
[DT_FINI
])();
1282 static void do_init_fini(struct dso
*p
)
1284 size_t dyn
[DYN_CNT
];
1285 int need_locking
= libc
.threads_minus_1
;
1286 /* Allow recursive calls that arise when a library calls
1287 * dlopen from one of its constructors, but block any
1288 * other threads until all ctors have finished. */
1289 if (need_locking
) pthread_mutex_lock(&init_fini_lock
);
1290 for (; p
; p
=p
->prev
) {
1291 if (p
->constructed
) continue;
1293 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1294 if (dyn
[0] & ((1<<DT_FINI
) | (1<<DT_FINI_ARRAY
))) {
1295 p
->fini_next
= fini_head
;
1298 #ifndef NO_LEGACY_INITFINI
1299 if ((dyn
[0] & (1<<DT_INIT
)) && dyn
[DT_INIT
])
1300 fpaddr(p
, dyn
[DT_INIT
])();
1302 if (dyn
[0] & (1<<DT_INIT_ARRAY
)) {
1303 size_t n
= dyn
[DT_INIT_ARRAYSZ
]/sizeof(size_t);
1304 size_t *fn
= laddr(p
, dyn
[DT_INIT_ARRAY
]);
1305 while (n
--) ((void (*)(void))*fn
++)();
1307 if (!need_locking
&& libc
.threads_minus_1
) {
1309 pthread_mutex_lock(&init_fini_lock
);
1312 if (need_locking
) pthread_mutex_unlock(&init_fini_lock
);
1315 void __libc_start_init(void)
1320 static void dl_debug_state(void)
1324 weak_alias(dl_debug_state
, _dl_debug_state
);
1326 void __init_tls(size_t *auxv
)
1330 hidden
void *__tls_get_new(tls_mod_off_t
*v
)
1332 pthread_t self
= __pthread_self();
1334 /* Block signals to make accessing new TLS async-signal-safe */
1336 __block_all_sigs(&set
);
1337 if (v
[0]<=(size_t)self
->dtv
[0]) {
1338 __restore_sigs(&set
);
1339 return (char *)self
->dtv
[v
[0]]+v
[1]+DTP_OFFSET
;
1342 /* This is safe without any locks held because, if the caller
1343 * is able to request the Nth entry of the DTV, the DSO list
1344 * must be valid at least that far out and it was synchronized
1345 * at program startup or by an already-completed call to dlopen. */
1347 for (p
=head
; p
->tls_id
!= v
[0]; p
=p
->next
);
1349 /* Get new DTV space from new DSO if needed */
1350 if (v
[0] > (size_t)self
->dtv
[0]) {
1351 void **newdtv
= p
->new_dtv
+
1352 (v
[0]+1)*a_fetch_add(&p
->new_dtv_idx
,1);
1353 memcpy(newdtv
, self
->dtv
,
1354 ((size_t)self
->dtv
[0]+1) * sizeof(void *));
1355 newdtv
[0] = (void *)v
[0];
1356 self
->dtv
= self
->dtv_copy
= newdtv
;
1359 /* Get new TLS memory from all new DSOs up to the requested one */
1361 for (p
=head
; ; p
=p
->next
) {
1362 if (!p
->tls_id
|| self
->dtv
[p
->tls_id
]) continue;
1363 mem
= p
->new_tls
+ (p
->tls
.size
+ p
->tls
.align
)
1364 * a_fetch_add(&p
->new_tls_idx
,1);
1365 mem
+= ((uintptr_t)p
->tls
.image
- (uintptr_t)mem
)
1367 self
->dtv
[p
->tls_id
] = mem
;
1368 memcpy(mem
, p
->tls
.image
, p
->tls
.len
);
1369 if (p
->tls_id
== v
[0]) break;
1371 __restore_sigs(&set
);
1372 return mem
+ v
[1] + DTP_OFFSET
;
1375 static void update_tls_size()
1377 libc
.tls_cnt
= tls_cnt
;
1378 libc
.tls_align
= tls_align
;
1379 libc
.tls_size
= ALIGN(
1380 (1+tls_cnt
) * sizeof(void *) +
1382 sizeof(struct pthread
) +
1387 /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
1388 * following stage 2 and stage 3 functions via primitive symbolic lookup
1389 * since it does not have access to their addresses to begin with. */
1391 /* Stage 2 of the dynamic linker is called after relative relocations
1392 * have been processed. It can make function calls to static functions
1393 * and access string literals and static data, but cannot use extern
1394 * symbols. Its job is to perform symbolic relocations on the dynamic
1395 * linker itself, but some of the relocations performed may need to be
1396 * replaced later due to copy relocations in the main program. */
1398 hidden
void __dls2(unsigned char *base
, size_t *sp
)
1401 void *p1
= (void *)sp
[-2];
1402 void *p2
= (void *)sp
[-1];
1404 size_t *auxv
, aux
[AUX_CNT
];
1405 for (auxv
=sp
+1+*sp
+1; *auxv
; auxv
++); auxv
++;
1406 decode_vec(auxv
, aux
, AUX_CNT
);
1407 if (aux
[AT_BASE
]) ldso
.base
= (void *)aux
[AT_BASE
];
1408 else ldso
.base
= (void *)(aux
[AT_PHDR
] & -4096);
1410 app_loadmap
= p2
? p1
: 0;
1411 ldso
.loadmap
= p2
? p2
: p1
;
1412 ldso
.base
= laddr(&ldso
, 0);
1416 Ehdr
*ehdr
= (void *)ldso
.base
;
1417 ldso
.name
= ldso
.shortname
= "libc.so";
1418 ldso
.phnum
= ehdr
->e_phnum
;
1419 ldso
.phdr
= laddr(&ldso
, ehdr
->e_phoff
);
1420 ldso
.phentsize
= ehdr
->e_phentsize
;
1421 kernel_mapped_dso(&ldso
);
1424 if (DL_FDPIC
) makefuncdescs(&ldso
);
1426 /* Prepare storage for to save clobbered REL addends so they
1427 * can be reused in stage 3. There should be very few. If
1428 * something goes wrong and there are a huge number, abort
1429 * instead of risking stack overflow. */
1430 size_t dyn
[DYN_CNT
];
1431 decode_vec(ldso
.dynv
, dyn
, DYN_CNT
);
1432 size_t *rel
= laddr(&ldso
, dyn
[DT_REL
]);
1433 size_t rel_size
= dyn
[DT_RELSZ
];
1434 size_t symbolic_rel_cnt
= 0;
1435 apply_addends_to
= rel
;
1436 for (; rel_size
; rel
+=2, rel_size
-=2*sizeof(size_t))
1437 if (!IS_RELATIVE(rel
[1], ldso
.syms
)) symbolic_rel_cnt
++;
1438 if (symbolic_rel_cnt
>= ADDEND_LIMIT
) a_crash();
1439 size_t addends
[symbolic_rel_cnt
+1];
1440 saved_addends
= addends
;
1447 /* Call dynamic linker stage-3, __dls3, looking it up
1448 * symbolically as a barrier against moving the address
1449 * load across the above relocation processing. */
1450 struct symdef dls3_def
= find_sym(&ldso
, "__dls3", 0);
1451 if (DL_FDPIC
) ((stage3_func
)&ldso
.funcdescs
[dls3_def
.sym
-ldso
.syms
])(sp
);
1452 else ((stage3_func
)laddr(&ldso
, dls3_def
.sym
->st_value
))(sp
);
1455 /* Stage 3 of the dynamic linker is called with the dynamic linker/libc
1456 * fully functional. Its job is to load (if not already loaded) and
1457 * process dependencies and relocations for the main application and
1458 * transfer control to its entry point. */
1460 _Noreturn
void __dls3(size_t *sp
)
1462 static struct dso app
, vdso
;
1463 size_t aux
[AUX_CNT
], *auxv
;
1465 char *env_preload
=0;
1466 char *replace_argv0
=0;
1469 char **argv
= (void *)(sp
+1);
1470 char **argv_orig
= argv
;
1471 char **envp
= argv
+argc
+1;
1473 /* Find aux vector just past environ[] and use it to initialize
1474 * global data that may be needed before we can make syscalls. */
1476 for (i
=argc
+1; argv
[i
]; i
++);
1477 libc
.auxv
= auxv
= (void *)(argv
+i
+1);
1478 decode_vec(auxv
, aux
, AUX_CNT
);
1479 __hwcap
= aux
[AT_HWCAP
];
1480 libc
.page_size
= aux
[AT_PAGESZ
];
1481 libc
.secure
= ((aux
[0]&0x7800)!=0x7800 || aux
[AT_UID
]!=aux
[AT_EUID
]
1482 || aux
[AT_GID
]!=aux
[AT_EGID
] || aux
[AT_SECURE
]);
1484 /* Setup early thread pointer in builtin_tls for ldso/libc itself to
1485 * use during dynamic linking. If possible it will also serve as the
1486 * thread pointer at runtime. */
1487 libc
.tls_size
= sizeof builtin_tls
;
1488 libc
.tls_align
= tls_align
;
1489 if (__init_tp(__copy_tls((void *)builtin_tls
)) < 0) {
1493 /* Only trust user/env if kernel says we're not suid/sgid */
1495 env_path
= getenv("LD_LIBRARY_PATH");
1496 env_preload
= getenv("LD_PRELOAD");
1499 /* If the main program was already loaded by the kernel,
1500 * AT_PHDR will point to some location other than the dynamic
1501 * linker's program headers. */
1502 if (aux
[AT_PHDR
] != (size_t)ldso
.phdr
) {
1503 size_t interp_off
= 0;
1504 size_t tls_image
= 0;
1505 /* Find load address of the main program, via AT_PHDR vs PT_PHDR. */
1506 Phdr
*phdr
= app
.phdr
= (void *)aux
[AT_PHDR
];
1507 app
.phnum
= aux
[AT_PHNUM
];
1508 app
.phentsize
= aux
[AT_PHENT
];
1509 for (i
=aux
[AT_PHNUM
]; i
; i
--, phdr
=(void *)((char *)phdr
+ aux
[AT_PHENT
])) {
1510 if (phdr
->p_type
== PT_PHDR
)
1511 app
.base
= (void *)(aux
[AT_PHDR
] - phdr
->p_vaddr
);
1512 else if (phdr
->p_type
== PT_INTERP
)
1513 interp_off
= (size_t)phdr
->p_vaddr
;
1514 else if (phdr
->p_type
== PT_TLS
) {
1515 tls_image
= phdr
->p_vaddr
;
1516 app
.tls
.len
= phdr
->p_filesz
;
1517 app
.tls
.size
= phdr
->p_memsz
;
1518 app
.tls
.align
= phdr
->p_align
;
1521 if (DL_FDPIC
) app
.loadmap
= app_loadmap
;
1522 if (app
.tls
.size
) app
.tls
.image
= laddr(&app
, tls_image
);
1523 if (interp_off
) ldso
.name
= laddr(&app
, interp_off
);
1524 if ((aux
[0] & (1UL<<AT_EXECFN
))
1525 && strncmp((char *)aux
[AT_EXECFN
], "/proc/", 6))
1526 app
.name
= (char *)aux
[AT_EXECFN
];
1529 kernel_mapped_dso(&app
);
1532 char *ldname
= argv
[0];
1533 size_t l
= strlen(ldname
);
1534 if (l
>= 3 && !strcmp(ldname
+l
-3, "ldd")) ldd_mode
= 1;
1536 while (argv
[0] && argv
[0][0]=='-' && argv
[0][1]=='-') {
1537 char *opt
= argv
[0]+2;
1538 *argv
++ = (void *)-1;
1541 } else if (!memcmp(opt
, "list", 5)) {
1543 } else if (!memcmp(opt
, "library-path", 12)) {
1544 if (opt
[12]=='=') env_path
= opt
+13;
1545 else if (opt
[12]) *argv
= 0;
1546 else if (*argv
) env_path
= *argv
++;
1547 } else if (!memcmp(opt
, "preload", 7)) {
1548 if (opt
[7]=='=') env_preload
= opt
+8;
1549 else if (opt
[7]) *argv
= 0;
1550 else if (*argv
) env_preload
= *argv
++;
1551 } else if (!memcmp(opt
, "argv0", 5)) {
1552 if (opt
[5]=='=') replace_argv0
= opt
+6;
1553 else if (opt
[5]) *argv
= 0;
1554 else if (*argv
) replace_argv0
= *argv
++;
1559 argv
[-1] = (void *)(argc
- (argv
-argv_orig
));
1561 dprintf(2, "musl libc (" LDSO_ARCH
")\n"
1563 "Dynamic Program Loader\n"
1564 "Usage: %s [options] [--] pathname%s\n",
1565 __libc_get_version(), ldname
,
1566 ldd_mode
? "" : " [args]");
1569 fd
= open(argv
[0], O_RDONLY
);
1571 dprintf(2, "%s: cannot load %s: %s\n", ldname
, argv
[0], strerror(errno
));
1574 Ehdr
*ehdr
= (void *)map_library(fd
, &app
);
1576 dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname
, argv
[0]);
1582 aux
[AT_ENTRY
] = (size_t)laddr(&app
, ehdr
->e_entry
);
1583 /* Find the name that would have been used for the dynamic
1584 * linker had ldd not taken its place. */
1586 for (i
=0; i
<app
.phnum
; i
++) {
1587 if (app
.phdr
[i
].p_type
== PT_INTERP
)
1588 ldso
.name
= laddr(&app
, app
.phdr
[i
].p_vaddr
);
1590 dprintf(1, "\t%s (%p)\n", ldso
.name
, ldso
.base
);
1594 libc
.tls_head
= tls_tail
= &app
.tls
;
1595 app
.tls_id
= tls_cnt
= 1;
1597 app
.tls
.offset
= GAP_ABOVE_TP
;
1598 app
.tls
.offset
+= -GAP_ABOVE_TP
& (app
.tls
.align
-1);
1599 tls_offset
= app
.tls
.offset
+ app
.tls
.size
1600 + ( -((uintptr_t)app
.tls
.image
+ app
.tls
.size
)
1601 & (app
.tls
.align
-1) );
1603 tls_offset
= app
.tls
.offset
= app
.tls
.size
1604 + ( -((uintptr_t)app
.tls
.image
+ app
.tls
.size
)
1605 & (app
.tls
.align
-1) );
1607 tls_align
= MAXP2(tls_align
, app
.tls
.align
);
1611 makefuncdescs(&app
);
1613 app
.loadmap
= (void *)&app_dummy_loadmap
;
1614 app
.loadmap
->nsegs
= 1;
1615 app
.loadmap
->segs
[0].addr
= (size_t)app
.map
;
1616 app
.loadmap
->segs
[0].p_vaddr
= (size_t)app
.map
1618 app
.loadmap
->segs
[0].p_memsz
= app
.map_len
;
1620 argv
[-3] = (void *)app
.loadmap
;
1623 /* Initial dso chain consists only of the app. */
1624 head
= tail
= syms_tail
= &app
;
1626 /* Donate unused parts of app and library mapping to malloc */
1628 reclaim_gaps(&ldso
);
1630 /* Load preload/needed libraries, add symbols to global namespace. */
1631 if (env_preload
) load_preload(env_preload
);
1633 for (struct dso
*p
=head
; p
; p
=p
->next
)
1636 /* Attach to vdso, if provided by the kernel, last so that it does
1637 * not become part of the global namespace. */
1638 if (search_vec(auxv
, &vdso_base
, AT_SYSINFO_EHDR
) && vdso_base
) {
1639 Ehdr
*ehdr
= (void *)vdso_base
;
1640 Phdr
*phdr
= vdso
.phdr
= (void *)(vdso_base
+ ehdr
->e_phoff
);
1641 vdso
.phnum
= ehdr
->e_phnum
;
1642 vdso
.phentsize
= ehdr
->e_phentsize
;
1643 for (i
=ehdr
->e_phnum
; i
; i
--, phdr
=(void *)((char *)phdr
+ ehdr
->e_phentsize
)) {
1644 if (phdr
->p_type
== PT_DYNAMIC
)
1645 vdso
.dynv
= (void *)(vdso_base
+ phdr
->p_offset
);
1646 if (phdr
->p_type
== PT_LOAD
)
1647 vdso
.base
= (void *)(vdso_base
- phdr
->p_vaddr
+ phdr
->p_offset
);
1650 vdso
.shortname
= "linux-gate.so.1";
1658 for (i
=0; app
.dynv
[i
]; i
+=2) {
1659 if (!DT_DEBUG_INDIRECT
&& app
.dynv
[i
]==DT_DEBUG
)
1660 app
.dynv
[i
+1] = (size_t)&debug
;
1661 if (DT_DEBUG_INDIRECT
&& app
.dynv
[i
]==DT_DEBUG_INDIRECT
) {
1662 size_t *ptr
= (size_t *) app
.dynv
[i
+1];
1663 *ptr
= (size_t)&debug
;
1667 /* The main program must be relocated LAST since it may contin
1668 * copy relocations which depend on libraries' relocations. */
1669 reloc_all(app
.next
);
1673 if (libc
.tls_size
> sizeof builtin_tls
|| tls_align
> MIN_TLS_ALIGN
) {
1674 void *initial_tls
= calloc(libc
.tls_size
, 1);
1676 dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
1677 argv
[0], libc
.tls_size
);
1680 if (__init_tp(__copy_tls(initial_tls
)) < 0) {
1684 size_t tmp_tls_size
= libc
.tls_size
;
1685 pthread_t self
= __pthread_self();
1686 /* Temporarily set the tls size to the full size of
1687 * builtin_tls so that __copy_tls will use the same layout
1688 * as it did for before. Then check, just to be safe. */
1689 libc
.tls_size
= sizeof builtin_tls
;
1690 if (__copy_tls((void*)builtin_tls
) != self
) a_crash();
1691 libc
.tls_size
= tmp_tls_size
;
1693 static_tls_cnt
= tls_cnt
;
1695 if (ldso_fail
) _exit(127);
1696 if (ldd_mode
) _exit(0);
1698 /* Determine if malloc was interposed by a replacement implementation
1699 * so that calloc and the memalign family can harden against the
1700 * possibility of incomplete replacement. */
1701 if (find_sym(head
, "malloc", 1).dso
!= &ldso
)
1702 __malloc_replaced
= 1;
1704 /* Switch to runtime mode: any further failures in the dynamic
1705 * linker are a reportable failure rather than a fatal startup
1710 debug
.bp
= dl_debug_state
;
1712 debug
.base
= ldso
.base
;
1716 if (replace_argv0
) argv
[0] = replace_argv0
;
1720 CRTJMP((void *)aux
[AT_ENTRY
], argv
-1);
1724 static void prepare_lazy(struct dso
*p
)
1726 size_t dyn
[DYN_CNT
], n
, flags1
=0;
1727 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1728 search_vec(p
->dynv
, &flags1
, DT_FLAGS_1
);
1729 if (dyn
[DT_BIND_NOW
] || (dyn
[DT_FLAGS
] & DF_BIND_NOW
) || (flags1
& DF_1_NOW
))
1731 n
= dyn
[DT_RELSZ
]/2 + dyn
[DT_RELASZ
]/3 + dyn
[DT_PLTRELSZ
]/2 + 1;
1732 if (NEED_MIPS_GOT_RELOCS
) {
1733 size_t j
=0; search_vec(p
->dynv
, &j
, DT_MIPS_GOTSYM
);
1734 size_t i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_SYMTABNO
);
1737 p
->lazy
= calloc(n
, 3*sizeof(size_t));
1739 error("Error preparing lazy relocation for %s: %m", p
->name
);
1740 longjmp(*rtld_fail
, 1);
1742 p
->lazy_next
= lazy_head
;
1746 void *dlopen(const char *file
, int mode
)
1748 struct dso
*volatile p
, *orig_tail
, *orig_syms_tail
, *orig_lazy_head
, *next
;
1749 struct tls_module
*orig_tls_tail
;
1750 size_t orig_tls_cnt
, orig_tls_offset
, orig_tls_align
;
1755 if (!file
) return head
;
1757 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE
, &cs
);
1758 pthread_rwlock_wrlock(&lock
);
1762 orig_tls_tail
= tls_tail
;
1763 orig_tls_cnt
= tls_cnt
;
1764 orig_tls_offset
= tls_offset
;
1765 orig_tls_align
= tls_align
;
1766 orig_lazy_head
= lazy_head
;
1767 orig_syms_tail
= syms_tail
;
1769 noload
= mode
& RTLD_NOLOAD
;
1772 if (setjmp(*rtld_fail
)) {
1773 /* Clean up anything new that was (partially) loaded */
1774 revert_syms(orig_syms_tail
);
1775 for (p
=orig_tail
->next
; p
; p
=next
) {
1777 while (p
->td_index
) {
1778 void *tmp
= p
->td_index
->next
;
1783 if (p
->rpath
!= p
->rpath_orig
)
1785 if (p
->deps
!= &nodeps_dummy
)
1790 if (!orig_tls_tail
) libc
.tls_head
= 0;
1791 tls_tail
= orig_tls_tail
;
1792 if (tls_tail
) tls_tail
->next
= 0;
1793 tls_cnt
= orig_tls_cnt
;
1794 tls_offset
= orig_tls_offset
;
1795 tls_align
= orig_tls_align
;
1796 lazy_head
= orig_lazy_head
;
1801 } else p
= load_library(file
, head
);
1805 "Library %s is not already loaded" :
1806 "Error loading shared library %s: %m",
1811 /* First load handling */
1812 int first_load
= !p
->deps
;
1815 if (!p
->relocated
&& (mode
& RTLD_LAZY
)) {
1817 for (i
=0; p
->deps
[i
]; i
++)
1818 if (!p
->deps
[i
]->relocated
)
1819 prepare_lazy(p
->deps
[i
]);
1822 if (first_load
|| (mode
& RTLD_GLOBAL
)) {
1823 /* Make new symbols global, at least temporarily, so we can do
1824 * relocations. If not RTLD_GLOBAL, this is reverted below. */
1826 for (i
=0; p
->deps
[i
]; i
++)
1827 add_syms(p
->deps
[i
]);
1833 /* If RTLD_GLOBAL was not specified, undo any new additions
1834 * to the global symbol table. This is a nop if the library was
1835 * previously loaded and already global. */
1836 if (!(mode
& RTLD_GLOBAL
))
1837 revert_syms(orig_syms_tail
);
1839 /* Processing of deferred lazy relocations must not happen until
1840 * the new libraries are committed; otherwise we could end up with
1841 * relocations resolved to symbol definitions that get removed. */
1850 pthread_rwlock_unlock(&lock
);
1851 if (p
) do_init_fini(orig_tail
);
1852 pthread_setcancelstate(cs
, 0);
1856 hidden
int __dl_invalid_handle(void *h
)
1859 for (p
=head
; p
; p
=p
->next
) if (h
==p
) return 0;
1860 error("Invalid library handle %p", (void *)h
);
1864 static void *addr2dso(size_t a
)
1868 if (DL_FDPIC
) for (p
=head
; p
; p
=p
->next
) {
1870 if (a
-(size_t)p
->funcdescs
< i
*sizeof(*p
->funcdescs
))
1873 for (p
=head
; p
; p
=p
->next
) {
1874 if (DL_FDPIC
&& p
->loadmap
) {
1875 for (i
=0; i
<p
->loadmap
->nsegs
; i
++) {
1876 if (a
-p
->loadmap
->segs
[i
].p_vaddr
1877 < p
->loadmap
->segs
[i
].p_memsz
)
1882 size_t phcnt
= p
->phnum
;
1883 size_t entsz
= p
->phentsize
;
1884 size_t base
= (size_t)p
->base
;
1885 for (; phcnt
--; ph
=(void *)((char *)ph
+entsz
)) {
1886 if (ph
->p_type
!= PT_LOAD
) continue;
1887 if (a
-base
-ph
->p_vaddr
< ph
->p_memsz
)
1890 if (a
-(size_t)p
->map
< p
->map_len
)
1897 void *__tls_get_addr(tls_mod_off_t
*);
1899 static void *do_dlsym(struct dso
*p
, const char *s
, void *ra
)
1902 uint32_t h
= 0, gh
= 0, *ght
;
1904 if (p
== head
|| p
== RTLD_DEFAULT
|| p
== RTLD_NEXT
) {
1905 if (p
== RTLD_DEFAULT
) {
1907 } else if (p
== RTLD_NEXT
) {
1908 p
= addr2dso((size_t)ra
);
1912 struct symdef def
= find_sym(p
, s
, 0);
1913 if (!def
.sym
) goto failed
;
1914 if ((def
.sym
->st_info
&0xf) == STT_TLS
)
1915 return __tls_get_addr((tls_mod_off_t
[]){def
.dso
->tls_id
, def
.sym
->st_value
});
1916 if (DL_FDPIC
&& (def
.sym
->st_info
&0xf) == STT_FUNC
)
1917 return def
.dso
->funcdescs
+ (def
.sym
- def
.dso
->syms
);
1918 return laddr(def
.dso
, def
.sym
->st_value
);
1920 if (__dl_invalid_handle(p
))
1922 if ((ght
= p
->ghashtab
)) {
1924 sym
= gnu_lookup(gh
, ght
, p
, s
);
1927 sym
= sysv_lookup(s
, h
, p
);
1929 if (sym
&& (sym
->st_info
&0xf) == STT_TLS
)
1930 return __tls_get_addr((tls_mod_off_t
[]){p
->tls_id
, sym
->st_value
});
1931 if (DL_FDPIC
&& sym
&& sym
->st_shndx
&& (sym
->st_info
&0xf) == STT_FUNC
)
1932 return p
->funcdescs
+ (sym
- p
->syms
);
1933 if (sym
&& sym
->st_value
&& (1<<(sym
->st_info
&0xf) & OK_TYPES
))
1934 return laddr(p
, sym
->st_value
);
1935 for (i
=0; p
->deps
[i
]; i
++) {
1936 if ((ght
= p
->deps
[i
]->ghashtab
)) {
1937 if (!gh
) gh
= gnu_hash(s
);
1938 sym
= gnu_lookup(gh
, ght
, p
->deps
[i
], s
);
1940 if (!h
) h
= sysv_hash(s
);
1941 sym
= sysv_lookup(s
, h
, p
->deps
[i
]);
1943 if (sym
&& (sym
->st_info
&0xf) == STT_TLS
)
1944 return __tls_get_addr((tls_mod_off_t
[]){p
->deps
[i
]->tls_id
, sym
->st_value
});
1945 if (DL_FDPIC
&& sym
&& sym
->st_shndx
&& (sym
->st_info
&0xf) == STT_FUNC
)
1946 return p
->deps
[i
]->funcdescs
+ (sym
- p
->deps
[i
]->syms
);
1947 if (sym
&& sym
->st_value
&& (1<<(sym
->st_info
&0xf) & OK_TYPES
))
1948 return laddr(p
->deps
[i
], sym
->st_value
);
1951 error("Symbol not found: %s", s
);
1955 int dladdr(const void *addr_arg
, Dl_info
*info
)
1957 size_t addr
= (size_t)addr_arg
;
1963 size_t besterr
= -1;
1965 pthread_rwlock_rdlock(&lock
);
1967 pthread_rwlock_unlock(&lock
);
1972 strings
= p
->strings
;
1973 nsym
= count_syms(p
);
1976 size_t idx
= (addr
-(size_t)p
->funcdescs
)
1977 / sizeof(*p
->funcdescs
);
1978 if (idx
< nsym
&& (sym
[idx
].st_info
&0xf) == STT_FUNC
) {
1979 best
= (size_t)(p
->funcdescs
+ idx
);
1980 bestsym
= sym
+ idx
;
1985 if (!best
) for (; nsym
; nsym
--, sym
++) {
1987 && (1<<(sym
->st_info
&0xf) & OK_TYPES
)
1988 && (1<<(sym
->st_info
>>4) & OK_BINDS
)) {
1989 size_t symaddr
= (size_t)laddr(p
, sym
->st_value
);
1990 if (symaddr
> addr
|| symaddr
<= best
)
1994 besterr
= addr
- symaddr
;
1995 if (addr
== symaddr
)
2000 if (bestsym
&& besterr
> bestsym
->st_size
-1) {
2005 info
->dli_fname
= p
->name
;
2006 info
->dli_fbase
= p
->map
;
2009 info
->dli_sname
= 0;
2010 info
->dli_saddr
= 0;
2014 if (DL_FDPIC
&& (bestsym
->st_info
&0xf) == STT_FUNC
)
2015 best
= (size_t)(p
->funcdescs
+ (bestsym
- p
->syms
));
2016 info
->dli_sname
= strings
+ bestsym
->st_name
;
2017 info
->dli_saddr
= (void *)best
;
2022 hidden
void *__dlsym(void *restrict p
, const char *restrict s
, void *restrict ra
)
2025 pthread_rwlock_rdlock(&lock
);
2026 res
= do_dlsym(p
, s
, ra
);
2027 pthread_rwlock_unlock(&lock
);
2031 int dl_iterate_phdr(int(*callback
)(struct dl_phdr_info
*info
, size_t size
, void *data
), void *data
)
2033 struct dso
*current
;
2034 struct dl_phdr_info info
;
2036 for(current
= head
; current
;) {
2037 info
.dlpi_addr
= (uintptr_t)current
->base
;
2038 info
.dlpi_name
= current
->name
;
2039 info
.dlpi_phdr
= current
->phdr
;
2040 info
.dlpi_phnum
= current
->phnum
;
2041 info
.dlpi_adds
= gencnt
;
2043 info
.dlpi_tls_modid
= current
->tls_id
;
2044 info
.dlpi_tls_data
= current
->tls
.image
;
2046 ret
= (callback
)(&info
, sizeof (info
), data
);
2048 if (ret
!= 0) break;
2050 pthread_rwlock_rdlock(&lock
);
2051 current
= current
->next
;
2052 pthread_rwlock_unlock(&lock
);
2057 hidden
void __dl_vseterr(const char *, va_list);
2059 static void error(const char *fmt
, ...)
2064 vdprintf(2, fmt
, ap
);
2070 __dl_vseterr(fmt
, ap
);